#ifndef __CMS_CLUSTER__H #define __CMS_CLUSTER__H /******************************************************************************/ /* */ /* X r d C m s C l u s t e r . h h */ /* */ /* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdCms/XrdCmsTypes.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdOuc/XrdOucEnum.hh" #include "XrdSys/XrdSysPthread.hh" class XrdLink; class XrdCmsDrop; class XrdCmsNode; class XrdCmsSelect; class XrdCmsSelector; class XrdNetAddr; namespace XrdCms { struct CmsRRHdr; } /******************************************************************************/ /* O p t i o n F l a g s */ /******************************************************************************/ namespace XrdCms { // Flags passed to Add() // static const int CMS_noStage = 1; static const int CMS_Suspend = 2; static const int CMS_Perm = 4; static const int CMS_isMan = 8; static const int CMS_Lost = 16; static const int CMS_isPeer = 32; static const int CMS_isProxy = 64; static const int CMS_noSpace =128; static const int CMS_isSuper =256; static const int CMS_isVers3 =0x01000000; static const int CMS_notServ =CMS_isMan|CMS_isPeer|CMS_isSuper; static const int CMS_hasAlts =CMS_isMan|CMS_isPeer; // Class passed to Space() // class SpaceData { public: long long Total; // Total space long long TotFr; // Total space free int wMinF; // Free space minimum to select wFree node int wFree; // Free space for nodes providing r/w access (largest one) int wNum; // Number of nodes providing r/w access int wUtil; // Average utilization (largest one) int sFree; // Free space for nodes providing staging (largest one) int sNum; // Number of nodes providing staging int sUtil; // Average utilization (largest one) SpaceData() : Total(0), TotFr(0),wMinF(0), wFree(0), wNum(0), wUtil(0), sFree(0), sNum(0), sUtil(0) {} ~SpaceData() {} }; } /******************************************************************************/ /* C l a s s X r d C m s C l u s t e r */ /******************************************************************************/ // This a single-instance global class // class XrdCmsBaseFR; class XrdCmsClustID; class XrdCmsSelected; class XrdOucTList; class XrdCmsCluster { public: friend class XrdCmsDrop; int NodeCnt; // Number of active nodes // Called to add a new node to the cluster. Status values are defined above. // XrdCmsNode *Add(XrdLink *lp, int dport, int Status, int sport, const char *theNID, const char *theIF); // Put nodes in or remove from a blacklist // virtual void BlackList(XrdOucTList *blP); // Sends a message to all nodes matching smask (three forms for convenience) // SMask_t Broadcast(SMask_t, const struct iovec *, int, int tot=0); SMask_t Broadcast(SMask_t smask, XrdCms::CmsRRHdr &Hdr, char *Data, int Dlen=0); SMask_t Broadcast(SMask_t smask, XrdCms::CmsRRHdr &Hdr, void *Data, int Dlen); // Sends a message to a single node in a round-robbin fashion. // int Broadsend(SMask_t smask, XrdCms::CmsRRHdr &Hdr, void *Data, int Dlen); // Returns the node mask matching the given IP address // SMask_t getMask(const XrdNetAddr *addr); // Returns the node mask matching the given cluster ID // SMask_t getMask(const char *Cid); // Extracts out node information. Opts are one or more of CmsLSOpts // enum CmsLSOpts {LS_NULL=0, LS_IPO=0x0100, LS_IDNT=0x0200, LS_ANY =0x0400, LS_IFMASK = 0x0f}; XrdCmsSelected *List(SMask_t mask, CmsLSOpts opts, bool &oksel); // Returns the location of a file // int Locate(XrdCmsSelect &Sel); // Always run as a separate thread to monitor subscribed node performance // void *MonPerf(); // Alwats run as a separate thread to maintain the node reference count // void *MonRefs(); // Return total number of redirect references (sloppy as we don't lock it) // long long Refs() {return SelWcnt+SelWtot+SelRcnt+SelRtot;} // Called to remove a node from the cluster // void Remove(XrdCmsNode *theNode); void Remove(const char *reason, XrdCmsNode *theNode, int immed=0); // Called to reset the node reference counts for nodes matching smask // void ResetRef(SMask_t smask); // Called to select the best possible node to serve a file (two forms) // static const int RetryErr = -3; int Select(XrdCmsSelect &Sel); int Select(SMask_t pmask, int &port, char *hbuff, int &hlen, int isrw, int isMulti, int ifWant); // Manipulate the global selection lock // void SLock(bool dolock) {if (dolock) STMutex.Lock(); else STMutex.UnLock(); } // Called to get cluster space (for managers and supervisors only) // void Space(XrdCms::SpaceData &sData, SMask_t smask); // Called to return statistics // int Stats(char *bfr, int bln); // Server int Statt(char *bfr, int bln); // Manager XrdCmsCluster(); virtual ~XrdCmsCluster() {} // This object should never be deleted private: XrdCmsNode *AddAlt(XrdCmsClustID *cidP, XrdLink *lp, int port, int Status, int sport, const char *theNID, const char *theIF); XrdCmsNode *calcDelay(XrdCmsSelector &selR); int Drop(int sent, int sinst, XrdCmsDrop *djp=0); void Record(char *path, const char *reason, bool force=false); bool maxBits(SMask_t mVec, int mbits); int Multiple(SMask_t mVec); enum {eExists, eDups, eROfs, eNoRep, eNoSel, eNoEnt}; // Passed to SelFail int SelFail(XrdCmsSelect &Sel, int rc); int SelNode(XrdCmsSelect &Sel, SMask_t pmask, SMask_t amask); XrdCmsNode *SelbyCost(SMask_t, XrdCmsSelector &selR); XrdCmsNode *SelbyLoad(SMask_t, XrdCmsSelector &selR); XrdCmsNode *SelbyRef (SMask_t, XrdCmsSelector &selR); int SelDFS(XrdCmsSelect &Sel, SMask_t amask, SMask_t &pmask, SMask_t &smask, int isRW); void sendAList(XrdLink *lp); void setAltMan(int snum, XrdLink *lp, int port); int Unreachable(XrdCmsSelect &Sel, bool none); int Unuseable(XrdCmsSelect &Sel); // Number of :Port characters per entry was INET6_ADDRSTRLEN+10 // static const int AltSize = 254; // We may revert to IP address XrdSysMutex XXMutex; // Protects cluster summary state variables XrdSysMutex STMutex; // Protects all node information variables XrdCmsNode *NodeTab[STMax]; // Current set of nodes int STHi; // NodeTab high watermark int doReset; // Must send reset event to Managers[resetMask] long long SelWcnt; // Curr number of r/w selections (successful) long long SelWtot; // Total number of r/w selections (successful) long long SelRcnt; // Curr number of r/o selections (successful) long long SelRtot; // Total number of r/o selections (successful) long long SelTcnt; // Total number of all selections // The following is a list of IP:Port tokens that identify supervisor nodes. // The information is sent via the try request to redirect nodes; as needed. // The list is alays rotated by one entry each time it is sent. // char AltMans[STMax*AltSize]; // ||123.123.123.123:12345|| = 21 char *AltMend; int AltMent; // The foloowing three variables are protected by the STMutex // SMask_t resetMask; // Nodes to receive a reset event SMask_t peerHost; // Nodes that are acting as peers SMask_t peerMask; // Always ~peerHost }; XRDOUC_ENUM_OPERATORS(XrdCmsCluster::CmsLSOpts) namespace XrdCms { extern XrdCmsCluster Cluster; } #endif