/******************************************************************************/
/* */
/* X r d C m s M e t e r . c c */
/* */
/* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */
/* All Rights Reserved */
/* Produced by Andrew Hanushevsky for Stanford University under contract */
/* DE-AC02-76-SFO0515 with the Department of Energy */
/* */
/* This file is part of the XRootD software suite. */
/* */
/* XRootD is free software: you can redistribute it and/or modify it under */
/* the terms of the GNU Lesser General Public License as published by the */
/* Free Software Foundation, either version 3 of the License, or (at your */
/* option) any later version. */
/* */
/* XRootD is distributed in the hope that it will be useful, but WITHOUT */
/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
/* License for more details. */
/* */
/* You should have received a copy of the GNU Lesser General Public License */
/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
/* COPYING (GPL license). If not, see . */
/* */
/* The copyright holder's institutional names and contributor's names may not */
/* be used to endorse or promote products derived from this software without */
/* specific prior written permission of the institution or contributor. */
/******************************************************************************/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "XrdCms/XrdCmsCluster.hh"
#include "XrdCms/XrdCmsConfig.hh"
#include "XrdCms/XrdCmsMeter.hh"
#include "XrdCms/XrdCmsNode.hh"
#include "XrdCms/XrdCmsState.hh"
#include "XrdCms/XrdCmsTrace.hh"
#include "XrdOss/XrdOss.hh"
#include "XrdSys/XrdSysPlatform.hh"
using namespace XrdCms;
/******************************************************************************/
/* G l o b a l s */
/******************************************************************************/
XrdCmsMeter XrdCms::Meter;
/******************************************************************************/
/* E x t e r n a l T h r e a d I n t e r f a c e s */
/******************************************************************************/
void *XrdCmsMeterRun(void *carg)
{XrdCmsMeter *mp = (XrdCmsMeter *)carg;
return mp->Run();
}
void *XrdCmsMeterRunFS(void *carg)
{XrdCmsMeter *mp = (XrdCmsMeter *)carg;
return mp->RunFS();
}
/******************************************************************************/
/* C o n s t r u c t o r */
/******************************************************************************/
XrdCmsMeter::XrdCmsMeter() : myMeter(&Say)
{
Running = 0;
dsk_calc = 0;
fs_nums = 0;
noSpace = 0;
MinFree = 0;
HWMFree = 0;
dsk_lpn = 0;
dsk_tot = 0;
dsk_free = 0;
dsk_maxf = 0;
lastFree = 0;
lastUtil = 0;
monpgm = 0;
monint = 0;
montid = 0;
rep_tod = time(0);
xeq_load = 0;
cpu_load = 0;
mem_load = 0;
pag_load = 0;
net_load = 0;
Virtual = 0;
VirtUpdt = 1;
}
/******************************************************************************/
/* D e s t r u c t o r */
/******************************************************************************/
XrdCmsMeter::~XrdCmsMeter()
{
if (monpgm) free(monpgm);
if (montid) XrdSysThread::Kill(montid);
}
/******************************************************************************/
/* c a l c L o a d */
/******************************************************************************/
int XrdCmsMeter::calcLoad(int pcpu, int pio, int pload, int pmem, int ppag)
{
return (Config.P_cpu * pcpu /100)
+ (Config.P_io * pio /100)
+ (Config.P_load * pload/100)
+ (Config.P_mem * pmem /100)
+ (Config.P_pag * ppag /100);
}
/******************************************************************************/
int XrdCmsMeter::calcLoad(int nowload, int pdsk)
{
return (Config.P_dsk * pdsk /100) + nowload;
}
/******************************************************************************/
/* F r e e S p a c e */
/******************************************************************************/
int XrdCmsMeter::FreeSpace(int &tot_util)
{
long long fsavail;
// If we are a virtual filesystem, do virtual stats
//
if (Virtual)
{if (Virtual == peerFS) {tot_util = 0; return 0x7fffffff;}
if (VirtUpdt) UpdtSpace();
tot_util = lastUtil;
return lastFree;
}
// The values are calculated periodically so use the last available ones
//
cfsMutex.Lock();
fsavail = dsk_maxf;
tot_util= dsk_util;
cfsMutex.UnLock();
// Now adjust the values to fit
//
if (fsavail >> 31LL) fsavail = 0x7fffffff;
// Return amount available
//
return static_cast(fsavail);
}
/******************************************************************************/
/* I n i t */
/******************************************************************************/
void XrdCmsMeter::Init()
{
XrdOssVSInfo vsInfo;
pthread_t monFStid;
char buff[1024], sfx1, sfx2, sfx3;
long maxfree, totfree, totDisk;
int rc;
// Get initial free space
//
if ((rc = Config.ossFS->StatVS(&vsInfo, 0, 1)))
{Say.Emsg("Meter", rc, "calculate file system space");
noSpace = 1;
}
else if (!(fs_nums = vsInfo.Extents))
{Say.Emsg("Meter", "Warning! No writable filesystems found.");
noSpace = 1;
}
else {dsk_tot = vsInfo.Total >> 20LL; // in MB
dsk_lpn = vsInfo.Large >> 20LL;
}
// Check if we should bother to continue
//
if (noSpace)
{if (!Config.asSolo()) CmsState.Update(XrdCmsState::Space, 0);
Say.Emsg("Meter", "Write access and staging prohibited.");
return;
}
// Set values (disk space values are in megabytes)
//
if (Config.DiskMinP) MinFree = dsk_lpn * Config.DiskMinP / 100;
if (Config.DiskMin > MinFree) MinFree = Config.DiskMin;
MinStype= Scale(MinFree, MinShow);
if (Config.DiskHWMP) HWMFree = dsk_lpn * Config.DiskHWMP / 100;
if (Config.DiskHWM > HWMFree) HWMFree = Config.DiskHWM;
HWMStype= Scale(HWMFree, HWMShow);
dsk_calc = (Config.DiskAsk < 5 ? 5 : Config.DiskAsk);
// Calculate the initial free space and start the FS monitor thread
//
calcSpace();
if ((noSpace = (dsk_maxf < MinFree)) && !Config.asSolo())
CmsState.Update(XrdCmsState::Space, 0);
if ((rc = XrdSysThread::Run(&monFStid, XrdCmsMeterRunFS, (void *)this, 0,
"FS meter"))) Say.Emsg("Meter", rc, "start filesystem meter.");
// Document what we have
//
sfx1 = Scale(dsk_maxf, maxfree);
sfx2 = Scale(dsk_tot, totDisk);
sfx3 = Scale(dsk_free, totfree);
sprintf(buff,"Found %d filesystem(s); %ld%cB total (%d%% util);"
" %ld%cB free (%ld%cB max)", fs_nums, totDisk, sfx2,
dsk_util, totfree, sfx3, maxfree, sfx1);
Say.Emsg("Meter", buff);
if (noSpace)
{sprintf(buff, "%ld%cB minimum", MinShow, MinStype);
Say.Emsg("Meter", "Warning! Available space <", buff);
}
}
/******************************************************************************/
/* M o n i t o r */
/******************************************************************************/
int XrdCmsMeter::Monitor(char *pgm, int itv)
{
char *mp, pp;
int rc;
// Isolate the program name
//
mp = monpgm = strdup(pgm);
while(*mp && *mp != ' ') mp++;
pp = *mp; *mp ='\0';
// Make sure the program is executable by us
//
if (access(monpgm, X_OK))
{Say.Emsg("Meter", errno, "find executable", monpgm);
return -1;
}
// Start up the program. We don't really need to serialize Restart() because
// Monitor() is a one-time call (otherwise unpredictable results may occur).
//
*mp = pp; monint = itv;
if ((rc = XrdSysThread::Run(&montid, XrdCmsMeterRun, (void *)this, 0,
"Perf meter"))) Say.Emsg("Meter", rc, "start performance meter.");
Running = 1;
return 0;
}
/******************************************************************************/
/* R e c o r d */
/******************************************************************************/
void XrdCmsMeter::Record(int pcpu, int pnet, int pxeq,
int pmem, int ppag, int pdsk)
{
int temp;
repMutex.Lock();
temp = cpu_load + cpu_load/2;
cpu_load = (cpu_load + (pcpu > temp ? temp : pcpu))/2;
temp = net_load + net_load/2;
net_load = (net_load + (pnet > temp ? temp : pnet))/2;
temp = xeq_load + xeq_load/2;
xeq_load = (xeq_load + (pxeq > temp ? temp : pxeq))/2;
temp = mem_load + mem_load/2;
mem_load = (mem_load + (pmem > temp ? temp : pmem))/2;
temp = pag_load + pag_load/2;
pag_load = (pag_load + (ppag > temp ? temp : ppag))/2;
repMutex.UnLock();
}
/******************************************************************************/
/* R e p o r t */
/******************************************************************************/
int XrdCmsMeter::Report(int &pcpu, int &pnet, int &pxeq,
int &pmem, int &ppag, int &pdsk)
{
int maxfree;
// Force restart the monitor program if it hasn't reported within 2 intervals
//
if (!Virtual && montid && (time(0) - rep_tod > monint*2)) myMeter.Drain();
// Format a usage line
//
repMutex.Lock();
maxfree = FreeSpace(pdsk);
if (!Running && !Virtual) pcpu = pnet = pmem = ppag = pxeq = 0;
else {pcpu = cpu_load; pnet = net_load; pmem = mem_load;
ppag = pag_load; pxeq = xeq_load;
}
repMutex.UnLock();
// All done
//
return maxfree;
}
/******************************************************************************/
/* R u n */
/******************************************************************************/
void *XrdCmsMeter::Run()
{
const struct timespec rqtp = {30, 0};
int i, myLoad, prevLoad = -1;
char *lp = 0;
// Execute the program (keep restarting and keep reading the output)
//
while(1)
{if (myMeter.Exec(monpgm) == 0)
while((lp = myMeter.GetLine()))
{repMutex.Lock();
i = sscanf(lp, "%d %d %d %d %d",
&xeq_load, &cpu_load, &mem_load, &pag_load, &net_load);
rep_tod = time(0);
repMutex.UnLock();
if (i != 5) break;
myLoad = calcLoad(cpu_load,net_load,xeq_load,mem_load,pag_load);
if (prevLoad >= 0)
{prevLoad = prevLoad - myLoad;
if (prevLoad < 0) prevLoad = -prevLoad;
if (prevLoad > Config.P_fuzz) XrdCmsNode::Report_Usage(0);
}
prevLoad = myLoad;
}
if (lp) Say.Emsg("Meter","Perf monitor returned invalid output:",lp);
else Say.Emsg("Meter","Perf monitor died.");
nanosleep(&rqtp, 0);
Say.Emsg("Meter", "Restarting monitor:", monpgm);
}
return (void *)0;
}
/******************************************************************************/
/* r u n F S */
/******************************************************************************/
void *XrdCmsMeter::RunFS()
{
const struct timespec rqtp = {dsk_calc, 0};
int noNewSpace;
int mlim = 60/dsk_calc, nowlim = 0;
while(1)
{nanosleep(&rqtp, 0);
calcSpace();
noNewSpace = dsk_maxf < (noSpace ? HWMFree : MinFree);
if (noSpace != noNewSpace)
{SpaceMsg(noNewSpace);
noSpace = noNewSpace;
if (!Config.asSolo()) CmsState.Update(XrdCmsState::Space,!noSpace);
}
else if (noSpace && !nowlim) SpaceMsg(noNewSpace);
nowlim = (nowlim ? nowlim-1 : mlim);
}
return (void *)0;
}
/******************************************************************************/
/* T o t a l S p a c e */
/******************************************************************************/
unsigned int XrdCmsMeter::TotalSpace(unsigned int &minfree)
{
long long fstotal, fsminfr;
// If we are a virtual filesystem, do virtual stats
//
if (Virtual)
{if (Virtual == peerFS) {minfree = 0; return 0x7fffffff;}
if (VirtUpdt) UpdtSpace();
}
// The values are calculated periodically so use the last available ones
//
cfsMutex.Lock();
fstotal = dsk_tot;
fsminfr = MinFree;
cfsMutex.UnLock();
// Now adjust the values to fit
//
if (fsminfr >> 31LL) minfree = 0x7fffffff;
else minfree = static_cast(fsminfr);
if (fstotal == 0) fstotal = 1;
else if (fstotal >> 31LL) fstotal = 0x7fffffff;
// Return amount available
//
return static_cast(fstotal);
}
/******************************************************************************/
/* P r i v a t e M e t h o d s */
/******************************************************************************/
/******************************************************************************/
/* c a l c S p a c e */
/******************************************************************************/
void XrdCmsMeter::calcSpace()
{
EPNAME("calcSpace")
XrdOssVSInfo vsInfo;
int old_util, rc;
long long fsutil;
// Get free space statistics. On error, all fields will be zero, which is
// what we really want to kill space allocation.
//
if ((rc = Config.ossFS->StatVS(&vsInfo, 0, 1)))
Say.Emsg("Meter", rc, "calculate file system space");
// Calculate the disk utilization (note that dsk_tot is in MB)
//
fsutil = (dsk_tot ? 100-(((vsInfo.Free >> 20LL)*100)/dsk_tot) : 100);
if (fsutil < 0) fsutil = 0;
else if (fsutil > 100) fsutil = 100;
// Update the stats and release the lock
//
cfsMutex.Lock();
old_util = dsk_util;
dsk_maxf = vsInfo.LFree >> 20LL; // In MB
dsk_free = vsInfo.Free >> 20LL; // In MB
dsk_util = static_cast(fsutil);
cfsMutex.UnLock();
if (old_util != dsk_util)
DEBUG("New fs info; maxfree=" < 1024; i++) inval = inval/1024;
outval = static_cast(inval);
return sfx[i];
}
/******************************************************************************/
/* S p a c e M s g */
/******************************************************************************/
void XrdCmsMeter::SpaceMsg(int why)
{
const char *What;
char sfx, buff[1024];
long maxfree;
sfx = Scale(dsk_maxf, maxfree);
if (why)
{What = "Insufficient space; ";
if (noSpace)
sprintf(buff, "%ld%cB available < %ld%cB high watermark",
maxfree, sfx, HWMShow, HWMStype);
else
sprintf(buff, "%ld%cB available < %ld%cB minimum",
maxfree, sfx, MinShow, MinStype);
} else {
What = " Sufficient space; ";
sprintf(buff, "%ld%cB available > %ld%cB high watermak",
maxfree, sfx, HWMShow, HWMStype);
}
Say.Emsg("Meter", What, buff);
}
/******************************************************************************/
/* U p d t S p a c e */
/******************************************************************************/
void XrdCmsMeter::UpdtSpace()
{
static const SMask_t allNodes(~0);
SpaceData mySpace;
// Get new space values for the cluser
//
Cluster.Space(mySpace, allNodes);
// Update out local information
//
cfsMutex.Lock();
if (mySpace.wFree > mySpace.sFree)
{lastFree = mySpace.wFree; lastUtil = mySpace.wUtil;
} else {
lastFree = mySpace.sFree; lastUtil = mySpace.sUtil;
}
dsk_tot = static_cast(mySpace.Total)<<10LL; // In MB
MinFree = mySpace.wMinF;
VirtUpdt = 0;
cfsMutex.UnLock();
}