#!/usr/bin/env python3

import htcondor
import classad

import sys
import os
import re
import datetime
import optparse

from operator import itemgetter
from time import sleep
from copy import deepcopy

### command-line options ###
parser = optparse.OptionParser(usage=(
    'Usage: %prog [options]\n'
    '       %prog [options] <classad-filename> <classad-filename>\n\n'
    'Only -c, -s, and --attrs options considered when passing ClassAds via files.'))

# daemons
parser.set_defaults(daemon=False)
group = optparse.OptionGroup(parser, title='Daemon Options', description=(
    'If none of these options are used, the local DAEMON_LIST will be searched '
    'in the order: SCHEDD, STARTD, COLLECTOR, NEGOTIATOR, MASTER. '
    'If -n NAME is not specified, the first ClassAd matching the constraint '
    '"Machine == $(FULL_HOSTNAME)" (for Schedd/Startd/Master ads) '
    'or "Machine == $(COLLECTOR_HOST)" (for Collector/Negotiator ads) '
    'will be monitored.'))
group.add_option('--collector', action='store_const', const='Collector',
                     dest='daemon', help=(
                         'Monitor condor_collector ClassAds.'))
group.add_option('--negotiator', action='store_const', const='Negotiator',
                     dest='daemon', help=(
                         'Monitor condor_negotiator ClassAds.'))
group.add_option('--master', action='store_const', const='Master',
                     dest='daemon', help=(
                         'Monitor condor_master ClassAds.'))
group.add_option('--schedd', action='store_const', const='Schedd',
                     dest='daemon', help=(
                         'Monitor condor_schedd ClassAds.'))
group.add_option('--startd', action='store_const', const='Startd',
                     dest='daemon', help=(
                         'Monitor condor_startd ClassAds.'))
parser.add_option_group(group)

# live mode
parser.add_option('-l', '--live', action='store_const', const=True, dest='live',
                      default=False, help=(
    'Display results in a live, top-like mode.'))

# pool
parser.add_option('-p', '--pool', dest='pool', default=None, help=(
    'Query the specified Central Manager instead of $(COLLECTOR_HOST). '
    'Use of this option implies --collector unless otherwise specified.'))

# name
parser.add_option('-n', '--name', dest='name', default=None, help=(
    'Query using the constraint Name == NAME. If omitted '
    'and multiple ClassAds are returned, only the daemon named '
    'in the first ClassAd returned will be monitored.'))

# delay
parser.add_option('-d', '--delay', type='int', dest='delay',
                      default=htcondor.param['STATISTICS_WINDOW_QUANTUM'],
                      help=(
    'Specifies the delay between ClassAd updates, in integer seconds. '
    'If omitted, the value of the configuration variable '
    'STATISTICS_WINDOW_QUANTUM is used.'))

# display columns
parser.add_option('-c', '--columns', dest='column_set', default='default', help=(
    'Choose the set of columns displayed. Valid column sets are: '
    'default, runtime, count, all.'))

# sort column
parser.add_option('-s', '--sort', dest='column', default='', help=(
    'Sort table by column name. See the man page for a list of column names '
    'and definitions.'))
    
# additional ClassAd attributes
parser.add_option('--attrs', dest='attrs', default='', help=(
    'Comma-delimited list of additional ClassAd attributes to monitor. '
    'Values will be considered as "counts".'))

# set global variables based on command-line options
opts, args = parser.parse_args()
FILES = args
DAEMON = opts.daemon
LIVE = opts.live
POOL = opts.pool
NAME = opts.name
DELAY = opts.delay
COL_SET = opts.column_set.lower()
SORT_COL = opts.column
ATTRIBUTES = [attr.strip() for attr in opts.attrs.split(',')]
    
# set default daemon from checking the DAEMON_LIST in specified order
if not DAEMON:
    if POOL is not None:
        DAEMON = 'Collector'
    else:
        for DAEMON in ['Schedd', 'Startd', 'Collector', 'Negotiator', 'Master']:
            if DAEMON.lower() in htcondor.param['DAEMON_LIST'].lower():
                break

# set machine name if name is not set
MACHINE = None
if (NAME is None) and (POOL is None):
    if DAEMON in ['Schedd', 'Startd', 'Master']:
        MACHINE = htcondor.param['FULL_HOSTNAME']
    else:
        MACHINE = htcondor.param['COLLECTOR_HOST']

# determine if we should do direct queries
DIRECT = DAEMON in ['Schedd', 'Startd']

# column names
COL_NAMES = [
    'TotalRt',  # 0. Total Runtime
    'InstRt',   # 1. Runtime between ads
    'InstAvg',  # 2. Runtime/Count between ads
    'TotAvg',   # 3. Mean Runtime/Count since daemon start
    'TotMax',   # 4. Max Runtime/Count since daemon start
    'TotMin',   # 5. Min Runtime/Count since daemon start
    'RtPctAvg', # 6. InstAvg / TotAvg ("percent of mean runtime")
    'RtPctMax', # 7. InstAvg-TotMin / TotMax-TotMin ("percent of max runtime")
    'RtSigmas', # 8. InstAvg-TotAvg / TotStd ("std devs from mean runtime")
    'TotalCt',  # 9. Total Count
    'InstCt',   # 10. Count between ads
    'InstRate', # 11. Count/time between ads
    'AvgRate',  # 12. Count/time since daemon start
    'CtPctAvg', # 13. InstRate / AvgRate ("percent of mean count rate")
    'Item'      # 14. Attribute name
]

# column sets
COL_SETS = {
    'all':     COL_NAMES,
    'default': ['InstRt', 'InstAvg', 'TotAvg', 'TotMax', 'RtPctAvg',
                    'InstRate', 'AvgRate', 'Item'],
    'runtime': ['InstRt', 'InstAvg', 'TotAvg', 'TotMax', 'RtPctAvg',
                    'RtPctMax', 'RtSigmas', 'Item'],
    'count':   ['TotalCt', 'InstCt', 'InstRate', 'AvgRate', 'CtPctAvg', 'Item'],
}

# use the default column set if not specified
if not (COL_SET in COL_SETS):
    COL_SET = 'default'

# check that SORT_COL is a valid column
if not (SORT_COL in COL_NAMES):
    for col_name in COL_NAMES:
        if SORT_COL.lower() == col_name.lower():
            SORT_COL = col_name
            break
    else: # if not a valid column, use the default
        if COL_SET != 'count':
            SORT_COL = 'InstRt'
        else:
            SORT_COL = 'InstRate'

# check if output being piped
PIPE = not sys.stdout.isatty()

# live mode does not apply when piped or using ClassAd files
if PIPE or FILES:
    LIVE = False

###

### ClassAd querying functions ###

def getConstraint(machine = None, name = None):
    """Figure out what the constraint string should be and return it"""

    constraints = []
    if machine is not None :
        constraints.append('(Machine =?= {0})'.format(classad.quote(machine)))
    if name is not None:
        constraints.append('(Name =?= {0})'.format(classad.quote(name)))

    constraint = ' && '.join(constraints)
    return constraint

def warnMultipleAds(n, daemon, pool):
    """Prints a warning to stderr if the Collector returns multiple ads"""
    
    if pool is None:
        pool = htcondor.param['COLLECTOR_HOST']
    sys.stderr.write((
        'Warning: Received {0} {1} ads from Collector {2}\n'
        'Consider specifying a daemon name with -n NAME.\n'
        ).format(n, daemon, pool))

def errNoAds(daemon, pool):
    """Prints an error to stderr and quits if the Collector returns no ClassAds"""
    
    if pool is None:
        pool = htcondor.param['COLLECTOR_HOST']
    sys.stderr.write((
        'Error: Did not receive any {0} ads from Collector {1}\n'
        ).format(daemon, pool))
    sys.exit(1)
    
def getDaemonName(pool, daemon, machine):
    """Returns Name attribute from the first ClassAd returned by the Collector"""

    # first try getting ads by constraining the Machine name
    collector = htcondor.Collector(pool)
    constraint = getConstraint(machine = machine)
    ads = collector.query(htcondor.AdTypes.names[daemon],
                                 constraint=constraint,
                                 projection=['Name'])

    # if that doesn't work, remove the constraint and try again
    if len(ads) == 0:
        ads = collector.query(htcondor.AdTypes.names[daemon],
                                  projection=['Name'])

    if len(ads) == 0:
        errNoAds(daemon, pool)
    if len(ads) > 1:
        warnMultipleAds(len(ads), daemon, pool)

    name = ads[0]['Name']
    return name

def queryClassAd(pool, daemon, name, direct = False, statistics = 'All:2'):
    """Returns the first ClassAd returned by the Collector"""

    collector = htcondor.Collector(pool)

    if direct:
        # direct queries take different arguments and return one ad
        kwargs = {
            'daemon_type': htcondor.DaemonTypes.names[daemon],
            'name': name,
            'statistics': statistics
        }
        ad = collector.directQuery(**kwargs)

    else:
        # indirect queries may return multiple ads
        kwargs = {
            'ad_type': htcondor.AdTypes.names[daemon],
            'constraint': getConstraint(name = name),
            'statistics': statistics
        }
        ads = collector.query(**kwargs)

        if len(ads) == 0:
            errNoAds(daemon, pool)
        if len(ads) > 1:
            warnMultipleAds(len(ads), daemon, pool)

        # return only the first ad
        ad = ads[0]

    return ad

###

### ClassAd parsing ###

def getRuntimeAttrs(ad):
    """Returns a list of runtime attributes"""
    
    re_runtime = re.compile('^(.*)Runtime$')

    # some attributes should always be ignored
    re_ignore = re.compile('^DC(Socket|Pipe)')
    ignored_attrs = ['SCGetAutoCluster_cchit']

    attrs = []
    for key in ad.keys():
        match = re_runtime.match(key)
        if match:
            attr = match.groups()[0]
            if not (re_ignore.match(attr) or (attr in ignored_attrs)):
                attrs.append(attr)

    return attrs

### calculations ###

def computeUpdateTimes(old, new):
    """Returns the ClassAds' update times (Unix epoch)"""

    if (('DaemonStartTime' in new) and ('StatsLifetime' in new) and
        ('DaemonStartTime' in old) and ('StatsLifetime' in old)):
        t_0 = new['DaemonStartTime']
        if t_0 != old['DaemonStartTime']:
            sys.stderr.write('Error: Daemon ads have different start times.\n')
            sys.exit(1)
        t_old = t_0 + old['StatsLifetime']
        t_new = t_0 + new['StatsLifetime']
    else:
        t_old = old['MyCurrentTime']
        t_new = new['MyCurrentTime']

    # check for same timestamp
    if (t_new - t_old) == 0:
        if new['MyCurrentTime'] > old['MyCurrentTime']:  # try using MyCurrentTime
            t_new = new['MyCurrentTime']
            t_old = old['MyCurrentTime']
        else:  # quit and notify if the ads have the same timestamp
            sys.stderr.write('Error: ClassAds have the same timestamp.\n')
            sys.exit(1)
        
    return (t_old, t_new)

def computeInstDutyCycle(old, new):
    """Returns the daemon core duty cycle computed between two ClassAds"""

    if not (('DCSelectWaittime' in new) and ('DCPumpCycleSum' in new)):
        return None

    dPumpCycleCount = new['DCPumpCycleCount'] - old['DCPumpCycleCount']
    dPumpCycleSum = new['DCPumpCycleSum'] - old['DCPumpCycleSum']

    # check if a cycle has occured between ads
    if (dPumpCycleCount == 0) or (dPumpCycleSum < 1e-6):
        sys.stderr.write(('Duty cycle undefined, try setting a larger delay (-d).\n'))
        return None
    
    dSelectWaittime = new['DCSelectWaittime'] - old['DCSelectWaittime']
    
    duty_cycle = 100*(1 - dSelectWaittime/float(dPumpCycleSum))
    return duty_cycle    

def computeInstOpsPerSec(old, new):
    """Returns the daemon core commands per second computed between two ClassAds"""

    if not ('DCCommands' in new):
        return None

    t_old, t_new = computeUpdateTimes(old, new)
    dt = t_new - t_old

    dCommands = new['DCCommands'] - old['DCCommands']
        
    ops_per_sec = dCommands / float(dt)
    return ops_per_sec   
    
def computeRuntimeStats(old, new):
    """Returns a list of derived runtime and count statistics
    computed between two ClassAds for each runtime attribute"""
    
    t_old, t_new = computeUpdateTimes(old, new)
    dt = t_new - t_old

    # StatsLifetime is not reported by Startd, so we can't compute
    # lifetime stats (e.g. average counts/sec) from Startd ads.
    if 'StatsLifetime' in new:
        t = new['StatsLifetime']
    else:
        t = None

    # get the list of runtime attributes
    attrs = getRuntimeAttrs(new)

    # add any additional attributes specified by --attr
    for attr in ATTRIBUTES:
        attrs.append(attr)

    # build a list with tuples containing values of each statistic
    table = []
    for attr in attrs:

        # check that attribute counts exist in both ads
        if (attr in new) and (attr in old):
            C = new[attr]
            dC = new[attr] - old[attr]
        else:
            # don't bother keeping this attribute if there are no counts
            continue

        # check that attribute runtimes exist in both ads
        if ((attr + 'Runtime') in new) and ((attr + 'Runtime') in old):
            R = new[attr + 'Runtime']
            dR = new[attr + 'Runtime'] - old[attr + 'Runtime']
        else:
            R, dR = (None, None)

        # compute runtime/count between ads
        if (dC > 0) and (dR is not None):
            R_curr = dR / float(dC)
        else:
            R_curr = None

        # grab the attribute's lifetime runtime stats
        R_ = {}
        for stat in ['Avg', 'Max', 'Min', 'Std']:
            if (attr + 'Runtime' + stat) in new:
                R_[stat] = new[attr + 'Runtime' + stat]
            else:
                R_[stat] = None

        R_pct_avg, R_pct_max, R_sigmas = (None, None, None)
        if R_curr is not None:

            # compare current runtime/count to lifetime runtime/count
            if R_['Avg'] is not None:
                R_pct_avg = 100. * R_curr / float(R_['Avg'])

            # compare current runtime/count to lifetime max/min range
            if (R_['Max'] is not None) and (R_['Min'] is not None):
                if R_['Max'] == R_['Min']:
                    R_pct_max = 100.
                else:
                    R_pct_max = 100. * ( (R_curr - R_['Min']) /
                                             float(R_['Max'] - R_['Min']) )

            # compare difference between current and lifetime runtime/count
            # to the lifetime standard deviation in runtime/count
            if (R_['Avg'] is not None) and (R_['Std'] is not None):
                R_sigmas = (R_curr - R_['Avg']) / float(R_['Std'])

        # compute count/sec between ads
        C_curr = dC / float(dt)

        # compute lifetime counts/sec
        if t:
            C_avg = C / float(t)
        else:
            C_avg = None

        # compare current counts/sec to lifetime counts/sec
        C_pct_avg = None
        if (dC > 0) and (C_avg is not None):
            C_pct_avg = 100. * C_curr / float(C_avg)

        # cleanup item name
        if attr[0:2] == 'DC':
            attr = attr[2:]
            
        # store stats in a list in the same order as COL_NAMES
        row = [None]*len(COL_NAMES)
        row[COL_NAMES.index('Item')]      = attr
        row[COL_NAMES.index('TotalRt')]   = R
        row[COL_NAMES.index('InstRt')]    = dR
        row[COL_NAMES.index('InstAvg')]   = R_curr
        row[COL_NAMES.index('TotAvg')]    = R_['Avg']
        row[COL_NAMES.index('TotMax')]    = R_['Max']
        row[COL_NAMES.index('TotMin')]    = R_['Min']
        row[COL_NAMES.index('RtPctAvg')]  = R_pct_avg
        row[COL_NAMES.index('RtPctMax')]  = R_pct_max
        row[COL_NAMES.index('RtSigmas')]  = R_sigmas
        row[COL_NAMES.index('TotalCt')]   = C
        row[COL_NAMES.index('InstCt')]    = dC
        row[COL_NAMES.index('InstRate')]  = C_curr
        row[COL_NAMES.index('AvgRate')]   = C_avg
        row[COL_NAMES.index('CtPctAvg')]  = C_pct_avg

        # tupleize the stats list and add them to the larger list
        table.append(tuple(row))
        
    return table

### view ###

def getTerminalSize():
    """Returns the height (lines) and width (characters) of the terminal"""

    try: # works on Linux
        height, width = [int(x) for x in
                                  os.popen('stty size', 'r').read().split()]
        width -= 1 # leave a blank column on the right
        
    except Exception: # otherwise assume standard console size
        height = 25
        width = 80 - 1 # leave a blank column on the right

    return (height, width)

def getRuntimeColFmts(col_name, table, fmt_type):
    """Returns the text and number format that best fits a column"""

    # the only string-type data displayed is at the end of the row
    # so it doesn't require any special formatting
    column_fmt = '{0}'
    number_fmt = '{0}'

    if fmt_type == 'str':
        return (column_fmt, number_fmt)

    # for numerical data, compare the length of the column name
    # to the length of the largest (in characters) number.
    name_width = len(col_name)

    # the "largest" number could be negative, so check both max and min
    i = COL_NAMES.index(col_name)
    max_val = max(table, key=lambda row: row[i] if row[i] is not None else 0)[i]
    min_val = min(table, key=lambda row: row[i] if row[i] is not None else 0)[i]

    try:
        max_width = len('{0:d}'.format(int(max_val)))
    except TypeError:
        max_width = 0

    try:
        min_width = len('{0:d}'.format(int(min_val)))
    except TypeError:
        min_width = 0

    n_width = max([max_width, min_width])

    # for floats, we want the largest number of decimal places
    # that we can get away with without needlessly expanding the column size
    if fmt_type in ['float', 'floatneg']:

        # add one to the length of columns that could have negative numbers
        if fmt_type == 'floatneg':
            n_width += 1 

        # compare against max int length + 1 because of decimal point
        if (n_width + 1) < name_width:
            decimals = name_width - (n_width + 1)
            n_width = (n_width + decimals + 1)
        else:
            decimals = 0

        col_width = max([n_width, name_width])

        column_fmt = '{0:>%ds}' % (col_width,)
        number_fmt = '{0:>%d.%df}' % (col_width, decimals)

    # for integers, we just need to make sure that we aren't needlessly
    # expanding the column size unless the numbers are really large
    if fmt_type == 'int':
        col_width = max([n_width, name_width])

        column_fmt = '{0:>%ds}' % (col_width,)
        number_fmt = '{0:>%dd}' % (col_width,)

    return (column_fmt, number_fmt)

def formatMemorySize(size):
    """Returns a string of human-readable memory size"""

    # ClassAds report memory in KB, so start with K
    suffixes = ['K', 'M', 'G', 'T', 'P']

    for i in range(len(suffixes)):
        if size/(10**(i*3)) < 1000:
            break

    size = size/float(10**(i*3))
    suffix = suffixes[i]

    return '{0:.1f} {1}'.format(size, suffix)

def printMonitorSelf(daemon, name, ad):
    """Prints the MonitorSelf stats and returns the number of lines printed"""
    
    out = []

    t = ad['MonitorSelfTime']
    out.append('{0} {1} status at {2}:'.format(daemon, name,
                    datetime.datetime.fromtimestamp(t)))
    out.append('\n')

    if 'MonitorSelfAge' in ad:
        uptime = datetime.timedelta(seconds = ad['MonitorSelfAge'])
        out.append('Uptime: {0}'.format(uptime))
        out.append('\n')
    if 'MonitorSelfSysCpuTime' in ad:
        sys_cpu_time = datetime.timedelta(seconds = ad['MonitorSelfSysCpuTime'])
        out.append('SysCpuTime: {0}'.format(sys_cpu_time))
        out.append(' '*4)
    if 'MonitorSelfUserCpuTime' in ad:
        user_cpu_time = datetime.timedelta(seconds=ad['MonitorSelfUserCpuTime'])
        out.append('UserCpuTime: {0}'.format(user_cpu_time))
        out.append('\n')
    if 'MonitorSelfImageSize' in ad:
        memory = formatMemorySize(ad['MonitorSelfImageSize'])
        out.append('Memory: {0}'.format(memory))
        out.append(' '*4)
    if 'MonitorSelfResidentSetSize' in ad:
        rss = formatMemorySize(ad['MonitorSelfResidentSetSize'])
        out.append('RSS: {0}'.format(rss))
        out.append('\n')
    out.append('\n')

    sys.stdout.write(''.join(out))

    lines = out.count('\n')
    
    return lines

def printRecentHealthStats(ad):
    """Prints the "recent" DC duty cycle and commands per second
    and returns the number of lines printed"""

    lines = 0

    # not all ClassAds report the RecentStatsTickTime,
    # but MonitorSelfTime is a close approximation
    if 'RecentStatsTickTime' in ad:
        t = datetime.datetime.fromtimestamp(ad['RecentStatsTickTime'])
    else:
        t = datetime.datetime.fromtimestamp(ad['MonitorSelfTime'])

    sys.stdout.write('Recent DC status at {0}:\n'.format(t))
    lines += 1
    
    if 'RecentDaemonCoreDutyCycle' in ad:
        duty_cycle = 100*ad['RecentDaemonCoreDutyCycle']
        if 'RecentStatsLifetime' in ad:
            duration = ad['RecentStatsLifetime']
            sys.stdout.write('Duty Cycle (last {0}s): {1:.2f}%    '.format(
                duration, duty_cycle))
        else:
            sys.stdout.write('Duty Cycle: {0:.2f}%    '.format(duty_cycle))

    for attr in ['DCCommandsPerSecond_' + l for l in ['4m', '20m', '4h']]:
        if attr in ad:
            ops_per_sec = ad[attr]
            duration = attr.split('_')[1]
            sys.stdout.write('Ops/second (last {0}): {1:.3f}'.format(
                duration, ops_per_sec))
            break

    sys.stdout.write('\n\n')
    lines += 2
        
    return lines    
    
def printInstHealthStats(old, new):
    """Prints the DC duty cycle and commands per second between two ads
    and returns the number of lines printed"""

    duty_cycle = computeInstDutyCycle(old, new)
    ops_per_sec = computeInstOpsPerSec(old, new)

    # in the case that we cannot get the values from between ads,
    # just print the recent DC stats
    if (duty_cycle is None) or (ops_per_sec is None):
        return printRecentHealthStats(new)

    lines = 0

    # print the time between ads
    t_old, t_new = computeUpdateTimes(old, new)
    t_old, t_new = [datetime.datetime.fromtimestamp(t) for t in [t_old, t_new]]
    sys.stdout.write('DC status from {0} to {1}:\n'.format(t_old, t_new))
    lines += 1

    # print the stats
    sys.stdout.write('Duty Cycle: {0:.2f}%    '.format(duty_cycle))
    sys.stdout.write('Ops/second: {0:.3f}\n'.format(ops_per_sec))
    lines += 1

    sys.stdout.write('\n')
    lines += 1

    return lines
    
    
def printRuntimeTable(old, new, col_set = COL_SET, sort_col = SORT_COL, lines = 0):
    """Prints the table of runtime statistics"""

    # check the terminal size
    height, width = getTerminalSize()

    # get the table of stats
    table = computeRuntimeStats(old, new)

    # get the text and number formats for each column
    col_fmts = {}
    for col in ['TotalCt', 'InstCt']:
        col_fmts[col] = getRuntimeColFmts(col, table, 'int')
    for col in ['RtSigmas']:
        col_fmts[col] = getRuntimeColFmts(col, table, 'floatneg')
    for col in ['Item']:
        col_fmts[col] = getRuntimeColFmts(col, table, 'str')
    for col in COL_NAMES:
        if not (col in col_fmts):
            col_fmts[col] = getRuntimeColFmts(col, table, 'float')

    # sort the table
    table.sort(key = itemgetter(COL_NAMES.index(sort_col)), reverse = True)

    # get the column names to be printed
    col_names = COL_SETS[col_set]

    # print the time between ads
    t_old, t_new = computeUpdateTimes(old, new)
    t_old, t_new = [datetime.datetime.fromtimestamp(t) for t in [t_old, t_new]]
    sys.stdout.write('Runtime stats from {0} to {1}:\n'.format(t_old, t_new))
    lines += 1
    
    # print the header
    cols = []
    for col_name in col_names:
        cols.append(col_fmts[col_name][0].format(col_name))
    col_string = ' '.join(cols)

    # crop the header to the terminal window if needed
    if (not PIPE) and (len(col_string) > width):
        col_string = col_string[:width]
    sys.stdout.write(col_string + '\n')
    lines += 1

    # print the rows
    for row in table:

        # make a list of columns
        cols = []

        # go in the order of col_names
        for col_name in col_names:
            col_value = row[COL_NAMES.index(col_name)]

            # if the value can't be formatted, it's probably None ("n/a")
            try:
                cols.append(col_fmts[col_name][1].format(col_value))
            except (ValueError, TypeError):
                cols.append(col_fmts[col_name][0].format('n/a'))

        # merge the column list into a single string
        row_string = ' '.join(cols)

        # crop the row to the terminal window if needed
        if (not PIPE) and (len(row_string) > width):
            row_string = row_string[:width]

        # print the row
        sys.stdout.write(row_string + '\n')
        lines += 1

        # leave a blank row at the bottom of the table
        if (not PIPE) and (lines >= height - 1):
            break

def printCountdown(delay = DELAY):
    """Prints countdown to the next update to stderr"""
    now = datetime.datetime.utcnow()
    refresh_time = now + datetime.timedelta(seconds=delay)
    time_left = refresh_time - now
    while time_left > datetime.timedelta(seconds=0):
        # rewind the line (\r) and print countdown each second
        sys.stderr.write('\rUpdating ClassAd in {0:>5d} s'.format(
            time_left.seconds))
        sys.stderr.flush()
        sleep(1)
        time_left = refresh_time - datetime.datetime.utcnow()
    sys.stderr.write('\rUpdating ClassAd ' + 10*'.' + '\n\n')
    sys.stderr.flush()

def resetTerminal():
    """Clears the current terminal output"""
    sys.stdout.write('\n\n') # add a few blank lines
    sys.stdout.flush()
    if os.name == 'nt':
        os.system('cls')
    else:
        os.system('clear')
        
if __name__ == '__main__':

    # Two modes of input:
    # 1. ClassAds parsed from two files
    # 2. ClassAds queried from the Collector

    # Two modes of output:
    # 1. stdout to the terminal window
    # 2. stdout to a pipe

    # wrap everything to catch Ctrl-C
    try:
    
        # if we have files, parse them into old and new ClassAds
        if FILES:
            if len(FILES) != 2:
                sys.stderr.write(('Error: Bad arguments or more/less '
                                      'than 2 files passed:\n'))
                sys.stderr.write(' '.join(FILES))
                sys.stderr.write('\n')
                sys.exit(1)
            try:
                with open(FILES[0]) as f:
                    old = classad.parseNext(f)
                    if not old:
                        sys.stderr.write((
                            'Error: {0} does not contain any (valid) ClassAds\n'
                            ).format(FILES[0]))

            except Exception as e:
                sys.stderr.write(str(e))
                sys.stderr.write('\n')
                sys.exit(1)
            try:
                with open(FILES[1]) as f:
                    new = classad.parseNext(f)
                    if not new:
                        sys.stderr.write((
                            'Error: {0} does not contain any (valid) ClassAds\n'
                            ).format(FILES[1]))
            except Exception as e:
                sys.stderr.write(str(e))
                sys.stderr.write('\n')
                sys.exit(1)

            # make sure new is newer than old
            if new['MyCurrentTime'] < old['MyCurrentTime']:
                old, new = (new, old)

            # get the daemon type
            types = {
                'Collector': 'Collector',
                'DaemonMaster': 'Master',
                'Negotiator': 'Negotiator',
                'Scheduler': 'Schedd',
                'Machine': 'Startd'
            }

            if new['MyType'] in types:
                DAEMON = types[new['MyType']]
            else:
                DAEMON = new['MyType']

            # get the daemon name
            NAME = new['Name']

        # if we're querying the Collector, first get the "old" ClassAd
        else:
            if NAME is None:
                NAME = getDaemonName(POOL, DAEMON, MACHINE)

            old = queryClassAd(POOL, DAEMON, NAME, DIRECT)

            # go ahead and print the information we have
            printMonitorSelf(DAEMON, NAME, old)
            printRecentHealthStats(old)

            # if there aren't any runtime attributes in the classad,
            # print the information we have and exit now
            if len(getRuntimeAttrs(old)) == 0:
                sys.stderr.write("Did not get any comparable runtime attributes from daemon ClassAd, exiting early.\n")
                sys.exit(0)

            # countdown and get the "new" ClassAd from the Collector
            printCountdown()
            new = queryClassAd(POOL, DAEMON, NAME, DIRECT)

            # if in live mode, clear the terminal
            if LIVE:
                resetTerminal()
                pass

        # at this point, we have both old and new ClassAds, so print the info
        lines = printInstHealthStats(old, new)
        printRuntimeTable(old, new, lines = lines)

        # if in live mode, continue getting new ClassAds
        if LIVE:
            while True:
                old = deepcopy(new)
                printCountdown()
                new = queryClassAd(POOL, DAEMON, NAME, DIRECT)
                resetTerminal()
                lines = printMonitorSelf(DAEMON, NAME, new)
                lines += printInstHealthStats(old, new)
                printRuntimeTable(old, new, lines = lines)

    except KeyboardInterrupt:
        sys.stdout.write('\n')
        sys.stdout.flush()