#!/usr/bin/env python2
'''Script to validate the DQ flags written into the db
Author F. R. Di Lodovico, QMUL, Oct, 2010
'''
import databaseUtils
import getopt
import os
import sys
import commands
import time

def usage():
    '''Function to print out the usage of the script
    '''
    print "Script to validate the DQ flags written into the db "
    print "Usage:"
    print "validateDQFlags.py [-h] [-v -p -l -s <subdetector> -d <subdetector division> -b <begin time> -e <end time>]"
    print ""
    print "Where:"
    print "<subdetector> is the subdetector."
    print "<subdetector division> is the division of the subdetector."
    print "<begin time> is the time in seconds the event was taken."
    print "<end time> is the time in seconds the event was taken."
    print ""
    print "Options:"
    print "-h, --help            Prints this help."
    print "-p, --production      Choosing the production tables for the chosen db (default is test tables)." 
    print "-s, --subdetector     Subdetector to query (default is all subdetectors)."
    print "                      A ':'-separated list of subdetectors can be supplied"
    print "                      ECAL:TPC:P0D to select ECAL, TPC and P0D."
    print "-d, --division=       The division of the subdetector to select. Allowed values"
    print "                      are TPC1, TPC2, TPC3, FGD1, FGD2, DSECAL, P0DECAL, BarECAL"
    print "-b, --begintime=      The start time interval in seconds."
    print "-e, --endtime=        The end time interval in seconds."
    print "-l, --log             Produces log-scale graphs. Note that the plot"
    print "                      may contain discontinuities as values of -1"
    print "                      are not plotted in the log graph." 
    print "-v, --verbose         Prints verbose output."
    print ""

def modifyInterval(starttime, endtime, inFlag, aboveIntervals):
    '''Function to modify the interval according to intervals above it
    '''
    valid = -10
    # print 'aboveIntervals: ', aboveIntervals 
    # print 'candidate interval: ', starttime, endtime, inFlag

    if (len(aboveIntervals) == 0):
        # print 'length is zero!'
        aboveIntervals.append((starttime, endtime, inFlag))
        valid = 1
        return (valid, aboveIntervals)
    
    for startTime, endTime, flag in aboveIntervals:
        
        # Completely contained in above interval
        # in which case set the interval to 0 and ignore it
        if (starttime >= startTime and
            endtime <= endTime):
            # print 'Completely contained!'
            valid = 0
            break

        # Partially contained in above interval (start inside, end outside)
        if (starttime >= startTime and 
            endtime > endTime and starttime < endTime):
            # print 'start in end out: ', startTime, endTime
            newStart = endTime
            newEnd = endtime
            # print 'newStart %s newEnd %s ' % (newStart, newEnd)

            # Check if new interval complete contained in an above interval
            if (1 == completelyContained(newStart, 
                                         newEnd, 
                                         aboveIntervals)):
                # print 'Completely contained 1'
                valid = 0
                break

            # If the interval spans a long period check if there are gaps
            # in above intervals and save the gaps
            gaps = checkGaps(newStart, newEnd, inFlag, aboveIntervals)
            if (len(gaps) > 0):
                for gap in gaps:
                    aboveIntervals.append(gap)
                valid = 0
                break
            # 
            # Check if new interval partially contained in an above interval
            # (start outside, end inside). Need to check the pared-down
            # interval until we find no intervals it's contained within
            savedValid = -1
            savedEnd = -1
            savedStart = -1
            while (1):
                newStart, newEnd, valid = \
                        partiallyContained1(newStart, 
                                            newEnd,
                                            aboveIntervals)
                print "newStart %s newEnd %s valid %s " % \
                        (newStart, newEnd, valid)

                if (valid == -1):
                    break
                else:
                    savedStart = newStart
                    savedEnd = newEnd
                    savedValid = valid

            if (savedValid == 1):
                aboveIntervals.append((savedStart, savedEnd, inFlag))

            # Check if new interval partially contained in above interval
            # (start inside, end outside). Need to check the pared-down
            # interval until we find no intervals it's contained within
            savedValid = -1
            savedStart = -1
            savedEnd = -1
            while (1):
                newStart, newEnd, valid = \
                        partiallyContained2(newStart,
                                            newEnd,
                                            aboveIntervals)
                # print 'newStart2 %s newEnd2 %s valid2 %s' % \
                #        (newStart, newEnd, valid)
                if (valid == -1):
                    break
                else:
                    savedStart = newStart
                    savedEnd = newEnd
                    savedValid = valid

            if (savedValid == 1):
                aboveIntervals.append((savedStart, savedEnd, inFlag))

    if (valid == -10):
       saveStart = -1
       saveEnd   = 10000000000000000
       for startTime, endTime, flag in aboveIntervals:
           # print 'startTime ', startTime, ' starttime ', starttime, ' endTime ', endTime, ' endtime ', endtime
           # Lower interval spans above interval and more
           # We have 2 intervals one at the beginning and one at the end
           if (starttime < startTime and endtime > endTime):
              if (startTime < saveEnd):
                 saveEnd = startTime
              if (endTime > saveStart):
                 saveStart = endTime
       
       # Check if completely contained in an existing interval
       contained1 = completelyContained(starttime, saveEnd, aboveIntervals)
       contained2 = completelyContained(saveStart, endtime, aboveIntervals)
       if (saveStart != -1 and saveEnd != 10000000000000000):
          if (contained1 == 0):
             aboveIntervals.append((starttime, saveEnd, inFlag))
          if (contained2 == 0):
             aboveIntervals.append((saveStart, endtime, inFlag))
          valid = 0

    # If the interval doesn't meet any overlap criteria keep it as is
    if (valid == -10):
        valid = 1
        aboveIntervals.append((starttime, endtime, inFlag))

    # print 'aboveIntervals is now: ', aboveIntervals
    return (valid, aboveIntervals)

def completelyContained(starttime, endtime, aboveIntervals):
    '''Function to check if an interval is completely contained in above
    intervals
    '''
    contained = 0
    for startt, endt, flag in aboveIntervals:
        if (starttime >= startt and endtime <= endt):
            contained = 1
            break
    return contained

def checkGaps(starttime, endtime, flag, aboveIntervals):
    '''Function to check if the above intervals have gaps in which case
    we resize the interval for just the gaps
    '''
    pieces = [(starttime, endtime, flag)]
    piecesIn = [(starttime, endtime, flag)]
    count = 0
    countOld = 100
    aboveCopy = sorted(aboveIntervals, key=lambda x: x[0])
    while (1):
        # print 'start pieces: ', pieces
        # print 'aboveCopy: ', aboveCopy
        for startPiece, endPiece, flagPiece in pieces:
            for startt, endt, flag in aboveCopy:
                if (startPiece < startt and endPiece > endt 
                    and endPiece > startt and startPiece < endt):
                    if (completelyContained(pieces[count][0], startt,
                                            aboveIntervals)):
                        pieces[count] = (endt, endPiece, flagPiece)
                    else:
                        pieces[count] = (pieces[count][0], startt, flagPiece)
                        pieces.append((endt, endPiece, flagPiece))
                        count += 1
            # If we have reached the end of the series check the last
            # interval to make sure we're not overlapping

        if (countOld == count):
            break
        countOld = count
    
    # print 'pieces: ',pieces
    
    if (len(pieces) == 1):
        if (pieces[0][0] == piecesIn[0][0] and pieces[0][1] == piecesIn[0][1]):
            pieces = []
    
    gaps = []
    for piece in pieces:
        if (0 == completelyContained(piece[0], piece[1], aboveCopy)):
            gaps.append(piece)

    return gaps 

def partiallyContained1(starttime, endtime, aboveIntervals):
    '''Function to check if an interval is partially contained in the
    interval (start outside end inside)
    '''
    contained = -1
    savedStart = -1
    savedEnd = -1
    for startt, endt, flag in aboveIntervals:
        if (starttime < startt and endtime < endt and endtime > startt):
            contained = 1
            savedStart = starttime
            savedEnd = startt
            break
    # Is the new interval completely contained within existing
    # intervals - in which case ignore it
    for sstarttime, sendtime, sflag in aboveIntervals:
        if (savedStart >= sstarttime and savedEnd <= sendtime):
            contained = 0
            savedStart = 0
            savedEnd = 0
            break
    return (savedStart, savedEnd, contained)
 
def partiallyContained2(starttime, endtime, aboveIntervals):
    '''Function to check if an interval is partially contained in the
    interval (start inside end outside)
    '''
    contained = -1
    savedStart = -1
    savedEnd = -1
    for startt, endt, flag in aboveIntervals:
        if (starttime > startt and starttime < endt and endtime > endt):
            contained = 1
            savedStart = starttime
            savedEnd = sstarttime
            break
    
    # Is the new interval completely contained within an existing
    # interval - in which case ignore it
    for sstarttime, sendtime, sflag in aboveIntervals:
        if (savedStart >= sstarttime and savedEnd <= sendtime):
            contained = 0
            savedStart = 0
            savedEnd = 0
            break

    return (savedStart, savedEnd, contained)

def getFlagIntervals(rows, begin, end, division):
    '''Function to return the flag and corresponding start time it is
    valid for (end time assumes infinity or endtime limit) for each
    subdetector
    '''
    detectorRows = {}
    subdetectors = []
    for row in rows[1:]:
        elements = row.split('\t')
        if (len(elements) > 5):
            subdetector = elements[0]
            starttime = int(elements[1])
            endtime = int(elements[2])
            createtime = int(elements[4])
            flag = int(elements[3])
            flags = []
            if (division != ""):
                if (subdetector == "TPC"):
                    flags = databaseUtils.unpackTPCValidateFlag(flag)
                    if (division == "TPC1"):
                        flag = flags[1]
                    if (division == "TPC2"):
                        flag = flags[2]
                    if (division == "TPC3"):
                        flag = flags[3]
                if (subdetector == "ECAL"):
                    flags = databaseUtils.unpackECALValidateFlag(flag)
                    if (division == "DSECAL"):
                        flag = flags[1]
                    if (division == "P0DECAL"):
                        flag = flags[2]
                    if (division == "BarECAL"):
                        flag = flags[3]
                if (subdetector == "FGD"):
                    flags = databaseUtils.unpackFGDValidateFlag(flag)
                    if (division == "FGD1"):
                        flag = flags[1]
                    if (division == "FGD2"):
                        flag = flags[2]

            if (subdetector not in subdetectors):
                subdetectors.append(subdetector)
            
            if (detectorRows.has_key(subdetector)):
                detectorRows[subdetector].append((createtime,
                                                  starttime,
                                                  endtime,
                                                  flag))
            else:
                detectorRows[subdetector] = [(createtime,
                                              starttime,
                                              endtime,
                                              flag)]
    subdetectorFlags = {}
    for subdetector in subdetectors:
        detectorRows[subdetector].sort(key=lambda x: x[0])
        detectorRows[subdetector].reverse()
        for createtime, starttime, endtime, flag in detectorRows[subdetector]:
            # print 'detectorRows: ',createtime,' ', starttime,' ', endtime,' ',flag
            # print 'detectorRows: %s %s %s %d \n' % \
            #        (time.strftime("%d/%m/%Y::%H:%M:%S",
            #                       time.gmtime(float(createtime))),
            #         time.strftime("%d/%m/%Y::%H:%M:%S",
            #                       time.gmtime(float(starttime))),
            #         time.strftime("%d/%m/%Y::%H:%M:%S",
            #                       time.gmtime(float(endtime))),
            #         flag)
            # Modify the interval according to the intervals above
            if (not subdetectorFlags.has_key(subdetector)):
                subdetectorFlags[subdetector] = []

            valid, subdetectorFlags[subdetector] = \
                    modifyInterval(starttime, endtime, flag, 
                                   subdetectorFlags[subdetector])
        
        subdetectorFlags[subdetector].sort(key=lambda x: x[0])        
    return subdetectorFlags


def parseOptions():
    '''Function to read in the command line options
    '''
    detectors = ["TPC", "FGD", "P0D", "ECAL", "SMRD", "MAGNET", "INGRID"]
    
    eventtime = None
    
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hs:d:b:e:c:r:f:a:pvl",
                                   ["help",
                                    "subdetector=",
                                    "division=",
                                    "flag=",
                                    "version=",
                                    "author=",
                                    "begintime=",
                                    "endtime=",
                                    "createtime=",
                                    "production", 
                                    "verbose",
                                    "log"])
    except getopt.GetoptError, err:
        print str(err)
        usage()
        sys.exit(2)
    
    mode = "test"
    begintime = -1
    endtime = -1
    subdetectors = []
    division = ""
    verboseFlag = False
    logFlag = False
    for opt, val in opts:
        if (opt in ("-h", "--help")):
            usage()
            sys.exit()
        if (opt in ("-p", "--production")):
            mode = "production"
        if (opt in ("-l", "--log")):
            logFlag = True
        if (opt in ("-v", "--verbose")):
            verboseFlag = True
        if (opt in ("-s", "--subdetector")):
            if (":" in val):
                subdetectors = val.split(":")
            else:
                subdetectors = [val]
        if (opt in ("-d", "--division")):
            division = val.strip()
        if (opt in ("-b", "--begintime")):
            begintime = int(val)
        if (opt in ("-e", "--endtime")):
            endtime = int(val)

    # Return the event time and the subdetectors selected 
    return (mode, subdetectors, division, begintime, endtime, verboseFlag, logFlag)

def createPlotMacro(starttime, endtime, flagmax, logFlag):
    # Function to create the gnuplot macro
    print "start ", starttime
    print "end ", endtime
    outputStartTime = time.strftime("%d/%m/%Y::%H:%M:%S", time.gmtime(float(starttime)))
    outputEndTime = time.strftime("%d/%m/%Y::%H:%M:%S", time.gmtime(float(endtime)))

    macroFile = 'plotDQFlag'
    try:
        fh = file(macroFile, 'w')
    except Exception, err:
        print 'Error creating macro file %s' % macroFile
        print str(err)
        sys.exit(1)
    #set macro \n
    if (logFlag):
        output1 = '''
        set logscale y \n
        set xdata time \n
        set timefmt "%%d/%%m/%%Y::%%H:%%M:%%S" \n
        set xrange["%s":"%s"] \n
        set yrange[1:%e] \n 
        ''' % (outputStartTime, outputEndTime, float(flagmax+flagmax*0.5))

    else:
        output1 = '''
        set xdata time \n
        set timefmt "%%d/%%m/%%Y::%%H:%%M:%%S" \n
        set xrange["%s":"%s"] \n
        set yrange[-1.5:%e] \n
        ''' % (outputStartTime, outputEndTime, float(flagmax+2))

    output2 = '''
    set format x "%d/%b" \n
    set grid \n
    plot '/tmp/mysql_results.tmp' using 2:1 with line lw 3 lc rgb "red" title "DQ Flag" \n
    set term gif small \n
    set output "validation.gif" \n
    replot \n
    set term postscript \n
    set output "validation.eps" \n
    replot \n
    set term X11 \n
    '''
    output = output1 + output2
    #print output
    fh.write(output)
    fh.close()

def queryDQ():

    # Read in the command-line options
    mode, subdetectors, division, begintime, \
    endtime, verboseFlag, logFlag = parseOptions()

    if (division != ""):
        validDivision = False
        for subdetector in subdetectors:
            if (subdetector in division):
                validDivision = True
                break
        if (validDivision == False):
            print "Error: Incompatible division for subdetector!"
            usage()
            sys.exit(1)

    # Read in the table schemas
    dqTableName, authorTableName, flagDefsTableName, versionTableName, subdetectorTableName = databaseUtils.dqTables(mode, verboseFlag)

    # Read in the schema
    dqSchema = databaseUtils.DQSchema(dqTableName, verboseFlag)
    
    # Get the database tables
    dqtable = databaseUtils.DQTable(dqSchema, verboseFlag)

    # Read the list of time intervals within the request time interval.
    # Query the dataquality table for the dq flag, creation time
    # Must pass the subdetector schema as an argument for the join query
    # Returned rows are of the format flag, starttime, endtime, createtime
    # Query the table if subdetector + begintime + endtime supplied
#    if (len(subdetectors) > 0 and begintime > 0 and endtime > 0 and
#       createtime == -1 and flag == -1 and len(versions) == 0 and len(authors) == 0):
    queryStartTime = begintime
    queryEndTime = endtime
    rows = dqtable.getRowsWithSubdetectorBeginEndTime(subdetectors, begintime, endtime)

    #in case the query is inside an interval:
    if ( len(rows) == 0 ):
        rows = dqtable.getRowsWithSubdetectorBeginEndTime2(subdetectors, begintime, endtime)

        # Process the returned rows to create flags for intervals
    flagIntervals = getFlagIntervals(rows, begintime, endtime, division)
    #print 'flagIntervals: ',flagIntervals
    flagmax = -1000
    for subdetector in flagIntervals.keys():
        results_file = '/tmp/mysql_results_' + subdetector + '.tmp' 
        fo = file(results_file, 'w')
        bad_file = '/tmp/mysql_badinterv_' + subdetector + '.txt' 
        fobad = file(bad_file, 'w')
        for starttime,endtime,flag in flagIntervals[subdetector]:
            if( flag != 0 ):
                stringb = str(starttime) +' '+ str(endtime) 
                fobad.write(stringb + '\n') 
            starttime = time.strftime("%d/%m/%Y::%H:%M:%S", time.gmtime(float(starttime))) 
            string0 = "%d %s" % (flag, starttime)
            fo.write(string0 + '\n')
            endtime = time.strftime("%d/%m/%Y::%H:%M:%S", time.gmtime(float(endtime))) 
            string1 = "%d %s" % (flag, endtime)
            print flag, starttime, endtime
            if (flag >  flagmax): flagmax = flag
            fo.write(string1 + '\n')
        fo.close()
        fobad.close()

        # Create the script to generate the plots
        createPlotMacro(queryStartTime, queryEndTime, flagmax, logFlag)

        string2 = 'mv ' + results_file + ' /tmp/mysql_results.tmp' 
        commands.getstatusoutput(string2) 
        string3 = "gnuplot plotDQFlag" 
        status3, output3 = commands.getstatusoutput(string3) 
        print 'status, command ', string3, ' ',status3,' ', output3
        string4 = subdetector
        if (division):
            string4 = subdetector + "_" + division
        string5 = 'mv validation.gif validation_' + string4 + '.gif'
        commands.getstatusoutput(string5) 
        string6 = 'mv validation.eps validation_' + string4 + '.eps'
        commands.getstatusoutput(string6) 
        #os.remove(results_file)

if __name__ == '__main__':
    queryDQ()