import os import sys import re import math def dqTables(mode, debugFlag): """Function to return the data quality tables for a given mode""" if (mode == "production"): tables = ["DataQuality", "Authors", "FlagDefs", "Versions", "SubDetectors"] elif (mode == "prodcosmics"): tables = ["DataQualityCosmics", "Authors", "FlagDefs", "Versions", "SubDetectors"] elif (mode == "test"): tables = ["DataQualityTest", "AuthorsTest", "FlagDefsTest", "VersionsTest", "SubDetectorsTest"] else: tables = ["DataQualityCosmicsTest", "AuthorsTest", "FlagDefsTest", "VersionsTest", "SubDetectorsTest"] if (debugFlag): print "dataquality tables: ", tables return tables def unpackFGDValidateFlag(flagWord): """Unpack the FGD word for validation """ npow = [] for x in range(0,3): npow.append(pow(2,x)) """The order of the flags is: AllFGD, FGD1, FGD2' The AllFGD is the full DQ flag, the other two have only values -1, 0, 1 """ flag = [-1, -1, -1] if ( flagWord <= 0 ): flag[0] = flag[1] = flag[2] = flagWord else: flag[1] = (flagWord & npow[0]) flag[2] = (flagWord & npow[1]) return (flag) def unpackFGDFlag(flagWord): """Unpack the FGD word """ npow = [] for x in range(0,3): npow.append(pow(2,x)) """The order of the flags is: AllFGD, FGD1, FGD2' The AllFGD is the full DQ flag, the other two have only values -1, 0, 1 """ flag = [-1, -1, -1] if ( flagWord <= 0 ): flag[0] = flag[1] = flag[2] = flagWord else: flag[1] = (flagWord & npow[0])/npow[0] flag[2] = (flagWord & npow[1])/npow[1] if (flag[1]>0): flag[1] = 1 if (flag[2]>0): flag[2] = 1 if ((flag[1] + flag[2] ) > 0): flag[0] = 1 return (flag) def unpackECALValidateFlag(flagWord): """Unpack the ECAL word for validation """ npow = [] for x in range(0,12): npow.append(pow(2,x)) """The order of the flags is: AllECAL, DSECal, P0DECal, BarECal The AllECAL is the full DQ flag, the other two have only values -1, 0, 1""" flag = [-1, -1, -1, -1] if ( flagWord <= 0 ): flag[0] = flag[1] = flag[2] = flag[3] = flagWord else: flag[1] = ((flagWord & npow[0]) + (flagWord & npow[1])) flag[2] = ((flagWord & npow[2]) + (flagWord & npow[7])) flag[3] = ((flagWord & npow[3]) + (flagWord & npow[4]) + (flagWord & npow[5]) + (flagWord & npow[6]) + (flagWord & npow[8]) + (flagWord & npow[9]) + (flagWord & npow[10]) + (flagWord & npow[11]) ) return (flag) def unpackECALFlag(flagWord): """Unpack the ECAL word """ npow = [] for x in range(0,12): npow.append(pow(2,x)) """The order of the flags is: AllECAL, DSECal, P0DECal, BarECal The AllECAL is the full DQ flag, the other two have only values -1, 0, 1""" flag = [-1, -1, -1, -1] if ( flagWord <= 0 ): flag[0] = flag[1] = flag[2] = flag[3] = flagWord else: flag[1] = ((flagWord & npow[0])/npow[0] + (flagWord & npow[1])/npow[1]) flag[2] = ((flagWord & npow[2])/npow[2] + (flagWord & npow[7])/npow[7]) flag[3] = ((flagWord & npow[3])/npow[3] + (flagWord & npow[4])/npow[4] + (flagWord & npow[5])/npow[5] + (flagWord & npow[6])/npow[6] + (flagWord & npow[8])/npow[8] + (flagWord & npow[9])/npow[9] + (flagWord & npow[10])/npow[10] + (flagWord & npow[11])/npow[11] ) if (flag[1]>0): flag[1] = 1 if (flag[2]>0): flag[2] = 1 if (flag[3]>0): flag[3] = 1 if ((flag[1] + flag[2] + flag[3] ) > 0): flag[0] = 1 return (flag) def unpackTPCValidateFlag(flagWord): """Unpack the TPC word for validation """ npow = [] for x in range(0,32): npow.append(pow(2,x)) """The order of the flags is: AllTPC, TPC1, TPC2, TPC3' The AllTPC is the full DQ flag, the other two have only values -1, 0, 1""" flag = [-1, -1, -1, -1] """flagWord <= 128 means old schema flagWord > 128 means new schema""" if ( flagWord <= 128 ): flag[0] = flag[1] = flag[2] = flag[3] = flagWord else: flag[1] = ((flagWord & npow[8]) + (flagWord & npow[9]) + (flagWord & npow[10]) + (flagWord & npow[11]) + (flagWord & npow[12]) + (flagWord & npow[13]) + (flagWord & npow[14]) + (flagWord & npow[15]) ) flag[2] = ((flagWord & npow[16]) + (flagWord & npow[17]) + (flagWord & npow[18]) + (flagWord & npow[19]) + (flagWord & npow[20]) + (flagWord & npow[21]) + (flagWord & npow[22]) + (flagWord & npow[23]) ) flag[3] = ((flagWord & npow[24]) + (flagWord & npow[25]) + (flagWord & npow[26]) + (flagWord & npow[27]) + (flagWord & npow[28]) + (flagWord & npow[29]) + (flagWord & npow[30]) + (flagWord & npow[31]) ) return (flag) def unpackTPCFlag(flagWord): """Unpack the TPC word """ npow = [] for x in range(0,32): npow.append(pow(2,x)) """The order of the flags is: AllTPC, TPC1, TPC2, TPC3' The AllTPC is the full DQ flag, the other two have only values -1, 0, 1""" flag = [-1, -1, -1, -1] """flagWord <= 128 means old schema flagWord > 128 means new schema""" if ( flagWord <= 128 ): flag[0] = flag[1] = flag[2] = flag[3] = flagWord else: flag[1] = ((flagWord & npow[8])/npow[8] + (flagWord & npow[9])/npow[9] + (flagWord & npow[10])/npow[10] + (flagWord & npow[11])/npow[11] + (flagWord & npow[12])/npow[12] + (flagWord & npow[13])/npow[13] + (flagWord & npow[14])/npow[14] + (flagWord & npow[15])/npow[15] ) flag[2] = ((flagWord & npow[16])/npow[16] + (flagWord & npow[17])/npow[17] + (flagWord & npow[18])/npow[18] + (flagWord & npow[19])/npow[19] + (flagWord & npow[20])/npow[20] + (flagWord & npow[21])/npow[21] + (flagWord & npow[22])/npow[22] + (flagWord & npow[23])/npow[23] ) flag[3] = ((flagWord & npow[24])/npow[24] + (flagWord & npow[25])/npow[25] + (flagWord & npow[26])/npow[26] + (flagWord & npow[27])/npow[27] + (flagWord & npow[28])/npow[28] + (flagWord & npow[29])/npow[29] + (flagWord & npow[30])/npow[30] + (flagWord & npow[31])/npow[31] ) if (flag[1]>0): flag[1] = 1 if (flag[2]>0): flag[2] = 1 if (flag[3]>0): flag[3] = 1 if ((flag[1] + flag[2] + flag[3] ) > 0): flag[0] = 1 return (flag) class DatabaseInterface: """A simple interface to a MySQL Database. Makes no assumption about the tables that the database holds.""" def __init__(self,parent): ## Initialise state self.parent = parent # The parent DatabaseUpdater self.debug = parent.debug self.access_string = "" # Access string needed for command: mysql --execute= self.SetAccessString() self.public_access_string = re.compile(r'--password=\S+').sub('--password=XXX',self.access_string) self.results_file = '/tmp/mysql_results_%d.tmp' % os.getpid() self.results = [] if self.IsOK(): print 'DatabaseInterface initialising with account info: ' + self.public_access_string else: print 'MySQL DBI connection failed with account info: ' + self.public_access_string #--------------------------------------------- DatabaseInterface -------------------------------------------------- def IsOK(self): """Return True if able to execute queries.""" if not self.access_string: return False #See if we can run a dummy statement return self.Query('select 1',False) #--------------------------------------------- DatabaseInterface -------------------------------------------------- def GetResults(self): """Return the results from the last query.""" return self.results #--------------------------------------------- DatabaseInterface -------------------------------------------------- def Query(self,sql,log_error = True): """Apply query and return True if successful. To get the results themselves call GetResults() Unless log_error is False, errors are logged.""" self.results = [] cmd = 'mysql %s --execute="%s" > %s 2>&1' % (self.access_string,sql,self.results_file) if self.debug: print 'About to execute: ' + cmd exit_code = os.system(cmd) if os.path.isfile(self.results_file): log_results = exit_code and log_error if log_results: print 'SQL query failed with error code %d' % exit_code file_results = open(self.results_file) for line in file_results: line = line.strip() if log_results: print ' SQL log:' + line self.results.append(line) os.remove(self.results_file) return exit_code == 0 #--------------------------------------------- DatabaseInterface -------------------------------------------------- def SetAccessString(self): """Prepare self.SetAccessString from TSQL environment.""" self.access_string = "" env_url = os.environ.get('ENV_TSQL_URL') env_user = os.environ.get('ENV_TSQL_USER') env_pswd = os.environ.get('ENV_TSQL_PSWD') if not env_url or not env_user or not env_pswd: print 'Cannot connect to database. One or more of the environmental variables:-' print ' ENV_TSQL_URL, ENV_TSQL_USER or ENV_TSQL_PSWD' print 'is not defined.' exit(1) # Collect server and db and port from from first element of ENV_TSQL_URL. mo = re.search(r'//(.*?)/(\w+)',env_url) if not mo: print 'Cannot parse the environmental variables ENV_TSQL_URL' exit(1) (server,db) = mo.groups() port_opt = '' mo = re.search(r'^(.*):(\d+)$',server) if mo: server = mo.group(1) port_opt = '--port=' + mo.group(2) # Collect user from first element of ENV_TSQL_USER user = env_user mo = re.search(r'(.*?);',env_user) if mo: user = mo.group(1) # Collect pswd from first element of ENV_TSQL_PSWD pswd = env_pswd mo = re.search(r'(.*?);',env_pswd) if mo: pswd = mo.group(1) pswd_opt = '--password=' + pswd if pswd == '\\0': pswd_opt = '' self.access_string = '%s --host %s %s --user=%s %s' % (db,server,port_opt,user,pswd_opt) #-------DatabaseInterface -------------------------------------- def TableExists(self,table_name): """Return True if table exists.""" return self.Query('describe ' + table_name,False) #--------------------------------------------- DQTable --------------------- class DQTable: '''Class for a Data Quality Table''' def __init__(self, dqSchema, debugFlag): self.debug = debugFlag self.dbi = DatabaseInterface(self) # Read in the schema (i.e. the queries) self.dqSchema = dqSchema self.lastRow = self.getLastRow() def getRowID(self, attribute): '''Method to return the row id for the attribute ''' if (type(attribute) == str): attrib = attribute.split('\'')[1] else: attrib = attribute self.dbi.Query(self.dqSchema.queryRowAttribute(attribute)) rows = self.dbi.GetResults() rowID = 0 if (self.debug): print "rows: ", rows print "attribute: ", attrib if (len(rows) > 0): for row in rows: rowElements = row.split('\t') if (self.debug): print "rowElements: ", rowElements for element in rowElements: if (self.debug): print "element: ", element, " attribute: ", attribute print "element == attribute: ", attrib.strip() == element.strip() if (attrib.strip() == element.strip()): rowID = rowElements[0] break if (self.debug): print "rowID is: ", rowID return int(rowID) def getLastRow(self): '''Method to return the last row number in the database table ''' self.dbi.Query(self.dqSchema.queryRow()) rows = self.dbi.GetResults() if (len(rows) == 0): lastRow = 0 else: lastRow = rows[-1].strip() return int(lastRow) def getRows(self): '''Method to return all rows for the DataQuality table ''' self.dbi.Query(self.dqSchema.queryRows()) rows = self.dbi.GetResults() return rows def getRowsWithBeginTime(self, begintime): '''Method to return the dataQuality table based on begin time selection ''' results = [] if (begintime > 0): self.dbi.Query(self.dqSchema.queryBeginTime(begintime)) results = self.dbi.GetResults() return results def getRowsWithEndTime(self, endtime): '''Method to return the dataQuality table based on end time selection ''' results = [] if (endtime > 0): self.dbi.Query(self.dqSchema.queryEndTime(endtime)) results = self.dbi.GetResults() return results def getRowsWithCreateTime(self, createtime): '''Method to return the dataQuality table based on create time selection ''' results = [] if (createtime > 0): self.dbi.Query(self.dqSchema.queryCreateTime(createtime)) results = self.dbi.GetResults() return results def getRowsWithSubdetector(self, subdetectors): '''Method to return the dataQuality table based on subdetector selection ''' results = [] if (len(subdetectors) == 1): self.dbi.Query(self.dqSchema.querySubdetector(subdetectors[0])) results = self.dbi.GetResults() else: for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetector(subdetector)) if (len(results) == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] return results def getRowsWithFlag(self, flag): '''Method to return the dataQuality table based on flag selection ''' results = [] self.dbi.Query(self.dqSchema.queryFlags(flag)) results = self.dbi.GetResults() return results def getRowsWithVersion(self, versions): '''Method to return the dataQuality table based on version selection ''' results = [] if (len(versions) == 1): self.dbi.Query(self.dqSchema.queryVersion(versions[0])) results = self.dbi.GetResults() else: for version in versions: self.dbi.Query(self.dqSchema.queryVersion(version)) if (len(results) == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] return results def getRowsWithAuthor(self, authors): '''Method to return the dataQuality table based on author selection ''' results = [] if (len(authors) == 1): self.dbi.Query(self.dqSchema.queryAuthor(authors[0])) results = self.dbi.GetResults() else: for author in authors: self.dbi.Query(self.dqSchema.queryAuthor(author)) if (len(results) == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] return results def getRowsWithSubdetectorBeginTime(self, subdetectors, begintime): '''Method to return the dataQuality table based on subdetector and begintime selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorBeginTime(subdetectors[0], begintime)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorBeginTime(subdetector, begintime)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorEndTime(self, subdetectors, endtime): '''Method to return the dataQuality table based on subdetector and endtime selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorEndTime(subdetectors[0], endtime)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorEndTime(subdetector, endtime)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorgtBeginTime(self, subdetectors, eventtime, begintime): '''Method to return the dataQuality table based on subdetector and begintime and eventtime selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorgtBeginTime(subdetectors[0], eventtime, begintime)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorgtBeginTime(subdetector, eventtime, begintime)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorltEndTime(self, subdetectors, eventtime, endtime): '''Method to return the dataQuality table based on subdetector and endtime and eventtime selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorltEndTime(subdetectors[0], eventtime, endtime)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorltEndTime(subdetector, eventtime, begintime)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorFlag(self, subdetectors, flag): '''Method to return the dataQuality table based on subdetector and flag selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorFlag(subdetectors[0], flag)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorFlag(subdetector, flag)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorCreateTime(self, subdetectors, createtime): '''Method to return the dataQuality table based on subdetector and createtime selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorCreateTime(subdetectors[0], createtime)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorCreateTime(subdetector, createtime)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorVersion(self, subdetectors, versions): '''Method to return the dataQuality table based on subdetector and version selection ''' results = [] if (len(subdetectors) == 1 and len(versions) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorVersion(subdetectors[0], versions[0])) results = self.dbi.GetResults() elif (len(subdetectors) > 1 and len(versions) == 1): counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorVersion(subdetector, versions[0])) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 elif (len(subdetectors) == 1 and len(versions) > 1): counter = 0 for version in versions: self.dbi.Query(self.dqSchema.querySubdetectorVersion(subdetectors[0], version)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 else: counter = 0 for subdetector in subdetectors: for version in versions: self.dbi.Query(self.dqSchema.querySubdetectorVersion(subdetector, version)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorAuthor(self, subdetectors, authors): '''Method to return the dataQuality table based on subdetector and author selection ''' results = [] if (len(subdetectors) == 1 and len(authors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorAuthor(subdetectors[0], authors[0])) results = self.dbi.GetResults() elif (len(subdetectors) > 1 and len(authors) == 1): counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorAuthor(subdetector, authors[0])) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 elif (len(subdetectors) == 1 and len(authors) > 1): counter = 0 for author in authors: self.dbi.Query(self.dqSchema.querySubdetectorAuthor(subdetectors[0], author)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 else: counter = 0 for subdetector in subdetectors: for author in authors: self.dbi.Query(self.dqSchema.querySubdetectorAuthor(subdetector, author)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorBeginEndTime(self, subdetectors, begintime, endtime): '''Method to return the dataQuality table based on subdetector and begin plus end time selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorBeginEndTime(subdetectors[0], begintime, endtime)) #print 'query: ', \ #self.dqSchema.querySubdetectorBeginEndTime(subdetectors[0], # begintime, # endtime) results = self.dbi.GetResults() #for result in results: # print 'result ', result else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorBeginEndTime(subdetector, begintime, endtime)) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 return results def getRowsWithSubdetectorBeginEndTime2(self, subdetectors, begintime, endtime): '''Method to return the dataQuality table based on subdetector and begin plus end time selection ''' results = [] if (len(subdetectors) == 1 ): self.dbi.Query(self.dqSchema.querySubdetectorBeginEndTime2(subdetectors[0], begintime, endtime)) results = self.dbi.GetResults() else: counter = 0 for subdetector in subdetectors: self.dbi.Query(self.dqSchema.querySubdetectorBeginEndTime2(subdetector, begintime, endtime)) print 'Query: ', \ self.dqSchema.querySubdetectorBeginEndTime2(subdetector, begintime, endtime) if (counter == 0): results += self.dbi.GetResults() else: resultsTemp = self.dbi.GetResults() results += resultsTemp[1:] counter += 1 print 'results: ', results return results def addRows(self, attributes): '''Method to add the input file rows to the database ''' if (self.debug): print "addrows attributes are: ", attributes for row in range(len(attributes)): newAttributes = [] rowNumber = row + self.lastRow + 1 if (type(attributes[row]) != list): newAttributes.append(attributes[row]) else: newAttributes = attributes[row] if (self.debug): print "type of newAttributes is: ", type(newAttributes) # Check of the row exists (must exclude the createtime as this will # always be updated in the case of the dqTable table only if (self.dqSchema.tableName == "DataQuality" or self.dqSchema.tableName == "DataQualityCosmics" or self.dqSchema.tableName == "DataQualityTest" or self.dqSchema.tableName == "DataQualityCosmicsTest"): self.dbi.Query(self.dqSchema.queryExists(newAttributes[0:-1])) else: self.dbi.Query(self.dqSchema.queryExists(newAttributes)) # Only fill the table if the row doesn't already exist results = self.dbi.GetResults() if (self.debug): print "getresults is: ", len(results) if (len(results) == 0): sql = self.dqSchema.loadQuery(rowNumber, newAttributes) self.dbi.Query(sql) def getFlag(self, eventtime, subdetectors): '''Method to return the DQ flag and the creation time corresponding to the event time. ''' if (self.debug): print "eventtime: ", eventtime print "subdetectors: ", subdetectors if (len(subdetectors) == 0): queryString = self.dqSchema.queryFlag(eventtime) self.dbi.Query(queryString) results = self.dbi.GetResults() else: results = [] for subdetector in subdetectors: self.dbi.Query(self.dqSchema.queryFlag(eventtime, subdetector)) tempresults = self.dbi.GetResults() results += tempresults if (self.debug): print "results are: ", len(results) return results #--------------------------------------------- DQSchema --------------------- class DQSchema: '''Class containing the schema for the database ''' def __init__(self, tableName, debugFlag): self.debugFlag = debugFlag self.tableName = tableName self.schemaDict = self.loadSchema() def loadSchema(self): '''Method to load the schemas into a dictionary ''' env_dqSchema = os.environ.get('OADATAQUALITYROOT') if (env_dqSchema): dqSchemaFile = env_dqSchema + "/src/dataQualityTableSchema.txt" else: print "Error: OADATAQUALITYROOT environmental variable must be set. Run Icedust_install/setup.sh" sys.exit(1) fh = file(dqSchemaFile, 'r') schemaDict = {} for line in fh.readlines(): if ("::" in line): keyTemp, query = line.split("::") key = (keyTemp.split("#")[1]).strip() if (self.debugFlag): print "key: %s query %s" % (key, query) schemaDict[key] = query if (self.debugFlag): print "keys: ", schemaDict.keys() return schemaDict def queryRows(self): '''Method to construct the query for all rows in the DataQuality table ''' queryKey = self.tableName + "_query_all" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryBeginTime(self, begintime): '''Method to construct the query for all rows in DataQuality constrained by begintime ''' queryKey = self.tableName + "_query_starttime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % begintime else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryEndTime(self, endtime): '''Method to construct the query for all rows in DataQuality constrained by endtime ''' queryKey = self.tableName + "_query_endtime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % endtime else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryCreateTime(self, createtime): '''Method to construct the query for all rows in DataQuality constrained by createtime ''' queryKey = self.tableName + "_query_createtime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % createtime else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetector(self, subdetector): '''Method to construct the query for all rows in DataQuality constrained by subdetector ''' queryKey = self.tableName + "_query_subdetector" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % subdetector else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryFlags(self, flag): '''Method to construct the query for all rows in DataQuality constrained by flag ''' queryKey = self.tableName + "_query_flag" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % flag else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryVersion(self, version): '''Method to construct the query for all rows in DataQuality constrained by version ''' queryKey = self.tableName + "_query_version" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % version else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryAuthor(self, author): '''Method to construct the query for all rows in DataQuality constrained by author ''' queryKey = self.tableName + "_query_author" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] % author else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorBeginTime(self, subdetector, begintime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and begintime ''' queryKey = self.tableName + "_query_subdetector_starttime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, begintime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorltEndTime(self, subdetector, eventtime, endtime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and begintime and eventtime ''' queryKey = self.tableName + "_query_subdetector_ltendtime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, eventtime, endtime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorEndTime(self, subdetector, endtime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and endtime ''' queryKey = self.tableName + "_query_subdetector_endtime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, endtime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorgtBeginTime(self, subdetector, eventtime, begintime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and endtime and eventtime ''' queryKey = self.tableName + "_query_subdetector_gtstarttime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, eventtime, begintime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorFlag(self, subdetector, flagtime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and flag ''' queryKey = self.tableName + "_query_subdetector_flag" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, flag) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorCreateTime(self, subdetector, createtime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and createtime ''' queryKey = self.tableName + "_query_subdetector_createtime" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, createtime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorVersion(self, subdetector, version): '''Method to construct the query for all rows in DataQuality constrained by subdetector and version ''' queryKey = self.tableName + "_query_subdetector_version" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, version) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorAuthor(self, subdetector, author): '''Method to construct the query for all rows in DataQuality constrained by subdetector and author ''' queryKey = self.tableName + "_query_subdetector_author" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, author) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorBeginEndTime(self, subdetector, begintime, endtime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and begintime + endtime ''' queryKey = self.tableName + "_query_subdetector_interval" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, begintime, endtime, begintime, endtime, begintime, endtime, begintime, endtime, endtime, begintime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def querySubdetectorBeginEndTime2(self, subdetector, begintime, endtime): '''Method to construct the query for all rows in DataQuality constrained by subdetector and begintime + endtime ''' queryKey = self.tableName + "_query_subdetector_interval2" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] %(subdetector, begintime, endtime) else: print "Error: Cannot find query in schema file with key %s \n" % queryKey sys.exit(1) return query def queryFlag(self, eventtime, subdetector=None): '''Method to construct the query to return the flag, subdetector, starttime, endtime and createtime ''' queryKey1 = self.tableName + "_query_flag_subdetector" queryKey2 = self.tableName + "_query_flag" query = "" if (self.debugFlag): print "subdetector: ", subdetector if (subdetector == None): if (queryKey2 in self.schemaDict.keys()): query = self.schemaDict[queryKey2] % (eventtime, eventtime) else: print "Error: Cannot find query in schema file with key %s" % queryKey1 sys.exist(1) else: if (queryKey1 in self.schemaDict.keys()): query = self.schemaDict[queryKey1] % (subdetector, eventtime, eventtime) else: print "Error: Cannot find query in schema file with key %s" % queryKey2 return query def loadQuery(self, row, attributes): '''Method to return the query to load the attributes. The attributes must be in exactly the same order as appear in the schema file. ''' insertKey = self.tableName + "_insert" query = "" attribs = [int(row)] for attrib in attributes: attribs.append(attrib) if (insertKey in self.schemaDict.keys()): if (self.debugFlag): print "query ", self.schemaDict[insertKey] print "attribs ", attribs query = self.schemaDict[insertKey] % tuple(attribs) else: print "Error cannot find query in schema file with key %s" % queryKey sys.exit(1) return query def queryRowAttribute(self, attribute): '''Method to return the row corresponding to the attribute. ''' queryKey = self.tableName + "_query_exists" query = "" if (queryKey in self.schemaDict.keys()): if (self.debugFlag): print "query: ", self.schemaDict[queryKey] print "attribs: ", attribute query = self.schemaDict[queryKey] % attribute else: print "Error: Cannot find query in schema file with key %s" % queryKey sys.exit(1) return query def queryRow(self): '''Method to return the query for the row number ''' queryKey = self.tableName + "_query_row" query = "" if (queryKey in self.schemaDict.keys()): query = self.schemaDict[queryKey] else: print "Error: Cannot find query in schema file with key %s" % queryKey sys.exit(1) return query def queryExists(self, attributes): '''Method to return the query for checking if a row exists. The attributes must be in the same order as in the schema file. ''' queryKey = self.tableName + "_query_exists" query = "" if (queryKey in self.schemaDict.keys()): if (self.debugFlag): print "query ", self.schemaDict[queryKey] print "attribs ", attributes query = self.schemaDict[queryKey] % tuple(attributes) else: print "Error: Cannot find query in schema file with key %s" % queryKey sys.exit(1) if (self.debugFlag): print "query is: ", query return query