#!/usr/bin/env python2 ''' Script to copy data quality files from the semioffline machines into iRODS The files to copy are the one produced for Data Quality and not running (ie a "done" file corresponding to the subrun exists). Necessarily this script needs to keep a track also of the files already copied through ascii files. What is needed a the moment is adding a configuration file (with the directories, file names etc to look at) and expecially some monitoring about the copied files and that they didnt crash. Author: Francesca Di Lodovico - Nov 2012 (moved to beam run 45 and added dq-fgd-rd) Francesca Di Lodovico - Oct 2012 (2012 run starting) Francesca Di Lodovico - Jan 2012 (removing dq-ecal-c) Francesca Di Lodovico - Jan 2012 (adding dq-ecal-stf, dq-ecal-cmd) Francesca Di Lodovico - Nov 2011 (adding dq-smrd-b, dq-p0d-b, dq-fgd-ht) Francesca Di Lodovico - Feb 2011 (adding dq-fgd-tm) Francesca Di Lodovico - Dec 2010 (major revision for how to get the files to copy) Francesca Di Lodovico - Nov 2010 (adding semaphore files) Francesca Di Lodovico - Sept 2010 (first version) ''' import sys, time, os, commands, shutil, glob, re def runsToCopy(beamRunNumber, indexrun, runstart, runend): '''Function to make a list of the runs _done_ but still to copy Steps: - Make a list of done files - From that list make another one with the proper files (takes care of adding .root .log .sh and the three executables for the dq) - Look at the files in iRODS and check that they are already in the above list - Make finally a list of the files to copy. This list will contain exactly the files (dirs and names) to copy - The next function will use that list, that will have to split to extract the run and subrun numbers, and copy the files. ''' # Define subDetectors subDetectorsOverall = ["p0d", "smrd", "dq", "ecal", "tpc", "fgd", "tript"] subDetectors = ["dq-p0d-li", "dq-p0d-b", "dq-smrd-c", "dq-smrd-b", "dq-bc", "dq-tpc-bc", "dq-ecal-mip", "dq-ecal-b", "dq-ecal-s", "dq-ecal-stf", "dq-ecal-cmd", "dq-tpc-bc", "dq-fgd-bc", "dq-fgd-dn", "dq-fgd-rd", "dq-fgd-tm", "dq-fgd-ht", "dq-tript-rmms"] r = re.compile("(log\.gz)") # Create a list with all the _done_ files outFile = '/home/irodsdq/cronjobs/RUNsOnSemiOff_done.dat' fo = file(outFile, 'w') filelist = glob.glob( '/t2k/dataquality/*/2012/*/*.done' ) for fdone in filelist: prestem = fdone.split(os.path.sep) week = prestem[5] stem = prestem[6].split(".") stemComponents = stem[0].split("_") if ( len(stemComponents) < 3 ) : print 'error: split file has too few comps, _done_ file ' + fdone +' '+ str(len(stemComponents)) + ' ' + stemComponents[0] print 'fdone: ',fdone,' ',week,' ',stem continue runNumber = int(stemComponents[1].strip()) subRunNumber = int(stemComponents[2].strip()) subDetector = str(stemComponents[0].strip()) if ( runNumber < runstart[indexrun] or runNumber > runend[indexrun] ): continue stringd = '%i %i %i %s %s %s ' %(beamRunNumber, runNumber, subRunNumber, subDetector, stem[0], week) if (subDetector != "dq-bc"): if (subDetector != "dq-tript-rmms"): if(subDetector != "dq-fgd-rd"): fo.write(stringd + ' root \n' + stringd + ' log \n' + stringd + ' sh \n') if(subDetector == "dq-tript-rmms" ): fo.write(stringd + ' dat \n' + stringd + ' log \n' + stringd + ' sh \n') if(subDetector == "dq-fgd-rd" ): fo.write(stringd + ' log \n' + stringd + ' sh \n') if (subDetector == "dq-bc" ): fo.write(stringd + ' log \n' + stringd + ' sh \n') rootstem = "dq-tpc-bc_" + stemComponents[1].strip() + "_" + stemComponents[2].strip() stringd = '%i %i %i %s %s %s ' %(beamRunNumber, runNumber, subRunNumber, "dq-tpc-bc", rootstem, week) fo.write(stringd + ' root \n') rootstem = "dq-fgd-bc_" + stemComponents[1].strip() + "_" + stemComponents[2].strip() stringd = '%i %i %i %s %s %s ' %(beamRunNumber, runNumber, subRunNumber, "dq-fgd-bc", rootstem, week) fo.write(stringd + ' root \n') if( runNumber > 6828 and runNumber < 7641): rootstem = "dq-ecal-mip_" + stemComponents[1].strip() + "_" + stemComponents[2].strip() stringd = '%i %i %i %s %s %s ' %(beamRunNumber, runNumber, subRunNumber, "dq-ecal-mip", rootstem, week) fo.write(stringd + ' root \n') fo.close() #print 'Created list of done files' # Now open the list doneRunFile = '/home/irodsdq/cronjobs/RUNsOnSemiOff_done.dat' fh = file(doneRunFile, 'r') hlines = fh.readlines() fh.close() ''' Print the list of files not copied into iRODS yet, as obtained by ils, and then remove the files that have been already copied into iRODS and put them into a new file ''' ilsworked = 0 string22 = "ils -rl /KEK-T2K/home/dataquality/data/ " status22, output22 = commands.getstatusoutput(string22) if ( status22 != 0 ): print 'status22 output22 : ' + status22 + ' ' + output22 ilsworked = 1 ilsfile = '' copiedFiles = [] skip = 0 for aline in output22.split('\n'): if (' C- ' in aline ): if (('/KEK-T2K/home/dataquality/data/summaryFiles/' in aline) or ('/KEK-T2K/home/dataquality/data/DPT/' in aline) ): skip = 1 else: skip = 0 if ( skip == 1 ): continue cpts = aline.split() if (len(cpts) > 5): if (cpts[-1] != ilsfile): ilsfile = cpts[-1] if ( 'root' in ilsfile or 'log' in ilsfile or 'log.gz' in ilsfile or 'sh' in ilsfile or 'dat' in ilsfile): stem = ilsfile.split(".") stemComponents = stem[0].split("_") if ( len(stemComponents) < 3 ) : print 'error: split file has too few comps, _done_ file ' + ilsfile +' '+ str(len(stemComponents)) + ' ' + stemComponents[0] continue runNumber = int(stemComponents[1].strip()) subRunNumber = int(stemComponents[2].strip()) subDetector = str(stemComponents[0].strip()) if ( runNumber < runstart[indexrun] or runNumber > runend[indexrun] ): continue stringd = '%i %i %i %s ' %(beamRunNumber, runNumber, subRunNumber, subDetector ) if ( 'root' in ilsfile ): copiedFiles.append(stringd + ' root ') if ( 'log' in ilsfile ): if(r.search(ilsfile) ): copiedFiles.append(stringd + ' log.gz ') else: copiedFiles.append(stringd + ' log ') if ( 'sh' in ilsfile ): copiedFiles.append(stringd + ' sh ') if ( 'dat' in ilsfile ): copiedFiles.append(stringd + ' dat ') if (ilsworked == 0 ): outFile = '/home/irodsdq/cronjobs/RUNsToCopy.txt' fout = file(outFile, 'w') for hline in hlines: ifound = 0 semioff = hline.split() for copiedFile in copiedFiles: irods = copiedFile.split() if ( semioff[0].strip() == irods[0].strip() and \ semioff[1].strip() == irods[1].strip() and \ semioff[2].strip() == irods[2].strip() and \ semioff[3].strip() == irods[3].strip() and \ semioff[6].strip() == irods[4].strip() ): ifound = 1 break if ( ifound == 0 ): fout.write(hline) #print 'Out of comparison list' fout.close() def doiPut(beamRunNumber, yearBeam): ''' Copy the files to be copied into iRODS as defined in the function runsToCopy ''' # Just opening a file to add the copied collections atime = time.time() # current time btime = time.localtime(atime) # convert current time into a tuple (y,m,d,h,m,s,timezone) ctime = time.strftime("%Y-%m-%d::%H:%M:%S", btime) # converting to human readable file copiedCollections = "/home/irodsdq/cronjobs/copiedFiles/RUNsOnSemiOff_iRODS_" + ctime + ".dat" fcopied = file(copiedCollections,"w") # Here start the real work of this function # Reading file created by runsToCopy with the list of files to copy runsToCopy = '/home/irodsdq/cronjobs/RUNsToCopy.txt' fh = file(runsToCopy, 'r') lines = fh.readlines() fh.close() # put the files that can be copied into iRODS subDetectorsOverall = ["p0d", "smrd", "dq", "ecal", "tpc", "fgd", "tript"] #subDetectors = ["dq-p0d-li", "dq-p0d-b", "dq-smrd-c", "dq-smrd-b", "dq-bc", "dq-tpc-bc", "dq-ecal-mip", "dq-ecal-b", "dq-ecal-s", "dq-ecal-stf", "dq-ecal-cmd", "dq-tpc-bc", "dq-fgd-bc", "dq-fgd-dn", "dq-fgd-rd" ,"dq-fgd-tm", "dq-fgd-ht", "dq-tript-rmms"] for line in lines[0:]: cpts = line.split() if ( len(cpts) < 7 ) : print 'error: split file has too few comps, _runs to copy_ file ' + cpts continue beamRunNumber = int(cpts[0]) runNumber = int(cpts[1].strip()) if ( runNumber < runstart[indexrun] or runNumber > runend[indexrun] ): continue subRunNumber = int(cpts[2].strip()) subDetector = str(cpts[3].strip()) fileType = cpts[6] stem = cpts[4] week = cpts[5] if ( subDetector == "dq-bc" ): subDetectorOverall = "dq" subDetectorOverallSoff = "log" if ( subDetector == "dq-tpc-bc" ): subDetectorOverall = "tpc" subDetectorOverallSoff = "tpc" if ( subDetector == "dq-fgd-bc" or subDetector == "dq-fgd-dn" or subDetector == "dq-fgd-tm" or subDetector == "dq-fgd-rd" or subDetector == "dq-fgd-ht"): subDetectorOverall = "fgd" subDetectorOverallSoff = "fgd" if ( subDetector == "dq-ecal-mip" or subDetector == "dq-ecal-b" or subDetector == "dq-ecal-s" or subDetector == "dq-ecal-stf" or subDetector == "dq-ecal-cmd" ): subDetectorOverall = "ecal" subDetectorOverallSoff = "ecal" if ( subDetector == "dq-p0d-li" or subDetector == "dq-p0d-b" ): subDetectorOverall = "p0d" subDetectorOverallSoff = "p0d" if ( subDetector == "dq-smrd-c" or subDetector == "dq-smrd-b"): subDetectorOverall = "smrd" subDetectorOverallSoff = "smrd" if ( subDetector == "dq-tript-rmms"): subDetectorOverall = "tript" subDetectorOverallSoff = "tript" fname = "/t2k/dataquality/" + subDetectorOverallSoff + "/" + str(yearBeam) + "/" + week + "/" + stem + "." + fileType fnameonly = stem + "." + fileType string0 = "ils /KEK-T2K/home/dataquality/data/" + subDetectorOverall + "/" + subDetector + \ "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" status0, output0 = commands.getstatusoutput(string0) if(status0 != 0): string1 = "imkdir -p /KEK-T2K/home/dataquality/data/" + subDetectorOverall + "/" + subDetector + \ "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" status1, output1 = commands.getstatusoutput(string1) if(status1 != 0): print string1,' ',status1,' ',output1 string2 = "iput " + fname + \ " /KEK-T2K/home/dataquality/data/" + \ subDetectorOverall + "/" + subDetector + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" status2, output2 = commands.getstatusoutput(string2) if(status2 != 0): print string2,' ',status2,' ',output2 string3 = "ils /KEK-T2K/home/dataquality/data/" + \ subDetectorOverall + "/" + subDetector + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnameonly status3, output3 = commands.getstatusoutput(string3) if( status3 != 0 ): print string3,' ',status3,' ',output3 continue # simply copy the list of copied files into a file stringCollection = str(beamRunNumber) + " " + str(runNumber) + " " + str(subRunNumber) + \ " " + subDetectorOverall + " " + subDetector + \ " /KEK-T2K/home/dataquality/data/" + subDetectorOverall + "/" + subDetector + "/" + \ str(beamRunNumber) + "/" + str(runNumber) + "/" + fnameonly atime = time.time() # current time btime = time.localtime(atime) # convert current time into a tuple (y,m,d,h,m,s,timezone) ctime = time.strftime("%Y-%m-%d::%H:%M:%S", btime) # converting to human readable file stringCollectionDone = stringCollection + " " + ctime + " " + str(status3) fcopied.write(stringCollectionDone + "\n") # write file with list of collections fcopied.close() ''' Start metadata ''' for line in lines[0:]: cpts = line.split() if ( len(cpts) < 7 ) : print 'error: split file has too few comps, _runs to copy_ file ' + cpts continue beamRunNumber = int(cpts[0]) runNumber = int(cpts[1].strip()) if ( runNumber < runstart[indexrun] or runNumber > runend[indexrun] ): continue subRunNumber = int(cpts[2].strip()) subDetector = str(cpts[3].strip()) fileType = cpts[6] stem = cpts[4] week = cpts[5] if ( subDetector == "dq-bc" ): subDetectorOverall = "dq" subDetectorOverallSoff = "log" if ( subDetector == "dq-tpc-bc" ): subDetectorOverall = "tpc" subDetectorOverallSoff = "tpc" if ( subDetector == "dq-fgd-bc" or subDetector == "dq-fgd-dn" or subDetector == "dq-fgd-tm" or subDetector == "dq-fgd-rd" or subDetector == "dq-fgd-ht"): subDetectorOverall = "fgd" subDetectorOverallSoff = "fgd" if ( subDetector == "dq-ecal-mip" or subDetector == "dq-ecal-b" or subDetector == "dq-ecal-s" or subDetector == "dq-ecal-stf" or subDetector == "dq-ecal-cmd" ): subDetectorOverall = "ecal" subDetectorOverallSoff = "ecal" if ( subDetector == "dq-p0d-li" or subDetector == "dq-p0d-b"): subDetectorOverall = "p0d" subDetectorOverallSoff = "p0d" if ( subDetector == "dq-smrd-c" or subDetector == "dq-smrd-b" ): subDetectorOverall = "smrd" subDetectorOverallSoff = "smrd" if ( subDetector == "dq-tript-rmms" ): subDetectorOverall = "tript" subDetectorOverallSoff = "tript" fname = "/t2k/dataquality/" + subDetectorOverallSoff + "/" + str(yearBeam) + "/" + week + "/" + stem + "." + fileType fnameonly = stem + "." + fileType if ( fileType == "log" ): fm = file(fname, 'r') mlines = fm.readlines() fm.close() for mline in mlines[0:]: if ( "CALIBOFFLINEROOT" in mline or "CALIBGLOBALROOT" in mline \ or "COMET_LEVEL" in mline \ or "OAFGDRAWDATAROOT" in mline \ or "RECONTPCROOT" in mline or "SIMECALUTILSROOT" in mline): envvar=mline.split("=") nameParam = envvar[0].strip() valueParam = envvar[1].strip() fnamelog = stem + ".log" stringlog = "imeta add -d /KEK-T2K/home/dataquality/data/" + subDetectorOverall + "/" + \ subDetector + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnamelog + " " + nameParam + " " + valueParam statuslog, outputlog = commands.getstatusoutput(stringlog) if(statuslog != 0): print stringlog,' ',statuslog,' ',outputlog fnamesh = stem + ".sh" stringsh = "imeta add -d /KEK-T2K/home/dataquality/data/" + subDetectorOverall + "/" + \ subDetector + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnamesh + " " + nameParam + " " + valueParam statussh, outputsh = commands.getstatusoutput(stringsh) if(statussh != 0): print stringsh,' ',statussh,' ',outputsh if (subDetector != "dq-bc" and subDetector != "dq-tript-rmms" and subDetector != "dq-fgd-rd"): fnameroot = stem + ".root" stringroot = "imeta add -d /KEK-T2K/home/dataquality/data/" + subDetectorOverall + "/" + \ subDetector + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnameroot + " " + nameParam + " " + valueParam statusroot, outputroot = commands.getstatusoutput(stringroot) if(statusroot != 0): print stringroot,' ',statusroot,' ',outputroot if (subDetector == "dq-bc" ): stemComponents = stem.split("_") if ( len(stemComponents) < 3 ) : print 'error: split file has too few comps, metadata ' + stem + ' ' + str(len(stemComponents)) + ' ' + stemComponents[0] continue fnameroot = "dq-tpc-bc_" + stemComponents[1].strip() + "_" + stemComponents[2].strip() + ".root" subDetectorRoot = "dq-tpc-bc" subDetectorOverallRoot = "tpc" stringroot = "imeta add -d /KEK-T2K/home/dataquality/data/" + subDetectorOverallRoot + "/" + \ subDetectorRoot + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnameroot + " " + nameParam + " " + valueParam statusroot, outputroot = commands.getstatusoutput(stringroot) if(statusroot != 0): print stringroot,' ',statusroot,' ',outputroot fnameroot = "dq-fgd-bc_" + stemComponents[1].strip() + "_" + stemComponents[2].strip() + ".root" subDetectorRoot = "dq-fgd-bc" subDetectorOverallRoot = "fgd" stringroot = "imeta add -d /KEK-T2K/home/dataquality/data/" + subDetectorOverallRoot + "/" + \ subDetectorRoot + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnameroot + " " + nameParam + " " + valueParam statusroot, outputroot = commands.getstatusoutput(stringroot) if(statusroot != 0): print stringroot,' ',statusroot,' ',outputroot if( runNumber > 6828 and runNumber < 7641): fnameroot = "dq-ecal-mip_" + stemComponents[1].strip() + "_" + stemComponents[2].strip() + ".root" subDetectorRoot = "dq-ecal-mip" subDetectorOverallRoot = "ecal" stringroot = "imeta add -d /KEK-T2K/home/dataquality/data/" + subDetectorOverallRoot + "/" + \ subDetectorRoot + "/" + str(beamRunNumber) + "/" + str(runNumber) + "/" + \ fnameroot + " " + nameParam + " " + valueParam statusroot, outputroot = commands.getstatusoutput(stringroot) if(statusroot != 0): print stringroot,' ',statusroot,' ',outputroot if __name__ == '__main__': # Define by hand the limits for the different neutrino run periods: beamRunNumber = 45 yearBeam = 2012 indexrun = beamRunNumber - 36 # RUN36: 6549 - 6967 # RUN37: 7397 - 7711 # RUN38: 7743 - 7754 # RUN39: 7755 - 7755 NOT USED # RUN40: 7997 - 8135 # RUN41: 8301 - 8549 # RUN42: 8550 - 8705 # RUN43: 8706 - 8783 # RUN44: 8844 - 9127 runstart = [6549, 7397, 7743, 7755, 7997, 8301, 8550, 8706, 8844, 9128] runend = [6967, 7711, 7754, 7755, 8135, 8549, 8705, 8783, 9127, 10000000000] # setup the environment variables os.environ["PATH"]="/home/irodsdq/irods/iRODS/clients/icommands/bin:"+os.environ["PATH"] os.environ["irodsEnvFile"]="/home/irodsdq/.irods/.irodsEnv" os.environ["irodsAuthFileName"]="/home/irodsdq/.irods/.irodsA" s0Time = time.time() # current time s1Time = time.localtime(s0Time) # convert current time into a tuple (y,m,d,h,m,s,timezone) s2Time = time.strftime("%Y-%m-%d::%H:%M:%S", s1Time) # converting to human readable file print 'starting cron job: ',s2Time runsToCopy(beamRunNumber, indexrun, runstart, runend) doiPut(beamRunNumber, yearBeam) e0Time = time.time() # current time e1Time = time.localtime(e0Time) # convert current time into a tuple (y,m,d,h,m,s,timezone) e2Time = time.strftime("%Y-%m-%d::%H:%M:%S", e1Time) # converting to human readable file print 'ending cron job: ',e2Time string0 = "iput -f /home/irodsdq/cronjobs/copyIntoIRODSCronJob.log /KEK-T2K/home/dataquality/bookkeeping/copyIntoIRODSCronJob.log " status0, output0 = commands.getstatusoutput(string0) if(status0 != 0): print string0,' ',status0,' ',output0