#!/usr/bin/env python from __future__ import print_function import os, sys, time import inspect, os.path import collections if True: if not 'ROOTSYS' in os.environ : print ("You do not have the ROOTSYS environment variable set" ) sys.exit() if not 'AADIR' in os.environ : print ("AADIR not set, using : ",) aadir = "/sps/km3net/users/heijboer/aanet" print (aadir) else : aadir = os.environ['AADIR'] print ("loading root.... ",os.environ['ROOTSYS']) sys.path.insert(0, os.environ['ROOTSYS']+'/lib') sys.path.insert(0, os.environ['ROOTSYS']+'/lib/root') import ROOT # do not pass command line options to root ROOT.PyConfig.IgnoreCommandLineOptions = True def loadlib(libfile) : fullpath = os.environ['AADIR']+"/lib/"+libfile print ("loading", fullpath, '...', end = "" ) sys.stdout.flush() t0 = time.time() r = ROOT.gSystem.Load( fullpath ) if r == 1 : print ("already loaded") return if r : print ("failed to load", fullpath, "error code =",r ) raise ImportError else : print (" ok (%3.2f s.)" % (time.time()-t0,) ) loadlib( 'libaaevt.so' ) loadlib( 'libaastro.so' ) print(" welcome to aanet", ROOT.get_git_version() ); # some symbols are now in namespace aa, but for bw compat, we need them in root for s in "water_index dndl c_light v_light cos_cherenkov_angle cherenkov_angle pi aaver".split() : ROOT.__dict__[ s ] = getattr( ROOT.aa, s ) # The print functions defined for the io classes defined in io_util.hh are # not used in interactive session, until io_uil.hh has been loaded. So # we trigger the loading of io_util.hh by referencing some random function # defined there. __dummy = ROOT.getclass_of_datamember # trigger loading of io_util.hh # The usual printing of objects via cling::PrintValue is leaking memory like mad, so let's try this: def cppprinter( obj ) : s = ROOT.ostringstream() obj.print(s) return s.str() for T in [ ROOT.Evt, ROOT.Hit, ROOT.Trk, ROOT.Head, ROOT.Vec, ROOT.Pmt, ROOT.Dom, ROOT.Det ] : T.__str__ = cppprinter def include(cpp_header_file): ROOT.gInterpreter.ProcessLine('#include "'+cpp_header_file+'"' ); def style(): ROOT.gStyle.SetOptStat(0) # ============================================= # start interactive python # ================================= def i() : os.environ['PYTHONINSPECT']="1" def ii() : i() raise EOFError # ================================================== # A few things that are simply missing from python # ================================================== import inspect, os def interpolate(template, **kwargs): if kwargs : return template.format(**kwargs) #http://stackoverflow.com/questions/4450592/string-interpolation-in-python calling_frame = inspect.currentframe().f_back names = dict( calling_frame.f_locals, **calling_frame.f_globals ) return interpolate( template, **names ) def run( string , **kwargs ): " version of os.system that does string interpolation " if not kwargs : calling_frame = inspect.currentframe().f_back kwargs = dict( calling_frame.f_locals, **calling_frame.f_globals ) return os.system( interpolate( string, **kwargs ) ); def autotype( obj ) : if obj=="False" : return False if obj=="True" : return True for T in (int, float, str) : try : return T(obj) except : pass return None def frange( start, stop, step): r = start while r < stop: yield r r += step def make_vector( L ): if type(L) != list or not L : return None T = type(L[0]) if T==float : T = ROOT.double r = ROOT.vector( T )() for elem in L : r.push_back( elem ) return r def __set( self, **kwargs ) : "Allows you do set a bunch of attributes using kwargs-style" for k,v in kwargs.items(): if not hasattr( self, k ) : print ("item",type(self),"has no attribute", k) raise AttributeError try : setattr( self, k, v ) except RuntimeError: print ("error in seting attribute", k , "to value ", v) raise return self # ============================================= # Pythonize Timer # ============================================= # def timer_enter(self) : # return self def timer_exit( self, type, value, traceback ): self.stop() ROOT.Timer.__exit__ = timer_exit # ============================================= # Pythonize memcounter # with get_mem_counter("some_name") : # checkme() # ============================================= def token_exit( self, type, value, traceback ): self.record() # or should i just call del? ROOT.MemCounter.Token.__exit__ = token_exit ROOT.MemCounter.Token.__enter__ = lambda s:s # ============================================= # Pythonize AAObject, EventFile, Table # ============================================= from . import pythonize_eventfile from . import pythonize_aaobject from . import pythonize_table del pythonize_aaobject del pythonize_eventfile del pythonize_table # ============================================= # Meta information # ============================================= def make_meta( options = None ) : import sys M = { "system" : "uname -a", "command" : " ".join(sys.argv), "application" : sys.argv[0].split("/")[-1], "GIT" : ROOT.get_git_version(), "ROOT" : ROOT.gROOT.GetVersion(), } if options : M["options"] = " ".join( [str(o) for o in options.parameters] ) s="" for k,v in M.items() : s += k +"="+v+"\n" return ROOT.TNamed( M['application'], s ) # ============================================= # Command line options # ============================================= import re class _Option( object ) : """ An option to use in the Options class. The idea is to have variables, with defaults and aliases (to allow short and long-form options) """ def __init__( self, S ) : # S is of the form # -f, -filename :\n description default=a.txt where the , and : are optional v = re.split("\n|:",S,1) ss = v[0].replace(","," ") self.desc, self.default = self._get_default_value( S ) self.value = self.default self.names = ss.split() self.type = type(self.default) self.source = "default" def _get_default_value( self, optionsline ): if not 'default' in optionsline : print (" No default value found in the line " , optionsline ) raise ValueError for f in ['default=','default ;','default:','default ='] : if not f in optionsline : continue return optionsline.split(f)[0], autotype( optionsline.split(f)[1].strip() ) print ("wrong format for default value", optionsline ) raise ValueError def hasname(self, name ): for n in self.names : if n == name or n.replace('-','') == name : return True return False def longname(self): return max(self.names, key = lambda s: len(s) ).replace("-","") def __str__(self) : return self.names[0] + "=" + str(self.value) + " ("+self.source+")" class Options( object ): """ The Options object collects user-settable options for python programs. The description of the options is human-readable and doubles as a man-page-like documentation. Recommended use is to make this the doc-string of the script. The Options objects will not allow to set attributes that are not defined as an option in the description -- this to hopefully catch typos early. """ def _parse_docstring( self, S ) : if 'options:' in S : S = S.split('options:')[1] import re m = re.split("\n\W*(-[a-zA-Z])", S ) # newline, followed by whitespace and "-x" (x=someletter) L = [ m[i] + m[i+1] for i in range( 1,len(m)-1,2) ] self.parameters = [ _Option(k) for k in L ] def __init__( self, desc, argv = None ): self._parse_docstring( desc ) self.__dict__['_desc'] = desc # the full docstring if argv : self.parse( argv ) self.process_standard_options() def __getattr__(self, name) : if name in self.__dict__ : return self.__dict__[name] for p in self.parameters : if p.hasname( name ): return p.value print ( "Options object has no attribute ", name ) raise AttributeError def __setattr__(self, name, value ): "Usually, only allow to set variables that are defined in the options" if name == "parameters" : self.__dict__["parameters"] = value #print ("xjxlj", self.__dict__); return self.__dict__["parameters"] p = self.get_par( name ) if p != None : p.value = value p.source = self._src return p.value # allow setting of (non-parameter) variables that are already # defined, AND allow the creation of new variables from within # this module (otherwise, we cannot set any attributes to self). if name not in self.__dict__ : print (" the variable ",name,"is not defined in the options") sys.exit() self.__dict__[name] = value return self.__dict__[name] def get_par( self, name ) : for p in self.parameters : if p.hasname( name ) : return p return None def parse( self, argv ): "Set options according to the command line." current = None D = self.__dict__ # print( "parsing", argv ) for o in argv : # things that look like negative numbers are values, not options if o.startswith("-") and o[1] not in "0123456789" : name = o.replace("-","") current = self.get_par( name ); if not current : print ("there is no option for:", name ) print (self) raise ValueError if current.value != current.default : print ("trying to set same options parameter more than once:", o ) raise ValueError else : if current.type == str : current.value = "" # so we can append else : current.value = None # it's comming soon # if bool, set to true (A falsy value may still follow) if current.type == bool : current.value = True current.source = "argv*" else : # o is a value if current == None : print ("error: expected an options starting with '-'' ", o ) raise ValueError try : # it must have the right type if current.type==str : if current.value != "" : current.value +=" " current.value += o else : if ( current.type == bool ) : if o.lower() in ['true','yes','1'] : current.value = True; elif o.lower() in ['false','0','no',""] : current.value = False else : print ("invalid value for Boolean option", " / ".join( current.names) ,":",o) print ("(should be True/true/False/false/1/0") raise ValueError else: # not a bool current.value = current.type( o ) current.source = "argv" except ValueError: print ("cannot convert",o, "to", current.type, "for option", current.names ) raise def process_standard_options( self ): try : if self.i : print ("set interactive") os.environ["PYTHONINSPECT"] = "1" except AttributeError : pass try : if self.h : print("help message:") print(self._desc) sys.exit() except AttributeError : pass def __str__( self ): from util.aautil import table T = table( "name(s) value source".split() ) for p in self.parameters : T.append( [" ".join(p.names), str(p.value), p.source ] ) return T.__str__() def check_exe( fname ): import ast docstring = None with open(fname, 'r') as f: tree = ast.parse(f.read()) docstring = ast.get_docstring(tree) return docstring import inspect, os.path class Job( object ) : """ The idea of a Job object is that you make one at when you submit a job and it will also be available when the job executes on the farm. Just set attributes at summission time, and call Job.submit. The values of the attributes will be transfereed to the object on the batch farm via environment variables """ def __init__( self, name = "aa" ) : self.jobscript = os.path.abspath(inspect.stack()[1][1]) self.jobname = name self.logfile = self.jobname+".log" self.verbose = True self.dry = 'dry' in sys.argv self.batch = '__AABATCH__' in os.environ self.cmd = 'qsub -notify -P P_antares -V -l sps=1 -l h_rss=8G' self.cmd += ' -N $jobname -o $logfile -j y $jobscript' self.internals = list( self.__dict__.keys()) + ['internals'] self.read_env(); def clear_env() : for k in list(os.environ.keys()) : if ( k.startswith( "AA__") ): del os.environ[k] def write_env( self ): Job.clear_env() D = { 'AA__'+k:str(v) for k,v in self.__dict__.items() if k not in self.internals } D = Job.split_long_variables( D ) os.environ.update( D ) def read_env( self ): #os.system("env") # almost always useful print ("reading...") D = { k.replace("AA__","",1):v for k,v in os.environ.items() if k.startswith("AA__") } print(D) D = Job.join_long_variables( D ) for k,v in D.items() : print (k,v ) self.__dict__.update( { k:autotype(v) for k,v in D.items() } ) def split_long_variables( D ) : """ split up environment variables that are very long strings to work around an apparent limitation of the batch queue system. """ def chop( s , maxlen = 400 ) : return [ s[i:i+maxlen] for i in range(0,len(s),maxlen) ] r = {} for name,v in D.items(): vv = chop(v) if len(vv)==1 : r[name] = vv[0] continue for i,vi in enumerate( vv ) : r[ name+"__%03d"%(i,) ] = vi return r def join_long_variables ( D ): ''' variables of the form AA__XXXX__NNN (NNN=number) are appended, in order, to AA__XXXX. returns a dict with name,value ''' DD = collections.defaultdict( dict ) for name,value in D.items(): if "__" in name : name, index = name.rsplit("__",1) index = int(index) else : name, index = name, -999 DD[name][index] = value def concat( dd ) : keys = sorted(dd.keys()) print (name, "joining", keys ) for k in keys : print (" ", k, dd[k]) return "".join( dd[k] for k in keys ) r = {} for name,v in DD.items(): r[name] = concat( v ); return r def submit( self ): self.write_env(); os.environ['__AABATCH__'] = "yes" for k,v in os.environ.items() : print(k,v) mycmd = self.cmd[:] if 'checkenv' in sys.argv : # thest that env variables get read in right j = Job("test") print(j) for f in j.infile.split() : print (f) if not os.path.exists( f ): print ("does not exist") sys.exit() return vars = [ x for x in mycmd.split() if x.startswith('$') ] for x in vars : mycmd = mycmd.replace( x, self.__dict__[ x[1:] ] ) print (mycmd) if not self.dry : os.system(mycmd) del os.environ['__AABATCH__'] def __str__( self ) : s = "Job with name " + self.jobname +"\n" for k,v in self.__dict__.items() : s += "%-10s"%(k,) + " : " + str(v) + "\n" return s # class WebFile( object ): # def __init__( self , # title = "some plots", # directory = "/afs/in2p3.fr/home/h/heijboer/www/rec/", # filename = "index.html", # imgdir = "imgdir" ) : # self.__dict__.update( locals() ) # if not os.path.exists(directory): # os.makedirs(directory) # if not os.path.exists(directory+"/"+imgdir): # os.makedirs(directory+"/"+imgdir) # self.filename = filename # self.directory = directory # self.content = "" # def __del__( self ) : # self.write() # def write(self) : # f = open( self.directory + "/" + self.filename , 'w' ) # print( self.content ) # f.write( self.content ) # del f # def __str__( self ) : # return str(self.__dict__) # def _add_header( self ): # s = """ #