from __future__ import with_statement import os,sys,copy,shutil import common,inout from xml.etree import ElementTree as ET from program import program from data import data_container try: import crvapi except: sys.exc_clear() crvapi = False class process(object): """ Base class for processes such as model building, density modification etc""" name="base class for process" references=() # if no_reporting then no information about the process will be printed in log/loggraph no_reporting=False # debug=0: no intermediate files are kept # 1: intermediate working files are kept # 2: all intermediate files are kept, incl. all the temporary scripts # THE DEBUG OPTION IS NOT IMPLEMENTED YET! debug=0 # list of supported programs (names) supported_progs = [] # list of supported processes (names) supported_procs = [] # list of supported (virtual) parameters supported_params = {} supported_params['no_output_to_next_step'] = common.parameter( desc='Disables propagation of output objects to the next step', typ=bool ) # log filename extension log_suffix='.log' # filehandle of the logfile - self.Info() will send output to this filehandle if not None logfilehandle=None # has *this process* opened logfilehandle or not (even if not opened_log, logfilehandle may be still opened eg by a parent process) opened_log=False opened_loggraph=False # rvapi report (for crank) or section (for other processes) rv_report=None def __init__(self, xmlelem=None, inpline=None, parent_process=None, rundir=None, dummy=False, no_support_check=False): self.nick = self.__class__.__name__ self.inp = inout.input_output(is_output=False,parent=self) self.out = inout.input_output(is_output=True,parent=self) self.rundir=rundir self.runname=self.nick # actual dictionary of all parameters self.param = {} # parent process using this process self.parent_process = parent_process # list of program objects used by the process self.programs = [] # list of process objects used by the process self.processes = [] self.dummy=dummy self.init_xmlelem = None # initialize by elementree if xmlelem is not None: self.XMLElemInit(xmlelem,dummy,no_support_check) elif inpline is not None: self.InputElemInit(inpline,dummy) if not dummy: self.Init() def Init(self): """Custom initialization of the process can be placed here""" pass def PrintParams(self): """Prints parameters of the process and their description.""" if self.supported_params: common.Info('The following parameters are defined for process {0}:'.format(self.nick)) for key,par in self.supported_params.iteritems(): common.Info(' {0:20} {1} (type {2})'.format(key, par.description, par.typ[0].__name__)) else: common.Info('No parameters exist for the process.') @classmethod def from_xml(cls, xml, parent_process=None, rundir=None, dummy=False, no_support_check=False): """Create process instance from XML file or XML tree object""" if isinstance(xml, file) or isinstance(xml, basestring): xml = common.ReadXML(xml) inst = cls( xml, None, parent_process, rundir, dummy=dummy, no_support_check=no_support_check ) return inst @classmethod def from_name(cls, procnick, parent, xmlelem=None, inpline=None, dummy=False, no_support_check=False): """Create specific (derived) process instance from the (nick)name of the process Parameters: procnick - the (nickname) process whose instance will be created parent - the parent process xmlelem - if specified then initialization is performed using this xml element inpline - if specified then initialization is performed using this input line (preprocessed list assumed) """ if procnick=='crank': from manager import crank return crank(xmlelem,inpline,parent,dummy=dummy) try: # import the specific process setattr(sys.modules[__name__], 'processes', __import__('processes.'+procnick)) except (AttributeError,ImportError): common.Error('Process {0} not supported (check the spelling)'.format(procnick)) try: # create an instance of the process inst = getattr(sys.modules['processes.'+procnick], procnick)(xmlelem,inpline,parent,dummy=dummy,no_support_check=no_support_check) except (AttributeError,KeyError): common.Error('Error when creating process {0} instance.'.format(procnick)) else: return inst def XMLElemInit(self, xelem, dummy=False, no_support_check=False): """Initialize this process from the inputted XML element""" param_set_by_crank=[] tags=["inp","out","param","program","process","param_set_by_crank"] for xchild in xelem: if xchild.tag in tags: if xchild.tag=="param_set_by_crank": param_set_by_crank = list(xchild.text) elif xchild.tag=="inp": self.inp.XMLElemInit(xchild,dummy=dummy) elif xchild.tag=="out": self.out.XMLElemInit(xchild,dummy=dummy) elif xchild.tag=="param": for xparam in list(xchild): input_by_user = not xparam.tag in param_set_by_crank self.SetParam( xparam.tag, common.AutoConvert(xparam.text), inputted=input_by_user ) elif xchild.tag=="program": pr_name=xchild.text.strip() if not pr_name in self.supported_progs and not no_support_check: common.Error('The program {0} is not supported by {1}.'.format(pr_name,self.name)) else: self.AddProg(pr_name,xchild,dummy=dummy) elif xchild.tag=="process": pr_name=xchild.text.strip() if not pr_name in self.supported_procs and not no_support_check: common.Error('The process {0} is not supported by {1}.'.format(pr_name,self.name)) else: self.AddProcess(pr_name,xchild,dummy=dummy,no_support_check=no_support_check) else: common.Error('Wrong tag when parsing XML for process {0}: {1}'.format(self.name,xchild.tag)) self.init_xmlelem = xelem def InputElemInit(self, input_line, dummy=False): """Initialize this process by parsing the input line Initializes until a token that cannot be used; consumes the tokens that were used """ first_proc = None while input_line: token=input_line[0] input_line.remove(token) if token in self.supported_procs: # special treatment needed to disable possibility of multiple crank subprocesses defined in one line by mistake if not first_proc: first_proc=token elif self.nick=='crank': common.Warning('Process {0} not supported by {1}'.format(token,first_proc)) input_line.insert(0,token) break pr=self.AddProcess(token, inpline=input_line, dummy=dummy) elif token in self.supported_progs: self.AddProg(token, inpline=input_line, dummy=dummy) elif token in [ sc.__name__ for sc in data_container.__subclasses__() ]: cont=self.inp.AddCopy( data_container.from_name(token, inpline=input_line), propagate=not dummy ) self.CheckInpContFile(cont) elif '::' in token: key,sep,val = token.partition('::') if val: self.SetParam( key, common.AutoConvert(val), inputted=True ) else: input_line.insert(0,token) break def CheckInpContFile(self,cont): #global hklin,seqin,xyzin # if file was not supplied then the previous mtz or hklin,seqin,xyzin are assumed if not cont.GetFile() and cont.GetAllLabels(): if hasattr(self,'hklin'): cont.AddFile(self.hklin, self.hklin.split('.')[-1]) else: common.Error("No data file associated with object {0} of type {1}".format(cont.nick,cont.typ)) if not cont.GetFile() and cont.nick=='sequence': if hasattr(self,'seqin'): cont.AddFile(self.seqin,self.seqin.split('.')[-1]) if not cont.GetFile() and cont.nick=='model' and cont.typ=='substr' and hasattr(self,'xyzin'): cont.AddFile(self.xyzin,self.xyzin.split('.')[-1]) # save hklin file if cont.GetFile() and cont.GetAllLabels(): self.hklin=cont.GetFileName() # copy cell/spgr from plus to minus import data if isinstance(cont,data.fsigf) and cont.GetType()=='minus' and \ self.inp.Get('fsigf',typ='plus',xname=cont.xname,dname=cont.dname): cont.spgr=self.inp.Get(typ='plus',xname=cont.xname,dname=cont.dname).spgr cont.cell=self.inp.Get(typ='plus',xname=cont.xname,dname=cont.dname).cell # create minus by copy if fsigf "anom" type was used if hasattr(cont,'minus_needs_to_be_added'): self.inp.AddCopy(cont,allow_duplicates=True).SetType('minus') del cont.minus_needs_to_be_added def Data2XML(self, ET_element=None): """Convert the actual process object into XML""" if ET_element is None: xroot = ET.Element('process') else: xroot = ET.SubElement(ET_element,'process') xroot.text=self.__class__.__name__ if self.param: param_set_by_crank = [ p for p in self.param if self.IsParam(p) and not self.IsInputtedParam(p) ] if param_set_by_crank: ET.SubElement(xroot,'param_set_by_crank').text = str(param_set_by_crank) xparam = ET.SubElement(xroot,'param') for key,par in self.param.iteritems(): ET.SubElement(xparam, key).text = str(par.value[0]) for obj in [self.inp, self.out] + self.processes + self.programs: obj.Data2XML(xroot) return xroot def GetCCP4Header(self): # the banner functionality is not (yet?) provided either by a ccp4 python module or # a standalone binary afaik, so this is a reimplementation # (calling static C libccp from python would be too much trouble for this single purpose) version_file = os.path.join(os.path.dirname(common.__file__), 'VERSION') with open(version_file) as f: version = f.readline() import datetime,getpass mdate=datetime.datetime.fromtimestamp( os.path.getmtime(sys.modules[self.__class__.__module__].__file__) ) mdate=str(mdate.day)+'/'+str(mdate.month)+'/'+str(mdate.year)[-2:] name='crank2.'+self.nick if self.nick!='crank' else 'crank2' rundate=datetime.datetime.today() runtime=str(rundate.hour)+':'+str(rundate.minute)+':'+str(rundate.second) rundate=str(rundate.day)+'/'+str(rundate.month)+'/'+str(rundate.year) bars=' ###############################################################\n' proc_info=' ### CCP4 {0:3}: {1:21} version {2:7}: {3:8}##\n'.format( self.GetCrankParent().GetCCP4Version(), name, version, mdate) run_info=' User: {0} Run date: {1} Run time: {2}\n'.format(getpass.getuser(),rundate,runtime) header=bars+bars+bars+proc_info+bars+run_info+'\n' return header def Info(self, message, eol=True, stdout=True): common.Info(message, self.logfilehandle, eol, also_stdout=stdout) def LGInfo(self, message, eol=True, stdout=False): if self.GetCrankParent(): common.Info(message, self.GetLogGraphHandle(), eol, also_stdout=stdout) else: common.Warning('Could not print loggraph: crank parent does not exist.') def GetSupportedParam(self,par,ignore_error=False): # checks whether inputted par is amongst supported and completes it if necessary and possible # if ignore_error is True then gracefully returns None on issue, otherwise prints the error if par in self.supported_params: return par if par in process.supported_params: self.supported_params[par] = process.supported_params[par] return par # complete or return error message sp_lst = [ sp for sp in self.supported_params if sp.startswith(par) ] if ignore_error and (not sp_lst or len(sp_lst)>1): return None if not sp_lst: common.Error('No such parameter {0} supported by process of {1}'.format(par,self.name)) if len(sp_lst)>1: common.Error('Multiple choices for parameter {0} by process of {1}: {2}'.format( par,self.name,', '.join(sp_lst))) return sp_lst[0] def IsInstance(self,value,typ): # somewhat "corrected" isinstance() - bool!=int if typ==bool and type(value)==int: return False if typ==int and type(value)==bool: return False return isinstance(value,typ) def CheckType(self,par,value): # checks type of the parameter to be set - called by SetParam() if not [ typ for typ in self.supported_params[par].typ if self.IsInstance(value,typ) ]: # we do not want to allow all conversions. just those specified here. # for example, I consider str an incorrect input for bool or bool an incorrect input for int. if bool in self.supported_params[par].typ and self.IsInstance(value,int): value=bool(value) elif float in self.supported_params[par].typ and self.IsInstance(value,int): value=float(value) else: common.Error( 'Wrong type {0} of value {1} for parameter {2}: must be {3}'.format( type(value).__name__, value, par, ' or '.join(t.__name__ for t in self.supported_params[par].typ)) ) def SetParam(self, par, value=True, no_type_check=False, inputted=False): # the same as self.SetVirtPar() par = self.GetSupportedParam(par) if not no_type_check: self.CheckType(par,value) if par not in self.param: self.param[par] = copy.deepcopy(self.supported_params[par]) self.param[par].Set(value,is_key=False,is_arg=False,append=False,inputted=inputted) # shares the parameter par with children of this process having the same parameter supported # if the par has a shared_with_children attribute (recursive) if self.param[par].shared_with_children: for p in self.GetProcesses(): if par in p.supported_params and not p.IsInputtedParam(par): p.SetParam(par, self.GetParam(par), inputted=inputted) def SetVirtPar(self, par, value=True): """Set program parameter 'par' to inputted value (only 'virtual' parameters allowed for process)""" self.SetParam(par,value) def GetParam(self, par, capitalized=None, ignore_error=False): # the same as self.GetVirtPar() par = self.GetSupportedParam(par,ignore_error) if par and par in self.param: self.param[par].accessed=1 return self.param[par].Get(capitalized=capitalized) else: return None def GetVirtPar(self, par): """Returns the value of parameter 'par', None if not defined""" return self.GetParam(par) def IsVirtPar(self,par): return self.IsParam(par) # is param defined? def IsParam(self,par): if par in self.param and self.GetParam(par) is not [] and self.GetParam(par) is not None: return True else: return False # was param inputted by user? def IsInputtedParam(self,par,test_parents=False): if self.IsParam(par) and self.param[par].inputted: return True else: if not test_parents or not self.parent_process or not self.parent_process.IsParam(par): return False else: return self.parent_process.IsInputtedParam(par,test_parents=True) def IsNonFalseVirtPar(self,par): return self.IsNonFalseParam(par) # is param defined and non-False? def IsNonFalseParam(self,par): if self.IsParam(par) and self.param[par] is not False: return True else: return False def IsTrueOrNoneVirtPar(self,par): return self.IsTrueOrNoneParam(par) # is param not defined or None or True? (usually used to test whether program defaults should be redefined - if False then orig.prog.def. are forced) def IsTrueOrNoneParam(self,par): if par not in self.param or self.GetParam(par) is True or self.GetParam(par) is None: return True else: return False def CapitalizeVirtPar(self, par): """Capitalize value of the inputted virtual parameter (if existing and if possible)""" self.CapitalizeParam(par) def CapitalizeParam(self, par): # Capitalize value of the inputted parameter (if existing and if possible) par = self.GetSupportedParam(par) self.SetParam( par, self.GetParam(par,capitalized=True) ) def TreatInOutPar(self,set_all_par=False): """processes the actual process input,output and/or parameters""" if set_all_par: for par in self.supported_params: if par not in self.param: self.SetParam(par, None) # accessing the parameter - the warning is not meant to be printed for this parameter. self.GetParam('no_output_to_next_step') def CheckParamAccess(self): """checks whether all the inputted 'virtual' parameters have been evaluated""" for parname,parinst in self.param.iteritems(): if not parinst.accessed: common.Warning('Virtual parameter {0} has not been used by process of {1}.'.format(parname,self.name)) def GetProgs(self,name=None,supported=False): """returns the list of programs (of this process) with specified (nick)name (empty list if there are none)""" return [ p for p in self.programs if (p.nick==name or name is None) and (not supported or p.nick in self.supported_progs) ] def GetProg(self,name=None,supported=False): """returns the first program (of this process) with specified name (None if there is none)""" try: return self.GetProgs(name,supported)[0] except IndexError: sys.exc_clear() return None def GetOrAddProg(self,name): """returns the first program object with the specified name or creates a new one if none found""" prog = self.GetProg(name) if prog is None: prog = self.AddProg(name) return prog def GetProcesses(self,name=None,supported=False): """returns the list of processes (of this process) with specified (nick)name(s) (empty list if there are none)""" if isinstance(name,basestring) or name is None: name=[name,] return [ p for p in self.processes for n in name if (p.nick==n or n is None) and (not supported or p.nick in self.supported_procs) ] def GetProcess(self,name=None,supported=False): """returns the first process (of this process) with specified name (None if there is none)""" try: return self.GetProcesses(name,supported)[0] except IndexError: sys.exc_clear() return None def GetOrAddProcess(self,name): """returns the first subprocess with the specified name or creates a new one if none found""" proc = self.GetProcess(name) if proc is None: proc = self.AddProcess(name) return proc def GetFlatSubTree(self,lst=None): if lst is None: lst=[] lst.append(self) for prog in self.programs: prog.GetFlatSubTree(lst) for proc in self.processes: proc.GetFlatSubTree(lst) return lst def AddProg(self, name, xmlelem=None, inpline=None, ind=None, propagate_inp=True, propagate_out=True, dummy=False): """Adds a new program to this process. Parameters: name - (nick)name of the program to be added xmlelem - xml element used for initialization of the added program object (None by default) inpline - input line (preprocessed list) used for initialization of the program (None by default) ind - add to this index in the programs list of this process (after the last by default) propagate_inp - if True then the added program's input is propagated from the self process (default) if False then nothing is propagated to the input propagate_out - if True then the output will be propagated (default) if False then the output is flagged as never_propagate dummy - create a 'dummy' object - without its specific initialization and turns propagate to False """ if dummy: propagate_inp=False propagate_out=False if ind is None: ind=len(self.programs) self.programs.insert(ind, program.from_name(name,self,xmlelem,inpline,dummy)) if propagate_inp: for container in self.inp.GetAll(): # propagating to the beginning of the list so that the program's own objects have priority self.programs[ind].inp.Add(container,ind=0) if not propagate_out: self.programs[ind].out.never_propagate=True if self.programs[ind].always_rundir: self.programs[ind].SetRunDir(error_on_failure=False) return self.programs[ind] def AddProcess(self, name, xmlelem=None, inpline=None, ind=None, propagate_inp=True, propagate_out=True, dummy=False, no_support_check=False): """Adds a new subprocess to this process. The same parameters as AddProg().""" if dummy: propagate_inp=False propagate_out=False if ind is None: ind=len(self.processes) self.processes.insert(ind, process.from_name(name,self,xmlelem,inpline,dummy,no_support_check)) if propagate_inp: for container in self.inp.GetAll(): # propagating to the beginning of the list so that the process's own objects have priority self.processes[ind].inp.Add(container,ind=0) if not propagate_out: self.processes[ind].out.never_propagate=True # share the value of the parameters with .shared_with_children attribute with the added child for par in self.processes[ind].supported_params: if self.IsParam(par) and self.param[par].shared_with_children and not self.processes[ind].IsInputtedParam(par): self.processes[ind].SetParam(par, self.GetParam(par)) return self.processes[ind] def AddProgCopy(self, prog, ind=None, propagate_inp=False, deeper_copy=False): """Adds a copy of the inputted prog program object to this process. No input objects are propagated to the added program from the parent process by default. The output propagation is the same as that of the program that was copied. Parameters: prog - the program object a copy of which will be added to this process ind - add to this index in the programs list of this process (after the last by default) propagate_inp - if True then the added program's input is propagated from the self process if False then nothing is propagated to the input (default) deeper_copy: if True then the input/output and args/keys are copied if False then the original proc's i/o,args/keys objects are kept (ie shared with prog) (False by default) """ assert isinstance(prog, program) if ind is None: ind=len(self.programs) self.programs.insert( ind, copy.copy(prog) ) self.programs[ind].process=self if deeper_copy: self.programs[ind].inp, self.programs[ind].out = copy.copy(prog.inp), copy.copy(prog.out) self.programs[ind].inp.parent = self.programs[ind].out.parent = self.programs[ind] self.programs[ind].param = copy.copy(prog.param) self.programs[ind].key, self.programs[ind].arg = copy.copy(prog.key), copy.copy(prog.arg) self.programs[ind].key_list = copy.copy(prog.key_list) self.programs[ind].arg_list = copy.copy(prog.arg_list) if propagate_inp: for container in self.inp.GetAll(): # propagating to the beginning of the list so that the process's own objects have priority self.programs[ind].inp.Add(container,ind=0) return self.programs[ind] def AddProcessCopy(self, proc, ind=None, propagate_inp=False, deeper_copy=False): """Adds a copy of the proc process object to this process. The same parameters as AddProgCopy() + deeper_copy: if True then all the subprocesses/subprograms,input/output,param are copied if False then the original proc's subprocesses/subprograms,i/o,par are kept (ie shared with proc) (False by default) """ assert isinstance(proc, process) if ind is None: ind=len(self.processes) self.processes.insert( ind, copy.copy(proc) ) self.processes[ind].parent_process=self if deeper_copy: self.processes[ind].inp = copy.copy(proc.inp) self.processes[ind].out = copy.copy(proc.out) self.processes[ind].inp.parent = self.processes[ind].out.parent = self.processes[ind] self.processes[ind].param = copy.copy(proc.param) self.processes[ind].processes, self.processes[ind].programs = [], [] for pr in proc.processes: self.processes[ind].AddProcessCopy(pr, deeper_copy=True) for pr in proc.programs: self.processes[ind].AddProgCopy(pr, deeper_copy=True) if propagate_inp: for container in self.inp.GetAll(): # propagating to the beginning of the list so that the process's own objects have priority self.processes[ind].inp.Add(container,ind=0) return self.processes[ind] def GetProcessCopy(self): # returns a copy of this process by conversion to and from xml. # does not copy all the meta-attributes: use with care! proc_copy = self.from_xml(self.Data2XML(), no_support_check=True) proc_copy.rundir = self.rundir proc_copy.logfilehandle = self.logfilehandle if hasattr(self,'run'): proc_copy.run = self.run.GetProcessCopy() #proc_copy = copy.copy(self) #proc_copy.param = {} #for key,par in self.param.iteritems(): # proc_copy.param[key] = par #proc_copy.processes=[] #for proc in self.processes: # proc_copy.processes.append( proc.GetProcessCopy() ) #proc_copy.programs=[] #for prog in self.programs: # proc_copy.programs.append( prog.GetProgramCopy() ) return proc_copy def SetRunDir(self,rundir=None): self.previous_cwd=os.getcwd() if rundir is not None: self.rundir=rundir else: if self.rundir is None: if self.parent_process and self.parent_process.rundir: self.rundir=self.parent_process.rundir if len(self.parent_process.processes+self.parent_process.programs)>1: self.rundir=os.path.join(self.rundir,self.nick) else: self.rundir=os.getcwd() if not os.path.isdir(self.rundir): os.mkdir(self.rundir) if not os.path.isabs(self.rundir): self.rundir=os.path.abspath(self.rundir) os.chdir(self.rundir) def GetLogFileName(self): verb = '' if self.nick!='crank' else '_verb' return os.path.join(self.rundir, self.runname + verb + self.log_suffix) def GetLogGraphFileName(self, actual=False): """Returns the loggraph filename for this process. For crank process, this is the 'merge' loggraph filename; otherwise the 'actual' loggraph filename. Returns None if crank parent does not exist or loggraph disabled by user using --lgout None If the parameter 'actual' is True then the actual filename in crank directory is retuned (used to be default with crank2<2.0.10 and still used for gui2 as symlink) """ crank = self.GetCrankParent() if crank: if actual: lgdir, lgname, lgsuffix = crank.rundir, crank.runname, '.loggraph' else: lgdir, lgname, lgsuffix = self.rundir, self.runname, '.loggraph' if hasattr(self,'lgout') or (hasattr(crank,'lgout') and actual): if crank.lgout == 'None': if self.nick=='crank': common.Info('Loggraph output disabled.'.format(lgdir)) return None lgdir=os.path.dirname(crank.lgout) lgname,lgsuffix=os.path.splitext(os.path.basename(crank.lgout)) actual_str='_act' if self.nick=='crank' or not actual: actual_str='' return os.path.join(lgdir, lgname + actual_str + lgsuffix) else: return None def GetLogGraphHandle(self): """Gets the loggraph handle of this process. For crank process, this is the 'merge' loggraphfile handle; otherwise the 'actual' handle. """ if self.nick=='crank': return self.loggraphfilehandle else: return self.GetCrankParent().loggraph_actfilehandle def SetLogGraphHandle(self, handle): """Sets the loggraph handle of this process to the inputted handle. For crank process, sets the 'merge' loggraphfile handle; otherwise the 'actual' handle """ if self.nick=='crank': self.loggraphfilehandle = handle else: self.GetCrankParent().loggraph_actfilehandle = handle def GetLogGraphReferences(self, crank_only=True, recursive=True, rvapi=False, bib_to_file=False): """Returns list of references strings for the current process if rvapi Returns a loggraph formatted string of references if not rvapi Returns empty list/string if no references are defined. Arguments: crank_only - if True then returns empty list/string for any other processes than crank (default True) recursive - if true then searches all the subprocesses recursively for references as well """ if crank_only and self.nick!='crank': return '' if not rvapi else [] try: instance=self.emulate.run except: instance=self pros = [instance,] + instance.programs if recursive: pros = instance.GetAllSubPrograms() + instance.GetAllSubProcesses() # the SHELX reference is currently the one of SHELXE if self.GetProcess('phdmmb'): pros.insert(0,program.from_name('shelxe',None)) #section.Text(' '+program.from_name('shelxe',None).references[0]) refs = [ p.references[0] for p in pros if p.references ] # make unique refs = [ r for i,r in enumerate(refs) if refs.index(r)==i ] if refs and bib_to_file and self.GetCrankParent(): ref_dir = os.path.join(os.path.dirname(__file__),'programs','references') done=[] with open(os.path.join(self.GetCrankParent().rundir,'crank2.bibtex.txt'),'w' if pros[0].nick=='crank' else 'a') as g1: with open(os.path.join(self.GetCrankParent().rundir,'crank2.medline.txt'),'w' if pros[0].nick=='crank' else 'a') as g2: for p in pros: if os.path.isfile(os.path.join(ref_dir,p.nick+'.bib')): with open(os.path.join(ref_dir,p.nick+'.bib')) as f: ref_bib = f.read() if not ref_bib or ref_bib in done: continue done.append( ref_bib ) g1.write( ref_bib+'\n' ) if os.path.isfile(os.path.join(ref_dir,p.nick+'.nbib')): with open(os.path.join(ref_dir,p.nick+'.nbib')) as f: g2.write( f.read()+'\n' ) if rvapi: return refs else: return '$TEXT:Reference: $$ Please reference $$\n{0}\n$$\n'.format('\n'.join(refs)) def GetLogGraphPrograms(self,rvapi=False): """Returns list of program (nickname) strings used by the process if rvapi Returns loggraph formatted string containing information about programs used by the process.""" try: programs=self.emulate.run.GetAllSubPrograms() except: programs=self.GetAllSubPrograms() # make list of unique program names programs = [p.name for p in programs] programs = [p for i,p in enumerate(programs) if programs.index(p)==i] if 'sftools' in programs: programs.remove('sftools') if rvapi: return programs else: if not programs: return '' return '$TEXT:Programs used: $$ Programs used in the run $$\n{0}\n$$\n'.format(', '.join(programs)) def GetAllSubPrograms(self, programs=None): """Returns list of all (currently defined) programs of this process or its subprocesses (recursively)""" if programs is None: programs=[] programs.extend(self.programs) for proc in self.processes: proc.GetAllSubPrograms(programs) return programs def GetAllSubProcesses(self, processes=None): """Returns list of all (currently defined) subprocesses of this process or its subprocesses (recursively)""" if processes is None: processes=[] processes.extend(self.processes) for proc in self.processes: proc.GetAllSubProcesses(processes) return processes def BackupAnyPars(self): # backups all actual parameters into private tmp arrays self.partmp = copy.deepcopy(self.param) return self.partmp class RestoreParamsError(Exception): pass def RestoreAnyPars(self,par=None): # restores the previously backuped parameters from private tmp arrays if par: self.param = copy.deepcopy(par) else: try: self.param = copy.deepcopy(self.partmp) except AttributeError: raise self.RestoreParamsError('Restoring backup parameters failed, backup does not exist.') def GetNumParents(self): # returns the number of parents of the process (the length of the shortest path to the top of the tree) numpar=0 proc=self.parent_process while proc: proc=proc.parent_process numpar+=1 return numpar def GetCrankParent(self,prep=None): # returns the (main) crank process proc=self while proc: proc_save=proc proc=proc.parent_process if proc_save.nick=='crank': if prep and hasattr(proc_save,'prep'): return proc_save.prep else: return proc_save return None def RunPreprocess(self, rundir=None, clear_out=True, save=True, *args, **kwargs): """Any preprocessing before the actual run of the process (including TreatInOutPar)""" setallpar = kwargs.pop('set_all_par', False) from_ccp4i2 = kwargs.pop('from_ccp4i2', False) emulate = kwargs.pop('emulate', '') info = kwargs.pop('info', '') global crvapi crvapi = kwargs.pop('crvapi', crvapi) self.SetRunDir(rundir) if self.GetCrankParent() and self.GetCrankParent().logfilehandle: self.GetCrankParent(prep=True).Info('{0}Starting process of {1} {2}'.format(' '*self.GetNumParents(), self.name, info), stdout=False) if not emulate and self.nick=='crank': self.CheckBinaries() if clear_out: self.out.ClearAll(propagate=False) if save: self.partmp=copy.copy(self.param) if self.nick!='crank': self.TreatInOutPar(set_all_par=setallpar) if not self.no_reporting: if self.logfilehandle is None: self.logfilehandle=open(self.GetLogFileName(),'a',0) self.opened_log=True if from_ccp4i2 and os.path.isfile(self.GetLogFileName()): # create symlinks to the logfile - as of now required by ccp4i2 to register the logfile... common.SymLink(self.GetLogFileName(), os.path.join(self.rundir, 'log.txt')) if crvapi and self.GetCrankParent(): section=None if self.nick=='crank': #self.rv_is_tree = self.run.rv_is_tree = crvapi.tree if hasattr(crvapi,'tree') else True #self.rv_separate_steps = self.run.rv_separate_steps = False viewer = self.rvapi_viewer if hasattr(self,'rvapi_viewer') else None # full layout for non i2; nothing for i2 tree; otherwise tabs in i2 (sections in grid do not work in rvapi) layout = 7 if not self.run.ccp4i2 else 0 if crvapi.tree else 4 crank2doc = crvapi.Document( ID='crank2doc', outdir=self.rundir, header='CRANK2 job viewer', \ viewer=viewer, i2=self.run.ccp4i2, layout=layout) if self.run.ccp4i2 and crvapi.tree: self.rv_body = self.rv_report = crank2doc.Grid("body") else: self.rv_body = self.rv_report = crank2doc.Tab("results_tab", "Results") if hasattr(self,'logout'): self.rv_log = crank2doc.Tab("log_tab", "Logfile") crvapi.AppendContent(self.logout, "log_tab") title = "SHELX (run via CRANK2)" if self.GetProcess('phdmmb') else "CRANK2" section = self.rv_report.Section('crank2', title=title, opened=False) section.Text('Please cite:') for r in self.GetLogGraphReferences(recursive=False,rvapi=True,bib_to_file=self.run.ccp4i2): section.Text(' '+r) section.Text('
') if crvapi.tree: self.rv_report = self.rv_report.Tree("Processes") elif self.parent_process and self.parent_process.nick=='crank' and self.GetCrankParent().rv_report: sec_name = self.short_name if crvapi.tree else self.name if crvapi.separate_steps: rv_proc = crvapi.Document( ID=self.nick+'doc', outdir=self.rundir, i2=True, layout=4 ).Tab(self.nick+"_tab", "Results") row = -1 else: rv_proc = self.GetCrankParent().rv_report row = self.GetCrankParent().processes.index(self)+1 #sec_name = self.short_name if crvapi.tree else self.name #section = self.rv_report = self.GetCrankParent().rv_report.Section(self.nick,sec_name[0].upper()+sec_name[1:]) section = self.rv_report = rv_proc.Section(self.nick,sec_name[0].upper()+sec_name[1:],row=row) if section is not None: section.Text("Programs used:") section.Text(' '+', '.join(self.GetLogGraphPrograms(rvapi=True))) if self.nick=='crank': section.Text("
Program references:") for r in self.GetLogGraphReferences(rvapi=True,bib_to_file=self.run.ccp4i2): section.Text(' '+r) if not self.run.ccp4i2: section.Text("
Input data:") for fname in set([o.GetFileName() for o in self.inp.GetAll('fsigf')]): section.DataFile(fname, "hkl:hkl", "Input data") else: crvapi.Flush(ignore_timing_restr=True) # at the moment, there is one "previous steps' merge" loggraphfile and the actual step loggraphfile # both of them are opened from here; the actual step one is only opened if not opened yet by another # process and if the main merge one is opened # currently, only the step processes (direct crank's children processes) have a loggraphfile if self.GetLogGraphFileName() and self.GetLogGraphHandle() is None and \ (self.nick=='crank' or self.GetCrankParent().opened_loggraph): self.SetLogGraphHandle( open(self.GetLogGraphFileName(),'w',0) ) self.opened_loggraph=True # create symlink as _act to the main crank directory (used by both ccp4i1 and ccp4i2) if self.nick!='crank': common.SymLink(self.GetLogGraphFileName(), self.GetLogGraphFileName(actual=True)) self.LGInfo(self.GetCCP4Header()) self.LGInfo(self.GetLogGraphReferences(recursive=False)) self.LGInfo(self.GetLogGraphPrograms()) self.LGInfo(self.GetLogGraphReferences()) if self.parent_process is self.GetCrankParent(): common.Info('\n*** Starting process of {0} ***\n'.format(self.name)) def RunBody(self,*args,**kwargs): """Actual run of the process, by default just calling the associated program(s, in sequence)""" prev_prog=None for prog in self.programs: if prev_prog: for container in prev_prog.out.GetAll(): prog.inp.AddCopy(container) prog.Run() prev_prog=prog def RunPostprocess(self,restore=True,*args,**kwargs): """Any postprocessing after the actual run of the process""" from_ccp4i2 = kwargs.pop('from_ccp4i2', False) info = kwargs.pop('info', '') if restore: self.param=copy.copy(self.partmp) self.CheckParamAccess() os.chdir(self.previous_cwd) if self.GetCrankParent(): self.GetCrankParent(prep=True).Info('{0}Process of {1} {2} has finished'.format(' '*self.GetNumParents(),self.name,info), stdout=False) if self.opened_log: if self.parent_process is self.GetCrankParent(): common.Info('\n*** Process of {0} has finished. ***\n\n'.format(self.name)) self.logfilehandle.close() self.logfilehandle=None self.opened_log=False if os.path.getsize(self.GetLogFileName())==0: os.remove(self.GetLogFileName()) if self.opened_loggraph: self.GetLogGraphHandle().close() self.SetLogGraphHandle(None) self.opened_loggraph=False # the content of the "actual" loggraphfile is copied to the main crank loggraphfile if self.nick!='crank': if os.path.getsize(self.GetLogGraphFileName())>0: with open(self.GetLogGraphFileName()) as act_loggraph: shutil.copyfileobj(act_loggraph, self.GetCrankParent().loggraphfilehandle) # fix for Windows: otherewise file not found and crank crashes fpath = self.GetLogGraphFileName(actual=True) if os.path.isfile(fpath): os.remove(fpath) if not self.no_reporting and crvapi and not crvapi.tree and not crvapi.separate_steps and self.rv_report: # backwards compatibility - as Eugene explained, the fallback should not be used, thus should be removed once distributed jsrview rev>=104 can be assumed if not self.rv_report.SetState(close=True): self.GetCrankParent().rv_report.Section(self.nick,"",row=20,opened=False) elif not self.no_reporting and crvapi: build = [p for p in self.run.processes if p.nick in ('comb_phdmmb','mbref')] if build: summary = self.rv_body.Section("summary","Job Summary",row=0) summary.Text(''+build[-1].result_str+'') summary.Text('Number of residues in the built model: {0}'.format(build[-1].mb_res_all[-1][1])) if hasattr(self.run.processes[-1],'i2_R'): summary.Text('Final R factor: {0}'.format(self.run.processes[-1].i2_R)) if hasattr(self.run.processes[-1],'i2_Rfree') and self.run.processes[-1].i2_Rfree: summary.Text('Final Rfree factor: {0}'.format(self.run.processes[-1].i2_Rfree)) def Run(self,rundir=None,*args,**kwargs): """Run the process!""" clearout=kwargs.pop('clear_out',True) restore=kwargs.pop('restore',False) self.RunPreprocess(rundir=rundir,clear_out=clearout,save=restore,*args,**kwargs) self.RunBody(*args,**kwargs) self.RunPostprocess(restore,*args,**kwargs)