""" This module generates C code that handles the reading and writing of options from ini files. The idea is that instead of writing C code when you add a new parameter you just add the parameter to a list here, and the code is generated for you. This module generates a series of generic (model-independent) options, and then also specialized options for the models selected on the command line. Each parameter belongs to a parameter set (to make it easier to read the parameter files), and every option has a default value (which it takes if it is not set in the parameter file) and some help text. To add a new parameter to an existing group: Find the group you want to add it to below - they are currently mcmc_options, image_options and basic_options. Add a new line to the list of parameters for your option set. Each line is a tuple, consisting of: ('name_of_parameter', parameter_type,parameter_default_value, 'Parameter help text'), Do not forget the comma at the end. This will generate a new option, which can be referred to in the options struct pointer as options->name_of_parameter. If you want to add a whole new group of parameters options: create a list, modelled after the existing ones described above, containing your new options append to the list below the tuple: ('Option Set Name',option_set_list) """ import StringIO import sys import os import importlib sys.path.append("./models/") FLT_MAX=1e30 default_parameter_filename = "ini/i3_options.ini" struct_name='i3_options' header_filename='tools/i3_options.h' code_filename='tools/i3_options.c' test_filename='tests/i3_options_test.c' python_filename='bindings/i3_options.py' script_name=__file__ script_dir=os.path.abspath(os.path.split(__file__)[0]) data_dir=os.path.abspath(os.path.join(script_dir,'..','data')) max_line_length=1024 #To make new parameters: #make a new list for your new parameter group or choose an existing group to attach your parameter to. #add to that list a tuple of: #(Parameter_name,Parameter_type,Default_value,Help_text) #parameter_name and help_text are strings #parameter_type is one of float,int,str,bool #default value is appropriate to parameter_type #If you created a new group then add it to the "options" list below., with a (name,my_list) tuple. basic_options=[ ('model_name',str,'sersics','Name of model to use.'), ('noise_sigma',float,1.0,'Assumed noise standard deviation'), ('image_mask', str, '', 'Image mask. If set, the weighting assigned to each pixel is scaled by the mask value (i.e. ignore any pixels with mask=0)'), ('background_subtract',bool,True,'Whether to subtract off the mean of the edge pixels from the whole stamp'), ('rescale_stamp',bool,True,'Whether to rescale the whole postage stamp so flux is 1'), ('verbosity',int,0,'Verbosity level [-1 to 4]'), ('save_images',bool,False,'Whether to save images of the model and fit in output_directory'), ('save_output',bool,False,'Whether to save im3shape output in output_directory'), ('output_directory',str,'.','Output directory in which to save model and data images.'), ('output_filename',str,'im3shape_output.txt','Output filename in which to save im3shape results.'), ('airy_psf',bool,False,'Use an Airy PSF instead of a Moffat.'), ('psf_truncation_pixels',float,40.0,'PSF truncation radius, in pixels (use 5.7 for GREAT08, or a large number e.g. 40 for no truncation)'), ] optimizer_options=[ ('use_computed_start',bool,True,'Use a computed starting position (if available). This (partially) overwrites ini settings. If not just use ini file settings.'), ('minimizer_tolerance',float,1.0e-9,'Tolerance of the minimizer (for levmar it is eps2>||dp||_2)'), ('levmar_eps1',float,-1,'Levmar stopping criterion: ||J^T e||_inf < eps1. Set to -1 if not used.'), ('levmar_eps2',float,1e-20,'Levmar stopping criterion: ||dp||_2 < eps2. Set to -1 if not used.'), ('levmar_eps3',float,-1,'Levmar stopping criterion: ||e||_2 < eps3. Set to -1 if not used.'), ('levmar_eps4',float,1e-7,'Levmar stopping criterion: D||e||_2*n < eps4. Set to -1 if not used.'), ('levmar_eps5',float,-1,'Levmar stopping criterion: ellipticty did not change by eps5 since last iteration.'), ('levmar_tau',float,1e-10,'Levmar size of the step used for finding finite difference derivatives. If <0 then use central differences.'), ('levmar_LM_INIT_MU',float,1e-8,'Levmar initial damping factor'), ('minimizer_max_iterations',int,500,'The maximum number of iterations of the minimizer'), ] image_options=[ ('stamp_size',int,32,'The size of a postage-stamp patch with a single object in to analyze.'), ('upsampling', int, 5, 'The upsampling to use when creating high-res models'), ('central_pixel_upsampling', bool, True, 'Whether to do extra upsampling in central pixels of galaxy image - N.B. Only implemented in milestone model'), ('n_central_pixel_upsampling', int, 9, 'Factor by which to upsample the high-resolution pixels, in the center of the galaxy image. Odd number.'), ('n_central_pixels_to_upsample', int, 5, 'If 1 it will upsample only central pixel, if 2 also its one neighbor on both sides, etc.'), ('padding', int, 2, 'Padding to use when generating model (in low-res pixels), has to be even, 0 allowed'), ('sersics_disc_truncation',float,0.0,'Number of pixels at which to truncate sersic discs'), ('sersics_bulge_truncation',float,0.0,'Number of pixels at which to truncate sersic bulge'), ] advanced_options=[ ('do_mcmc',bool,False,'Whether to perform a full MCMC. If not, just get the maximum posterior solution.'), ('minimizer_loops',int,1,'Number of loops of the minimizer to perform'), ('save_sersics',bool,False,'Whether to save images of the sersic image within the likelihood function'), ('psf_input',str,'moffat_catalog','Model of the PSF to use, currently supported "moffat_catalog", "psf_image_cube", "psf_image_single", "psfex" and "shapelet" '), ('perform_pixel_integration',bool,True,'Whether to perform pixel integration during convolution. If effective PSF is provided, set this option to False.'), ('use_segmentation_mask',bool,False,'Whether to use a galaxy segmentation mask for fitting.'), ('segmentation_mask_filename',str,'','Filename of galaxy segmentation mask.'), ('use_hankel_transform',bool,False,'Whether to use Hankel transform trick to speed up model generation'), ('sersics_beermat_amplitudes_positive',bool,False,"If yes, then constrain the amplitudes of bulge and disc to be positive, using beermat"), ('sersic_lookup_file',str,os.path.join(data_dir,'HankelSersic_table.fits'),'Path to a data file containing a Sersic Fourier lookup table'), ('levmar_use_analytic_gradients',bool,False,'Use analytic gradients'), ('levmar_check_analytic_gradients',bool,False,'Check analytic gradients'), ('image_generator_resolution_high',bool,False,'If im3generate creates high resolution images'), ('minimizer_verbosity',int,0,'Minimizer verbosity level [0-4]'), ('minimizer_method',int,0,'Minimizer method 0=levmar 1=simplex 2=powell 3=praxis 4=golden'), ('poisson_errors',bool,False,'Use Poisson errors for the xray model'), ('timeit',bool,False,'Time parameter estimation time'), ('circularize_mask',bool,False,'Make the post stamp circular by masking out pixels around the edge'), ] mcmc_options=[ ('number_samples',int,20000,'The number of MCMC samples to generate'), ('burn_length',int,100,'The length of the MCMC burn-in phase. Must be < number_samples'), ('tuning_phase',int,100,'The length of the MCMC tuning phase. This comes after the burn period and tunes the covariance.'), ('scaling',float,2.4,'The MCMC proposal scaling factor.'), ('do_rotation',bool,True,'Whether or not to rotate proposals'), ('mcmc_tune_covmat', bool, True, 'Periodically tune the proposal covariance during the tuning phase'), ('max_annealing_temperature',float,2.0,'Initial temperature of annealing.'), ('annealing_one_point',float,0.5,'Fraction of the way through the chain that the temperature should reach one. Schedule is linear'), ('adaptive_metropolis_interval',int,100,'Interval between adaptive metropolis calls'), ('climbing_only',bool,False,'Climb the hill only; reject jumps that reduce likelihood'), ('mcmc_filename_base',str,'','If set, must include a %ld somewhere. Save MCMC output to this with the galaxy identifier'), ('use_mcmc_mean',bool,False,'Whether to use the mean of the MCMC distribution instead of the maximum likelihood'), ('use_mcmc_sample',bool,False,'Whether to use the final sample of the MCMC instead of the maximum likelihood'), ('print_mcmc_samples',bool,False,'Print all the MCMC samples to stdout.'), ] meds_options = [ ('starting_params_only',bool,False,'Do not optimize; just use initial params.'), ('rescale_weight',bool,False,'Rescale the weights so that the edge pixels have the correct std dev.'), ('uberseg',bool,True,'Use the ubserseg masking algorithm.'), ('use_image_flags',bool,True,'Use the MEDS image flags to select exposures.'), ('reject_outliers',bool,True,'Use the ubserseg masking algorithm.'), ('replace_psf_path',str,'','Replace the path in the PSF using this comma-separated pair old,new'), ('psfex_rerun_version',str,'','Use the Jarvis re-run PSF with this version'), ('database_output', bool, False, 'Output to database instead of file'), ] generic_options=[ ("Basic Options",basic_options), ('Optimizer Options',optimizer_options), ('Image Options',image_options), ] experimental_options = [ ('Advanced Options - THESE ARE EXPERIMENTAL - USE AT OWN RISK',advanced_options), ('MCMC Options - EXPERIMENTAL',mcmc_options), ('MEDS Options - Probably DES only', meds_options), ] def build_model_option_list(model_name,param_names,param_types, default_widths, default_starts,default_mins, default_maxs, fixed_by_default): options=[] for name,ptype,start in zip(param_names,param_types, default_starts): option_name = "%s_%s_start" % (model_name,name) option_help = "Initial value of parameter %s in model %s" % (name,model_name) option=(option_name,ptype,start,option_help) options.append(option) for name,ptype,width in zip(param_names,param_types,default_widths): option_name = "%s_%s_width" % (model_name,name) option_help = "Initial value of parameter %s in model %s" % (name,model_name) option=(option_name,ptype,width,option_help) options.append(option) for name,ptype,minval in zip(param_names,param_types,default_mins): option_name = "%s_%s_min" % (model_name,name) option_help = "Min value of parameter %s in model %s" % (name,model_name) option=(option_name,ptype,minval,option_help) options.append(option) for name,ptype,maxval in zip(param_names,param_types,default_maxs): option_name = "%s_%s_max" % (model_name,name) option_help = "Max value of parameter %s in model %s" % (name,model_name) option=(option_name,ptype,maxval,option_help) options.append(option) for name in param_names: option_name = "%s_%s_fixed" % (model_name,name) option_help = "Whether to fix the value of parameter %s in model %s" % (name,model_name) option=(option_name,bool,name in fixed_by_default,option_help) options.append(option) return options def generate_model_options(name, compiling=True): if compiling: module = __import__('i3_'+name) else: module = importlib.import_module('.i3_'+name,'py3shape.models') param_types = [module.types[param] for param in module.parameters] param_widths = [module.widths[param] for param in module.parameters] param_starts = [module.starts[param] for param in module.parameters] param_mins = [module.min[param] for param in module.parameters] param_maxs = [module.max[param] for param in module.parameters] try: fixed_by_default=module.fixed_by_default except AttributeError: fixed_by_default=[] class_name = 'Options for model %s'%name option_tuple = class_name,build_model_option_list(name,module.parameters,param_types,param_widths,param_starts,param_mins,param_maxs,fixed_by_default) return option_tuple def c_declaration(ptype,struct, name, default): if ptype==list: return 'float %s[%d]' % (name,len(default)) return{ int:'int %s'%name, float:'float %s'%name, str:'char %s[%s_max_line_length] '%(name,struct), bool:'bool %s'%name, }[ptype] def c_literal(ptype,value): if ptype is list: return "{%s}" % (', '.join(str(v) for v in value)) if ptype in [int,float]: return str(value) elif ptype is bool: if value: return 'true' else: return 'false' elif ptype is str: return '"%s"'%value else: raise ValueError("Unknown parameter type %r"%ptype) def generate_option_struct(struct_name,options): S=StringIO.StringIO() S.write('typedef struct %s{\n'%struct_name) for group_name,option_group in options: S.write('/*\n * %s\n */\n\n'%group_name) for name,ptype,default,help in option_group: S.write("/* %s */\n%s;\n\n"%(help,c_declaration(ptype,struct_name,name,default))) S.write('} %s;\n\n'%struct_name) S.seek(0) return S.read() def list_initializer(default): middle = ', '.join(str(d) for d in default) return '{' + middle + "}" def generate_default_option_setter(struct_name,options): S=StringIO.StringIO() S.write("void %s_set_defaults(%s * options){\n"%(struct_name,struct_name)) for group_name,option_group in options: S.write(' /*\n * %s\n*/\n'%group_name) for name,ptype,default,help in option_group: if ptype==str: S.write(" strcpy(options->%s,%s);\n"%(name, c_literal(ptype,default))) elif ptype==list: temp_declaration = c_declaration(ptype,struct_name,"tmp_value",default) temp_initializer = list_initializer(default) S.write("{ %s = %s;\n memcpy(options->%s,tmp_value,sizeof(options->%s));}\n"%(temp_declaration,temp_initializer, name, name)) else: S.write(" options->%s = %s;\n"%(name, c_literal(ptype,default))) S.write('}\n\n') S.seek(0) declaration = "\n/*Set default values for all the parameters.*/\nvoid %s_set_defaults(%s * options);\n"%(struct_name,struct_name) return S.read(), declaration def guard_tags(struct_name): tag = '_H_%s'%struct_name.upper() top = '#ifndef %s\n#define %s\n\n'%(tag,tag) bottom = '\n#endif\n' return top,bottom def generate_header(struct_name,options): S=StringIO.StringIO() S.write("""/* This code is auto-generated by the script %s. There is no point in changing it. Adding new parameters is easy - just modify the top of that script. Adding new functions is a little more involved. */ """%script_name) S.write('#include "stdbool.h"\n') S.write('#include "stdio.h"\n') S.write('#include "stdlib.h"\n') S.write('#include "ctype.h"\n') S.write('#include "string.h"\n\n') total_options=sum([len(group[1]) for group in options]) S.write('#define %s_max_line_length %d\n'%(struct_name,max_line_length)) S.write('\n\n/* This is the main option structure that this code acts on.*/\n') S.write(generate_option_struct(struct_name,options)) S.write('\n\n') S.write("i3_options * i3_options_default();\n\n") S.write("void i3_options_destroy(i3_options * options);\n\n") # S.write('const int %s_number_parameters = %d;\n'%(struct_name,total_options)) # S.write('char * %s_full_parameter_list[%s_number_parameters] = {\n' % (struct_name,struct_name)) # for group_name,option_group in options: # for name,ptype,default,help in option_group: # S.write('"%s",\n'%name) # S.write('};\n\n') S.seek(0) return S.read() def generate_parameter_parsers(struct_name): S=StringIO.StringIO() declaration_list = [] S.write(""" void %s_report_parameter_fail(char * parameter,char * type){ fprintf(stderr,"ERROR:\\nFailed to parse string '%%s' into parameter of type %%s.\\n",parameter,type); } """ % struct_name) declaration_list.append('\n/*Internal: Called when a parameter value cannot be read.*/\nvoid %s_report_parameter_fail(char * parameter,char * type);'%struct_name) parameter_types = [('int','%d'),('float','%f'),('double','%lf')] for (parameter_type,parameter_code) in parameter_types: S.write(""" %s %s_parse_%s(char * parameter, int * status){ %s result; int count; count = sscanf(parameter,"%s",&result); *status=0; if (count!=1) { %s_report_parameter_fail(parameter,"%s"); *status=1; } return result; } """ % (parameter_type,struct_name,parameter_type,parameter_type,parameter_code,struct_name,parameter_type)) declaration_list.append('\n/* Internal: option value parser */\n%s %s_parse_%s(char * parameter, int * status);'%(parameter_type,struct_name,parameter_type)) #Handle bool specially S.write(""" bool %s_parse_bool(char * parameter, int * status){ bool result; char text[%s_max_line_length]; int count; bool found=false; int n_variants = 10; char * true_variants[] = {"T","True","true","TRUE","YES","Yes","yes","Y","y","1"}; char * false_variants[] = {"F","False","false","FALSE","NO","No","no","N","n","0"}; count = sscanf(parameter,"%%s",text); for (int i=0;i%s_max_line_length){parameter_name[0]='\\0';parameter_value[0]='\\0'; return 5;} strncpy(parameter_value,lstripped_value_part,length_of_stripped_value); parameter_value[length_of_stripped_value]='\\0'; return 0; } """ % (struct_name, struct_name), ("\n/*Internal: Split the line into two parts, stripping whitespace and comments*/\nint %s_split_parts(char * line, char * parameter_name, char * parameter_value);\n"%struct_name) def parser_for_type(ptype,struct_name,): if ptype in [int,float,bool]: return '%s_parse_%s' % (struct_name,ptype.__name__) elif ptype == str: return '%s_parse_string'% struct_name else: raise ValueError("No parser for parameter type: %r"%ptype) def generate_name_value_handler(struct_name,options): S=StringIO.StringIO() S.write(""" int %s_handle_name_value(%s * options, char * name, char * value){ int status = 0; %s_handle_name_value_body(options, name,value,true, &status); return status; } int %s_parse_float_array_element(i3_options * options, char * name, char * value, int * status){ char name_part[1024]; int element_number; char * bracket_pos = strstr(name, "["); int len_start = bracket_pos - name; strncpy(name_part, name, 1024); name_part[len_start]='\\0'; int count = sscanf(name+len_start, "[%%d]", &element_number); if (count!=1) { %s_report_parameter_fail(name,"array"); *status = 1; return 0; } """%(struct_name,struct_name,struct_name,struct_name,struct_name) ) for group_name,option_group in options: for name,ptype,default,help in option_group: if ptype==list: S.write(""" if (!strcmp(name_part,"%s")){ if(element_number>=sizeof(options->%s)/sizeof(float)){ fprintf(stderr,"Element index too large for array option %s!\\n"); return 0; } else{ options->%s[element_number]=%s(value, status); return 1; } } """ % (name,name,name,name,parser_for_type(float, struct_name))) S.write("\treturn 0;\n}") S.write("int %s_handle_name_value_body(%s * options, char * name, char * value, bool try_model, int * status){\n"%(struct_name,struct_name)) S.write(" bool found=false;\n") S.write(" if (strstr(name,\"[\")) {found = %s_parse_float_array_element(options, name, value, status);}\n"%struct_name) for group_name,option_group in options: for name,ptype,default,help in option_group: if ptype==list: S.write(' if (!strcmp(name,"%s")) {found=true; %s_parse_float_array(options->%s, value, %d, status);}\n'%(name,struct_name,name,len(default))) elif ptype==str: S.write(' if (!strcmp(name,"%s")) {found=true;strcpy(options->%s,value);}\n'%(name,name)) else: S.write(' if (!strcmp(name,"%s")) {found=true;options->%s=%s(value, status);}\n'%(name,name,parser_for_type(ptype,struct_name))) S.write(""" if (!found){ if (try_model){ char extended_name[%s_max_line_length]; snprintf(extended_name, %s_max_line_length,"%%s_%%s",options->model_name,name); found=%s_handle_name_value_body(options, extended_name, value, false, status); if (!found) fprintf(stderr,"Unkown parameter ignored: '%%s'\\n(Also tried '%%s_%%s', based on chosen model name)\\n",name,options->model_name,name); } return found; } """ % (struct_name,struct_name,struct_name)) S.write(' return found;') S.write("}\n") S.seek(0) declaration="\n/* Internal: Sets structure parameter from name and value */\nint %s_handle_name_value(%s * options, char * name, char * value);\nint %s_handle_name_value_body(%s * options, char * name, char * value, bool try_model, int * status);"%(struct_name,struct_name,struct_name,struct_name) return S.read(), declaration def generate_option_command_line_reader(struct_name,options): code = """ void %s_read_command_line(%s * options, int argc, int start, char *argv[]){ char parameter_name[%s_max_line_length]; char parameter_value[%s_max_line_length]; char line[%s_max_line_length]; for (int i=start;i%s[%d], '%(name,i)) S.seek(-2,1) #cut off the last comma again. S.write(');\n') else: typecode=typecodes[ptype] S.write(' fprintf(outfile,"%s = %s\\n",options->%s);\n'%(name,typecode,name)) S.write(' fprintf(outfile,"\\n");\n') S.write('}\n') S.write("""void %s_printf(%s * options){ %s_fprintf(options, stdout); } void %s_save(%s * options, char * filename){ FILE * outfile = fopen(filename, "w"); %s_fprintf(options, outfile); fclose(outfile); } """ % (struct_name,struct_name,struct_name,struct_name,struct_name,struct_name) ) S.seek(0) declaration_list=[ "\n/* Print the option set to an already opened file. The file is saved such that it can be re-read directly. */", "void %s_fprintf(%s * options, FILE * outfile);"%(struct_name,struct_name), "\n/* Print the option set to screen */", "void %s_printf(%s * options);"%(struct_name,struct_name), "\n/*Save the option set to the named file. */", "void %s_save(%s * options, char * filename);"% (struct_name,struct_name) ] return S.read(), '\n'.join(declaration_list)+'\n' def generate_cfile_top(struct_name): return """/* This code is auto-generated by the script %s. There is no point in changing it. Adding new parameters is easy - just modify the top of that script. Adding new functions is a little more involved. */ #include "%s" #include "i3_logging.h" void i3_options_destroy(i3_options * options){ free(options); } %s * i3_options_default(){ i3_options * options = malloc(sizeof(%s)); i3_options_set_defaults(options); return options; } """ % (script_name,header_filename,struct_name,struct_name) def generate_test_program(struct_name): test_file = file(test_filename,'w') test_file.write('/* This test file was auto-generated by %s to test the %s mechanisms */\n' % (script_name,struct_name) ) test_file.write('#include "%s"\n'%header_filename) test_file.write(""" int main(int argc, char *argv[]){ %s options, options2; char * parameter_filename = "%s"; char * temporary_filename = "%s.temporary_copy"; %s_set_defaults(&options); %s_read(&options, parameter_filename); printf("\\nHave loaded parameter file: %%s:\\n",parameter_filename); %s_printf(&options); %s_save(&options, temporary_filename); %s_read(&options2, temporary_filename); printf("Have saved and re-read parameter file. Should be the same:\\n"); %s_read_command_line(&options, argc,1,argv); printf("Now read from command line\\n"); %s_printf(&options); remove(temporary_filename); return 0; } """ % (struct_name,default_parameter_filename,default_parameter_filename,struct_name,struct_name,struct_name,struct_name,struct_name,struct_name,struct_name)) test_file.close() def main(model_names): options = generic_options[:] for model in model_names: options.append(generate_model_options(model)) options += experimental_options header_file=file(header_filename,'w') c_file = file(code_filename,'w') guard_top,guard_bottom=guard_tags(struct_name) header_file.write(guard_top) header_file.write(generate_header(struct_name,options)) # generate_python(python_filename,options) def write_parts(code_decl): code,decl=code_decl c_file.write(code) header_file.write(decl) c_file.write(generate_cfile_top(struct_name)) write_parts(generate_option_reader(struct_name,options) ) write_parts(generate_option_command_line_reader(struct_name,options)) write_parts(generate_default_option_setter(struct_name,options) ) write_parts(generate_option_printer(struct_name,options) ) # Internal bits header_file.write('\n\n') write_parts(generate_parameter_parsers(struct_name) ) write_parts(generate_word_splitter(struct_name) ) write_parts(generate_name_value_handler(struct_name,options)) header_file.write(guard_bottom) header_file.close() c_file.close() default_parameter_file = file(default_parameter_filename,'w') default_parameter_file.write(generate_parameter_file(options)) default_parameter_file.write('\n') default_parameter_file.close() generate_test_program(struct_name) if __name__=="__main__": names = sys.argv[1:] main(names)