package ConfigCentral; # Builds an intermediate config structure used by infoproviders. # OBS: the structure is NOT the same as arc.conf. ## RESTRUCTURING PHASE ################ ## changes are identified by the tags ## #C changenumber ####################################### use strict; use warnings; use File::Basename; use Sys::Hostname; # added to parse JSON stuff binmode STDOUT, ":utf8"; use utf8; use XML::Simple; use Data::Dumper qw(Dumper); use JSON::XS; #use Data::Dumper::Concise; use IniParser; use InfoChecker; use LogUtils; # while parsing, loglevel is WARNING (the default) our $log = LogUtils->getLogger(__PACKAGE__); ####################################################################### ## Block dependencies ####################################################################### my $blockdependencies = { 'gridftpd/jobs' => ['gridftpd'], 'infosys/cluster' => ['infosys'], 'infosys/ldap' => ['infosys'], 'infosys/nordugrid' => ['infosys/ldap'], 'infosys/glue1' => ['infosys/nordugrid'], 'infosys/glue2' => ['infosys'], 'infosys/glue2/ldap' => ['infosys/glue2','infosys/ldap'], }; ###################################################################### # Legacy Internal representation of configuration data after parsing # ###################################################################### my $lrms_options = { pbs_bin_path => '*', pbs_log_path => '*', dedicated_node_string => '*', condor_bin_path => '*', condor_config => '*', condor_rank => '*', sge_bin_path => '*', sge_root => '*', sge_cell => '*', sge_qmaster_port => '*', sge_execd_port => '*', lsf_bin_path => '*', lsf_profile_path => '*', ll_bin_path => '*', slurm_bin_path => '*', slurm_wakeupperiod => '*', boinc_db_host => '*', boinc_db_port => '*', boinc_db_name => '*', boinc_db_user => '*', boinc_db_pass => '*', }; my $lrms_share_options = { queue_node_string => '*', condor_requirements => '*', sge_jobopts => '*', lsf_architecture => '*', ll_consumable_resources => '*', }; my $xenv_options = { Platform => '*', Homogeneous => '*', PhysicalCPUs => '*', LogicalCPUs => '*', CPUVendor => '*', CPUModel => '*', CPUVersion => '*', CPUClockSpeed => '*', CPUTimeScalingFactor => '*', WallTimeScalingFactor => '*', MainMemorySize => '*', VirtualMemorySize => '*', OSFamily => '*', OSName => '*', OSVersion => '*', VirtualMachine => '*', NetworkInfo => '*', ConnectivityIn => '*', ConnectivityOut => '*', Benchmark => [ '*' ], OpSys => [ '*' ], nodecpu => '*', }; my $share_options = { MaxVirtualMemory => '*', MaxSlotsPerJob => '*', SchedulingPolicy => '*', Preemption => '*', totalcpus => '*', defaultmemory => '*', AdvertisedVO => [ '*' ], maxcputime => '*', maxwalltime => '*', mincputime => '*', minwalltime => '*' }; my $cache_options = { cachedir => [ '*' ], cachesize => '*' }; my $gmuser_options = { controldir => '', sessiondir => [ '' ], defaultttl => '*', %$cache_options }; my $gmcommon_options = { arcversion => '', arexhostport => '', # lrms => '', C 10 removed from here to the lrms hash gmconfig => '*', endpoint => '*', hostname => '*', maxjobs => '*', maxload => '*', maxloadshare => '*', wakeupperiod => '*', gridmap => '*', #x509_user_key => '*', # C 5 x509_host_key => '*', # C 5 #x509_user_cert => '*', # C 6 x509_host_cert => '*', # C 6 x509_cert_dir => '*', runtimedir => '*', shared_filesystem => '*', shared_scratch => '*', scratchdir => '*', enable_perflog_reporting => '*', perflogdir => '*' }; my $sshcommon_options = { remote_user => '*', remote_host => '*', remote_sessiondir => '*', private_key => '*', }; # C 141 my $ldap_infosys_options = { enabled => '', port => '*', infosys_ldap_run_dir => '*', validity_ttl => '*', # These values have been checked to be used by infoproviders. user => '*', bdii_run_dir => '*', bdii_log_dir => '*', bdii_tmp_dir => '*', bdii_var_dir => '*', bdii_update_pid_file => '*' ## TODO: fix this elsewere, the value should be set according to the formula below ## not used directly by infoproviders, maybe by bdii config? ## this is probably done in the startup script, is called max_cycle ## bdii_read_timeout = number - Sets BDII_READ_TIMEOUT in bdii configuration file ## default: $bdii_provider_timeout + $infoproviders_timelimit + $wakeupperiod #bdii_read_timeout=300 }; # C 125-130 renaming these properly direcly in the gridftpd block #my $gridftpd_options = { # GridftpdEnabled => '*', # GridftpdPort => '*', # GridftpdMountPoint => '*', # GridftpdAllowNew => '*', # GridftpdPidFile => '*', #}; # [arex] subblocks my $wsemies_options = { # C 71 allownew => '*', # C 72 enabled => '' }; # C 66 my $ws_options = { enabled => '', wsurl => '*', emies => { %$wsemies_options } # C 71 }; my $admindomain_options = { Name => '*', Description => '*', WWW => '*', Distributed => '*', Owner => '*', OtherInfo => '*', }; # C 156 my $glue2_options = { enabled => '', computingservice_qualitylevel => '*' # 157 }; # # # # # # # # # # # # # # my $config_schema = { defaultLocalName => '*', #ProviderLog => '*', #C 133 ttl => '*', admindomain => { %$admindomain_options }, %$gmcommon_options, %$sshcommon_options, # moved all these to their own subtrees to be in sync with new config # %$gridftpd_options, # %$ldap_infosys_options, # %$lrms_options, # %$lrms_share_options, control => { '*' => { %$gmuser_options } }, service => { OtherInfo => [ '*' ], StatusInfo => [ '*' ], Downtime => '*', ClusterName => '*', ClusterAlias => '*', ClusterComment => '*', ClusterOwner => [ '*' ], Middleware => [ '*' ], AdvertisedVO => [ '*' ], LocalSE => [ '*' ], InteractiveContactstring => [ '*' ], # TODO: This causes odd/unsupported options to pass the checks %$xenv_options, %$share_options, QualityLevel => '' }, location => { Name => '*', Address => '*', Place => '*', Country => '*', PostCode => '*', Latitude => '*', Longitude => '*', }, contacts => [ { Name => '*', OtherInfo => [ '*' ], Detail => '', Type => '', } ], accesspolicies => { '*' => { Rule => [ '' ], UserDomainID => [ '' ] } }, mappingpolicies => { '*' => { ShareName => [ '' ], Rule => [ '' ], UserDomainID => [ '' ], } }, xenvs => { '*' => { OtherInfo => [ '*' ], NodeSelection => { Regex => [ '*' ], Command => [ '*' ], Tag => [ '*' ], }, %$xenv_options, } }, shares => { '*' => { Description => '*', OtherInfo => [ '*' ], MappingQueue => '', ExecutionEnvironmentName => [ '' ], %$share_options, %$lrms_share_options, } }, # start of newly added items for arcconf restructuring # C 10 lrms => { lrms => '', defaultqueue => '*', lrmsconfig => '*', %$lrms_options, %$lrms_share_options }, infosys => { enabled => '', nordugrid => { enabled => '' }, # C 155 logfile => '*', #C 133 loglevel => '*', #C 134, replaces ProviderLog validity_ttl => '*', ldap => { # C141 %$ldap_infosys_options, # C 141 }, glue1 => { enabled => '' }, # C 161 glue2 => { # C 156 %$glue2_options, ldap => { # C158 enabled => '', showactivities => '*' } } }, arex => { enabled => '', defaultttl => '*', logfile => '*', loglevel => '*', # replaces $config_schema->{debugLevel}, C 37 infoproviders_timelimit => '*', # C 42 port => '*', shared_filesystem => '*', ws => { %$ws_options } # C 66 }, gridftpd => { enabled => '', port => '*', mountpoint => '*', allownew => '*', pidfile => '*' } }; # TODO: remove some or change because of the enable-by-block strategy my $allbools = [ qw( Homogeneous VirtualMachine ConnectivityIn ConnectivityOut Preemption showactivities shared_filesystem enabled allownew Distributed) ]; ############################ Generic functions ########################### # walks a tree of hashes and arrays while applying a function to each hash. sub hash_tree_apply { my ($ref, $func) = @_; if (not ref($ref)) { return; } elsif (ref($ref) eq 'ARRAY') { map {hash_tree_apply($_,$func)} @$ref; return; } elsif (ref($ref) eq 'HASH') { &$func($ref); map {hash_tree_apply($_,$func)} values %$ref; return; } else { return; } } # Strips namespace prefixes from the keys of the hash passed by reference sub hash_strip_prefixes { my ($h) = @_; my %t; while (my ($k,$v) = each %$h) { next if $k =~ m/^xmlns/; $k =~ s/^\w+://; $t{$k} = $v; } %$h=%t; return; } # Verifies that a key is an HASH reference and returns that reference sub hash_get_hashref { my ($h, $key) = @_; my $r = ($h->{$key} ||= {}); $log->fatal("badly formed '$key' element in XML config") unless ref $r eq 'HASH'; return $r; } # Verifies that a key is an ARRAY reference and returns that reference sub hash_get_arrayref { my ($h, $key) = @_; my $r = ($h->{$key} ||= []); $log->fatal("badly formed '$key' element in XML config") unless ref $r eq 'ARRAY'; return $r; } # Set selected keys to either 'true' or 'false' sub fixbools { my ($h,$bools) = @_; for my $key (@$bools) { next unless exists $h->{$key}; my $val = $h->{$key}; if ($val eq '0' or lc $val eq 'false' or lc $val eq 'no' or lc $val eq 'disable') { $h->{$key} = '0'; } elsif ($val eq '1' or lc $val eq 'true' or lc $val eq 'yes' or lc $val eq 'enable' or lc $val eq 'expert-debug-on') { $h->{$key} = '1'; } else { $log->error("Invalid value for $key"); } } return $h; } sub move_keys { my ($h, $k, $names) = @_; for my $key (@$names) { next unless exists $h->{$key}; $k->{$key} = $h->{$key}; delete $h->{$key}; } } sub rename_keys { my ($h, $k, $names) = @_; for my $key (keys %$names) { next unless exists $h->{$key}; my $newkey = $names->{$key}; $k->{$newkey} = $h->{$key}; delete $h->{$key}; } } # Takes two hash references and merges values # the value of hash2 is taken if the value in # hash1 is not defined # usage: merge_hash_values(hash1,hash2) sub merge_hash_values { my ($hash1,$hash2) = @_; for my $key (keys %{$hash2}) { $hash1->{$key} = $hash2->{$key} if ((not defined $hash1->{$key}) || ($hash1->{$key} eq '')); # attempt to merge recursively # merge_hash_values($hash1->{$key},$hash2->{$key}) if (ref $key eq ref {}); } } ##################### Read config via arcconfig-parser ################ # execute parser and get json data sub read_json_config { my ($arcconf) = @_; # get the calling script basepath. Will be used to # find external scripts like arcconfig-parser. my $libexecpath = ($ENV{'ARC_LOCATION'} || '/cvmfs/dirac.egi.eu/dirac/v9.0.0a28/Linux-x86_64') . '/libexec/arc'; my $jsonconfig=''; { local $/; # slurp mode open (my $jsonout, "$libexecpath/arcconfig-parser -e json --load -r $arcconf |") || $log->error("Python config parser error: $! at line: ".__LINE__." libexecpath: $libexecpath"); $jsonconfig = <$jsonout>; close $jsonout; } my $config = decode_json($jsonconfig); #print Dumper($config); return $config; } # # Removes spaces at beginning and end from all config values # and config names (such as queue:\s\squeuename\s*) # sub strip_spaces { my ($jsonconf) = @_; for my $key (keys %{$jsonconf}) { # recur if inner values if (ref($jsonconf->{$key}) eq 'HASH') { strip_spaces($jsonconf->{$key}); } elsif (ref($jsonconf->{$key}) eq 'ARRAY') { # strip spaces from array elements for my $item (@{$jsonconf->{$key}}) { $item =~ s/^\s+|\s+$//g; } } elsif ($jsonconf->{$key} =~ /^\s*(.*)\s*$/) { my $newvalue = $1; $newvalue =~ s/^\s+|\s+$//g; $jsonconf->{$key} = $newvalue; } # change key once back from recursion if it contains spaces if ($key =~ /\s*(queue|authgroup)\s*:\s*(.*)\s*/) { my $prefix = "$1:"; my $stripstring = $2; $stripstring =~ s/^\s+|\s+$//g; my $newkey="$prefix$stripstring"; $jsonconf->{$newkey} = $jsonconf->{$key}; # not needed as the reference is copied. But I don't trust it. #delete $jsonconf->{$key}; } } } # # Reads the json config file passed as the first argument and produces a config # hash conforming to $config_schema. # sub build_config_from_json { my ($file) = @_; my $jsonconf = read_json_config($file); strip_spaces($jsonconf); set_defaults($jsonconf); # Those values that are the same as in arc.conf will # be copied and checked. my $config ||= {}; $config->{service} ||= {}; $config->{control} ||= {}; ## TODO: maybe move all glue2 stuff into some GLUE2 subset? $config->{location} ||= {}; $config->{contacts} ||= []; $config->{accesspolicies} ||= {}; $config->{mappingpolicies} ||= {}; $config->{xenvs} ||= {}; $config->{shares} ||= {}; # start of restructured pieces of information $config->{infosys} ||= {}; # use service for first implementation $config->{arex} ||= {}; $config->{lrms} ||= {}; # end of restructured pieces of information my $common = $jsonconf->{'common'}; ## TODO: remove everything that should not be in $config move_keys $common, $config, [keys %$gmcommon_options]; # TODO: create a mapping hash eventually # new mapping block (just move to the top config as it was before) my $mapping = $jsonconf->{'mapping'}; move_keys $mapping, $config, [keys %$gmcommon_options]; # C 10 new lrms block my $lrms = $jsonconf->{'lrms'}; # some options in lrms, moved to $config for backward compatibility, # should be moved to {lrms} instead #move_keys $lrms, $config, [keys %$lrms_options, keys %$lrms_share_options]; move_keys $lrms, $config->{'lrms'}, [keys %$lrms]; move_keys $lrms, $config->{'lrms'}, [keys %$lrms_options, keys %$lrms_share_options]; # Parsing for default queue my ($lrmsname, $defaultqueue) = split /\s+/, $config->{lrms}{lrms} || ''; # C 10 $config->{'lrms'}{'lrms'} = $lrmsname; $config->{'lrms'}{'defaultqueue'} = $defaultqueue if defined $defaultqueue; # C 173 my $arex = $jsonconf->{'arex'}; move_keys $arex, $config, [keys %$gmcommon_options]; my $ssh = $jsonconf->{'ssh'}; move_keys $ssh, $config, [keys %$sshcommon_options]; #C 133 134 my $infosys = $jsonconf->{'infosys'}; move_keys $infosys, $config->{'infosys'}, [keys %$infosys]; rename_keys $infosys, $config, {port => 'SlapdPort'}; move_keys $infosys, $config, [keys %$ldap_infosys_options]; # only one grid manager user, formerly represented by a dot $config->{control}{'.'} ||= {}; move_keys $arex, $config->{control}{'.'}, [keys %$gmuser_options]; # C 173 move_keys $arex, $config->{arex}, [keys %$arex]; # use service for first implementation #move_keys $arex, $config->{service}, [keys %$service]; # check that the above generated required information $log->error("No control directory configured") unless %{$config->{control}} or $config->{remotegmdirs}; # TODO: remove multiple control dirs feature while (my ($user, $control) = each %{$config->{control}}) { $log->error("No control directory configured for user $user") unless $control->{controldir}; $log->error("No session directory configured for user $user") unless $control->{sessiondir}; } # NEW: arex subblocks # C 71 if (defined $jsonconf->{'arex/ws'}) { my $arexws = $jsonconf->{'arex/ws'}; $config->{arex}{ws} ||= {}; move_keys $arexws, $config->{arex}{ws}, [keys %$ws_options]; } # handle ws endpoint information for use inside infoproviders if (defined $config->{arex}{ws}{wsurl}) { $config->{arex}{ws}{wsurl} =~ m{^(https?)://([^:/]+)(?::(\d+))?(.*)}; my ($proto,$host,$port,$mountpoint) = ($1,$2,$3,$4); $port ||= 80 if $proto eq "http"; $port ||= 443 if $proto eq "https"; $config->{arex}{port} = $port; $config->{arexhostport} = "$host:$port"; } $config->{arex}{ws}{emies} ||= {}; if (defined $jsonconf->{'arex/ws/jobs'}) { my $arexwsemies = $jsonconf->{'arex/ws/jobs'}; move_keys $arexwsemies, $config->{arex}{ws}{emies}, [keys %$wsemies_options]; $config->{arex}{ws}{emies}{enabled} = 1; } else { ## TODO: find a better strategy for the "enabled" defaults, blocks/hashes can't be used. $config->{arex}{ws}{emies}{enabled} = 0; } # cache handling, mixes info from different blocks into control $config->{arex}{cache} ||= {}; if (defined $jsonconf->{'arex/cache'}) { my $cacheopts = $jsonconf->{'arex/cache'}; # moved to control for legacy reasons move_keys $cacheopts, $config->{control}{'.'}, [keys %$cache_options]; } $config->{arex}{cache}{cleaner} ||= {}; if (defined $jsonconf->{'arex/cache/cleaner'}) { my $cacheopts = $jsonconf->{'arex/cache/cleaner'}; # moved to control for legacy reasons move_keys $cacheopts, $config->{control}{'.'}, [keys %$cache_options]; } # gridftpd config if ((defined $jsonconf->{'gridftpd/jobs'}) || (defined $jsonconf->{'gridftpd'})) { $config->{gridftpd} ||= {}; my $gconf = $jsonconf->{'gridftpd'}; my $gjconf = $jsonconf->{'gridftpd/jobs'}; move_keys $gconf, $config->{gridftpd}, [keys %{$config_schema->{gridftpd}}]; move_keys $gjconf, $config->{gridftpd}, [keys %{$config_schema->{gridftpd}}]; } else { $log->verbose("[gridftpd/jobs] block not found in arc.conf. Disabling gridftpd job interface information generation."); } # TODO: review BDII config, block disappeared but maybe options not moved back. # LDAP config if (defined $jsonconf->{'infosys/ldap'}) { $config->{infosys}{ldap} ||= {}; my $ldapconf = $jsonconf->{'infosys/ldap'}; move_keys $ldapconf, $config->{infosys}{ldap}, [keys %$ldap_infosys_options]; } else { $log->verbose("[infosys/ldap] block not found in arc.conf Disabling LDAP/LDIF information generation."); } # information schemas # NorduGRID if ($jsonconf->{'infosys/nordugrid'}{enabled}) { $config->{infosys}{nordugrid} ||= {}; move_keys $jsonconf->{'infosys/nordugrid'}, $config->{infosys}{nordugrid}, [keys %{$config_schema->{infosys}{nordugrid}}]; } # Glue 1.2/1.3 # this is only used to enable rendering here, no need to parse. # glue-generator.pl actually processes that information. if (defined $jsonconf->{'infosys/glue1'}) { $config->{infosys}{glue1} ||= {}; } ####### GLUE2 if ( $jsonconf->{'infosys/glue2'}{enabled} ) { $config->{infosys}{glue2} ||= {}; my $glue2conf = $jsonconf->{'infosys/glue2'}; move_keys $glue2conf, $config->{infosys}{glue2}, [keys %$glue2_options]; rename_keys $config->{infosys}{glue2}, $config->{service}, {computingservice_qualitylevel => 'QualityLevel'}; # GLUE2 ldap if ( $jsonconf->{'infosys/glue2/ldap'}{enabled} ) { $config->{infosys}{glue2}{ldap} ||= {}; my $glue2ldapconf = $jsonconf->{'infosys/glue2/ldap'}; move_keys $glue2ldapconf, $config->{infosys}{glue2}{ldap}, [keys %{$config_schema->{infosys}{glue2}{ldap}}]; } # AdminDomain $config->{admindomain} ||= {}; $log->warning('[infosys/glue2] section missing admindomain_name information. Default will be set to GLUE2 default UNDEFINEDVALUE.') if ($jsonconf->{'infosys/glue2'}{admindomain_name} eq 'UNDEFINEDVALUE' ); my $admindomainconf = $jsonconf->{'infosys/glue2'}; rename_keys $admindomainconf, $config->{'admindomain'}, { admindomain_name => 'Name', admindomain_description => 'Description', admindomain_www => 'WWW', admindomain_distributed => 'Distributed', admindomain_owner => 'Owner', admindomain_otherinfo => 'OtherInfo' } } else { $log->error('Infoproviders cannot continue without the [infosys/glue2] block. Please add it. Exiting...') } ### Process infosys/cluster my $cluster = $jsonconf->{'infosys/cluster'}; if (%$cluster) { # Ignored: cluster_location, lrmsconfig rename_keys $cluster, $config->{location}, { cluster_location => 'PostCode' }; rename_keys $cluster, $config->{service}, { interactive_contactstring => 'InteractiveContactstring', cluster_owner => 'ClusterOwner', localse => 'LocalSE', advertisedvo => 'AdvertisedVO', homogeneity => 'Homogeneous', architecture => 'Platform', opsys => 'OpSys', benchmark => 'Benchmark', nodememory => 'MaxVirtualMemory', middleware => 'Middleware', alias => 'ClusterAlias', comment => 'ClusterComment'}; if ($cluster->{clustersupport} and $cluster->{clustersupport} =~ /(.*)@/) { my $contact = {}; push @{$config->{contacts}}, $contact; $contact->{Name} = $1; $contact->{Detail} = "mailto:".$cluster->{clustersupport}; $contact->{Type} = 'usersupport'; } if (defined $cluster->{nodeaccess}) { $config->{service}{ConnectivityIn} = 0; $config->{service}{ConnectivityOut} = 0; for (split '\[separator\]', $cluster->{nodeaccess}) { $config->{service}{ConnectivityIn} = 1 if lc $_ eq 'inbound'; $config->{service}{ConnectivityOut} = 1 if lc $_ eq 'outbound'; } } # TODO: this causes possibly unsupported values like OSName, OSVersion to be moved and not renamed move_keys $cluster, $config->{service}, [keys %$share_options, keys %$xenv_options]; # TODO: check if this is needed move_keys $cluster, $config->{lrms}, [keys %$lrms_options, keys %$lrms_share_options]; } ## use hostname as cluster alias if not defined my $hostname = $config->{hostname}; my @dns = split /\./, $hostname; my $shorthost = shift @dns; my $dnsdomain = join ".", @dns; unless (defined $config->{service}{ClusterAlias}) { $log->info("[infosys/cluster] alias= in arc.conf missing. Defaulting to $shorthost"); chomp ($config->{service}{ClusterAlias} ||= $shorthost); } ## use cluster alias as cluster name. Currently we have no way to specify cluster name directly. $config->{service}{ClusterName} = $config->{service}{ClusterAlias}; # remove useless objects if not set delete $config->{location} unless $config->{location} and %{$config->{location}}; delete $config->{contacts} unless $config->{contacts} and @{$config->{contacts}}; # Some checks about contacts if ($config->{contacts}) { for (@{$config->{contacts}}) { $log->warning("Contact is missing Type") and next unless $_->{Type}; $log->warning("Contact is missing Detail") and next unless $_->{Detail}; $log->warning("Contact Detail is not an URI: ".$_->{Detail}) and next unless $_->{Detail} =~ m/^\w+:/; } } # Generate initial shares and execution environments array, using configured queues my @qnames=(); for my $keyname (keys %{$jsonconf}) { push(@qnames,$1) if $keyname =~ /queue\:(.*)/; } for my $name (@qnames) { my $queue = $jsonconf->{"queue:$name"}; # at first every bare queue is a share my $sconf = $config->{shares}{$name} ||= {}; $config->{shares}{$name}{MappingQueue} = $name; my $xeconf = $config->{xenvs}{$name} ||= {}; push @{$sconf->{ExecutionEnvironmentName}}, $name; rename_keys $queue, $sconf, {nodememory => 'MaxVirtualMemory', comment => 'Description', advertisedvo => 'AdvertisedVO', maxslotsperjob => 'MaxSlotsPerJob'}; move_keys $queue, $sconf, [keys %$share_options, keys %$lrms_share_options]; # TODO: change opsys here if needed rename_keys $queue, $xeconf, {homogeneity => 'Homogeneous', architecture => 'Platform', opsys => 'OpSys', osname => 'OSName', osversion => 'OSVersion', osfamily => 'OSFamily', benchmark => 'Benchmark'}; move_keys $queue, $xeconf, [keys %$xenv_options]; # TODO: possibly remove. This option was meant to select a group of nodes for a queue # Since arc 6.6 this is autodiscovered from the batch system for some LRMS. # It may be useful in cloud context, so kept for now $xeconf->{NodeSelection} = {}; } # At least one queue must be defined. Maybe this can be relaxed. $log->error("No queue or ComputingShare configured") unless %{$config->{shares}}; $log->error("GLUE2: defaultqueue set to nonexistent ComputingShare") if $config->{lrms}{defaultqueue} and not $config->{shares}{$config->{lrms}{defaultqueue}}; # fire warning if GLUE2 Service Quality Level is not good if (defined $config->{service}{QualityLevel}) { my $qualitylevelstring = $config->{service}{QualityLevel}; my $closedenumeration = {'development' => '1', 'pre-production' => '1', 'production' => '1', 'testing' => '1' }; unless (defined $closedenumeration->{$config->{service}{QualityLevel}}) { my @enum = keys %$closedenumeration; $log->error("computingservice_qualitylevel contains \"$qualitylevelstring\" which is an invalid value. Allowed value is one of: @enum"); } } # This only happens when no queue is configured. At the moment every queue has its own execution env $log->error("No ExecutionEnvironment configured") unless %{$config->{xenvs}}; # ExecutionEnvironments are automatically discovered in some LRMS since arc6.6 # However manual selection could be useful in the future (for example cloud environments) so this code is kept. # Cross-check ExecutionEnvironment references for my $s (values %{$config->{shares}}) { next unless $s->{ExecutionEnvironmentName}; for my $group (@{$s->{ExecutionEnvironmentName}}) { $log->error("ComputingShare associated with non-existent ExecutionEnvironment: $group") unless $config->{xenvs}{$group}; } } for my $s (values %{$config->{xenvs}}) { delete $s->{NodeSelection} unless %{$s->{NodeSelection}}; } ## TODO: Check that this code below is currently not used. Access/mapping policies are currently # generated in ARC1ClusterInfo.pm, but this information is mostly static and # can be generated already now, simplifying ARC1ClusterInfo.pm. However simplification is not much. # Also, this is strictly GLUE2, so maybe it should be confined in ARC1ClusterInfo.pm after all... # TODO: move this below to check_config # TODO: create shares based on configured VOs in ConfigCentral? Is this possible/good?) # why are these checks not done by InfoChecker? # Cross-check MappingPolicy references and move them to the share wehre they belong # Initialize policies data based on authorization information. # This might be extended in the future for more complex scenarios. For now # only VO based ones are built. #my %queuenamesset = map { $_ => '1' } (keys %{$config->{shares}}); # #if (defined $config->{service}{AuthorizedVO}) { # for my $policy (@{$config->{service}{AuthorizedVO}}) { # $config->{mappingpolicies}{$policy} ||= {}; # $config->{mappingpolicies}{$policy}{Scheme} = 'basic;'; # $config->{mappingpolicies}{$policy}{Rule} = [ "vo:$policy" ]; # $config->{mappingpolicies}{$policy}{queues} = Storable::dclone(\%queuenamesset); # # $config->{accesspolicies}{$policy} ||= {}; # $config->{accesspolicies}{$policy}{Scheme} = 'basic;'; # $config->{accesspolicies}{$policy}{Rule} = [ "vo:$policy" ]; # } #} #for my $queue (keys %queuenamesset) { # if (defined $config->{shares}{$queue}{authorizedvo}) { # for my $policy (@{$config->{shares}{$queue}{authorizedvo}}) { # #$log->debug("$policy".Dumper($config->{mappingpolicies})); # $config->{mappingpolicies}{$policy} ||= {}; # $config->{mappingpolicies}{$policy}{Scheme} = 'basic'; # $config->{mappingpolicies}{$policy}{Rule} = [ "vo:$policy" ]; # $config->{mappingpolicies}{$policy}{queues}{$queue} = '1'; # # $config->{accesspolicies}{$policy} ||= {}; # $config->{accesspolicies}{$policy}{Scheme} = 'basic'; # $config->{accesspolicies}{$policy}{Rule} = [ "vo:$policy" ]; # } # } #} # Disable this for now, but it should replace similar code in ARC1ClusterInfo.pm #for my $s (keys %{$config->{mappingpolicies}}) { ## generate new share names and their data #for my $queuename (keys %{$config->{mappingpolicies}{$s}{queues}}) { #my $newsharename = $queuename.'_'.$s; ## temporary until we fix mappingqueues properly #$config->{shares}{$newsharename} = Storable::dclone($config->{shares}{$queuename}); ## TODO: add this to the other shares too! #$config->{shares}{$newsharename}{MappingQueue} = $queuename; #}; #} # Create a list with all multi-valued options based on $config_schema. my @multival = (); hash_tree_apply $config_schema, sub { my $h = shift; for (keys %$h) { next if ref $h->{$_} ne 'ARRAY'; next if ref $h->{$_}[0]; # exclude deep structures push @multival, $_; } }; # Transform multi-valued options into arrays hash_tree_apply $config, sub { my $h = shift; while (my ($k,$v) = each %$h) { next if ref $v; # skip anything other than scalars $h->{$k} = [split '\[separator\]', $v]; unless (grep {$k eq $_} @multival) { $h->{$k} = pop @{$h->{$k}}; # single valued options, remember last defined value only } } }; hash_tree_apply $config, sub { fixbools shift, $allbools }; return $config; } # # Infoproviders config parser. It takes in input a json file which # represents arc.conf and manipulates it to prepare information. # sub parseConfig { my ($file,$arc_location) = @_; my $config; $config = build_config_from_json($file); #print Dumper($config); # C 134 LogUtils::level($config->{infosys}{loglevel}) if $config->{infosys}{loglevel}; my $checker = InfoChecker->new($config_schema); my @messages = $checker->verify($config,1); $log->verbose("config key config->$_") foreach @messages; $log->verbose("Some required config options are missing or not used by infosys") if @messages; return $config; } ## subblock_check_required(jsonconfig, block, requiredblock) # Checks if the parent block is enabled and sets the enable flags accordingly sub subblock_check_required { my ($config, $block) = @_; my @dependencyarray = @{$blockdependencies->{$block}}; if ($config->{$block}{enabled}) { $config->{$block}{enabled} = 0; for my $requiredblock (@dependencyarray) { $log->debug("Checking $block, enabled=$config->{$block}{enabled}, $requiredblock, enabled=$config->{$requiredblock}{enabled}"); unless ($config->{$requiredblock}{enabled}) { $log->error("required [$requiredblock] block not found but [$block] defined. Please define [$requiredblock]. Exiting"); } else { $config->{$block}{enabled} = 1; } } } } ## The defaults come from the parser, but due to a PERL side effect in scanning # hashes here we add information about enable/disable blocks. sub set_defaults { my ($config) = @_; $log->debug("Applying defaults"); # blocknames that are relevant for infosys and should be set to enable my @blocknames = ('arex', 'arex/ws', 'arex/ws/jobs', 'gridftpd', 'infosys', 'infosys/cluster', 'infosys/glue1', 'infosys/glue2', 'infosys/glue2/ldap', 'infosys/ldap', 'infosys/nordugrid' ); ## TODO: change logic to scan config and add enabled to all enabled blocks, then do checks for my $block (@blocknames) { $config->{$block}{enabled} = (defined $config->{$block}) ? 1 : 0; }; #$log->debug(Dumper($config)); # Check for dependencies for my $block (keys %$blockdependencies) { subblock_check_required($config,$block); } ## Other defaults not covered by arcconfig-parser. Might be done with an hash at some point #default ARC version taken from build. Artificiously placed in common $config->{common}{arcversion}='master'; # gridftpd hardcoded defaults $config->{gridftpd}{port} = '2811' if ($config->{'gridftpd'}{enabled}); $config->{gridftpd}{mountpoint} = '/jobs' if ($config->{'gridftpd'}{enabled}); # force ldap defaults if ldap renderings are enabled in config $config->{infosys}{ldap}{port}='2135' if ($config->{'infosys/ldap'}{enabled}); ## TODO after Andrii changes, might be removed ## force BDII defaults if LDAP configured #if (defined $config->{'infosys/ldap'}) { #$log->verbose("[infosys/ldap/bdii] block not found in arc.conf, using default values") unless (defined $config->{'infosys/ldap/bdii'}); #$config->{'infosys/ldap/bdii'} ||= {}; #} # Set default hostname #$confdefaults->{common}{hostname} = (defined $config->{common}{hostname}) ? $config->{common}{hostname} : hostname(); #$confdefaults->{'infosys/ldap'}{hostname} = (defined $config->{'infosys/ldap'}{hostname}) ? $config->{'infosys/ldap'}{hostname} : hostname(); # Define a-rex endpoint. Hardcoded after ARC6. if (defined $config->{'arex/ws'}) { $config->{'arex/ws'}{wsurl} ||= 'https://'.$config->{common}{hostname}.':443/arex'; } else { $log->verbose("WS interface disabled. Add [arex/ws] block to enable"); } ### end of default blocks creation #for (keys %$confdefaults) { #if (defined $config->{$_}) { #$config->{$_} ||= {}; #merge_hash_values($config->{$_},$confdefaults->{$_}); #} else { #$log->debug("\[$_\] block missing, defaults not applied."); #} #} } ## getValueOf: Cherry picks arc.conf values ## Perl wrapper for the python parser ## input: configfile,configblock, configoption ## TODO: maybe use read_json_config for security reasons? sub getValueOf ($$$){ my ($arcconf,$block,$option) = @_; # get the calling script basepath. Will be used to # find external scripts like arcconfig-parser. my $libexecpath = ($ENV{'ARC_LOCATION'} || '/cvmfs/dirac.egi.eu/dirac/v9.0.0a28/Linux-x86_64') . '/libexec/arc'; my $value=''; { local $/; # slurp mode open (my $parserout, "$libexecpath/arcconfig-parser --load -r $arcconf -b $block -o $option |") || $log->error("Python config parser error: $! at line: ".__LINE__." libexecpath: $libexecpath"); $value = <$parserout>; close $parserout; } # remove blank spaces before and after $value =~ s/^\s+|\s+$//g; # strip trailing newline chomp $value; # TODO: this should be removed after Andrii's default in place # get defaults #if ( !(defined $value) || $value eq 'None') { # $value = $confdefaults->{$block}{$option} if (defined $confdefaults->{$block}{$option}); #} return $value; } ## isBlockPresent: returns true if block exists in config sub isBlockPresent ($$) { my ($arcconf,$block) = @_; my $jsonconf = read_json_config($arcconf); if (defined $jsonconf->{$block}) { return 1 } else { return 0 } ; } sub dumpInternalDatastructure ($){ my ($config) = @_; print Dumper($config); } 1; __END__