package ConfigCentral;

# Builds an intermediate config structure used by infoproviders.
# OBS: the structure is NOT the same as arc.conf.

## RESTRUCTURING PHASE ################
## changes are identified by the tags
## #C changenumber
#######################################

use strict;
use warnings;
use File::Basename;
use Sys::Hostname;

# added to parse JSON stuff
binmode STDOUT, ":utf8";
use utf8;

use XML::Simple;
use Data::Dumper qw(Dumper);
use JSON::XS;
#use Data::Dumper::Concise;

use IniParser;
use InfoChecker;
use LogUtils;

# while parsing, loglevel is WARNING (the default)
our $log = LogUtils->getLogger(__PACKAGE__);

#######################################################################
## Block dependencies
#######################################################################

my $blockdependencies = {
    'gridftpd/jobs' => ['gridftpd'],
    'infosys/cluster' => ['infosys'],
    'infosys/ldap' => ['infosys'],
    'infosys/nordugrid' => ['infosys/ldap'],
    'infosys/glue1' => ['infosys/nordugrid'],
    'infosys/glue2' => ['infosys'],
    'infosys/glue2/ldap' => ['infosys/glue2','infosys/ldap'],
};



######################################################################
# Legacy Internal representation of configuration data after parsing #
######################################################################

my $lrms_options = {
    pbs_bin_path => '*',
    pbs_log_path => '*',
    dedicated_node_string => '*',
    condor_bin_path => '*',
    condor_config => '*',
    condor_rank => '*',
    sge_bin_path => '*',
    sge_root => '*',
    sge_cell => '*',
    sge_qmaster_port => '*',
    sge_execd_port => '*',
    lsf_bin_path => '*',
    lsf_profile_path => '*',
    ll_bin_path => '*',
    slurm_bin_path => '*',
    slurm_wakeupperiod => '*',
    boinc_db_host => '*',
    boinc_db_port => '*',
    boinc_db_name => '*',
    boinc_db_user => '*',
    boinc_db_pass => '*',
};
my $lrms_share_options = {
    queue_node_string => '*',
    condor_requirements => '*',
    sge_jobopts => '*',
    lsf_architecture => '*',
    ll_consumable_resources => '*',
};
my $xenv_options = {
    Platform => '*',
    Homogeneous => '*',
    PhysicalCPUs => '*',
    LogicalCPUs => '*',
    CPUVendor => '*',
    CPUModel => '*',
    CPUVersion => '*',
    CPUClockSpeed => '*',
    CPUTimeScalingFactor => '*',
    WallTimeScalingFactor => '*',
    MainMemorySize => '*',
    VirtualMemorySize => '*',
    OSFamily => '*',
    OSName => '*',
    OSVersion => '*',
    VirtualMachine => '*',
    NetworkInfo => '*',
    ConnectivityIn => '*',
    ConnectivityOut => '*',
    Benchmark => [ '*' ],
    OpSys => [ '*' ],
    nodecpu => '*',
};
my $share_options = {
    MaxVirtualMemory => '*',
    MaxSlotsPerJob => '*',
    SchedulingPolicy => '*',
    Preemption => '*',
    totalcpus => '*',
    defaultmemory => '*',
    AdvertisedVO =>  [ '*' ],
    maxcputime => '*',
    maxwalltime => '*',
    mincputime => '*',
    minwalltime => '*'
};
my $cache_options = {
   cachedir => [ '*' ],
   cachesize => '*'
};
my $gmuser_options = {
    controldir => '',
    sessiondir => [ '' ],
    defaultttl => '*',
    %$cache_options
};
my $gmcommon_options = {
    arcversion => '',
    arexhostport => '',
    # lrms => '', C 10 removed from here to the lrms hash
    gmconfig => '*',
    endpoint => '*',
    hostname => '*',
    maxjobs => '*',
    maxload => '*',
    maxloadshare => '*',
    wakeupperiod => '*',
    gridmap => '*',
    #x509_user_key => '*',  # C 5
    x509_host_key => '*',   # C 5
    #x509_user_cert => '*', # C 6
    x509_host_cert => '*',  # C 6
    x509_cert_dir => '*',
    runtimedir => '*',
    shared_filesystem => '*',
    shared_scratch => '*',
    scratchdir => '*',
    enable_perflog_reporting => '*',
    perflogdir => '*'
};
my $sshcommon_options = {
    remote_user => '*',
    remote_host => '*',
    remote_sessiondir => '*',
    private_key => '*',
};

# C 141
my $ldap_infosys_options = {
    enabled => '',
    port => '*',
    infosys_ldap_run_dir => '*',
    validity_ttl => '*',
    # These values have been checked to be used by infoproviders.
    user => '*',
    bdii_run_dir => '*',
    bdii_log_dir => '*',
    bdii_tmp_dir => '*',
    bdii_var_dir => '*',
    bdii_update_pid_file => '*'
    ## TODO: fix this elsewere, the value should be set according to the formula below
    ## not used directly by infoproviders, maybe by bdii config?
    ## this is probably done in the startup script, is called max_cycle
    ## bdii_read_timeout = number - Sets BDII_READ_TIMEOUT in bdii configuration file
    ## default: $bdii_provider_timeout + $infoproviders_timelimit + $wakeupperiod
    #bdii_read_timeout=300
};

# C 125-130 renaming these properly direcly in the gridftpd block
#my $gridftpd_options = {
#    GridftpdEnabled => '*',
#    GridftpdPort => '*',
#    GridftpdMountPoint => '*',
#    GridftpdAllowNew => '*',
#    GridftpdPidFile => '*',
#};

# [arex] subblocks
my $wsemies_options = { # C 71
    allownew => '*',     # C 72
    enabled => ''
}; 

# C 66
my $ws_options = {
    enabled => '',
    wsurl => '*',
    emies => { %$wsemies_options } # C 71
};

my $admindomain_options = {
    Name => '*',
    Description => '*',
    WWW => '*',
    Distributed => '*',
    Owner => '*',
    OtherInfo => '*',
};

# C 156
my $glue2_options = {
    enabled => '',
    computingservice_qualitylevel => '*' # 157
};


# # # # # # # # # # # # # #

my $config_schema = {
    defaultLocalName => '*',
    
    #ProviderLog => '*', #C 133
    ttl => '*',

    admindomain => { %$admindomain_options },   
    
    %$gmcommon_options,
    %$sshcommon_options,
    # moved all these to their own subtrees to be in sync with new config
    # %$gridftpd_options,
    # %$ldap_infosys_options,
    # %$lrms_options,
    # %$lrms_share_options,    
    control => {
        '*' => {
            %$gmuser_options
        }
    },
    service => {
        OtherInfo => [ '*' ],
        StatusInfo => [ '*' ],
        Downtime => '*',
        ClusterName => '*',
        ClusterAlias => '*',
        ClusterComment => '*',
        ClusterOwner => [ '*' ],
        Middleware => [ '*' ],
        AdvertisedVO => [ '*' ],
        LocalSE => [ '*' ],
        InteractiveContactstring => [ '*' ],
        # TODO: This causes odd/unsupported options to pass the checks
        %$xenv_options,
        %$share_options,
        QualityLevel => ''
    },
    location => {
        Name => '*',
        Address => '*',
        Place => '*',
        Country => '*',
        PostCode => '*',
        Latitude => '*',
        Longitude => '*',
    },
    contacts => [ {
        Name => '*',
        OtherInfo => [ '*' ],
        Detail => '',
        Type => '',
    } ],
    accesspolicies => {
        '*' => {
            Rule => [ '' ],
            UserDomainID => [ '' ]
        }
    },
    mappingpolicies => {
        '*' => {
           ShareName => [ '' ],
           Rule => [ '' ],
           UserDomainID => [ '' ],
        }
    },
    xenvs => {
        '*' => {
            OtherInfo => [ '*' ],
            NodeSelection => {
                Regex => [ '*' ],
                Command => [ '*' ],
                Tag => [ '*' ],
            },
            %$xenv_options,
        }
    },
    shares => {
        '*' => {
            Description => '*',
            OtherInfo => [ '*' ],
            MappingQueue => '',
            ExecutionEnvironmentName => [ '' ],
            %$share_options,
            %$lrms_share_options,
        }
    },
    # start of newly added items for arcconf restructuring
    # C 10
    lrms => {
        lrms => '',
        defaultqueue => '*',
        lrmsconfig => '*',
        %$lrms_options,
        %$lrms_share_options
    },
    infosys => {
        enabled => '',
        nordugrid => {
            enabled => ''
        }, # C 155
        logfile => '*',  #C 133
        loglevel => '*', #C 134, replaces ProviderLog
        validity_ttl => '*',
        ldap => { # C141
            %$ldap_infosys_options, # C 141
        },
        glue1 => {
            enabled => ''
            }, # C 161
        glue2 => { # C 156
           %$glue2_options,
           ldap => { # C158
                enabled => '',
                showactivities => '*'
           }
        }
    },
    arex => {
        enabled => '',
        defaultttl => '*',
        logfile => '*',
        loglevel => '*', # replaces $config_schema->{debugLevel}, C 37
        infoproviders_timelimit => '*', # C 42
        port => '*',
        shared_filesystem => '*',
        ws => { %$ws_options } # C 66
    },
    gridftpd => {
        enabled => '',
        port => '*',
        mountpoint => '*',
        allownew => '*',
        pidfile => '*'
    }
};

# TODO: remove some or change because of the enable-by-block strategy
my $allbools = [ qw(
                 Homogeneous VirtualMachine
                 ConnectivityIn ConnectivityOut Preemption
                 showactivities shared_filesystem
                 enabled allownew Distributed) ];

############################ Generic functions ###########################

# walks a tree of hashes and arrays while applying a function to each hash.
sub hash_tree_apply {
    my ($ref, $func) = @_;
    if (not ref($ref)) {
        return;
    } elsif (ref($ref) eq 'ARRAY') {
        map {hash_tree_apply($_,$func)} @$ref;
        return;
    } elsif (ref($ref) eq 'HASH') {
        &$func($ref);
        map {hash_tree_apply($_,$func)} values %$ref;
        return;
    } else {
        return;
    }
}

# Strips namespace prefixes from the keys of the hash passed by reference
sub hash_strip_prefixes {
    my ($h) = @_;
    my %t;
    while (my ($k,$v) = each %$h) {
        next if $k =~ m/^xmlns/;
        $k =~ s/^\w+://;
        $t{$k} = $v;
    }
    %$h=%t;
    return;
}

# Verifies that a key is an HASH reference and returns that reference
sub hash_get_hashref {
    my ($h, $key) = @_;
    my $r = ($h->{$key} ||= {});
    $log->fatal("badly formed '$key' element in XML config") unless ref $r eq 'HASH';
    return $r;
}

# Verifies that a key is an ARRAY reference and returns that reference
sub hash_get_arrayref {
    my ($h, $key) = @_;
    my $r = ($h->{$key} ||= []);
    $log->fatal("badly formed '$key' element in XML config") unless ref $r eq 'ARRAY';
    return $r;
}

# Set selected keys to either 'true' or 'false'
sub fixbools {
    my ($h,$bools) = @_;
    for my $key (@$bools) {
        next unless exists $h->{$key};
        my $val = $h->{$key};
        if ($val eq '0' or lc $val eq 'false' or lc $val eq 'no' or lc $val eq 'disable') {
            $h->{$key} = '0';
        } elsif ($val eq '1' or lc $val eq 'true' or lc $val eq 'yes' or lc $val eq 'enable' or lc $val eq 'expert-debug-on') {
            $h->{$key} = '1';
        } else {
            $log->error("Invalid value for $key");
        }
    }
    return $h;
}

sub move_keys {
    my ($h, $k, $names) = @_;
    for my $key (@$names) {
        next unless exists $h->{$key};
        $k->{$key} = $h->{$key};
        delete $h->{$key};
    }
}

sub rename_keys {
    my ($h, $k, $names) = @_;
    for my $key (keys %$names) {
        next unless exists $h->{$key};
        my $newkey = $names->{$key};
        $k->{$newkey} = $h->{$key};
        delete $h->{$key};
    }
}

# Takes two hash references and merges values
# the value of hash2 is taken if the value in 
# hash1 is not defined
# usage: merge_hash_values(hash1,hash2)
sub merge_hash_values {
    my ($hash1,$hash2) = @_;
    for my $key (keys %{$hash2}) {
        $hash1->{$key} = $hash2->{$key} if ((not defined $hash1->{$key}) || ($hash1->{$key} eq ''));
        # attempt to merge recursively
        # merge_hash_values($hash1->{$key},$hash2->{$key}) if (ref $key eq ref {});
    }
}

##################### Read config via arcconfig-parser ################

# execute parser and get json data
sub read_json_config {
    my ($arcconf) = @_;
    
    # get the calling script basepath. Will be used to
    # find external scripts like arcconfig-parser.
    my $libexecpath = ($ENV{'ARC_LOCATION'} || '/cvmfs/dirac.egi.eu/dirac/v8.0.22/Linux-x86_64') . '/libexec/arc';
    
    my $jsonconfig='';
    { 
      local $/; # slurp mode
       open (my $jsonout, "$libexecpath/arcconfig-parser -e json --load -r $arcconf |") || $log->error("Python config parser error: $! at line: ".__LINE__." libexecpath: $libexecpath");
       $jsonconfig = <$jsonout>;
       close $jsonout;
    }
    my $config = decode_json($jsonconfig);
    #print Dumper($config);
    
    return $config;
}

#
# Removes spaces at beginning and end from all config values
# and config names (such as queue:\s\squeuename\s*)    
# 
sub strip_spaces {
    my ($jsonconf) = @_;
    
    for my $key (keys %{$jsonconf}) {
        # recur if inner values
        if (ref($jsonconf->{$key}) eq 'HASH') {
            strip_spaces($jsonconf->{$key});
        } elsif (ref($jsonconf->{$key}) eq 'ARRAY') {
              # strip spaces from array elements
              for my $item (@{$jsonconf->{$key}}) {
                  $item =~ s/^\s+|\s+$//g;
              }
        } elsif ($jsonconf->{$key} =~ /^\s*(.*)\s*$/) {
                 my $newvalue = $1;
                 $newvalue =~ s/^\s+|\s+$//g;
                 $jsonconf->{$key} = $newvalue;
        }
        # change key once back from recursion if it contains spaces
        if ($key =~ /\s*(queue|authgroup)\s*:\s*(.*)\s*/) {
           my $prefix = "$1:";
           my $stripstring = $2;
           $stripstring =~ s/^\s+|\s+$//g;
           my $newkey="$prefix$stripstring";
           $jsonconf->{$newkey} = $jsonconf->{$key};
           # not needed as the reference is copied. But I don't trust it.
           #delete $jsonconf->{$key};
        }
    }
}

#
# Reads the json config file passed as the first argument and produces a config
# hash conforming to $config_schema. 
#
sub build_config_from_json {
    my ($file) = @_;
    
    my $jsonconf = read_json_config($file);

    strip_spaces($jsonconf);
    
    set_defaults($jsonconf);

    
    # Those values that are the same as in arc.conf will 
    # be copied and checked.
    my $config ||= {};
    $config->{service} ||= {};
    $config->{control} ||= {};
    ## TODO: maybe move all glue2 stuff into some GLUE2 subset?
    $config->{location} ||= {};
    $config->{contacts} ||= [];
    $config->{accesspolicies} ||= {};   
    $config->{mappingpolicies} ||= {};
    $config->{xenvs} ||= {};
    $config->{shares} ||= {};

    # start of restructured pieces of information
    $config->{infosys} ||= {};
    # use service for first implementation
    $config->{arex} ||= {};
    $config->{lrms} ||= {};
    #  end of restructured pieces of information
    

    my $common = $jsonconf->{'common'};    
    ## TODO: remove everything that should not be in $config
    move_keys $common, $config, [keys %$gmcommon_options];

    # TODO: create a mapping hash eventually
    # new mapping block (just move to the top config as it was before)
    my $mapping = $jsonconf->{'mapping'};
    move_keys $mapping, $config, [keys %$gmcommon_options];
    
    # C 10 new lrms block
    my $lrms = $jsonconf->{'lrms'};
    # some options in lrms, moved to $config for  backward compatibility, 
    # should be moved to {lrms} instead
    #move_keys $lrms, $config, [keys %$lrms_options, keys %$lrms_share_options];
    move_keys $lrms, $config->{'lrms'}, [keys %$lrms];
    move_keys $lrms, $config->{'lrms'}, [keys %$lrms_options, keys %$lrms_share_options];
    
    # Parsing for default queue
    my ($lrmsname, $defaultqueue) = split /\s+/, $config->{lrms}{lrms} || ''; # C 10
    $config->{'lrms'}{'lrms'} = $lrmsname; 
    $config->{'lrms'}{'defaultqueue'} = $defaultqueue if defined $defaultqueue;

    # C 173
    my $arex = $jsonconf->{'arex'};
    move_keys $arex, $config, [keys %$gmcommon_options];
    
    my $ssh = $jsonconf->{'ssh'};
    move_keys $ssh, $config, [keys %$sshcommon_options];
        
    #C 133 134
    my $infosys = $jsonconf->{'infosys'};
        
    move_keys $infosys, $config->{'infosys'}, [keys %$infosys];
        
    rename_keys $infosys, $config, {port => 'SlapdPort'};
    move_keys $infosys, $config, [keys %$ldap_infosys_options];
    
    # only one grid manager user, formerly represented by a dot
    $config->{control}{'.'} ||= {};
    move_keys $arex, $config->{control}{'.'}, [keys %$gmuser_options];
    # C 173
    move_keys $arex, $config->{arex}, [keys %$arex];
    # use service for first implementation
    #move_keys $arex, $config->{service}, [keys %$service];

    # check that the above generated required information
    $log->error("No control directory configured") unless %{$config->{control}} or $config->{remotegmdirs};
    
    # TODO: remove multiple control dirs feature
    while (my ($user, $control) = each %{$config->{control}}) {
        $log->error("No control directory configured for user $user") unless $control->{controldir};
        $log->error("No session directory configured for user $user") unless $control->{sessiondir};
    }


    # NEW: arex subblocks
    # C 71
    if (defined $jsonconf->{'arex/ws'}) {
       my $arexws = $jsonconf->{'arex/ws'};
       $config->{arex}{ws} ||= {};
       move_keys $arexws, $config->{arex}{ws}, [keys %$ws_options];
    }
    
     # handle ws endpoint information for use inside infoproviders
     if (defined $config->{arex}{ws}{wsurl}) {
        $config->{arex}{ws}{wsurl} =~ m{^(https?)://([^:/]+)(?::(\d+))?(.*)};
        my ($proto,$host,$port,$mountpoint) = ($1,$2,$3,$4);
        $port ||= 80 if $proto eq "http";
        $port ||= 443 if $proto eq "https";
        $config->{arex}{port} = $port;
        $config->{arexhostport} = "$host:$port";
    } 
    
    $config->{arex}{ws}{emies} ||= {};
    if (defined $jsonconf->{'arex/ws/jobs'}) {
        my $arexwsemies = $jsonconf->{'arex/ws/jobs'};
        move_keys $arexwsemies, $config->{arex}{ws}{emies}, [keys %$wsemies_options];
        $config->{arex}{ws}{emies}{enabled} = 1;
    } else {
        ## TODO: find a better strategy for the "enabled" defaults, blocks/hashes can't be used.
        $config->{arex}{ws}{emies}{enabled} = 0;
    }

   # cache handling, mixes info from different blocks into control
   $config->{arex}{cache} ||= {};
    if (defined $jsonconf->{'arex/cache'}) {
       my $cacheopts = $jsonconf->{'arex/cache'};
       # moved to control for legacy reasons
       move_keys $cacheopts, $config->{control}{'.'}, [keys %$cache_options];
    }
    $config->{arex}{cache}{cleaner} ||= {};
    if (defined $jsonconf->{'arex/cache/cleaner'}) {
       my $cacheopts = $jsonconf->{'arex/cache/cleaner'};
       # moved to control for legacy reasons
       move_keys $cacheopts, $config->{control}{'.'}, [keys %$cache_options];
    }

    # gridftpd config
    if ((defined $jsonconf->{'gridftpd/jobs'}) || (defined $jsonconf->{'gridftpd'})) {
        $config->{gridftpd} ||= {};
        my $gconf = $jsonconf->{'gridftpd'};
        my $gjconf = $jsonconf->{'gridftpd/jobs'};
        move_keys $gconf, $config->{gridftpd}, [keys %{$config_schema->{gridftpd}}];
        move_keys $gjconf, $config->{gridftpd}, [keys %{$config_schema->{gridftpd}}];
    } else {
        $log->verbose("[gridftpd/jobs] block not found in arc.conf. Disabling gridftpd job interface information generation.");
    }

    # TODO: review BDII config, block disappeared but maybe options not moved back.
    # LDAP config
    if (defined $jsonconf->{'infosys/ldap'}) {
        $config->{infosys}{ldap} ||= {};
        my $ldapconf = $jsonconf->{'infosys/ldap'};
        move_keys $ldapconf, $config->{infosys}{ldap}, [keys %$ldap_infosys_options];
    } else {
        $log->verbose("[infosys/ldap] block not found in arc.conf Disabling LDAP/LDIF information generation.");
    }

    # information schemas
    # NorduGRID
    if ($jsonconf->{'infosys/nordugrid'}{enabled}) {
        $config->{infosys}{nordugrid} ||= {};
        move_keys $jsonconf->{'infosys/nordugrid'}, $config->{infosys}{nordugrid}, [keys %{$config_schema->{infosys}{nordugrid}}];
    }

    # Glue 1.2/1.3
    # this is only used to enable rendering here, no need to parse.
    # glue-generator.pl actually processes that information.
    if (defined $jsonconf->{'infosys/glue1'}) {
        $config->{infosys}{glue1} ||= {};
    }    

    ####### GLUE2
    if ( $jsonconf->{'infosys/glue2'}{enabled} ) {
        $config->{infosys}{glue2} ||= {};
        my $glue2conf = $jsonconf->{'infosys/glue2'};
        move_keys $glue2conf, $config->{infosys}{glue2}, [keys %$glue2_options];
        rename_keys $config->{infosys}{glue2}, $config->{service}, {computingservice_qualitylevel => 'QualityLevel'};
        # GLUE2 ldap
        if ( $jsonconf->{'infosys/glue2/ldap'}{enabled} ) {
           $config->{infosys}{glue2}{ldap} ||= {};
           my $glue2ldapconf = $jsonconf->{'infosys/glue2/ldap'};
           move_keys $glue2ldapconf, $config->{infosys}{glue2}{ldap}, [keys %{$config_schema->{infosys}{glue2}{ldap}}];
        }    

        # AdminDomain
        $config->{admindomain} ||= {};
        $log->warning('[infosys/glue2] section missing admindomain_name information. Default will be set to GLUE2 default UNDEFINEDVALUE.') if ($jsonconf->{'infosys/glue2'}{admindomain_name} eq 'UNDEFINEDVALUE' );
        my $admindomainconf = $jsonconf->{'infosys/glue2'};
        rename_keys $admindomainconf, $config->{'admindomain'}, {
                admindomain_name => 'Name',
                admindomain_description => 'Description',
                admindomain_www => 'WWW',
                admindomain_distributed => 'Distributed',
                admindomain_owner => 'Owner',
                admindomain_otherinfo => 'OtherInfo'
        }
    } else {
        $log->error('Infoproviders cannot continue without the [infosys/glue2] block. Please add it. Exiting...')
    }

    ### Process infosys/cluster
    my $cluster = $jsonconf->{'infosys/cluster'};
    if (%$cluster) {
        # Ignored: cluster_location, lrmsconfig
        rename_keys $cluster, $config->{location}, { cluster_location => 'PostCode' };
        rename_keys $cluster, $config->{service}, {
                                 interactive_contactstring => 'InteractiveContactstring',
                                 cluster_owner => 'ClusterOwner', localse => 'LocalSE',
                                 advertisedvo => 'AdvertisedVO', homogeneity => 'Homogeneous',
                                 architecture => 'Platform', opsys => 'OpSys', benchmark => 'Benchmark',
                                 nodememory => 'MaxVirtualMemory', middleware => 'Middleware',
                                 alias => 'ClusterAlias', comment => 'ClusterComment'};
        if ($cluster->{clustersupport} and $cluster->{clustersupport} =~ /(.*)@/) {
            my $contact = {};
            push @{$config->{contacts}}, $contact;
            $contact->{Name} = $1;
            $contact->{Detail} = "mailto:".$cluster->{clustersupport};
            $contact->{Type} = 'usersupport';
        }
        if (defined $cluster->{nodeaccess}) {
            $config->{service}{ConnectivityIn} = 0;
            $config->{service}{ConnectivityOut} = 0;
            for (split '\[separator\]', $cluster->{nodeaccess}) {
                $config->{service}{ConnectivityIn} = 1 if lc $_ eq 'inbound';
                $config->{service}{ConnectivityOut} = 1 if lc $_ eq 'outbound';
            }
        }
        # TODO: this causes possibly unsupported values like OSName, OSVersion to be moved and not renamed
        move_keys $cluster, $config->{service}, [keys %$share_options, keys %$xenv_options];
        # TODO: check if this is needed
        move_keys $cluster, $config->{lrms}, [keys %$lrms_options, keys %$lrms_share_options];
    }

    ## use hostname as cluster alias if not defined
    my $hostname = $config->{hostname};
     
    my @dns = split /\./, $hostname;
    my $shorthost = shift @dns;
    my $dnsdomain = join ".", @dns;
        
    unless (defined $config->{service}{ClusterAlias}) {
        $log->info("[infosys/cluster] alias= in arc.conf missing. Defaulting to $shorthost");
        chomp ($config->{service}{ClusterAlias} ||= $shorthost);
    }
    ## use cluster alias as cluster name. Currently we have no way to specify cluster name directly.
    $config->{service}{ClusterName} = $config->{service}{ClusterAlias};

    # remove useless objects if not set
    delete $config->{location} unless $config->{location} and %{$config->{location}};
    delete $config->{contacts} unless $config->{contacts} and @{$config->{contacts}};
    
    # Some checks about contacts
    if ($config->{contacts}) {
        for (@{$config->{contacts}}) {
            $log->warning("Contact is missing Type") and next unless $_->{Type};
            $log->warning("Contact is missing Detail") and next unless $_->{Detail};
            $log->warning("Contact Detail is not an URI: ".$_->{Detail}) and next
                unless $_->{Detail} =~ m/^\w+:/;
        }
    }
    
    # Generate initial shares and execution environments array, using configured queues
    my @qnames=();
    for my $keyname (keys %{$jsonconf}) {
       push(@qnames,$1) if $keyname =~ /queue\:(.*)/;
    }
    for my $name (@qnames) {
        my $queue = $jsonconf->{"queue:$name"};

        # at first every bare queue is a share
        my $sconf = $config->{shares}{$name} ||= {};
        $config->{shares}{$name}{MappingQueue} = $name;
        
        my $xeconf = $config->{xenvs}{$name} ||= {};
        push @{$sconf->{ExecutionEnvironmentName}}, $name;

        rename_keys $queue, $sconf, {nodememory => 'MaxVirtualMemory',
                                     comment => 'Description', 
                                     advertisedvo => 'AdvertisedVO',
                                     maxslotsperjob => 'MaxSlotsPerJob'};
        move_keys $queue, $sconf, [keys %$share_options, keys %$lrms_share_options];

        # TODO: change opsys here if needed
        rename_keys $queue, $xeconf, {homogeneity => 'Homogeneous', architecture => 'Platform',
                                      opsys => 'OpSys', osname => 'OSName',
                                      osversion => 'OSVersion', osfamily => 'OSFamily',
                                      benchmark => 'Benchmark'};
        move_keys $queue, $xeconf, [keys %$xenv_options];
        
        # TODO: possibly remove. This option was meant to select a group of nodes for a queue 
        # Since arc 6.6 this is autodiscovered from the batch system for some LRMS.
        # It may be useful in cloud context, so kept for now
        $xeconf->{NodeSelection} = {};
    }

    # At least one queue must be defined. Maybe this can be relaxed.
    $log->error("No queue or ComputingShare configured") unless %{$config->{shares}};
    $log->error("GLUE2: defaultqueue set to nonexistent ComputingShare") 
      if $config->{lrms}{defaultqueue} and not $config->{shares}{$config->{lrms}{defaultqueue}};


    # fire warning if GLUE2 Service Quality Level is not good
    if (defined $config->{service}{QualityLevel}) {
        my $qualitylevelstring = $config->{service}{QualityLevel};
        my $closedenumeration = {'development' => '1',
                                 'pre-production' => '1',
                                 'production' => '1',
                                 'testing' => '1'
                                 };
        unless (defined $closedenumeration->{$config->{service}{QualityLevel}}) {
             my @enum = keys %$closedenumeration;
             $log->error("computingservice_qualitylevel contains \"$qualitylevelstring\" which is an invalid value. Allowed value is one of: @enum");
        }
    }

    # This only happens when no queue is configured. At the moment every queue has its own execution env
    $log->error("No ExecutionEnvironment configured") unless %{$config->{xenvs}};

    # ExecutionEnvironments are automatically discovered in some LRMS since arc6.6
    # However manual selection could be useful in the future (for example cloud environments) so this code is kept.
    # Cross-check ExecutionEnvironment references
    for my $s (values %{$config->{shares}}) {
        next unless $s->{ExecutionEnvironmentName};
        for my $group (@{$s->{ExecutionEnvironmentName}}) {
            $log->error("ComputingShare associated with non-existent ExecutionEnvironment: $group")
                unless $config->{xenvs}{$group};
        }
    }
    for my $s (values %{$config->{xenvs}}) {
        delete $s->{NodeSelection} unless %{$s->{NodeSelection}};
    }

    

    ## TODO: Check that this code below is currently not used. Access/mapping policies are currently
    #        generated in ARC1ClusterInfo.pm, but this information is mostly static and
    #        can be generated already now, simplifying ARC1ClusterInfo.pm. However simplification is not much.
    #        Also, this is strictly GLUE2, so maybe it should be confined in ARC1ClusterInfo.pm after all...
    # TODO: move this below to check_config
    # TODO: create shares based on configured VOs in ConfigCentral? Is this possible/good?)
    # why are these checks not done by InfoChecker?
    # Cross-check MappingPolicy references and move them to the share wehre they belong
    
    # Initialize policies data based on authorization information.
    # This might be extended in the future for more complex scenarios. For now
    # only VO based ones are built.
    
    #my %queuenamesset = map { $_ => '1' } (keys %{$config->{shares}});
    #
    #if (defined $config->{service}{AuthorizedVO}) {
    #    for my $policy (@{$config->{service}{AuthorizedVO}}) {
    #        $config->{mappingpolicies}{$policy} ||= {};
    #       $config->{mappingpolicies}{$policy}{Scheme} = 'basic;';
    #       $config->{mappingpolicies}{$policy}{Rule} = [ "vo:$policy" ];
    #       $config->{mappingpolicies}{$policy}{queues} = Storable::dclone(\%queuenamesset);
    #
    #        $config->{accesspolicies}{$policy} ||= {};
    #        $config->{accesspolicies}{$policy}{Scheme} = 'basic;';
    #        $config->{accesspolicies}{$policy}{Rule} = [ "vo:$policy" ];
    
    #    }
    #}
    
    #for my $queue (keys %queuenamesset) {
    #    if (defined $config->{shares}{$queue}{authorizedvo}) {
    #        for my $policy (@{$config->{shares}{$queue}{authorizedvo}}) {
    #            #$log->debug("$policy".Dumper($config->{mappingpolicies}));
    #            $config->{mappingpolicies}{$policy} ||= {};
    #            $config->{mappingpolicies}{$policy}{Scheme} = 'basic';
    #            $config->{mappingpolicies}{$policy}{Rule} = [ "vo:$policy" ];
    #            $config->{mappingpolicies}{$policy}{queues}{$queue} = '1';
    #
    #            $config->{accesspolicies}{$policy} ||= {};            
    #            $config->{accesspolicies}{$policy}{Scheme} = 'basic';
    #            $config->{accesspolicies}{$policy}{Rule} = [ "vo:$policy" ];
    #        }
    #    }
    #}
    
    # Disable this for now, but it should replace similar code in ARC1ClusterInfo.pm    
    #for my $s (keys %{$config->{mappingpolicies}}) {
        ## generate new share names and their data
        #for my $queuename (keys %{$config->{mappingpolicies}{$s}{queues}}) {
            #my $newsharename = $queuename.'_'.$s;
            ## temporary until we fix mappingqueues properly
            #$config->{shares}{$newsharename} = Storable::dclone($config->{shares}{$queuename});
            ## TODO: add this to the other shares too!
            #$config->{shares}{$newsharename}{MappingQueue} = $queuename;
        #};
    #}



    # Create a list with all multi-valued options based on $config_schema.
    my @multival = ();
    hash_tree_apply $config_schema, sub { my $h = shift;
                                           for (keys %$h) {
                                               next if ref $h->{$_} ne 'ARRAY';
                                               next if ref $h->{$_}[0]; # exclude deep structures
                                               push @multival, $_;
                                           }
                                     };
    # Transform multi-valued options into arrays
    hash_tree_apply $config, sub { my $h = shift;
                                   while (my ($k,$v) = each %$h) {
                                       next if ref $v; # skip anything other than scalars
                                       $h->{$k} = [split '\[separator\]', $v];
                                       unless (grep {$k eq $_} @multival) {
                                           $h->{$k} = pop @{$h->{$k}}; # single valued options, remember last defined value only
                                       }
                                   }
                             };

    hash_tree_apply $config, sub { fixbools shift, $allbools };

    return $config;
}


#
# Infoproviders config parser. It takes in input a json file which 
# represents arc.conf and manipulates it to prepare information.
#
sub parseConfig {
    my ($file,$arc_location) = @_;
    my $config;
    
    $config = build_config_from_json($file);

    #print Dumper($config);

    # C 134
    LogUtils::level($config->{infosys}{loglevel}) if $config->{infosys}{loglevel};

    my $checker = InfoChecker->new($config_schema);
    my @messages = $checker->verify($config,1);
    $log->verbose("config key config->$_") foreach @messages;
    $log->verbose("Some required config options are missing or not used by infosys") if @messages;

    return $config;
}

## subblock_check_required(jsonconfig, block, requiredblock)
# Checks if the parent block is enabled and sets the enable flags accordingly

sub subblock_check_required {
    my ($config, $block) = @_;
    my @dependencyarray = @{$blockdependencies->{$block}};
    if ($config->{$block}{enabled}) {
        $config->{$block}{enabled} = 0;
        for my $requiredblock (@dependencyarray) {
            $log->debug("Checking $block, enabled=$config->{$block}{enabled}, $requiredblock, enabled=$config->{$requiredblock}{enabled}");
            unless ($config->{$requiredblock}{enabled}) {
                $log->error("required [$requiredblock] block not found but [$block] defined. Please define [$requiredblock]. Exiting");
            } else {
            $config->{$block}{enabled} = 1;
            } 
        }
    }
}

## The defaults come from the parser, but due to a PERL side effect in scanning
# hashes here we add information about enable/disable blocks.
sub set_defaults {
    my ($config) = @_;
    
    $log->debug("Applying defaults");
    
    # blocknames that are relevant for infosys and should be set to enable
    my @blocknames = ('arex',
                      'arex/ws',
                      'arex/ws/jobs',
                      'gridftpd',
                      'infosys',
                      'infosys/cluster',
                      'infosys/glue1',
                      'infosys/glue2',
                      'infosys/glue2/ldap',
                      'infosys/ldap',
                      'infosys/nordugrid'
                      );

    ## TODO: change logic to scan config and add enabled to all enabled blocks, then do checks
    for my $block (@blocknames) {
        $config->{$block}{enabled} = (defined $config->{$block}) ? 1 : 0;
    };

    #$log->debug(Dumper($config));

    # Check for dependencies
    
    for my $block (keys %$blockdependencies) {
        subblock_check_required($config,$block);
    }
    
    ## Other defaults not covered by arcconfig-parser. Might be done with an hash at some point

    #default ARC version taken from build. Artificiously placed in common
    $config->{common}{arcversion}='master';

    # gridftpd hardcoded defaults
    $config->{gridftpd}{port} = '2811' if ($config->{'gridftpd'}{enabled});
    $config->{gridftpd}{mountpoint} = '/jobs' if ($config->{'gridftpd'}{enabled});

    
    # force ldap defaults if ldap renderings are enabled in config
    $config->{infosys}{ldap}{port}='2135' if ($config->{'infosys/ldap'}{enabled});
    
    ## TODO after Andrii changes, might be removed
    ## force BDII defaults if LDAP configured    
    #if (defined $config->{'infosys/ldap'}) {
        #$log->verbose("[infosys/ldap/bdii] block not found in arc.conf, using default values") unless (defined $config->{'infosys/ldap/bdii'});
        #$config->{'infosys/ldap/bdii'} ||= {};
    #}
    
    # Set default hostname
    #$confdefaults->{common}{hostname} = (defined $config->{common}{hostname}) ? $config->{common}{hostname} : hostname();
    #$confdefaults->{'infosys/ldap'}{hostname} = (defined $config->{'infosys/ldap'}{hostname}) ? $config->{'infosys/ldap'}{hostname} : hostname();
    
    # Define a-rex endpoint. Hardcoded after ARC6.
    if (defined $config->{'arex/ws'}) {
        $config->{'arex/ws'}{wsurl} ||= 'https://'.$config->{common}{hostname}.':443/arex';
    } else {
        $log->verbose("WS interface disabled. Add [arex/ws] block to enable");
    }
    
    ### end of default blocks creation
    
    
    #for (keys %$confdefaults) {
        #if (defined $config->{$_}) {
            #$config->{$_} ||= {};
            #merge_hash_values($config->{$_},$confdefaults->{$_});
        #} else {
          #$log->debug("\[$_\] block missing, defaults not applied.");
        #}
    #}
}

## getValueOf: Cherry picks arc.conf values
## Perl wrapper for the python parser
## input: configfile,configblock, configoption
## TODO: maybe use read_json_config for security reasons?
sub getValueOf ($$$){
    my ($arcconf,$block,$option) = @_;
    
    # get the calling script basepath. Will be used to
    # find external scripts like arcconfig-parser.
    my $libexecpath = ($ENV{'ARC_LOCATION'} || '/cvmfs/dirac.egi.eu/dirac/v8.0.22/Linux-x86_64') . '/libexec/arc';
    
    my $value='';
    { 
      local $/; # slurp mode
      open (my $parserout, "$libexecpath/arcconfig-parser --load -r $arcconf -b $block -o $option |") || $log->error("Python config parser error: $! at line: ".__LINE__." libexecpath: $libexecpath");
      $value = <$parserout>;
      close $parserout;
    }

    # remove blank spaces before and after
    $value =~ s/^\s+|\s+$//g;
    # strip trailing newline
    chomp $value;

    # TODO: this should be removed after Andrii's default in place
    # get defaults
    #if ( !(defined $value) || $value eq 'None') {
    #    $value = $confdefaults->{$block}{$option} if (defined $confdefaults->{$block}{$option});
    #}
  
    return $value;

}

## isBlockPresent: returns true if block exists in config
sub isBlockPresent ($$) {
    my ($arcconf,$block) = @_;

    my $jsonconf = read_json_config($arcconf);

    if (defined $jsonconf->{$block}) { return 1 } else { return 0 } ;
}

sub dumpInternalDatastructure ($){
    my ($config) = @_;
    print Dumper($config);
}

1;

__END__