#!/bin/bash
# set -x
#
#  Input: path to grami file (same as Globus).
#  This script creates a temporary job script and runs it
DEBUG=0
echo "----- starting submit_boinc_job -----" 1>&2
joboption_lrms="boinc"
lrms_options="boinc_app_id boinc_db_host boinc_db_port boinc_db_user boinc_db_pass boinc_db_name boinc_project_dir"

# ARC1 passes first the config file.
if [ "$1" = "--config" ]; then shift; ARC_CONFIG=$1; shift; fi
GRAMI_FILE=$1

# define paths and config parser
basedir=`dirname $0`
basedir=`cd $basedir > /dev/null && pwd` || exit $?
. "${basedir}/lrms_common.sh"

# include common submit functions
. "${pkgdatadir}/submit_common.sh" || exit $?

# run common init 
#  * parse grami
#  * parse config
#  * load LRMS-specific env
#  * set common variables
common_init

joboption_stdout='`pwd`/'`basename $joboption_stdout`
joboption_stderr='`pwd`/'`basename $joboption_stderr`

RUNTIME_NODE_SEES_FRONTEND=yes

##############################################################
# Zero stage of runtime environments
##############################################################
RTE_stage0

joboption_directory_orig=$joboption_directory
joboption_directory='`pwd`'
# make sure session is world-writable
chmod 777 $joboption_directory_orig

PROJECT_ROOT="$CONFIG_boinc_project_dir"
echo "project_root=$PROJECT_ROOT" 1>&2

if [ -z "$PROJECT_ROOT" ]; then
  echo "No project directory configured" 1>&2
  exit 1
elif [ ! -d "$PROJECT_ROOT" ]; then
  echo "Project dir ${PROJECT_ROOT} does not exist" 1>&2
  exit 1
elif [ ! -x "${PROJECT_ROOT}/bin/create_work" ]; then
  echo "Project dir ${PROJECT_ROOT} does appear to be a BOINC project" 1>&2
  exit 1
fi

cd $PROJECT_ROOT

##############################################################
# create job script
##############################################################
mktempscript
LRMS_JOB_BOINC="${LRMS_JOB_SCRIPT}.boinc"
touch $LRMS_JOB_BOINC
chmod u+x ${LRMS_JOB_SCRIPT}


##############################################################
# Start job script
##############################################################

N=0
x=$joboption_directory_orig
while [ "$x" != "/" ]
do
 x=`dirname $x`
N=$((N+1))
done

echo '#!/bin/sh' > $LRMS_JOB_SCRIPT

echo "#job script built by grid-manager and input file for BOINC job" >> $LRMS_JOB_SCRIPT
echo "" >> $LRMS_JOB_SCRIPT
cat >> $LRMS_JOB_SCRIPT <<"FMARK"
set -x 
export RUNTIME_CONFIG_DIR=`pwd`/
#rename root file
FMARK
echo tar --strip-components=$N -xvf *input.tar.gz >> $LRMS_JOB_SCRIPT

##############################################################
# non-parallel jobs
##############################################################

set_count
sourcewithargs_jobscript

##############################################################
# Override umask
##############################################################
echo "" >> $LRMS_JOB_SCRIPT
echo "# Overide umask of execution node (sometime values are really strange)" >> $LRMS_JOB_SCRIPT
echo "umask 077" >> $LRMS_JOB_SCRIPT

##############################################################
# Init accounting
##############################################################
accounting_init

##############################################################
# Add environment variables
##############################################################
add_user_env

##############################################################
# Check for existance of executable,
##############################################################
if [ -z "${joboption_arg_0}" ] ; then
  echo 'Executable is not specified' 1>&2
  exit 1
fi

setup_runtime_env

##############################################################
# Add std... to job arguments
##############################################################
include_std_streams

##############################################################
#  Move files to local working directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
move_files_to_node 

##############################################################
#  Runtime configuration
##############################################################
echo RUNTIME_JOB_DIAG='`pwd`/'`basename $joboption_directory_orig`.diag >>$LRMS_JOB_SCRIPT
RTE_stage1 
echo "echo \"runtimeenvironments=\$runtimeenvironments\" >> \"\$RUNTIME_JOB_DIAG\"" >> $LRMS_JOB_SCRIPT

#####################################################
# Accounting (WN OS Detection)
#####################################################
detect_wn_systemsoftware

#####################################################
#  Go to working dir and start job
#####################################################
cd_and_run

# Add nodename username@hostname and fix core count for accounting
# Add exit code here since accounting_end does it after the diag file has been tarred up
cat >> $LRMS_JOB_SCRIPT <<"FMARK"
sed -i -e '/nodename=/d' $RUNTIME_JOB_DIAG
hostname=` grep domain_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
username=` grep user_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
nodename=$username@$hostname
echo "nodename=$nodename" >> "$RUNTIME_JOB_DIAG"
[ -n "$ATHENA_PROC_NUMBER" ] && sed -i -e 's/Processors=1/Processors='"$ATHENA_PROC_NUMBER"'/' $RUNTIME_JOB_DIAG
echo "exitcode=$RESULT" >> "$RUNTIME_JOB_DIAG"
FMARK

echo "" >> $LRMS_JOB_SCRIPT

##############################################################
#  Runtime (post)configuration at computing node
##############################################################
RTE_stage2


##############################################################
#  Move files back to session directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
#move_files_to_frontend

#############################################################
# zip the result files into 1 file zip.tar.gz
#############################################################
notnull ()
{
   if [ -z $1 ];then
	echo 0
   else
	echo 1
   fi
}
result_list=
i=0
eval opt=\${joboption_outputfile_$i}
ret=`notnull $opt`
while [ $ret = "1" ]
do
   output_file=$(echo $opt|sed -e "s#^/#./#")
   output_file=$(echo $output_file|sed -e "s#@##")
   result_list=$result_list" "$output_file
   i=$((i+1))
   eval opt=\${joboption_outputfile_$i}
   ret=`notnull $opt`
   echo "ret="$ret
done

files=$(echo $result_list|tr " " "\n")

cat >> $LRMS_JOB_SCRIPT <<'EOF'
echo "zip all output files"
flist="*.diag "
EOF
echo for f in $files >>$LRMS_JOB_SCRIPT

cat >> $LRMS_JOB_SCRIPT <<'EOF'
do
 if [  -e $f ];then
 flist=$flist" "$f
fi
done
EOF
#echo $flist
cat <<'EOF' >>$LRMS_JOB_SCRIPT
if [ -f output.list ];then
ol=$(awk '{print $1}' output.list)
for i in $ol
do
 if [[ -f $i && ! $i =~ HITS.* ]];then
 flist=$flist" "$i
 fi
done
sed -i -e 's/HITS.* /HITS.pool.root.1 /' output.list
fi
EOF

echo 'tar cvf result.tar.gz $flist' >>$LRMS_JOB_SCRIPT

chmod a+r $LRMS_JOB_SCRIPT

##############################################################
# Finish accounting and exit job
##############################################################
accounting_end

#######################################
#  Submit the job
#######################################

echo "job script ${LRMS_JOB_SCRIPT} built" 1>&2
JobId=`basename $joboption_directory_orig`
JobInput=$joboption_directory_orig/$JobId"_input.tar.gz"
wu=$JobId

echo "#!/bin/bash" >> $LRMS_JOB_BOINC
echo "set -x" >> $LRMS_JOB_BOINC

tflist=""
Root_basename=()
RootFile=()
## RootFile keeps the orginal path of the root files
## Root_basename keeps the basename of the root files, with adding JobID to make them unique
i=0
for file in `ls $joboption_directory_orig`
  do
    echo $file|grep ".root" > /dev/null
    ret=$?
    if [ $ret -eq 0 ]; then
      echo skip root file $file
      Root_basename[$i]=$JobId"_"$file
      RootFile[$i]=$joboption_directory_orig/$file 
      sed -i -e "/#rename root file/a\mv ATLAS.root_$i $file" $LRMS_JOB_SCRIPT
      let i=$i+1
      continue
    else
      tflist=$tflist" "$joboption_directory_orig/$file
    fi
  done

echo tar zhcvf $JobInput $tflist
echo "tar zhcvf $JobInput $tflist" >> $LRMS_JOB_BOINC

echo "cd $PROJECT_ROOT " >>$LRMS_JOB_BOINC

JobInput_basename=`basename $JobInput`
Script_basename=`basename $LRMS_JOB_SCRIPT` 

echo "cp $JobInput "'`bin/dir_hier_path '$(basename $JobInput)'`' >> $LRMS_JOB_BOINC
echo "chmod a+r "'`bin/dir_hier_path '$(basename $JobInput)'`' >> $LRMS_JOB_BOINC
echo "cp $LRMS_JOB_SCRIPT " '`bin/dir_hier_path' $(basename $LRMS_JOB_SCRIPT)'`' >>$LRMS_JOB_BOINC
echo "chmod a+r " '`bin/dir_hier_path' $(basename $LRMS_JOB_SCRIPT)'`' >>$LRMS_JOB_BOINC

[ -n "$PROJECT_DOWNLOAD_ROOT" ] && echo "cd $PROJECT_DOWNLOAD_ROOT" >> $LRMS_JOB_BOINC

## process the root files as remote files
cd $PROJECT_ROOT
echo "current directory is the project_root: "$PWD
remote_url=()
fsize=()
md5=()
i=0
while [ $i -lt ${#RootFile[@]} ]
do
  [ -L ${RootFile[$i]} ] && RootFile[$i]=`ls -l ${RootFile[$i]}|awk '{print $11}'`
  echo "ln -s ${RootFile[$i]} "'`bin/dir_hier_path' ${Root_basename[$i]} '`' >> $LRMS_JOB_BOINC
  if [ -n "$PROJECT_DOWNLOAD_ROOT" ]; then
    download_dir=`bin/dir_hier_path ${Root_basename[$i]} | awk -F/ '{print $(NF-1)}'`
    remote_url[$i]="${PROJECT_DOWNLOAD_URL}/${download_dir}/${Root_basename[$i]}"
    fsize[$i]=`stat -c %s ${RootFile[$i]}`
    md5[$i]=`md5sum ${RootFile[$i]} | awk '{print $1}'`
    echo "Using remote file ${remote_url[$i]} ${fsize[$i]} ${md5[$i]}" 1>&2
  fi
  let i=$i+1
done

[ -n "$PROJECT_DOWNLOAD_ROOT" ] && echo "cd $PROJECT_ROOT" >> $LRMS_JOB_BOINC

## generate the input template file
let ifileno=2+${#RootFile[@]}
i=0
intmp=""
while [ $i -lt $ifileno ]
do
  intmp="$intmp
<file_info>
   <number>$i</number>
</file_info>"

  let i=$i+1
done
intmp="$intmp
<workunit>"
i=0
while [ $i -lt ${#RootFile[@]} ]
do
  intmp="$intmp
  <file_ref>
     <file_number>$i</file_number>
     <open_name>shared/ATLAS.root_$i</open_name>
     <copy_file/>
   </file_ref>"
  
  let i=$i+1
done

intmp="$intmp
  <file_ref>
     <file_number>$i</file_number>
     <open_name>shared/input.tar.gz</open_name>
     <copy_file/>
   </file_ref>"

let i=$i+1
intmp="$intmp
   <file_ref>
        <file_number>$i</file_number>
        <open_name>shared/start_atlas.sh</open_name>
        <copy_file/>
    </file_ref>"

intmp_res=$(cat $WU_TEMPLATE)
intmp="$intmp
$intmp_res
"

intmp="$intmp
   </workunit>
"

WU_TEMPLATE_tmp=$(mktemp /tmp/${BOINC_APP}_XXXXXX)
cat << EOF  > $WU_TEMPLATE_tmp
$intmp
EOF

#######################################
if [ -z $joboption_memory ];then
  memreq=2000000000
else
  memreq=$((joboption_memory*1000000))
fi

if [ -z $joboption_cputime ];then
  maxcputime=$((2*3600*3000000000))
else
  maxcputime=$((joboption_cputime*3000000000))
fi

priority=
if [ ! -z "$joboption_priority" ]; then
  priority="--priority $joboption_priority"
fi

batchid=
if [ -f "${joboption_directory_orig}/pandaJobData.out" ]; then
  taskid=$(grep -E -m1 -o "taskID=[[:digit:]]+" ${joboption_directory_orig}/pandaJobData.out|cut -d = -f 2 )
  if [ ! -z "$taskid" ]; then
    batchid="--batch $taskid"
  fi
fi

cmd="bin/create_work \
      --appname $BOINC_APP \
      --wu_name $wu \
      --wu_template $WU_TEMPLATE_tmp \
      --result_template $RESULT_TEMPLATE \
      --rsc_memory_bound $memreq \
      --rsc_fpops_est $maxcputime \
      $batchid \
      $priority"
j=0
while [ $j -lt ${#RootFile[@]} ] 
  do
  if [ -n "$PROJECT_DOWNLOAD_ROOT" ]; then
    cmd="$cmd \
    --remote_file ${remote_url[$j]} ${fsize[$j]} ${md5[$j]}"
  else
    cmd="$cmd \
    ${Root_basename[$j]}"
  fi
  let j=$j+1
done

cmd="$cmd \
      $(basename $JobInput) \
      $(basename $LRMS_JOB_SCRIPT)"
echo $cmd >> $LRMS_JOB_BOINC
echo 'ret=$?' >>$LRMS_JOB_BOINC

echo 'exit $ret' >>$LRMS_JOB_BOINC
if [ $DEBUG -eq 2 ];then
  cat $LRMS_JOB_BOINC 1>&2
else
  sh $LRMS_JOB_BOINC 1>&2
fi

rc=$?

if [ $rc -eq 0 ];then
  echo "job $wu submitted successfully!" 1>&2
  echo "joboption_jobid=$wu" >> $GRAMI_FILE
fi
echo "----- removing intermediate files ----" 1>&2
if [ $DEBUG -ne 1 ];then
  rm -fr $WU_TEMPLATE_tmp
  rm -fr $LRMS_JOB_BOINC $LRMS_JOB_ERR $LRMS_JOB_OUT $LRMS_JOB_SCRIPT
  rm -fr $JobInput
fi

echo "----- exiting submit_boinc_job -----" 1>&2
echo "" 1>&2
exit $rc