#! /bin/bash

# Given a batch config file
# Summarize which jobs are still running
# Summarize how long jobs took to finish
# Summarize any errors
# Summarize the condition of output files

usage () {
cat<<EOF
usage: `basename $0` [options] [BatchConfig]

Options:
-h, --help     This help message
-d, --debug    Enable debug output
EOF
  exit $1
}

VERBOSE=1
VERBOSE_STREAM=/dev/stderr
DEBUG_STREAM=/dev/null
BatchConfig="BatchConfig"
while [ "$#" != 0 ]; do
  case "$1" in
    -h | --help )
      usage 0;;
    -d | --debug )
      shift;Debug=true; set -x; DEBUG_STREAM=/dev/stderr ;;
    *)
      BatchConfig="$1"; shift || true;;
  esac
done

if [ ! -r "$BatchConfig" ]; then
  echo "Error: Batch config file, '$BatchConfig' doesn't seem to exist."
  exit 1
fi
BatchConfig="`readlink -e "$BatchConfig"`"
MainDirectory="`dirname "$BatchConfig"`"
TimerDir="$MainDirectory"/timers
OutputDir="$MainDirectory"/output
BatchDir="$MainDirectory"/batch_output
HadErrors=false

# ======================================
# Process BatchConfig
# ======================================
JobId=""
JobName=""
NJobsSent=""
function ProcessConfig(){
    local _file="$1"
    local _qsubResponse="`grep 'qsub response' "$_file" `"
    JobName="`sed -e 's!.*"\(.*\)".*!\1!' <<<"$_qsubResponse"`"
    JobId="`sed -e 's!.*\s\([0-9]\+\).*!\1!' <<<"$_qsubResponse"`"
    NJobsSent="`sed -e 's!.*[0-9]\+\.[0-9]-\([0-9]*\):.*!\1!' <<<"$_qsubResponse"`"
}

QstatOutput=""
JobsStillRunning=( )
function QueryQstat(){
  local _jobId="$1"
  QstatOutput="`qstat 2> $DEBUG_STREAM |grep "^[ \t]*${_jobId}"`"
  JobsStillRunning=( `awk '{print $10}' <<<"$QstatOutput" ` )
}

function Contains(){
 local _item="$1"; shift
 local _options=( "$@" )
 for trial in "${_options[@]}";do
  if [ "$trial" == "$_item" ];then
    return 0
  fi
 done
 return 1
}

IncompleteJobIds=( )
CompleteJobIds=( )
RunAnalysis=( )
AnalyseRun(){
  local _index="`sed -e 's!^0*!!' <<< $1`";shift
  local _start_file="$1";shift
  local tmp="`sed -e 's!\(.*\)*:[^:]*$!\1!' <"$_start"`"
  RunAnalysis=( "$_index" "$tmp" )
  if [ -e "$_stop" ];then 
    IncompleteJobIds=( "${IncompleteJobIds[@]}" "$_index" )
    tmp="`sed -e 's!\(.*\)*:[^:]*$!\1!' <"$_stop"`"
  elif `Contains "$_index" ${JobsStillRunning[@]}`;then
    tmp=running
  else
    CompleteJobIds=( "${CompleteJobIds[@]}" "$_index" )
    tmp=error
    HadErrors=true
  fi
  RunAnalysis=( "${RunAnalysis[@]}" "$tmp" )
}

function PrintJobTable(){
  local _prefix="$1"
  local format_str="% 5s | % 15s | % 15s\n"
  printf "$format_str"  job start stop  
  printf "$format_str"  --- ----- ----  
  for _start in "$TimerDir"/*start;do
     local _stop="`sed -e "s!.start!.stop!" <<<"$_start" `"
     local _index="`sed -e "s!$TimerDir/\([0-9]*\).start!\1!" <<<"$_start" `"
     AnalyseRun "$_index" "$_start"
     printf "$format_str" ${RunAnalysis[@]}
  done
}

# ======================================
# Main body
# ======================================
ProcessConfig "$BatchConfig"
QueryQstat "$JobId"

echo "== Summary for job '$JobName' with ID: $JobId =="
echo = Batch Engine Status =
if $NoJobsRunning; then
  echo " All jobs on batch engine have ended."
else
  echo "$QstatOutput"
fi
echo 
echo = Status table =
PrintJobTable "  "
echo 
echo = Generally =
$HadErrors && echo "  We got issues" || echo "  We look ok"