#!/bin/sh
# Copyright (c) 2002-08 Peter Guntert. All rights reserved.

# Batch job submission script
# Peter Guntert, 12-03-2001
# Modified version, Peter Guntert, 26-11-2004

dir=`pwd`
name=
prog=${CYANA:-cyana}
nproc=1
queue=
opt=
after=
submit='nice $name.job >$name.out &'
nodes=
llheader=
beforeprog=
afterprog=
 
 
# ------ Defaults for specific computers ------

  # Add to this section a new entry that the fits 
  # your compute server and batch system.

case `uname -n` in

  # Demo (explanation of options)
  demo)  nproc=5                   # default number of processors to use (can be changed with -n option)
         queue=batch               # default batch queue (can be changed with -q option)
         opt="-t demo-mpi -c 'mpirun -np NPROCPLUSONE'"  # options to start CYANA
         submit='qsub $name.job'   # command to submit batch job
         ;;
         
  # Linux cluster with Open MPI
  kume | yaku | iriomote | rebun | rishiri | hirado | ishigaki | taketomi | shikotan) 
         nproc=20
	 # Look for hostfile first in the current directory, then in ~/.openmpi
	 nohostfile=y
	 for hostfile in hostfile $HOME/.openmpi/hostfile; do
	   if [ -f $hostfile ]; then
             opt="-t intel-openmpi -c 'mpirun --hostfile $hostfile -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
	     nohostfile=""
	     break
	   fi
	 done
	 if [ "$nohostfile" ]; then
           opt="-t intel-openmpi -c 'mpirun -x CYANALIB -np NPROCPLUSONE'"
	 fi
         #opt="-t intel-openmpi -c 'mpirun --wdir $PWD -x CYANALIB -np NPROCPLUSONE'"
         submit='$name.job > $name.out &'
         ;;

  # Linux cluster with Open MPI
  kyushu | node* | linux) 
         nproc=20
	 # Look for hostfile first in the current directory, then in ~/.openmpi
	 nohostfile=y
	 for hostfile in hostfile $HOME/.openmpi/hostfile; do
	   if [ -f $hostfile ]; then
	     sortnodes $hostfile > openmpi.hosts
             opt="-t intel-openmpi -c 'mpirun --hostfile openmpi.hosts -x CYANALIB -np NPROCPLUSONE --mca btl tcp,self'"
	     nohostfile=""
	     break
	   fi
	 done
	 if [ "$nohostfile" ]; then
           opt="-t intel-openmpi -c 'mpirun -x CYANALIB -np NPROCPLUSONE'"
	 fi
         #opt="-t intel-openmpi -c 'mpirun --wdir $PWD -x CYANALIB -np NPROCPLUSONE'"
         submit='$name.job > $name.out &'
         ;;

  # Linux cluster with LAM-MPI
  gandin | gurnemanz | gurzgri | parzival | anfortas | gawan | hardiz | ither | iwanet | kyot | ilinot | galoes ) 
         nproc=12
         opt="-t intel-lam -c 'mpirun -x CYANALIB -np NPROCPLUSONE'"
         submit='$name.job > $name.out &'
         ;;

  # Linux cluster with LAM-MPI (interactive and PBS batch system)
  artus | control | node* ) 
         nproc=25
	 if [ `echo " $*" | awk '{ print index($0," -q ") }'` -eq 0 ]; then
	   nodes=y
           opt="-t intel-lam -c 'mpirun -x CYANALIB NODES'"
           submit='$name.job > $name.out &'
	 else
           opt="-t intel-lam -c 'mpirun -x CYANALIB -np NPROCPLUSONE'"
           submit='qsub -q $queue -N $name -j oe -o $dir/$name.out -l nodes=NPROCHALF:ppn=2 $name.job'
           beforeprog='lamboot -v $PBS_NODEFILE'
           afterprog=lamhalt
	 fi
         ;;
	 
  # IBM cluster system (interactive and LoadLeveler batch system)
  ibmc* ) 
         nproc=8
	 if [ `echo " $*" | awk '{ print index($0," -q ") }'` -eq 0 ]; then
           submit='$name.job > $name.out &'
	 else
           llheader=y
           opt="-t ibm-mpi"
           submit='llsubmit $name.job'
	 fi
         ;;

  # HP cluster using MPI and lsNQS batch system
  xcc98) 
         nproc=20
         queue=tall20
         opt="-t pgi-mpi -c 'mpirun -srun'"
         #submit='qsub -q $queue -eo -o $dir/$name.out $dir/$name.job'
         submit='bsub -q $queue -J $name -oo $dir/$name.out -n NPROCPLUSONE $dir/$name.job'
         ;;
         
	 
  # Compaq Alpha shared-memory system using LSF batch system
  c4-3*) nproc=4
         queue=s
         submit='bsub -q $queue -o $name.out $dir/$name.job'
         ;;
         
  # HP shared-memory system using LSF batch system
  tornado | stardust) 
         nproc=4
         queue=normal
         submit='bsub -q $queue -o $name.out $dir/$name.job'
	 ;;
         
  # HP cluster using MPI and LSF batch system
  albula | bernina | maloja | julier) 
         nproc=4
         queue=normal
         opt="-t hp-mpi -c 'mpirun -np NPROCPLUSONE'"
         submit='bsub -q $queue -o $dir/$name.out $dir/$name.job'
         ;;
         
  # SGI cluster using MPI
  lodur | mani | skinfaxi) 
         nproc=2
         opt="-t sgi-mpi -c 'mpirun -np NPROCPLUSONE -machinefile $HOME/machines'"
         submit='$name.job > $name.out'
         ;;
	 
  # By default, assume a shared-memory multiprocessor machine without batch system
  *)     nproc=5
         opt=""
         submit='$name.job > $name.out &'
         ;;
esac

 
# ------ Command line options ------

options=a:j:n:p:q:o:s:Sh

optind=1
usage=""
eval par=\$$optind
while true; do
  case $par in
  --) optind=`expr $optind + 1`; break;;
  -?*) option=`echo X$par | sed 's/^X-\(.\).*/\1/'`
       if [ `echo $options | sed 's/.*'$option'.*/+/'` != "+" ]; then
         echo "$0: -$option: unknown option"; usage=1
       elif [ `echo $options | sed 's/.*'$option':.*/+/'` = "+" ]; then
         optind=`expr $optind + 1`
         eval $option=\$$optind
       else
         eval $option=1
       fi
       par=`echo X$par | sed 's/^X-./-/'`
       if [ "$par" = "-" ]; then 
         optind=`expr $optind + 1`
	 eval par=\$$optind
       fi
       ;;
  *) break;;
  esac
done
shift `expr $optind - 1`

if [ "$h" ]; then usage=1; fi
if [ "$usage" ]; then
  echo "Usage: cyanajob [-hajnopqsS] macro parameters"			   
  echo  								   
  echo "       -h	     help"					   
  echo "       -a file       start job only after <file> exists"
  echo "       -j name       job name (default: <macro>)"		   
  echo "       -n nproc      number of processors (default: $nproc)"	   
  echo "       -o options    options for the program (default: $opt)"	   
  echo "       -p program    program name (default: $prog)"		   
  echo "       -q queue      batch queue (default: $queue)"		   
  echo "       -s command    submit command (default: $submit)" 	   
  echo "       -S            run in restriced (safe) mode" 	   
  exit 2								   
fi
if [ "$a" ]; then after="$a"; fi
if [ "$j" ]; then name="$j"; fi
if [ "$n" ]; then nproc="$n"; fi
if [ "$p" ]; then prog="$p"; fi
if [ "$q" ]; then queue="$q"; fi
if [ "$o" ]; then opt="$o"; fi
if [ "$s" ]; then submit="$s"; fi
if [ "$S" ]; then safe="-S"; fi


# ------ Macro name ------

if [ $# -lt 1 ]; then
  echo "Missing macro file."
  exit 2
fi
macro=`echo $1 | sed 's/\.cya$//'`
if [ ! -f $macro.cya ]; then
  echo "Macro file $macro.cya not found."
  exit 2
fi
shift
if [ "$name" = "" ]; then name=`basename $macro`; fi


# ------ Start options and submit command ------

nproc1=`expr $nproc + 1`
nproc2=`expr $nproc1 / 2`

# Find least loaded nodes
if [ "$nodes" ]; then
  lamnodes 2>/dev/null >/dev/null
  if [ $? -ne 0 ]; then lamboot -v; fi
  tmp=sortnodes$$
#  dsh -a uptime >/dev/null 2>$tmp
  dsh -w `lamnodes |\
          awk 'NR>1 { if (NR>2) printf(", ");
                      printf("%s",substr($2,1,index($2,".")-1)); }
              '` uptime >/dev/null 2>$tmp
  nodes=`awk '/^node/ { s=substr($0,index($0,"load average")+13)
  			l=substr(s,1,index(s,",")-1)
  			print NR,l }
  	     ' $tmp |\
  	 sort -n -k 2,2 | head -n $nproc2 | sort -n |\
  	 awk -F: '{ if (NR==1) printf("n0");
  		    if (NR<=nproc2) printf(",%d,%d",$1,$1)}' nproc2=$nproc2`
  rm -f $tmp
#  echo "nodes=$nodes"
  opt=`echo $opt | sed "s/NODES/$nodes/"`
fi

opt=`echo $opt | sed "s/NPROCPLUSONE/$nproc1/"`
submit=`echo $submit | sed -e "s,NPROCHALF,$nproc2," -e "s/NPROCPLUSONE/$nproc1/"`
#echo "opt=$opt"
#echo "submit=$submit"


# ------ Create job script ------

echo "#!/bin/sh" > $name.job

# Header for IBM LoadLeveler
if [ "$llheader" ]; then
  nodes=`expr $nproc1 / 8`
  if [ `expr $nproc1 % 8` -ne 0 ]; then nodes=`expr $nodes + 1`; fi
  echo "\
# @ job_name = $name
# @ initialdir = `pwd`
# @ output = \$(initialdir)/\$(job_name).out
# @ error = \$(output)
# @ job_type = parallel
# @ node = $nodes
# @ total_tasks = $nproc1
# @ class = $queue
#PG @ resources = ConsumableCpus($nproc)
# @ queue
NLSPATH=/usr/lib/nls/msg/%L/%N:/usr/lib/nls/msg/%L/%N.cat
LC_MESSAGES=en_US
echo \$LOADL_PROCESSOR_LIST
" >> $name.job
fi

# Job script body
echo "\
cd `pwd`
date +'%d-%b-%Y %H:%M:%S'
$beforeprog
$prog $safe $opt << EOF
erract:=abort
nproc=$nproc
$macro $* 
quit
EOF
$afterprog
echo
date +'%d-%b-%Y %H:%M:%S'
" >> $name.job
chmod +x $name.job


# ------ Submit job ------

rm -f $name.out
#echo "after=$after"
if [ "$after" ]; then
  (while [ ! -f $after ]; do sleep 10; done
   eval ./$submit
  ) &
else
  eval ./$submit
fi