#! /bin/sh
# This is the LHEA perl script: /cvmfs/extras-fp7.egi.eu/extras/heasoft/ftools/x86_64-unknown-linux-gnu-libc2.19-0/bin/pcamergeobsids
# The purpose of this special block is to make this script work with
# the user's local perl, regardless of where that perl is installed.
# The variable LHEAPERL is set by the initialization script to
# point to the local perl installation.
#-------------------------------------------------------------------------------
eval '
if [ "x$LHEAPERL" = x ]; then
  echo "Please run standard LHEA initialization before attempting to run /cvmfs/extras-fp7.egi.eu/extras/heasoft/ftools/x86_64-unknown-linux-gnu-libc2.19-0/bin/pcamergeobsids."
  exit 3
elif [ "$LHEAPERL" = noperl ]; then
  echo "During LHEA initialization, no acceptable version of Perl was found."
  echo "Cannot execute script /cvmfs/extras-fp7.egi.eu/extras/heasoft/ftools/x86_64-unknown-linux-gnu-libc2.19-0/bin/pcamergeobsids."
  exit 3
elif [ `$LHEAPERL -v < /dev/null 2> /dev/null | grep -ic "perl"` -eq 0 ]; then
  echo "LHEAPERL variable does not point to a usable perl."
  exit 3
else
  # Force Perl into 32-bit mode (to match the binaries) if necessary:
  if [ "x$HD_BUILD_ARCH_32_BIT" = xyes ]; then
    if [ `$LHEAPERL -V 2> /dev/null | grep -ic "USE_64_BIT"` -ne 0 ]; then
      VERSIONER_PERL_PREFER_32_BIT=yes
      export VERSIONER_PERL_PREFER_32_BIT
    fi
  fi
  exec $LHEAPERL -x $0 ${1+"$@"}
fi
'
if(0);
# Do not delete anything above this comment from an installed LHEA script!
#-------------------------------------------------------------------------------
#!/usr/bin/perl
#
# File: pcamergeobsids
#
# Description:  Merge one or more PCA data sets
#
# Author: C. B. Markwardt
#

use Getopt::Std;

getopts(':h');
if (defined $opt_h) {
    print <<EOHELP;

NAME
    pcamergeobsids - merge multiple 'pcaprepobsid' directories into one

USAGE
    pcamergeobsids indir outdir

DESCRIPTION

    The task pcamergeobsids is a high-level convenience task that
    combines multiple Standard2 directories into a single directory.
    This task eases the job of merging multiple observations into a
    single observation, and thus extracting spectra or light curves.

    The task essentially concatenates list files, which list filter
    files and Standard2 files, into a single list file for the
    combined set.  It also creates a single RXTE filter file covering
    all observations, which has been merged and time-sorted.

    As an alternative to using pcamergeobsids, the user can collect
    the desired files by their own methods.

INPUTS and OUTPUTS

    The input to the task is a list of input directories, either
    comma-separated or as a @filename.lis list file.  These
    directories must have the directory and file structure produced by
    pcaprepobsid.  Thus, you should run pcaprepobsid at least once for
    each observation of interest, and then merge those outputs with
    pcamergeobids.

    Your input list may optionally point to each subdirectory's
    "pcaprepobsid_done.txt" file instead of the parent subdirectory.
    This is a convenient way to search for only those observation
    directories that have successfully been processed by pcaprepobsid.
    For example the shell command:

        ls 96443-*/pcaprepobsid_done.txt > indirs.lis

    will automatically find any 96443-* subdirectory that has been
    successfully processed by pcaprepobsid.  pcamergeobsids will
    automatically use the parent subdirectory for further processing.

    The output of pcamergeobsids is a new results directory, also of
    the same format as produced by pcaprepobsid, which can be used for
    further scientific processing.

    The output of pcamergeobsids is suitable to be used as input to
    itself.  Thus, it is possible to merge sub-groups of data with
    pcamergeobsids, and then use the same task to merge those
    sub-grouped results into a single master data set, ad infinitum.

COPY or LINK

    pcamergeobsids gives you a choice to LINK or COPY the raw data.
    The setting copymeth=COPY will copy all relevant inputs to the
    output directory, including Standard2 files, backgrounds, and
    files with dead-time quantities.  Use this setting if you plan on
    deleting the individual observation data, or plan on moving
    observation data around.  This option has the down side of using
    twice the amount of disk space as the original data.

    The setting copymeth=LINK will create new list files, and a merged
    filter file, but none of the Standard2 files, backgrounds or files
    with dead-time quantities are copied into the output directory.
    The output list files point to the input files in the original
    location.  Use this setting if you wish to save disk space, or are
    experimenting with merging different data sets.  This option has
    the down side of forcing you to preserve the input directories
    where the original data lives.

PARAMETERS

    indir [filename]
        Name of one or more input directories containing
        pcaprepobsid-style data.  This parameter should be either a
        comma-separated list of directories given on the command line,
        or a @filename.lis list file (ASCII file with one subdirectory
        per line).

    outdir [filename]
        Name of output directory which will contain output products.
        If the directory does not exist, then it will be created.  See
        the help for 'pcaprepobsid' for description of output products.

    (abspath = 'NO') [boolean]
        If YES, then absolute path names will be written to *.lis
        files.  If NO, then relative path names will be written,
        relative to the current working directory when the task is
        run.  Any existing relative paths will be converted to
        absolute paths.

    (copymeth = 'LINK') [string]
        Either 'LINK' or 'COPY'.  If COPY, then all files are copied
        to the output directory.  If LINK, then new list files are created
        by other files are not copied.  See above.

    (skipbad = no) [boolean]
        If skipbad = YES, then input directories with missing or
        partial information are ignored.  If skipbad = no, a single
        missing or partial observation will cause the task to stop
        with an error message.

    (cleanup = yes) [boolean]
        Clean up scratch files?

    (chatter = 1) [int] range 0-5
        Verbosity level of output

    (clobber = yes) [boolean]
        Overwrite output files?  By default this is YES, and the task does
        not function properly if you set it to NO.

    (history = yes) [boolean]
        Write standard HEADAS parameter history into output file?

EXAMPLES

   # Merging results for 96443-01-01-16 and 96443-01-01-17, which have been
   # created by running pcaprepobsid.

   pcamergeobsids indir=96443-01-01-16-result,96443-01-01-17-result \
       outdir=merged-96443


   # Same example but using an @-file
   # Create the @ file  
   ls 96443-01-01-*/pcaprepobsid_done.txt > merged-96443.lis
   # Merge the data
   pcamergeobsids @merged-96443.lis  outdir=merged-96443

CAVEATS

    The user is strongly recommended to use an updated RXTE
    Calibration database (CALDB), and to set breakfile=CALDB,
    modelfile=CALDB, saahfile=CALDB.

BUGS
    Please report problems to xtehelp\@athena.gsfc.nasa.gov.

SEE ALSO
    pcamergeobsids

EOHELP
exit;
    }

use HEACORE::HEAINIT;



# ==================================================================
# Call the main task subroutine with an exception handler
$status = 0;
$cleanup = 1;
@scratchfiles = ();
@scratchfiles_badonly = ();
eval {
    $status = headas_main(\&pcamergeobsids);
};

# ===================================
# Check for errors and report them to the console
if ($@) {
    if ($status == 0) { $status = -1; }
    warn $@;
    exit $status;
}
exit 0;


# ===================================
# Main subroutine
sub pcamergeobsids {

    $taskname = "pcamergeobsids";
    $taskvers = "1.0";

    # Use the standard HEAdas methods for registering the toolname and version number to be
    # used in error reporting and in the record of parameter values written by HDpar_stamp
    set_toolname($taskname);
    set_toolversion($taskvers);

    eval {
	$status = &pcamergeobsids_work();
    };

    # Special effort to remove scratch files
    if ($cleanup) {
	foreach $scratchfile (@scratchfiles) {
	    unlink "$scratchfile";
	}

	# Delete some product files only if we exited with an error
	if ($@) { 
	  foreach $scratchfile (@scratchfiles_badonly) {
	    unlink "$scratchfile";
	  }
	}
    }

    if ($@) {
	if ($status == 0) { $status = -1; }
	warn $@;
	return $status;
    }


    return $status;
}

# ==================================================================
# Working subroutine
sub pcamergeobsids_work {

    # The HEAUTILS module provides access to HDpar_stamp()
    # set_toolname(), set_toolversion(), and headas_clobberfile()
    use HEACORE::HEAUTILS;
    use HEACORE::PIL;
    # include the file specification functions
    use Astro::FITS::CFITSIO qw( :shortnames :constants :longnames);
    # User defined module which contains the Perl-CFITSIO interface functions 
    use SimpleFITS;

    use Cwd 'abs_path';

    my ($infile, $outfile, $chatter);
    my ($fits1, $fits2, $handle1, $handle2);
    my ($mode, $tddes5);

    ($status = PILGetFname('indir', $indir)) == 0 || die "error getting indir parameter";
    ($status = PILGetFname('outdir', $outdir)) == 0 || die "error getting indir parameter";
    ($status = PILGetBool('abspath', $abspath)) == 0 || die "error getting abspath parameter";
    ($status = PILGetString('copymeth', $copymeth)) == 0 || die "error getting copymeth parameter";
    ($status = PILGetBool('skipbad', $skipbad)) == 0 || die "error getting skipbad parameter";
    ($status = PILGetBool('clobber', $clobber)) == 0 || die "error getting clobber parameter";
    ($status = PILGetBool('cleanup', $cleanup)) == 0 || die "error getting cleanup parameter";
    ($status = PILGetInt('chatter', $chatter)) == 0 || die "error getting chatter parameter";

    print "Running $taskname v$taskvers\n" if ($chatter >= 1);
    print "----------------------------------------\n" if ($chatter >= 2);

    $clobstr = $clobber ? "YES" : "NO";
    $cleanstr = $cleanup ? "YES" : "NO";

    if (! -d "$outdir" ) {
      mkdir("$outdir") or die "ERROR: could not create $outdir";
    }
    die "ERROR: $outdir is not writable" if (! -w "$outdir" );

    # =======================
    # List of standard output files
    $okfile   = "$outdir/pcaprepobsid_done.txt";
    unlink($okfile);
    $filtlis  = "$outdir/FP_xtefilt.lis";
    $filtfile = "$outdir/FP_merged.xfl";
    $std2lis  = "$outdir/FP_std2.lis";
    $bkg2lis  = "$outdir/FP_bkg2.lis";
    $dtstd2lis  = "$outdir/FP_dtstd2.lis";
    $dtbkg2lis  = "$outdir/FP_dtbkg2.lis";

    # =======================
    # Remove most of them.  This task does not take too kindly to
    # having previous versions lying around so warn against this.
    &headas_clobberfile($filtlis);
    &headas_clobberfile($filtfile);
    &headas_clobberfile($std2lis);
    &headas_clobberfile($bkg2lis);
    &headas_clobberfile($dtstd2lis);
    &headas_clobberfile($dtbkg2lis);
    die "ERROR: $filtlis already exists and clobber not set" if ( -f $filtlis);
    die "ERROR: $filtfile already exists and clobber not set" if ( -f $filtfile);
    die "ERROR: $hklis already exists and clobber not set" if ( -f $hklis);
    die "ERROR: $std2lis already exists and clobber not set" if ( -f $std2lis);
    die "ERROR: $bkg2lis already exists and clobber not set" if ( -f $bkg2lis);
    die "ERROR: $dtstd2lis already exists and clobber not set" if ( -f $dtstd2lis);
    die "ERROR: $dtbkg2lis already exists and clobber not set" if ( -f $dtbkg2lis);

    if ($copymeth =~ m/^COPY$/i) {
      $copymeth = "COPY";
    } elsif ($copymeth =~ m/^LINK$/i) {
      $copymeth = "LINK";
    } else {
      die "ERROR: unrecognized copymeth=$copymeth";
    }
    my $copy = ($copymeth eq "COPY");

    # =======================
    # Initial parsing and checking of input directories
    # Parse "indir" as either an @listfile or comma-separated list.
    if ("$indir" =~ /^@(.*)$/) {
      my $infile = $1;
      @indirs = ();
      open(IN,"<$infile") or die "ERROR: could not open $infile";
      while (my $line=<IN>) {
	chomp($line);
	push @indirs, $line;
      }
      close(IN);
    } else {
      @indirs = split(/,/,$indir);
    }

    # Quick error checking of input directories
    die "ERROR: no input directories found" if ($#indirs < 0);
    $nindirs = $#indirs + 1; print "  Listed $nindirs input directories\n" if ($chatter >= 5);
    @indirs2 = ();
    foreach $indir1 (@indirs) {
      # Break off the optional pcaprepobsids_done.txt suffix
      if ($indir1 =~ m|^(.*)/pcaprepobsid_done.txt$|) {
	$indirs1 = "$1";
      }
      if (! (-d "$indir1" && -r "$indir1")) {
	$warnmsg = "$indir1 is not a readable directory";
	if ($skipbad) {
	  warn "WARNING: $warnmsg"; next;
	}
	die "ERROR: $warnmsg";
      }
      if ((! -f "$indir1/pcaprepobsid_done.txt") ||
	  (! -f "$indir1/FP_xtefilt.lis") ||
	  (! -f "$indir1/FP_std2.lis") ||
	  (! -f "$indir1/FP_bkg2.lis") ||
	  (! -f "$indir1/FP_dtstd2.lis") ||
	  (! -f "$indir1/FP_dtbkg2.lis")) {
	$warnmsg = "$indir1 does not have required FP_{xtefilt,std2,bkg2,dtstd2,dtbkg2}.lis files";
	if ($skipbad) {
	  warn "WARNING: $warnmsg"; next;
	}
	die "ERROR: $warnmsg";
      }

      # Passed all quality checks
      push @indirs2, $indir1;
    }
    @indirs = @indirs2;
    die "ERROR: no verified input directories" if ($#indirs < 0);
    $nindirs = $#indirs + 1; print "  Found $nindirs input directories\n" if ($chatter >= 5);

    # =======================
    # Create new filter file by 'ftmerge'ing the individual ones
    &merge_list_files(\@indirs,$outdir,$filtlis,"FP_xtefilt.lis",$abspath,0,1,$chatter);
    die "ERROR: no valid input filter files" if (! -f "$filtlis");

    if ($nindirs > 1) {
      $cmd = "ftmerge infile='"."@"."$filtlis' outfile='$filtfile' columns='*' ".
	"lastkey='TSTOP' copyall=YES skipbadfiles=NO clobber=YES chatter=0 ";

    } else {
      # Grr, ftmerge is so stupid it doesn't know how to merge a single file.
      my $filtfile1 = `head -1 '$filtlis'`; chomp($filtfile1);
      $cmd = "ftcopy infile='$filtfile1' outfile='$filtfile' ".
	"clobber=YES chatter=0 ";
    }

    print "COMMAND: $cmd\n" if ($chatter >= 5);
    system("$cmd");
    die "ERROR: ftmerge of $filtfile failed" if ($? || ! -f "$filtfile");

    # =======================
    # Sort the filter file by time
    $tmp = "$filtfile"."1";
    rename($filtfile,$tmp);
    push @scratchfiles, "$tmp";
    $cmd = "ftsort infile='$tmp"."[1]' outfile='$filtfile' ".
      "columns=TIME copyall=YES clobber=YES chatter=0 ";
    print "COMMAND: $cmd\n" if ($chatter >= 5);
    system("$cmd");
    die "ERROR: ftsort of $filtfile failed" if ($? || ! -f "$filtfile");

    # Create .lis file for this filter file
    print "  FILT: $filtfile\n" if ($chatter >= 2);
    add_list_item($filtlis,$filtfile,$abspath,1);

    # =======================
    # Now work on Standard2 files
    &merge_list_files(\@indirs,$outdir,$std2lis,"FP_std2.lis",$abspath,$copy,1,$chatter);
    print "  STD2: $std2lis\n"  if ($chatter >= 2);
    &merge_list_files(\@indirs,$outdir,$bkg2lis,"FP_bkg2.lis",$abspath,$copy,1,$chatter);
    print "  BKG2: $bkg2lis\n"  if ($chatter >= 2);
    &merge_list_files(\@indirs,$outdir,$dtstd2lis,"FP_dtstd2.lis",$abspath,$copy,1,$chatter);
    print "  DST2: $dtstd2lis\n"  if ($chatter >= 2);
    &merge_list_files(\@indirs,$outdir,$dtbkg2lis,"FP_dtbkg2.lis",$abspath,$copy,1,$chatter);
    print "  DBK2: $dtbkg2lis\n"  if ($chatter >= 2);

    # Final clean up
    if ($status == 0) {
      add_list_item($okfile,localtime()."",0,1);
    }
    return $status;
}



# ==================================================================
# Append item to list
#   $listfile - output file name
#   $item - string to be appended to file (newline is added)
#   $abspath - if true, then convert relative to absolute path
#   $clobber - if true 
sub add_list_item {
  my ($listfile,$item,$abspath,$clobber) = (@_);
  unlink($listfile) if ($clobber);
  
  open(OUT,">>$listfile") or die "ERROR: could not append to $listfile";
  # print "item=$item abspath=$abspath\n";
  print OUT ($abspath ? abs_path($item) : $item)."\n";
  close(OUT);
}


# ==================================================================
sub merge_list_files {
  my ($rdirs,$outdir,$lisfile,$suffix,$abspath,$copy,$check,$chatter) = (@_);
  my ($clobber1, $indir, $line);
  use File::Copy;

  $clobber1 = 1;
  foreach $indir (@$rdirs) {
    my $lisfile1 = "$indir/$suffix";
    print "   List file: $lisfile1\n" if ($chatter >= 5);
    open(IN,"<$lisfile1") or die "ERROR: could not open $lisfile1";
    while ($line = <IN>) {
      chomp($line);
      print "    $line\n" if ($chatter >= 5);
      if ($check || $copy) {
	die "ERROR: file $line is not readable" if (! -f "$line" );
      }
      if ($copy) {
	$line =~ m|([^/]*)$|; 
	my $root = "$1";             # file name without path components
	my $outfile = "$outdir/$root";
	copy($line,$outfile) or die "ERROR: couldn't copy $line to $outfile";
	print "       (copy to $outfile)\n" if ($chatter >= 5);
	$line = "$outfile";
      }

      add_list_item($lisfile,$line,$abspath,$clobber1);
      $clobber1 = 0;
    }
    close(IN);
  }
}