#include <RAT/TPMuonFollowerCut.hh>
#include <RAT/DBJsonLoader.hh>
#include <RAT/DB.hh>
#include <RAT/BitManip.hh>

#include <RAT/DBTable.hh>
#include <RAT/DBTextLoader.hh>
#include <RAT/DU/Utility.hh>
#include <RAT/DU/PMTInfo.hh>
#include <RAT/DU/DataCleaningBits.hh>
#include <RAT/DU/TrigBits.hh>
#include <RAT/Log.hh>
#include <RAT/json.hh>

#include <Byteswap.h> // From ROOT

#include <iostream>
#include <sstream>
#include <glob.h>
#include <algorithm>
#include <sys/types.h>
#include <sys/stat.h>
#include <sstream>

namespace RAT {

  void TPMuonFollowerCut::BeginOfRun(DS::Run& run)
  {
    // ECACal is used in the neck cut part of the muon tag
    fECACal.BeginOfRun(run);

    fLClean = DB::Get()->GetLink("DATACLEANING",fName);
    fShortBit = DU::Utility::Get()->GetDataCleaningBits().GetBitIndex("tpmuonfollowercut-short");
    fLongBit = DU::Utility::Get()->GetDataCleaningBits().GetBitIndex("tpmuonfollowercut-long");

    fOwlLockoutTime = fLClean->GetI("owl_lockout_ns");
    fNhitMin = static_cast<unsigned int>( fLClean->GetI("nhit_min") );
    fTrmsMax = fLClean->GetD("trms_max");
    fMaxNeck = static_cast<unsigned int>( fLClean->GetI("max_neck") );
    fMinOwls = static_cast<unsigned int>( fLClean->GetI("min_owls") );
    fNeckNhitMax = static_cast<unsigned int>( fLClean->GetI("neck_cut_nhit_max") );

    fMaxQ = fLClean->GetI("max_q");
    fMinQ = fLClean->GetI("min_q");
    fTimeDiff = fLClean->GetD("muon_time_diff");
    fShortFollowerTime = fLClean->GetD("short_follower_time");
    fLongFollowerTime = fLClean->GetD("long_follower_time");

    DBLinkPtr fRun = DB::Get()->GetLink("RUN");
    fRunType = fRun->GetI("runtype");
    validGTID = run.GetValidGTID();

    fLastMuonPreviousRunDay = 0;
    fLastMuonPreviousRunSec = 0;
    fLastMuonPreviousRunNs = 0;
    fCutLastMuonDay = 0;
    fCutLastMuonSec = 0;
    fCutLastMuonNs = 0;
    fFirstEvent = true;
    foundDBFile = false;
    foundJsonFile = false;
    foundLastFile = false;
    fFreeMuon = true;
    fOwlTime = 0;

    try{
      // Last muon in the previous run
      DBLinkPtr fLastMuonData = DB::Get()->GetLink("LAST_MUON");
      fCutLastMuonDay = fLastMuonData->GetI("day_last_muon");
      fCutLastMuonSec = fLastMuonData->GetI("sec_last_muon");
      fCutLastMuonNs  = fLastMuonData->GetI("ns_last_muon");
      fRunTypePreviousRun = fLastMuonData->GetI("runtype");
      foundLastFile = true;
      detail << "TPMuonFollower::BeginOfRun: Got last muon in previous run." << endl;
    }
    catch(DBNotFoundError &e){
      detail << "TPMuonFollower::BeginOfRun: Could not find LAST_MUON ratdb file. Inserting fake muon at start of run." << endl;
    }

    if (fProcessPass== 2){
      // Try and get the muon table, assuming the entries an array
      try{
        // List of muon and owl events in current run
        DBLinkPtr fMuonData = DB::Get()->GetLink("TPMUONFOLLOWER");
        fCutMuonGTIDs = fMuonData->GetIArray("gtid_muons");
        fCutDayMuons  = fMuonData->GetIArray("day_muons");
        fCutSecMuons  = fMuonData->GetIArray("sec_muons");
        fCutNsMuons   = fMuonData->GetIArray("ns_muons");
        foundDBFile = true;
      }
      catch(DBNotFoundError &e){
        warn << "TPMuonFollower::BeginOfRun: Could not find TPMUONFOLLOWER, "
                "but its possible the arrays are empty." << endl;
      }
      if(!foundDBFile){
        // Try and get the muon table, assuming it was empty and get set to invalid
        try{
          DBLinkPtr fMuonData = DB::Get()->GetLink("TPMUONFOLLOWER");
          fCutMuonGTIDs.push_back(fMuonData->GetI("gtid_muons"));
          fCutDayMuons.push_back(fMuonData->GetI("day_muons"));
          fCutSecMuons.push_back(fMuonData->GetI("sec_muons"));
          fCutNsMuons.push_back(fMuonData->GetI("ns_muons"));
          foundDBFile = true;
          warn << "TPMuonFollower::BeginOfRun: Found empty TPMUONFOLLOWER table." << endl;
        }
        catch(DBNotFoundError &e){
          warn << "TPMuonFollower::BeginOfRun: Could not find TPMUONFOLLOWER ratdb file. "
                  "Using local json file, if it exists." << endl;
          foundDBFile = false;
        }
      }
      // Only use the local file if you can't find the ratdb file
      if(!foundDBFile){
        try {
          // FIXME, T Kaptanoglu. This is the OWLN retrigger cut. I included this
          // in the first pass, but I'll leave it in for now to keep the existing
          // structure untouched
          std::ostringstream fname;
          fname << "tpmuonfollowercut_" << run.GetRunID() << ".json";
          std::vector<DBTable *> contents = DBJsonLoader::parse(fname.str());
          json::Value fMuonsTemp = contents[0]->GetJSON("muons");
          fOwls = contents[0]->GetJSON("owls");
          foundJsonFile = true;
          for (int i=0;i<(int)fMuonsTemp.getArraySize();i++){
            // want to check that none are too close to an owl event
            // also calculate deadtime
            int itimes[3];
            for (int j=0;j<3;j++){
              itimes[j] = fMuonsTemp[i]["time"][j].cast<int>();
            }
            int pass = 1;
            for (int j=0;j<(int)fOwls.getArraySize();j++){
              if (i==j) continue; // FIXME, T. Kaptanoglu: why?
              int jtimes[3];

              for (int k=0;k<3;k++){
                jtimes[k] = fOwls[j]["time"][k].cast<int>();
              }
              double timeSinceOwl = (double) (((double)((itimes[0]-jtimes[0])*86400) + (double)(itimes[1]-jtimes[1]))*1E9) + (double)(itimes[2]-jtimes[2]);
              if (timeSinceOwl > 0 && timeSinceOwl < fOwlLockoutTime){
                pass = 0;
                break;
              }
            }
            if (pass)
              fMuons.push_back(fMuonsTemp[i]);
          }
        } catch (FileError &e) {
          warn << "TPMuonFollowerCut::BeginOfRun: Couldn't find JSON file." << "\n";
        }
      }
      if(!foundJsonFile && !foundDBFile){
        Log::Die("TPMuonFollowerCut::BeginOfRun: Could not find RATDB or JSON "
                 "file specifing muon event times for pass 2.");
      }
    }
  }

  void TPMuonFollowerCut::EndOfRun(DS::Run& run)
  {
    // Build the json file and RATDB tables for the first pass
    if (fProcessPass == 1){
      DBTable table;
      table.SetJSON("muons",fMuonOutput);
      table.SetJSON("owls",fOwlOutput);
      std::ostringstream fname;
      fname << "tpmuonfollowercut_" << run.GetRunID() << ".json";
      table.SaveAs(fname.str());

      // Table listing the muons in the current run
      RAT::DBTable muonTable("TPMUONFOLLOWER");
      muonTable.SetI("version",1);
      muonTable.SetIndex("");
      muonTable.SetPassNumber(-1);
      muonTable.SetRunRange(run.GetRunID(), run.GetRunID());
      if(fGTIDMuons.size() == 0){
        muonTable.SetI("gtid_muons", -1);
        muonTable.SetI("day_muons", -1);
        muonTable.SetI("sec_muons", -1);
        muonTable.SetI("ns_muons", -1);
      }
      else{
        muonTable.SetIArray("gtid_muons", fGTIDMuons);
        muonTable.SetIArray("day_muons", fDayMuons);
        muonTable.SetIArray("sec_muons", fSecMuons);
        muonTable.SetIArray("ns_muons", fNsMuons);
      }
      muonTable.SaveAs("TPMUONFOLLOWER.ratdb");

      // Put the last muon and the run type in a table that is valid for the next run
      RAT::DBTable lastMuonTable("LAST_MUON");
      lastMuonTable.SetI("version",1);
      lastMuonTable.SetIndex("");
      lastMuonTable.SetPassNumber(-1);
      lastMuonTable.SetRunRange(run.GetRunID()+1, run.GetRunID()+1);
      if(fGTIDMuons.size() > 0){
        lastMuonTable.SetI("day_last_muon", fDayMuons.back());
        lastMuonTable.SetI("sec_last_muon", fSecMuons.back());
        lastMuonTable.SetI("ns_last_muon",  fNsMuons.back());
      }
      else{
        lastMuonTable.SetI("day_last_muon", -1);
        lastMuonTable.SetI("sec_last_muon", -1);
        lastMuonTable.SetI("ns_last_muon",  -1);
      }
      lastMuonTable.SetI("runtype", fRunType);
      lastMuonTable.SaveAs("LAST_MUON.ratdb");
    }
  }

  Processor::Result TPMuonFollowerCut::DSEvent(DS::Run&, DS::Entry& ds)
  {
    bool pass = true; // If any triggered event fails, they all fail
    for( size_t iEV = 0; iEV < ds.GetEVCount(); iEV++ )
    {
      // First event gets a FREE muon tag
      if( fFreeMuon && fProcessPass == 1 ){
        fFreeMuon = false;
        DS::EV& ev = ds.GetEV(iEV);
        int gtid = -1;
        std::vector<int> TimeNow(3);
        TimeNow[0] = ev.GetUniversalTime().GetDays();
        TimeNow[1] = ev.GetUniversalTime().GetSeconds();
        TimeNow[2] = ev.GetUniversalTime().GetNanoSeconds();
        json::TObject oneMuon;
        oneMuon["gtid"] = gtid;
        oneMuon["time"] = TimeNow;
        fMuonOutput.push_back(oneMuon);
        // First event, check the GTID
        if(fFirstEvent){
          fFirstEvent = false;
          gtid = ev.GetGTID();
          // Condition that a free muon is not needed
          // This is here just to make the condition explicit:
          // 1. first_gtid == valid_gtid -> rollover run
          // 2. current run is physics run
          // 3. previous run is physics run
          // 4. the ratdb file specifying the muon information in the previous run was read
          if(gtid == validGTID && (fRunType & kPhysicsRun) && (fRunTypePreviousRun & kPhysicsRun) && foundLastFile){
            // Do nothing
          }
          // One of the above conditions failed, so need to add a free muon at the beginning of the run
          else{
            fDayMuons.push_back(TimeNow[0]);
            fSecMuons.push_back(TimeNow[1]);
            fNsMuons.push_back(TimeNow[2]);
            fGTIDMuons.push_back(-1);
          }
        }
      }
      if( Event(ds, ds.GetEV(iEV)) != OKTRUE )
        pass = false;
    }
    return pass ? OKTRUE : OKFALSE;
  }


  Processor::Result TPMuonFollowerCut::Event(DS::Entry&, DS::EV& ev)
  {
    // lets tag muon candidates in the first pass
    if (fProcessPass== 1){

      int gtid = ev.GetGTID();

      // Retrigger lockout after a NOWL event, remove these as muon events
      // DeltaT between most revent OWL event and current event
      int64_t owlDeltaT = ((ev.GetClockCount50() - fOwlTime) & 0x7FFFFFFFFFF)*20; //43-bit rollover
      if(owlDeltaT > 0 && owlDeltaT < fOwlLockoutTime){
        return fPassFlag ? OKFALSE : OKTRUE;
      }

      // check number of owl tubes and total nhit
      if (ev.GetUncalPMTs().GetOWLCount() < fMinOwls){
        return fPassFlag ? OKFALSE : OKTRUE;
      }

      // we need a list of owl hit times so we can reject events soon afterwards that might look like muons
      std::vector<int> TimeNow(3);
      TimeNow[0] = ev.GetUniversalTime().GetDays();
      TimeNow[1] = ev.GetUniversalTime().GetSeconds();
      TimeNow[2] = ev.GetUniversalTime().GetNanoSeconds();
      json::TObject oneOwl;
      oneOwl["gtid"] = json::Value(gtid);
      oneOwl["time"] = TimeNow;
      fOwlOutput.push_back(oneOwl);
      fOwlTime = ev.GetClockCount50();

      // Minimum nhit for a muon candidate
      if (ev.GetUncalPMTs().GetCount() < fNhitMin){
        return fPassFlag ? OKFALSE : OKTRUE;
      }

      // check trigger word to make sure there
      // was a valid physics event. throw out
      // pulser, pedestal, and ext_a triggers
      if (BitManip::TestBit(ev.GetTrigType(), DU::TrigBits::PulseGT) ||
          BitManip::TestBit(ev.GetTrigType(), DU::TrigBits::Pedestal) ||
          BitManip::TestBit(ev.GetTrigType(), DU::TrigBits::EXTASY)){
        return fPassFlag ? OKFALSE : OKTRUE;
      }

      // Event is an orphan
      if(ev.GetTrigType() == 0){
        return fPassFlag ? OKFALSE : OKTRUE;
      }


      // FIXME T. Kaptanoglu this uses a version of the neck cut which might need
      // modifying. Since its not a critical part of the code, I'm leaving the existing
      // structure in place
      unsigned int num_neck = 0;
      for (unsigned int i=0;i<ev.GetUncalPMTs().GetNeckCount();i++){
        num_neck++;
        if (num_neck >= fMaxNeck){
          // this is a neck event, return
          return fPassFlag ? OKFALSE : OKTRUE;
        }
      }

      double t_total = 0.;
      double t2_total = 0.;
      double tneck_total = 0;
      double t_avg = 0;
      double t_rms = 0;
      int t_count = 0;
      int tneck_count = 0;

      fCalType = ev.GetPMTCalType();
      // If only ECA calibrations
      if (fCalType == 1){
        // loop over pmts, check if they are in the bottom half
        // of the detector, if so, add up the time
        for (size_t ipmt=0;ipmt<ev.GetCalPMTs().GetCount();ipmt++){
          const DS::PMTCal& cpmt = ev.GetCalPMTs().GetPMT(ipmt);
          float time = cpmt.GetTime();
          if (time > -100){
            t_total += time;
            t2_total += time*time;
            t_count++;
            if (DU::Utility::Get()->GetPMTInfo().GetPosition(cpmt.GetID()).Z() < 0){
              tneck_total += time;
              tneck_count++;
            }
          }
        } // end loop over all pmts

      }else{
        // We have PCA calibrated, so take uncalibrated times
        // and manually ECA calibrate them
        for (size_t ipmt=0;ipmt<ev.GetUncalPMTs().GetCount();ipmt++){
          const DS::PMTUncal& upmt = ev.GetUncalPMTs().GetPMT(ipmt);
          fECACal.CalibrateHit(upmt.GetQHS(),upmt.GetQHL(),upmt.GetQLX(),upmt.GetTime(),upmt.GetCrateCardChannelCell());
          // check that the calibration is ok still
          double time = fECACal.GetTime();
          if (std::abs(time) < 8000){
            t_total += time;
            t2_total += time*time;
            t_count++;
            if (DU::Utility::Get()->GetPMTInfo().GetPosition(upmt.GetID()).Z() < 0){
              tneck_total += time;
              tneck_count++;
            }
          }
        }
      } // end if/else ECA or PCA

      if (t_count >= 2)
        t_rms = sqrt((t2_total-t_total*t_total/(double)t_count) / (double)(t_count-1.));

      if (t_rms > fTrmsMax){
        // this is a neck event, return
        return fPassFlag ? OKFALSE : OKTRUE;
      }

      // only bother with the rest of the neck cut if nhit is small enough
      // if its too large, definitely a muon and not a neck event
      if (ev.GetCalPMTs().GetCount() <= fNeckNhitMax){
        for (size_t ipmt=0;ipmt<ev.GetCalPMTs().GetNeckCount();ipmt++){
          const DS::PMTCal& cneck = ev.GetCalPMTs().GetNeckPMT(ipmt);
          int q_lx = cneck.GetQLX();
          int q_hl = cneck.GetQHL();
          int q_hs = cneck.GetQHS();
          int time = cneck.GetTime();
          // if there was a high charge hit on a neck tube
          if (q_hs < fMinQ || q_hs > fMaxQ ||
              q_hl < fMinQ || q_hl > fMaxQ ||
              q_lx < fMinQ || q_lx > fMaxQ){
            if (time < 0.5)
              t_avg = -9999;
            else
              t_avg = tneck_total/(double)tneck_count;
            // and it was separated in time from bottom hits
            if (time < t_avg - fTimeDiff)
              return fPassFlag ? OKFALSE : OKTRUE;
          } // end if bad charge
        } // end loop over neck tubes
      } // end if nhit not too high

      // since we got here, it is a muon candidate, so lets mark it
      json::TObject oneMuon;
      oneMuon["gtid"] = gtid;
      oneMuon["time"] = TimeNow;
      fGTIDMuons.push_back(gtid);
      fMuonOutput.push_back(oneMuon);
      fDayMuons.push_back(TimeNow[0]);
      fSecMuons.push_back(TimeNow[1]);
      fNsMuons.push_back(TimeNow[2]);
      return fPassFlag ? OKFALSE : OKTRUE;

    }else{  // (fProcessPass ==2)
      // pass two - lets go through and mark muon long and short followers,
      // unmark those too close to other owl events
      // by default lets update the mask now as "clean" so the applied bits get set
      // the logic below will update the mask if needed
      fPassFlag = true;
      fBit = fShortBit;
      UpdateMask(ev);
      fBit = fLongBit;
      UpdateMask(ev);

      int gtid = ev.GetGTID();
      std::vector<int> TimeNow(3);
      TimeNow[0] = ev.GetUniversalTime().GetDays();
      TimeNow[1] = ev.GetUniversalTime().GetSeconds();
      TimeNow[2] = ev.GetUniversalTime().GetNanoSeconds();

      if(!foundDBFile){
        for (int i=0;i<(int)fMuons.size();i++){
          if (gtid == fMuons[i]["gtid"].cast<int>()){
            // dont want to flag muons as followers
            // untag as both a long and short follower
            fBit = fShortBit;
            UpdateMask(ev);
            fBit = fLongBit;
            UpdateMask(ev);
            return fPassFlag ? OKFALSE : OKTRUE;
          }
        }
        for (int i=0;i<(int)fMuons.size();i++){
          int times[3];
          for (int j=0;j<3;j++){
            times[j] = fMuons[i]["time"][j].cast<int>();
          }

          double timeSinceMuon = (double) (((double)((TimeNow[0]-times[0])*86400) + (double)(TimeNow[1]-times[1]))*1E9) + (double)(TimeNow[2]-times[2]);
          if (timeSinceMuon > 0){
            if (timeSinceMuon < (fLongFollowerTime*1E9)){
              fBit = fLongBit;
              fPassFlag = false;
              UpdateMask(ev);
              if (timeSinceMuon < (fShortFollowerTime*1E9)){
                fBit = fShortBit;
                fPassFlag = false;
                UpdateMask(ev);
              }
            }
          }
        }
        // Wait to look at all the muons before returning
        return fPassFlag ? OKFALSE : OKTRUE;
      }
      else{
        for(size_t i = 0; i < fCutMuonGTIDs.size(); i++){
          if(gtid == fCutMuonGTIDs[i]){
            // dont want to flag muons as followers
            // untag as both a long and short follower
            fBit = fShortBit;
            UpdateMask(ev);
            fBit = fLongBit;
            UpdateMask(ev);
            return fPassFlag ? OKFALSE : OKTRUE;
          }
        }
        // First look at last muon from the previous run
        int times[3];
        times[0] = fCutLastMuonDay;
        times[1] = fCutLastMuonSec;
        times[2] = fCutLastMuonNs;
        double timeSinceMuon = (double)(((double)((TimeNow[0]-times[0])*86400) + (double)(TimeNow[1]-times[1]))*1E9) + (double)(TimeNow[2]-times[2]);
        if (timeSinceMuon > 0){
          if (timeSinceMuon < (fLongFollowerTime*1E9)){
            fBit = fLongBit;
            fPassFlag = false;
            UpdateMask(ev);
            if (timeSinceMuon < (fShortFollowerTime*1E9)){
              fBit = fShortBit;
              fPassFlag = false;
              UpdateMask(ev);
            }
          }
        }
        // Loop over the Muons in the run and get the times of each one. If the current time of the event is
        // close enough to any of the muons, than we tag it as either a long or short follower.
        for(size_t i = 0; i < fCutMuonGTIDs.size(); i++){
          times[0] = fCutDayMuons[i];
          times[1] = fCutSecMuons[i];
          times[2] = fCutNsMuons[i];
          timeSinceMuon = (double)(((double)((TimeNow[0]-times[0])*86400) + (double)(TimeNow[1]-times[1]))*1E9) + (double)(TimeNow[2]-times[2]);
          if (timeSinceMuon > 0){
            if (timeSinceMuon < (fLongFollowerTime*1E9)){
              fBit = fLongBit;
              fPassFlag = false;
              UpdateMask(ev);
              if (timeSinceMuon < (fShortFollowerTime*1E9)){
                fBit = fShortBit;
                fPassFlag = false;
                UpdateMask(ev);
              }
            }
          }
        }
        // Wait to look at all the muons before returning
        return fPassFlag ? OKFALSE : OKTRUE;
      }
    }

    return fPassFlag ? OKFALSE : OKTRUE;
  }

} // namespace RAT