// -- RAT headers
#include <RAT/DB.hh>
#include <RAT/DBTable.hh>
#include <RAT/DBTextLoader.hh>
#include <RAT/Log.hh>
#include <RAT/json.hh>
#include <RAT/StringUtil.hh>
// -- G4 headers
#include <G4SystemOfUnits.hh>
// -- ROOT headers
#include <Byteswap.h> // From ROOT

#include <fstream>
#include <sstream>
#include <algorithm>


extern "C" {
#include <glob.h>
#include <sys/stat.h>
}

using std::list;
using std::string;
using std::vector;

namespace RAT {

// The maximum amount of memory downloaded from the DB at any time
// If this amount is reached, RAT::DB will release older tables to make up memory.
// While this value might be adjusted in the future, we should try to keep it
// relatively small
const int maxServerBytes = 209715200; // 200 MB

std::vector<std::string> DB::fDBSetList;

const std::string RAT::DB::default_server = "23 21 3c 5f 34 3c 2a 58 69 61 60 58 3d 21 3f 47 26 3d 75 4f 3c 20 3b 4e 20 3a 20 5b 3e 2b 21 44 24 e 3f 4c 20 3f 23 5 20 20 20 5b 3f 60 3a 58 69 7b 7b 1b 63 61 3d 4a 27 2a 2d";

DB::DB() : server(""),fDbBackend(0), fRunID(0), fDbTag(""), fAirplaneMode(false), fDefaultPlaneLockEnabled(true), fDefaultsLoaded(false)
{
  fAllowDanger = false;

  // construct a list of tables for allowdanger to ignore
  // unfortunately you can't put this in a table itself (snake eats tail)
  fAllowDangerWhitelist.push_back("GEO");
}

DB::~DB()
{
  // Smart pointers will free tables automatically, nothing needed here anymore

  // Since this database is going away, disable all the existing links.
  // Prevents links from trying to remove themselves from nonexistent
  // databases
  list<DBLink *>::iterator l;
  for (l = links.begin(); l != links.end(); ++l)
    (*l)->Unlink();

  if (fDbBackend) delete fDbBackend;
}

void DB::BeginOfRun(DS::Run &run)
{
#ifdef RATDB_DEBUG
  info << "DB::BeginOfRun : Establishing connection to the server...(if not already opened)" << newline;
#endif
  fRunID = run.GetRunID();

  // -- initialize only if there is effectively a connection.
  if (!fAirplaneMode) {
    if (fDbBackend) {
      fDbBackend->Connect();
      fDbBackend->SetBoRCompleted(false);

      // Refresh the list of available objects in the remote server that are available for
      // this run
      warn << "DB::BeginOfRun : Refreshing list of valid tables on server..." << newline;
      tableNamesOnServer.clear();
      tableNamesOnServer = fDbBackend->GetListTypes(fRunID);
      info << "DB::BeginofRun : Tables on server include";
      for (std::set<std::string>::const_iterator it = tableNamesOnServer.begin(); it != tableNamesOnServer.end(); ++it) {
        info << " " << *it;
      }
      info << "\n";
    } else if (fRunID != 0) {
      // There is no backend defined, but the run is not zero, so it should be loaded
      if (!server.size()) {
        info << BYELLOW << "DB::BeginOfRun:: Setting PgSQL server to default..." << CLR << newline;
        SetServer(decode_server(default_server),false);
      }
      InitBackend();
    }
  }
  // Propagate to all the DBLinks which is the current run
  for (std::list<RAT::DBLink *>::iterator it = links.begin() ; it != links.end(); ++it) {
#ifdef RATDB_DEBUG
    info << "DB::BeginOfRun : Updating [" << (*it)->GetName() << "]"<< newline;
#endif

    (*it)->SetCurrentRun(fRunID);
  }
#ifdef RATDB_DEBUG
  info << "DB::BeginOfRun : Completed beginOfRun..." << newline;
#endif

}

void DB::EndOfBeginOfRun()
{
  // Force the connection to close.
  // If necessary, the connection can be re-opened, but then a warning is sent to tell
  // Everyone that the database should not be accessed from outside BeginOfRun

  if (fDbBackend && !fDbBackend->GetBoRCompleted()) {
    warn << "DB::EndOfBeginOfRun : BeginOfRun stage completed. Closing remote DB connection." << newline;
    // Stop the connection
    fDbBackend->Disconnect();
    fDbBackend->SetBoRCompleted(true);
  }
}

void DB::EndOfRun(DS::Run &/*run*/)
{
  //TODO: Perhaps we could include here something to clean up tables that will no
  // longer be valid
  if (fDbBackend) {
    fDbBackend->SetBoRCompleted(false);
  }
}

void DB::CheckSetFieldUsage()
{
  TMap* dbtrace = Log::GetDBTraceMap();
  vector<string>::iterator it;
  for (it = fDBSetList.begin(); it<fDBSetList.end(); ++it)
    if (!dbtrace->GetValue((*it).c_str()))
      warn << "DB::CheckSetFieldUsage: Warning: Field " << *it << " was set in the macro but never used." << newline;
}

int DB::Load(std::string filename, bool printPath)
{
  // Try to get file info assuming the name is literal
  struct stat s;
  if (stat(filename.c_str(), &s) == 0) {
    if (printPath)
      info << "DB::Load: Loading " << filename << "\n";
    if (S_ISDIR(s.st_mode))
      return LoadAll(filename);
    else
      return LoadFile(filename);
  } else {
    string newfilename = string(getenv("GLG4DATA")) + "/" + filename;
    if (printPath)
      info << "DB::Load: Loading " << newfilename << "\n";
    if (stat(newfilename.c_str(), &s) == 0) {
      if (S_ISDIR(s.st_mode))
        return LoadAll(newfilename);
      else
        return LoadFile(newfilename);
    }
  }

  Log::Die("DB::Load: Cannot open " + filename);
  return 0; // Never get here
}

int DB::LoadFile(std::string filename)
{
  vector<DBTable *> contents;
  try {
    contents = ReadRATDBFile(filename);
  } catch (FileError &e) {
    warn << "DB::LoadFile: Error! Cannot open " << e.filename << "\n";
    return 0;
  }

  // NOTE: Smart pointers in tables mean that we do not need to
  // explicitly delete old table before replacing with new one below!

  vector<DBTable *>::iterator itbl;
  for (itbl=contents.begin(); itbl != contents.end(); ++itbl) {
    DBTable *table = (*itbl);  // Get rid of iterator mess
    //DBTableKey id(table->GetName(), table->GetIndex());
    // Grab the
    // Careful with runs
    DBTableKey id(table->GetName(),table->GetIndex(),table->GetRunBegin(),table->GetRunEnd(),table->GetPassNumber());

    if (table->IsDefault()) {

      if (tables_default.find(id) != tables_default.end()) {
        warn << "DB::LoadFile: Replacing default " << id.name << "[" << id.index
            << "] in database.\n"
            << newline;
      }
#ifdef RATDB_DEBUG
      info << "DB::LoadFile : Setting table " << id.stream_id() << " in default plane." << newline;
#endif
      tables_default[id] = table;

    } else if (table->IsUser()) {
      Log::Die("DB::LoadFile : Attempting to load a user plane table from a file. This is not allowed. Sneaky, aren't we?");
      /**
      // This should not be valid
      // FIXME: Add user plane table checking: Do not allow a user plane table to be loaded
      //        from anywhere but a file
      id.run = -1;
      id.pass = -1;

      DBTable *oldtable = FindTable(id.name, id.index, id.run,id.pass);
      if (oldtable) {
        warn << "DB::LoadFile: Replacing user " << id.name << "[" << id.index
             << "] in database.\n"
             << newline;
      }
      tables[id] = table;
       */
    }
    else { /* Actual run range */
      // Explicitly create smart pointer here to ensure that
      // storage is shared among all copies.
#ifdef RATDB_DEBUG
      info << "DB::LoadFile : Setting table " << id.stream_id() << " in run plane." << newline;
#endif

      simple_ptr_nocopy<DBTable> tablePtr(table);

      // -- NFB: This logic has to be reversed. Instead of adding a reference for each valid run (breaking
      //         when an object is valid until infinity, it is better to do the validity checks per entry
      //          The rationale is that we will have far less objects in memory, than the range of possible run numbers

      // Loop over all objects in tables and check if there is overlap
      for (DBTableSet::const_iterator it = tables_run.begin(); it != tables_run.end(); ++it) {
        // Check if the table with the same id already exists
        if (it->first == id) {
          warn << "DB::LoadFile: Replacing " << id.name << "[" << id.index
              << "]" << " with index " << id.stream_id()
              << " in database.\n";
        }


        // The table being loaded overlaps with an existing run plane table
        if ((it->first).overlaps_range(id)) {
          // The new table overlaps with an existing table in the cache
          // This should never happen
          std::ostringstream msg;
          msg << "DB::LoadFile : Table " << id.stream_id() << " overlaps with existing run table loaded from file" << (it->first).stream_id();
          Log::Die(msg.str());
        }
      }
      // if it didn't fail by this point, it is good to go
      tables_run[id] = tablePtr;
    }
  }

  return 1;
}

int DB::LoadAll(std::string dirname, std::string pattern)
{
  pattern = dirname + "/" + pattern;
  glob_t g;

  int ntables = 0;
  if (glob(pattern.c_str(), 0, 0, &g) == 0) {
    for (unsigned i=0; i < g.gl_pathc; i++) {
      string path(g.gl_pathv[i]);
      detail << "DB::LoadAll: Loading " << path << " ... ";
      if (Load(path)) {
        detail << "Success!" << newline;
        ntables++;
      }
      else {
        detail << "DB::LoadAll: Load Failed!" << newline;
        warn << "DB::LoadAll: Error loading " << path << newline;
        globfree(&g);
        return 0;
      }
    }
  }
  info << "DB::LoadAll: Loaded " << ntables << " tables" << newline;

  globfree(&g);
  return 1;
}

int DB::LoadDefaults()
{
  int ret_default = 0;
  if ( getenv("GLG4DATA") != NULL )
    ret_default = LoadAll(string(getenv("GLG4DATA")));
  else
    ret_default = LoadAll(string("data"));
  if (!ret_default)
    return ret_default;

  // -- Load the RemoteDB
  if (!fAirplaneMode && fRunID != 0) {
    if (!server.size()) {
      info << BYELLOW << "DB::LoadDefaults:: Setting PgSQL server to default..." << CLR << newline;
      SetServer(decode_server(default_server),false);
    }
    InitBackend();
  }
  fDefaultsLoaded = true;
  return ret_default;
}

void DB::Clear()
{
  tables_user.clear();
  tables_run.clear();
  tables_default.clear();
}

void DB::InitBackend() {

  if (!fAirplaneMode) {
    if (server.find("postgres") != std::string::npos) {

      // PgSQL backend
      info << "DB::Loading PgSQL backend" << newline;
      fDbBackend = new PgSQLBackend();
      fDbBackend->SetConnURL(server);
      fDbBackend->Connect();
    } else if (server.size() != 0) {
      // There is no specified server....do nothing
      Log::Die("Unknown backend type " + server);
    }
  } else {
    warn << "DB::InitBackend : Attempting to initialize backend in 'airplane mode'. Aborting." << newline;
  }
  if (!fDbBackend) return;

  tableNamesOnServer.clear();
  if (fDbTag.size()) {
    info << "DB::SetServer : Requested tag found. Setting tag to " << fDbTag << newline;
    fDbBackend->SetDbTag(fDbTag);
  }
  tableNamesOnServer = fDbBackend->GetListTypes();
  info << "DB::SetServer : Tables on server include";
  for (std::set<std::string>::const_iterator it = tableNamesOnServer.begin(); it != tableNamesOnServer.end(); ++it) {
    info << BWHITE << " " << *it;
  }
  info << CLR << "\n";

}

void DB::LoadDefaultPlaneLocks() {
  // Is the plane locks were disabled
  if (!fDefaultPlaneLockEnabled) {
    warn << BRED << "DB::LoadDefaultPlaneLocks : Default plane locks have been disabled. Bypassing." << CLR << newline;
    return;
  }
  // reset the locks and reload them
  debug << "DB::LoadDefaultPlaneLocks : Loading default plane locks" << newline;
  fTblNoDefaultPlane.clear();
  try {
    DBLinkPtr tbl = GetLink("IO","");
    std::vector<std::string> tables_locked = tbl->GetSArray("tables_no_default");
    // -- Add iteratively to remove duplicated entries
    std::string type,index;
    std::pair<std::string,std::string> entry;
    for (size_t i = 0; i < tables_locked.size(); ++i) {
        ParseTableName(tables_locked.at(i),type,index);
        entry = std::make_pair(type,index);
        if (std::find(fTblNoDefaultPlane.begin(),fTblNoDefaultPlane.end(),entry) == fTblNoDefaultPlane.end()) {
        fTblNoDefaultPlane.push_back(entry);
      }
    }
  }
  catch (DBNotFoundError &e) {
    Log::Die("Couldn't find list of tables to override default plane access. Expected in IO[].tables_no_default'");
  }

  if (fTblNoDefaultPlaneUser.size()) {
    fTblNoDefaultPlane.insert(fTblNoDefaultPlane.end(),fTblNoDefaultPlaneUser.begin(),fTblNoDefaultPlaneUser.end());
  }
  // -- Now subtract the exceptions that have been added
  for (size_t i = 0; i < fTblNoDefaultPlaneExceptions.size(); ++i) {
    debug << "Adding default plane lock exception : " << fTblNoDefaultPlaneExceptions.at(i) << newline;
    RemoveTblNoDefaultPlane(fTblNoDefaultPlaneExceptions.at(i));
  }

  for (size_t i = 0; i < fTblNoDefaultPlane.size(); ++i) {
    if (i==0) {
      warn << RED << "Denying default plane access for the following tables: " << CLR << newline;
    }
    warn << RED << " :: " << fTblNoDefaultPlane.at(i).first << "[" <<  fTblNoDefaultPlane.at(i).second << "]" << CLR << newline;
  }
}

void DB::AddTblNoDefaultPlane(std::string tblname) {
  // -- Split the input into table and index
  std::string type, index = "";
  ParseTableName(tblname,type,index);
  std::pair<std::string,std::string> entry = std::make_pair(type,index);
  if (std::find(fTblNoDefaultPlane.begin(),fTblNoDefaultPlane.end(),entry) == fTblNoDefaultPlane.end()) {
    fTblNoDefaultPlane.push_back(entry);
  }
  if (std::find(fTblNoDefaultPlaneUser.begin(),fTblNoDefaultPlaneUser.end(),entry) == fTblNoDefaultPlaneUser.end()) {
    fTblNoDefaultPlaneUser.push_back(entry);
  }
}

void DB::RemoveTblNoDefaultPlane(const std::string& tblname) {
  std::string type, index = "";
  ParseTableName(tblname,type,index);
  if (std::find(fTblNoDefaultPlaneExceptions.begin(),fTblNoDefaultPlaneExceptions.end(),tblname) == fTblNoDefaultPlaneExceptions.end()) {
    fTblNoDefaultPlaneExceptions.push_back(tblname);
  }

  std::vector<std::pair<std::string,std::string> >::iterator it = std::find(fTblNoDefaultPlane.begin(),fTblNoDefaultPlane.end(),std::make_pair(type,index));
  if (it != fTblNoDefaultPlane.end()) {
    warn << RED << "DB::RemoveTblNoDefaultPlane : Table [" << tblname << "] removed list of locked default plane tables." << CLR << newline;
    fTblNoDefaultPlane.erase(it);
  }
#ifdef RATDB_DEBUG
  else {
    warn << RED << "DB::RemoveTblNoDefaultPlane : Table [" << tblname << "] not found in list of locked tables." << CLR << newline;
  }
#endif
}

void DB::SetDefaultPlaneLockStatus(bool enable) {
  if (!enable) {
    warn << BRED << "\nWARNING!\nWARNING! DB::SetDefaultPlaneLockStatus : Disabling default plane locks. You better know what you're doing!\nWARNING!" << CLR << newline;
    fTblNoDefaultPlane.clear();
    fDefaultPlaneLockEnabled = false;
  } else {
    fDefaultPlaneLockEnabled = true;
  }
}

void DB::SetServer(const std::string& url, bool init)
{
  if (fAirplaneMode) {
    warn << "DB::SetServer:: Setting a remote DB URL, but airplane mode is enabled...disabling airplane mode." << newline;
    fAirplaneMode = false;
  }
  server = url;
  if (init) {
    InitBackend();
  }
}

DBLinkPtr DB::GetLink(std::string tblname, std::string index, int pass)
{
  // By using smart pointer here, user does not have to worry about
  // memory management
  DBLink *real_ptr = new DBLink(this, tblname, index, pass);
  links.push_back(real_ptr);
  return DBLinkPtr(real_ptr);
}

DBLinkGroup DB::GetLinkGroup(std::string tblname)
{
  DBTableSet::iterator it_tbl;
  DBLinkGroup group;

  // To figure out which tables exist, we have to scan all available
  // planes
  // However, the RUN plane should be refreshed, since there might be already a table
  // locally for an index, and unfetched tables for different indexes

  // Start with the user plane
  for (it_tbl = tables_user.begin(); it_tbl != tables_user.end(); ++it_tbl) {
    DBTableKey id = it_tbl->first;
    if (id.name == tblname) {
#ifdef RATDB_DEBUG
      info << "DB::GetLinkGroup : Getting link for existing 'user' " << tblname << "[" << id.index << "]" << newline;
#endif
      group[id.index] = GetLink(tblname, id.index);
    }
  }

  // Next go through the run plane and check what we already have
  for (it_tbl = tables_run.begin(); it_tbl != tables_run.end(); ++it_tbl) {
    DBTableKey id = it_tbl->first;
    // user plane supersedes the run plane... if the index already exists, do not return
    if ((id.name == tblname) && (group.find(id.index) == group.end())) {
#ifdef RATDB_DEBUG
      info << "DB::GetLinkGroup : Getting link for existing 'run' " << tblname << "[" << id.index << "]" << newline;
#endif
      group[id.index] = GetLink(tblname, id.index);
    }
  }

  // Also include tables from the server that are not yet listed in the run plane
  if (tableNamesOnServer.count(tblname) != 0) {
    std::set<std::string> idxset = fDbBackend->GetListIndexes(tblname,fRunID);

    for (std::set<std::string>::const_iterator it = idxset.begin(); it != idxset.end(); ++it) {
      if (group.count(*it) == 0) {
#ifdef RATDB_DEBUG
      info << "DB::GetLinkGroup : Getting link for non existing 'remote' " << tblname << "[" << *it << "]" << newline;
#endif
        group[*it] = GetLink(tblname, *it);
      }
    }
  }

  // If default plane is locked for this table, do not return
  for (size_t i = 0 ; i < fTblNoDefaultPlane.size(); ++i) {
    if (fTblNoDefaultPlane.at(i).first == tblname && fTblNoDefaultPlane.at(i).second == "*")
      return group;
  }

  // Finally go through the default plane
  // Next go through the run plane
  for (it_tbl = tables_default.begin(); it_tbl != tables_default.end(); ++it_tbl) {
    DBTableKey id = it_tbl->first;
    if ((id.name == tblname) && (group.find(id.index) == group.end())
        && (std::find(fTblNoDefaultPlane.begin(),fTblNoDefaultPlane.end(),std::make_pair(id.name,id.index)) == fTblNoDefaultPlane.end())) {
#ifdef RATDB_DEBUG
      info << "DB::GetLinkGroup : Getting link for existing 'default' " << tblname << "[" << id.index << "]" << newline;
#endif
      group[id.index] = GetLink(tblname, id.index);
    }
  }

  return group;
}

bool DB::GetTblNoDefaultPlaneStatus(const std::string &tblname, const std::string &index) {
#ifdef RATDB_DEBUG
  debug << "DB::GetTblNoDefaultPlaneStatus : Checking " << tblname << "[" << index << "]" << newline;
#endif
  std::vector<std::pair<std::string,std::string> >::iterator it;
  for (it = fTblNoDefaultPlane.begin(); it != fTblNoDefaultPlane.end(); ++it) {
    if ((*it).first == tblname && ((*it).second == index || (*it).second == "*")) {
      return true;
    }
  }
  // no match was found
  return false;
}

void DB::UploadTableOnline(DBTable &tbl) {
  if (!fDbBackend) {
    Log::Die("DB::UploadTableOnline : Trying to push a table before the DB is initialized.");
  }
  // Wrap the insertion into a try block so we can catch and send the appropriate error
  try {
    fDbBackend->InsertObject(tbl);
  }
  catch(RAT::DBBackendError &e) {
    warn << "Backend error caught." << newline;
    warn << e.what() << newline;
    throw;
  }
  catch(std::exception &e) {
    warn << "DB::UploadTableOnline : Caught std exception." << newline;
    warn << e.what() << newline;
    throw;
  }
  catch(...) {
    warn << "DB::UploadTableOnline : Unidentified exception caught. Re-throwing." << newline;
    throw;
  }
}

void DB::SetDbTag(const std::string& tag) {
  info << BCYAN << "DB::SetDBTag : Setting RATDB tag to " << tag << CLR << newline;
  if (fDbTag.size()) {
    warn << "DB::SetDBTag : Overriding existing RATDB tag. Old : " << fDbTag << " New : " << tag << newline;
  }
  if (fDbBackend) {
    fDbBackend->SetDbTag(tag);
    fDbTag = tag;
  } else {
    warn << "DB::SetDBTag : Setting a tag without an active remote backend...caching it for when the backend is initialized." << newline;
    fDbTag = tag;
    return;
  }

  // This means that some tables might no longer be available.
  // Try it out
  detail << "DB::SetDBTag : Refreshing list of remote tables." << newline;
  tableNamesOnServer.clear();

  tableNamesOnServer = fDbBackend->GetListTypes();
  info << "DB::SetDBTag : Tables on server include : ";
  for (std::set<std::string>::const_iterator it = tableNamesOnServer.begin(); it != tableNamesOnServer.end(); ++it) {
    info << BCYAN << " " << *it;
  }
  info << CLR << "\n";


}


// This is actually simpler to do from 3 specialized functions

// Finds a table in the User Plane
DBTable *DB::FindTableUser(const std::string tblname,const std::string index) {
  DBTableKey id(tblname, index, -1, -1);
  DBTableSet::iterator table;

  // user table was asked for explicitly
  table = tables_user.find(id);
  if (table != tables_user.end())
    return (*table).second.pointer(); // Found in cache, return now
  else
    return 0;
}


// Finds a table in the User Plane
DBTable *DB::FindTableDefault(const std::string tblname,const std::string index) {
#ifdef RATDB_VERBOSE
  info << "DB::FindTableDefault : Trying to find a table in default plane : " << tblname << "[" << index<< "]" << newline;
#endif

  DBTableKey id(tblname, index,0,0,0);
  DBTableSet::iterator table;

  // If default plane is locked for this table, do not return
  if (GetTblNoDefaultPlaneStatus(tblname,index))
    return 0;

  // if specific pass is requested, unless the pass is 0, there is no match
  if (fPassMap.find(std::make_pair(tblname,index)) != fPassMap.end()) {
    // There is an entry. Set pass to that value
    int pass = fPassMap.at(std::make_pair(tblname,index));
    std::string msg = dformat("DB::FindTableDefault : Default plane access for table with fixed pass:  \'passmap\' [%s][%s] (pass %d)",tblname.c_str(),index.c_str(),pass);
    if (pass != 0) {
      Log::Die(msg);
    } else {
      warn << msg << newline;
    }
  }

  // user table was asked for explicitly
  table = tables_default.find(id);
  if (table != tables_default.end()) {
#ifdef RATDB_DEBUG
  info << "DB::FindTableDefault : Returning default plane: " << tblname << "[" << index<< "]" << newline;
#endif
    return (*table).second.pointer(); // Found in cache, return now
  } else {
    return 0;
  }
}


// Finds the first table of a given type and index.
// Quick access without caring for pass or run number
DBTable *DB::FindAnyTableRemote(const std::string tblname, const std::string tblindex) {
  size_t newTableBytes;
  bool found_object = false;

  // 2. If we haven't found it yet, then let's check if this type exists in the remote server
  // Now check if the server has tables of this type
  // Return if the server does not have tables with this name
  // or if no server has been specified (the set will be empty)
  if (tableNamesOnServer.count(tblname) == 0) {
    return 0;
  }

  json::Value jsonDoc = fDbBackend->FetchObjFast(tblname,tblindex,found_object,newTableBytes);

  if (!found_object)
    return 0;

  DBTable *newTable = new DBTable(jsonDoc,newTableBytes);

#ifdef RATDB_DEBUG
  info << "DB::FindAnyTableRemote : Table created " << newline;
#endif

  // Get the ID of this table
  // -- It can be different from the requested ID, as the run_range can be larger

  DBTableKey id = DB::GetTableKey(newTable);

#ifdef RATDB_DEBUG
  info << "DB::FindAnyTableRemote : Table created with id " << id.stream_id() << newline;
#endif

  // 5) Flush out some tables if we need the space

#ifdef RATDB_DEBUG
  info << "DB::FindAnyTableRemote : Table " << newTable->GetName() <<  "[" << newTable->GetIndex()<< "] has size "<<  newTableBytes << " bytes"<< newline;
#endif

  // This should probably be done only once
  size_t serverTableBytes = 0;
  for (unsigned i=0; i < tablesFromServer.size(); i++) {
    serverTableBytes += tablesFromServer[i].size_bytes;
  }

  while (tablesFromServer.size() > 0 && serverTableBytes + newTableBytes > maxServerBytes) {
    DBTableKey keyForDeletion = tablesFromServer.front();
    tablesFromServer.pop_front();
    size_t deleteSize = keyForDeletion.size_bytes;
    tables_run.erase(keyForDeletion);
    serverTableBytes -= deleteSize;
    warn << "DB::FindAnyTableRemote: Evicting object from run cache (" << keyForDeletion.stream_id() << ") : [" << deleteSize << " bytes]" << newline;
  }

  // 6) Add this table to the memory cache
  simple_ptr_nocopy<DBTable> newTablePtr(newTable);
  tables_run[id] = newTablePtr;
  tablesFromServer.push_back(id);

  return newTable;
}

DBTable *DB::FindTableRun(const std::string tblname, const std::string index, const int runNumber, int passNumber ) {

  // run plane does not have tables with "special" run numbers
  if (runNumber == 0 || runNumber == -1) return 0;

  DBTableKey id;
  DBTableSet::iterator table;
  int pass;
#ifdef RATDB_DEBUG
  id = DBTableKey(tblname, index, runNumber, runNumber,passNumber);
  info << "DB::FindTableRun : Finding table with parameters : " << id.stream_id() << newline;
#endif


  // if the pass number was not specified, check if there is an entry
  // from the macro input
  // Check if this particular object is part of the pass map
  // and the command was called with a default value
  // this would mean that we are looking for a specific pass

  if ((fPassMap.find(std::make_pair(tblname,index)) != fPassMap.end()) && passNumber == -1) {
    // There is an entry. Set pass to that value
    pass = fPassMap.at(std::make_pair(tblname,index));

#ifdef RATDB_DEBUG
    debug << "DB::FindTableRun : Found in \'passmap\' [" << tblname << "][" << index << "] : " << pass << newline;
#endif
  } else {
    pass = passNumber;
  }


  //-- 1. Search the run table cache already in memory
  for (table = tables_run.begin(); table != tables_run.end(); ++table) {
#ifdef RATDB_VERBOSE
    info << "DB::FindTableRun : Testing validity of " << (table->first).stream_id() << newline;
#endif
    if ((table->first).is_valid(tblname,index,runNumber,pass)) {
#ifdef RATDB_VERBOSE
      info << "DB::FindTableRun : Found valid table in run plane : " << (table->first).stream_id() << newline;
#endif
      return (*table).second.pointer();
    } else if ((table->first).is_valid(tblname,index,runNumber,-1)) {
      warn << "DB::FindTableRun : Found table in cache but with pass number mismatch [local = " <<  (table->first).pass << "][requested="<< pass << "]" << newline;
    }
  }

  // 2. If we haven't found it yet, then let's check if this type exists in the remote server
  // Now check if the server has tables of this type
  // Return if the server does not have tables with this name
  // or if no server has been specified (the set will be empty)
  if ((tableNamesOnServer.count(tblname) == 0) || (fDbBackend == NULL)) {
    return 0;
  }

  // Define the ID with the runNumber and pass number that we should now have
  id = DBTableKey(tblname, index, runNumber, runNumber,pass);

  // Return if we already performed this query before
  if (tablesNotOnServer.count(id) > 0) {
    return 0;
  }

  bool found_object = false;

#ifdef RATDB_DEBUG
  info << "DB::FindTableRun : Fetching object " << tblname << "["
      << index << "][run=" << runNumber << "][pass=" << pass << "]" << newline;
#endif
  size_t newTableBytes;
  json::Value jsonDoc = fDbBackend->FetchObject(tblname,index,runNumber,pass,found_object,newTableBytes);

  if (!found_object) {
    tablesNotOnServer.insert(id); // Remember this result for later
    return 0;
  }

#ifdef RATDB_DEBUG
  info << "DB::FindTableRun : Creating table from json doc " << newline;
#endif

  DBTable *newTable = new DBTable(jsonDoc,newTableBytes);

#ifdef RATDB_DEBUG
  info << "DB::FindTableRun : Table created " << newline;
#endif

  // Get the ID of this table
  // -- It can be different from the requested ID, as the run_range can be larger

  id = DB::GetTableKey(newTable);

#ifdef RATDB_DEBUG
  info << "DB::FindTableRun : Table created with id " << id.stream_id() << newline;
#endif

  // 5) Flush out some tables if we need the space

#ifdef RATDB_DEBUG
  info << "DB::FindTableRun : Table " << newTable->GetName() <<  "[" << newTable->GetIndex()<< "] has size "<<  newTableBytes << " bytes"<< newline;
#endif
  static bool first = true;
  if (first) {
    detail << "DB::FindTableRun : Many bothans died to bring you this table..." << newline;
    first = false;
  }

  // This should probably be done only once
  size_t serverTableBytes = 0;
  for (unsigned i=0; i < tablesFromServer.size(); i++) {
    serverTableBytes += tablesFromServer[i].size_bytes;
  }

  while (tablesFromServer.size() > 0 && serverTableBytes + newTableBytes > maxServerBytes) {
    DBTableKey keyForDeletion = tablesFromServer.front();
    tablesFromServer.pop_front();
    size_t deleteSize = keyForDeletion.size_bytes;
    tables_run.erase(keyForDeletion);
    serverTableBytes -= deleteSize;
    warn << "DB::FindTableRun: Evicting object from run cache (" << keyForDeletion.stream_id() << ") : [" << deleteSize << " bytes]" << newline;
  }

  // 6) Add this table to the memory cache
  simple_ptr_nocopy<DBTable> newTablePtr(newTable);
  tables_run[id] = newTablePtr;
  tablesFromServer.push_back(id);

  info << dformat("DB::FindTable: Loaded table %s[%s] from server\n", newTable->GetName().c_str(),
      newTable->GetIndex().c_str());

  return newTable;
}

/// Wrapper function to grab a table to parse its structure.
/// The logic is reversed from usual, as we want to prioritize on
/// avoiding DB access unless strictly necessary
/// Therefore we first check the default plane
DBTable *DB::FindTableStructure(const std::string tblname,const std::string index, int runNumber)
{
  DBTableKey id(tblname, index, runNumber, runNumber,-1);

  DBTableSet::iterator table;
  DBTable *tbl = NULL;

  // -- First try to find the table in the default plane
  // This will cover most of the requests and is the fastest way
  tbl = FindTableDefault(tblname,index);

  if (tbl)
    return tbl;

  /// -- Table does not exist in the default plane
  /// Check if it exists in the run plane. The problem here is that we actually don't care about the run number
  /// So search it manually
  for (table = tables_run.begin(); table != tables_run.end(); ++table) {
    if ((table->first).is_family(tblname,index)) {
      return (*table).second.pointer();
    }
  }

  // If it reached this point, we should attempt to grab from the DB
  // The problem is a choice of run, since a run is not yet guaranteed
  // to have been defined yet
  // -- Use a tailored method that grabs any object of this (type,index)
  // -- If the run number is not yet defined the first object found is fine
  // NFB: How can we know if a run number is already defined?

  if (fRunID == 0 && runNumber <= 0) {
    // Run is not defined yet, which means that the DB connection might not yet be open either
    // Tentatively grab the first object that matches
    tbl = FindAnyTableRemote(tblname,index);
    if (tbl)
      return tbl;
  } else if (runNumber > 0) {
    // Properly fetch a run valid table for the requested run number
    tbl = FindTableRun(tblname,index,runNumber);
    if (tbl)
      return tbl;
  } else {
    // We know internally what is the run
    tbl = FindTableRun(tblname,index,fRunID);
    if (tbl)
      return tbl;
  }

  // If we haven't found it yet, check the user plane
  // In this plane the table might not be completely defined, but we could be lucky and have there exactly
  // the field we are looking for
  tbl = FindTableUser(tblname,index);
  if (tbl)
    return tbl;

  // If we reach this point, there is nothing to return
  return 0;

}


bool DB::IsValidField(std::string tblname, std::string index, std::string fieldname)
{
  // Check the allowdanger table whitelist first
  if (find(fAllowDangerWhitelist.begin(), fAllowDangerWhitelist.end(), tblname) != fAllowDangerWhitelist.end())
    return true;

  // Ensure this field not been accessed already
  // If a field has been read before the macro is parsed (as in constructors
  // such as PMTCharge), setting it will have unexpected results
  //
  //FIXME: Use a better trace mapping
  std::string key = tblname + "[" + index + "]." + fieldname;
  if (Log::GetDBTraceMap()->GetValue(key.c_str())) {
    warn << "DB::IsValidField: Cannot set field " << key << " which has already been accessed.\n";
    return false;
  }

  int fieldtype = 0;
  // FIXME: This logic is flawed. Should be changed
  //        There is no particular reason why a table should not exist in the run plane only.
  //        Or even in the user plane only
  DBTable *table = FindTableStructure(tblname, index);
  if (table)
    fieldtype = table->GetFieldType(fieldname);

  // If fieldtype == 0, field (/table) doesn't exist in default
  if (fieldtype == DBTable::NOTFOUND && !fAllowDanger)
  {
    warn  << "DB::IsValidField: Cannot set field " << key << " which does not exist in database.\n"
        << "DB::IsValidField: If must set this variable, see the documentation on DB allowdanger command.\n";
    return false;
  }

  return true;
}

bool DB::IsValidFieldType(std::string tblname, std::string index, std::string fieldname, int valfieldtype)
{
  // Check the allowdanger table whitelist first
  if (find(fAllowDangerWhitelist.begin(), fAllowDangerWhitelist.end(), tblname) != fAllowDangerWhitelist.end())
    return true;

  DBTable *table = FindTableStructure(tblname, index);
  if (table){
    int fieldtype = table->GetFieldType(fieldname);
    if (fieldtype != 0 && fieldtype != valfieldtype)
      return false;
    else
      return true;
  }

  return true;
}

void DB::SetI(std::string tblname, std::string index, std::string fieldname, int val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetI: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::INTEGER),
      "DB::SetI: Wrong type: DB field " + key + " is not an integer");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetI(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetD(std::string tblname, std::string index, std::string fieldname, double val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetD: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::DOUBLE),
      "DB::SetD: Wrong type: DB field " + key + " is not a double");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetD(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetS(std::string tblname, std::string index, std::string fieldname, std::string val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetS: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::STRING),
      "DB::SetS: Wrong type: DB field " + key + " is not a string");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetS(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetZ(std::string tblname, std::string index, std::string fieldname, bool val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetS: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::BOOLEAN),
      "DB::SetS: Wrong type: DB field " + key + " is not a boolean");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetZ(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetIArray(std::string tblname, std::string index, std::string fieldname, const std::vector<int> &val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetIArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::INTEGER_ARRAY),
      "DB::SetIArray: Wrong type: DB field " + key + " is not an integer array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetIArray(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetIValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, int val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetIValInArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::INTEGER_ARRAY),
      "DB::SetIValInArray: Wrong type: DB field " + key + " is not an integer array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  DBLinkPtr ltab = DB::Get()->GetLink(tblname,index);
  std::vector<int> temp = ltab->GetIArray(fieldname);
  Log::Assert(temp.size()>=position, "DB::SetIValInArray: Cannot set entry, array too small " + key);
  temp[position] = val;
  t->SetIArray(fieldname, temp);
  fDBSetList.push_back(key);
}

void DB::SetDArray(std::string tblname, std::string index, std::string fieldname, const std::vector<double> &val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetDArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::DOUBLE_ARRAY),
      "DB::SetDArray: Wrong type: DB field " + key + " is not a double array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetDArray(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetDValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, double val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetDValInArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::DOUBLE_ARRAY),
      "DB::SetDValInArray: Wrong type: DB field " + key + " is not a double array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  DBLinkPtr ltab = DB::Get()->GetLink(tblname,index);
  std::vector<double> temp = ltab->GetDArray(fieldname);
  Log::Assert(temp.size()>=position, "DB::SetDValInArray: Cannot set entry, array too small " + key);
  temp[position] = val;
  t->SetDArray(fieldname, temp);
  fDBSetList.push_back(key);
}

void DB::SetSArray(std::string tblname, std::string index, std::string fieldname, const std::vector<std::string> &val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetSArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::STRING_ARRAY),
      "DB::SetSArray: Wrong type: DB field " + key + " is not a string array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetSArray(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetSValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, std::string val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetSValInArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::STRING_ARRAY),
      "DB::SetSValInArray: Wrong type: DB field " + key + " is not a string array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  DBLinkPtr ltab = DB::Get()->GetLink(tblname,index);
  std::vector<std::string> temp = ltab->GetSArray(fieldname);
  Log::Assert(temp.size()>=position, "DB::SetSValInArray: Cannot set entry, array too small " + key);
  temp[position] = val;
  t->SetSArray(fieldname, temp);
  fDBSetList.push_back(key);
}

void DB::SetZArray(std::string tblname, std::string index, std::string fieldname, const std::vector<bool> &val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetZArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::BOOLEAN_ARRAY),
      "DB::SetSArray: Wrong type: DB field " + key + " is not a boolean array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  t->SetZArray(fieldname, val);
  fDBSetList.push_back(key);
}

void DB::SetZValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, bool val)
{
  std::string key = tblname + "[" + index + "]." + fieldname;
  Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetZValInArray: Unable to set DB field " + key);
  Log::Assert(IsValidFieldType(tblname,index,fieldname,DBTable::BOOLEAN_ARRAY),
      "DB::SetSValInArray: Wrong type: DB field " + key + " is not a boolean array");
  DBTable *t = FindOrCreateTable(tblname, index, -1);
  DBLinkPtr ltab = DB::Get()->GetLink(tblname,index);
  std::vector<bool> temp = ltab->GetZArray(fieldname);
  Log::Assert(temp.size()>=position, "DB::SetSValInArray: Cannot set entry, array too small " + key);
  temp[position] = val;
  t->SetZArray(fieldname, temp);
  fDBSetList.push_back(key);
}

// -- Tables can only be created in the user plane
DBTable *DB::FindOrCreateTable(std::string tblname, std::string index, int runNumber)
{
  DBTable *table = NULL;

  if (runNumber == -1)
    table = FindTableUser(tblname, index);

  if (table)
    return table;

  //FIXME: Finish this logic for table searching

  // If it reached this point it means that it does not exist and therefore it
  // should belong to the user plane
  DBTableKey id(tblname, index, -1, -1, -1);
  table = new DBTable(tblname, index);
  table->SetUser();

  tables_user[id] = table;

  return table;
}

void DB::RemoveLink(DBLink *link)
{
  links.remove(link);
}


bool DB::ParseTableName(const std::string& descriptor,
    std::string &table, std::string &index)
{
  // -- There 2 possible patterns:
  // table in the format "TYPE"
  // table in the format "TYPE[INDEX]"

  std::string tmptype, tmpindex = "";
  size_t pos_start, pos_end;
  pos_start =descriptor.find("[");
  pos_end = descriptor.find("]");
  if (descriptor.length() == 0) {
    return false;
  }
  if (pos_start != std::string::npos) {
    // We have a pattern
    if (pos_end == std::string::npos) {
      Log::Die(dformat("DB::ParseTableName : Malformed table name %s",descriptor.c_str()));
      //return false;
    }
    tmptype = descriptor.substr(0,pos_start);
    tmpindex =descriptor.substr(pos_start+1,(pos_end-pos_start-1));
  } else {
    tmptype = descriptor;
  }

  table = tmptype;
  index = tmpindex;
  return true;
}

bool DB::ParseFieldName(std::string descriptor,
    std::string &field, int &index)
{
  Tokenizer t(descriptor);
  index = -9999;

  try {
    // Get field name
    if (t.Next() != Tokenizer::TYPE_IDENTIFIER)
      return 0;

    field = t.Token();

    // Check for index in brackets
    Tokenizer::Type toktype = t.Next();
    if (toktype == Tokenizer::TYPE_EOF) {
      // No index, assume 0
      index = -9876;
      return 1;
    } else if (toktype == Tokenizer::TYPE_SYMBOL && t.Token() == "[") {
      // Look for index in brackets
      if (t.Next() == Tokenizer::TYPE_INTEGER) {
        index = t.AsInt();

        // Make sure we got the rest
        if (t.Next() == Tokenizer::TYPE_SYMBOL && t.Token() == "]"
            && t.Next() == Tokenizer::TYPE_EOF)
          return 1;
        else
          return 0;
      } else
        return 0;
    } else
      return 0;
  } catch (ParseError &p) {
    return 0;
  }
}

std::vector<DBTable *> DB::ReadRATDBFile(const std::string &filename) {
  std::ifstream f;
  f.open(filename.c_str());
  json::Reader reader(f);
  json::Value next;
  std::vector<DBTable*> tables;
  // catch any errors from the
  try{
    while (reader.getValue(next)) {
      tables.push_back(new DBTable(next));
    }
  }
  catch(const std::exception& e){
    throw FastJsonError(filename, e.what());
  }
  return tables;
}

void DB::SetPassDBEntry(const std::string &tblname, const std::string &index,int pass) {

  if (fPassMap.find(std::make_pair(tblname,index)) != fPassMap.end()) {
    warn << "DB::SetPassDBEntry : Pass for entry " << tblname << "[" << index << "] already exists (" << fPassMap.at(std::make_pair(tblname,index))
            << "). Overwriting with new value (" << pass << ")" << newline;
  } else {
    info << "DB::SetPassDBEntry : Fixing pass [" << pass << "] for entry " << tblname << "[" << index << "]" << newline;
  }
  fPassMap[std::make_pair(tblname,index)] = pass;
}

int DB::GetPassDBEntry(std::string tblname, std::string index) {
  if (fPassMap.find(std::make_pair(tblname,index)) != fPassMap.end()) {
    return fPassMap.at(std::make_pair(tblname,index));
  } else {
    return -1; // if it was not defined, use the standard for highest pass
  }
}

void DB::SetAirplaneModeStatus(bool enable) {
  if (enable)
    warn << BCYAN << "DB::SetAirplaneModeStatus : Airplane mode activated....wooooosh!" << CLR << newline;
  else
    warn << BCYAN << "DB::SetAirplaneModeStatus : Airplane mode deactivated!" << CLR << newline;
  fAirplaneMode = enable;
}

std::string DB::decode_server(const std::string & original) {
  std::string decoded = "";
  const std::string sep = " ";
  std::vector<std::string> split = util_split(original,sep);
  for (size_t i = 0; i < split.size(); ++i) {
    decoded += static_cast<char>(util_to_int(split.at(i), 16));
  }
  decoded = xor_encdec(decoded);
#ifdef RATDB_DEBUG
  info << "DB::decode_server : Server set to [" << decoded << "]" << newline;
#endif
  return decoded;
}

} // namespace RAT