//////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////// /// \class RAT::DB /// /// \brief DB object /// /// \author Stan Seibert /// /// REVISION HISTORY:\n /// 24 Feb 2011 : Gabriel Orebi Gann - New bool: fAllowDanger: whether to /// allow the user to set DB values that do not exist in the /// default tables. Default: false i.e. not permitted. /// New fatal error: RAT will die (with an error msg) if this is /// attempted, unless fAllowDanger is set to `true' in the macro. /// 27 May 2015 : N Barros - RunID: /// - DB is now aware of the RunID, and propagates that into /// the DBLinks so that the database is "runID aware". /// 04 Dec 2016 : N Barros - Changed the underlying table validity logic to accommodate pass /// numbers and infinite run ranges /// 24 Jan 2017 : N Barros - Introduced method to push a DBTable into the run level database /// 26 Jan 2017 : N Barros - Introduced methods to use DB tags (setter) /// 15 May 2017 : N Barros - Introduced automatic remote DB connection. /// - Modified the default plane locks to index both type and index. /// 09 Jun 2017 : N Barros - Introduced a vector for default plane lock exceptions, so that they can be reloaded /// on new runs /// /// \details follows below (courtesy of S. Seibert): /// /// Database of constants /// /// @author Stan Seibert /// /// RATDB is the database of constants that will be used by RAT for /// all adjustable parameters. "Parameters" in this sense includes /// many things, such as: /// /// - physical properties of materials (density, optics, etc.) /// - geometric configuration of parts of the detector (dimensions and /// locations of the acrylic vessel, PMTs, ...) /// - calibration constants /// - lookup tables /// - control parameters which specify how processing should be done /// /// Note that RATDB is NOT intended to hold actual event information, /// just everything else. /// /// Despite the suggestive name, RATDB is not really a proper /// Database, like MySQL or Oracle. Rather it is just a simple set of /// C++ classes used to read parameters from various backends (text /// files and CouchDB servers) and make the data easily /// available to other parts of the RAT application. /// /// @section Org Data Organization /// /// Information in RATDB is organized into a two-level namespace of /// tables and fields. Tables have uppercase names like @c DAQ, @c /// @c IBD, and @c PMTLOCATION. Related tables can be given the same /// name and distinguished by an index string like "vacuum" or /// "acrylic". The convention for writing this out is to put the /// index in brackets, after the table name: @c GEO[tank], @c /// MEDIA[acrylic]. Tables without an index (like @p IBD) implicitly /// have an index which is the empty string. /// /// Tables contain fields, which have lowercase names like @p /// sampling_time and @p emin. A field contains value, which can have /// one of 6 data types: /// - integer, float, string /// - array of integers, array of doubles, array of strings /// /// @section VRange Run Ranges /// /// Frequently, you want to have several sets of the same constants /// which have different validity ranges. RATDB organizes the notion /// of validity range into 3 planes (think of a stack of transparencies): /// - user plane: This sits on top, and is where constants that /// users want to override in their macros go. /// - run plane: This is where constants that change with time /// (like attenuation lengths) go. /// - default plane: This is where a baseline set of constants /// live. Fields are only retrieved from the default plane if they /// cannot be found on the user plane or the time plane. /// /// The plane on which a table lives is specified by a few special /// fields in the table. The range of run numbers (inclusive) are /// contained the field @c run_range. The special value [0,0] is /// used to indicate the default plane and [-1,-1] is used for /// the user plane. /// /// @section DB Usage /// /// To use RATDB, you obtain a pointer to the global database (unless /// you want to create your own RATDB with some private data). Then /// you obtain a link to the table you wish to access. Finally you /// query the link for specific fields. The shortest example of this /// is: /// @code /// DB *db = DB::Get(); /// DBLinkPtr lmedia = db->GetLink("MEDIA", "acrylic"); // Link to MEDIA[acrylic] /// double rindex = lmedia->GetD("index_of_refraction"); /// @endcode /// /// You can hold onto DBLinkPtr objects as long as you like. If /// you expect you will need to access the same table at various /// points in your code, you can obtain a link pointer in your /// constructor and reuse it in other methods. /// //////////////////////////////////////////////////////////////////////// #ifndef __RAT_DB__ #define __RAT_DB__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace RAT { class DB; // Forward decl to allow for static DB member inside itself class DBTable; class PgSQLBackend; class DBTableKey { public: DBTableKey() : name(""), index(""), run_begin(0), run_end(0), pass(-1), size_bytes(0) {}; DBTableKey (const std::string &_name, const std::string &_index, const int _run_begin=0,const int _run_end=0, const int _pass = -1) : name(_name), index(_index), run_begin(_run_begin),run_end(_run_end), pass(_pass), size_bytes(0) {}; std::string name; std::string index; int run_begin; int run_end; int pass; size_t size_bytes; // note: this is only used for tables loaded from the DB // bool operator< (const DBTableKey &rhs) const { return (name < rhs.name) || (name == rhs.name && index < rhs.index) || (name == rhs.name && index == rhs.index && run_begin < rhs.run_begin) || (name == rhs.name && index == rhs.index && run_begin == rhs.run_begin && run_end < rhs.run_end) || (name == rhs.name && index == rhs.index && run_begin == rhs.run_begin && run_end == rhs.run_end && pass < rhs.pass); }; bool operator== (const DBTableKey &rhs) const { return (name == rhs.name) && (index == rhs.index) && (run_begin == rhs.run_begin) && (run_end == rhs.run_end) && (pass == rhs.pass); }; // Checks if the rhs object overlaps with this // True contidions: // 1. Same name and index and pass // AND // 2. this.run_begin <= rhs.run_end && this.run_end >= rhs.run_begin // RHS : **** *** *** ***** ***** // THIS: oooo ooooo o ooo ooooo bool overlaps(const DBTableKey &rhs) const { return ((name == rhs.name) && (index == rhs.index) && (pass == rhs.pass) && (run_begin <= rhs.run_end) && (run_end >= rhs.run_begin)); }; // check if there is overlap once we ignore the pass bool overlaps_range(const DBTableKey &rhs) const { return ((name == rhs.name) && (index == rhs.index) && (run_begin <= rhs.run_end) && (run_end >= rhs.run_begin)); }; std::string stream_id() const {return stlplus::dformat("%s[%s][%d,%d][%d]",name.c_str(),index.c_str(),run_begin,run_end,pass);} bool is_valid(const std::string &tblname, const std::string &tblindex, const int &run, const int &tblpass) const { if (tblpass == -1) { return ((name == tblname) && (index == tblindex) && (run_begin <= run) && (run_end >= run)); } else { return ((name == tblname) && (index == tblindex) && (run_begin <= run) && (pass == tblpass) && (run_end >= run)); } }; bool is_family(const std::string& tbltype, const std::string& tblindex) const { return ((name == tbltype) && (index == tblindex)); } }; typedef std::map > DBTableSet; typedef simple_ptr_nocopy DBLinkPtr; typedef std::map DBLinkGroup; class DB { public: /** Obtain pointer to global database * * Unless you want a private database, this is the method you * should use to obtain a pointer to DB. If the global database * has not yet been constructed, it will be constructed when you * call this function. */ static inline DB *Get() { static DB db; return &db; } /** Parse a table name written in the "TABLE" or "TABLE[index]" format. * * Handy method for when you are getting a table name from user * input. Separates the table name from the index name. * * @param[in] descriptor String in the form "TABLE" or * "TABLE[index]" * @param[out] table Just the table name * @param[out] index Just the index name. Empty string if no * index. * * @returns True if @p descriptor was a properly formatted table name. */ static bool ParseTableName(const std::string &descriptor, std::string &table, std::string &index); /** Parse a field name written in the "FIELD" or "FIELD[index]" format * * Handy method for when you are getting a field name from user * input. The index option is for setting single parameters in arrays. * Separates the field from the index. * * @param[in] descriptor String in the form "FIELD" or * "FIELD[index]" * @param[out] field Just the field name * @param[out] index Just the integer index. Negative -9999 if no * index. * * @returns True if @p descriptor was a properly formatted field name. */ static bool ParseFieldName(std::string descriptor, std::string &field, int &index); /** Reads a file and returns all the RATDB tables found inside. * Works with both RATDB native and JSON files */ static std::vector ReadRATDBFile(const std::string &filename); static const int MAXPASS = -1; static const int DBINFINITY = 2147483647; /** Generate a DBTableKey from a DBTable * */ static DBTableKey GetTableKey(const DBTable *table) { DBTableKey key; key.name = table->GetName(); key.index = table->GetIndex(); const std::vector &run_range = table->GetIArray("run_range"); key.run_begin = run_range.at(0); key.run_end = run_range.at(1); key.pass = table->GetPassNumber(); if (!table->IsDefault() && !table->IsUser()) key.size_bytes = table->GetBytes(); return key; } ClassDef(DB,7); protected: bool fAllowDanger; std::vector fAllowDangerWhitelist; /** Create a list of all explicitly set db values for comparison against * the DBTrace access log. */ static std::vector fDBSetList; public: /** Create new database in memory with no tables. */ DB(); virtual ~DB(); /** Initialize the backend. **/ void InitBackend(); /** Refresh all tables on the server. **/ void RefreshTablesOnServer(const unsigned int run = 0); /** Define a method to propagate the run number to all loaded tables * This should be one of the first BeginOfRun methods that should be called in RAT * So that others benefit from the updated information */ void BeginOfRun(DS::Run &run); /** Define a method that should be called at the end of the BeginOfRun * This should be the very last method called before the run starts * @param run */ void EndOfBeginOfRun(); /** Define a mthod to close up execution at the end of a run * */ void EndOfRun(DS::Run &run); /** Check that fields that were set were actually accessed by comparing * fDBSetList against Log's DBTrace */ void CheckSetFieldUsage(void); /** Check whether or not a vector of tables are in memory. * Returns false is any table is not found. */ bool CheckLoadedTables(const std::vector& contents); /** Load a DB text file or directory of text files in search path. * * If passed the name of a file, this function will load all the * DB tables it contains into memory. If passed the name of a * directory, all of the files in that directory ending in .ratdb * will be loaded. * * The current directory is searched first, then the $GLG4DATA * directory. * * If printPath is true, then an info message is printed * to stdout (and logged) indicating the path of the * file that is loaded. Shows whether $GLG4DATA was used. */ int Load(const std::string filename, bool printPath=false); /** Load DB text file of tables into memory. * * This function does not search in $GLG4DATA automatically, so it * must be given a file name relative to the current directory or * an absolute path. */ int LoadFile(const std::string filename); /** Load all of the DB files in a directory. * * Searches @p dirname for all files matching @p pattern. Does not * search $GLG4DATA automatically, so a path relative to current * directory or an absolute path much be given. */ int LoadAll(const std::string dirname, std::string pattern="*.ratdb"); /** Load standard tables into memory. * * Currently, the standard tables are $GLG4DATA/ *.ratdb. */ int LoadDefaults(); /** Delete all tables from memory. */ void Clear(); /** Allow self to set values that don't exist in tables */ void AllowDanger(){fAllowDanger = true;}; /** Load tables for which default plane should not be allowed **/ void LoadDefaultPlaneLocks(); /** Set URL of CouchDB server containing RATDB tables */ void SetServer(const std::string& url, bool init = true); /** Obtain a link to a particular table and index. * * Note that even if your table does not have an index value, * you should still use this method, but with an empty index. * (The default value) */ DBLinkPtr GetLink(const std::string tblname, std::string index="", int pass = -1); /** Obtain a link to all tables with the same name, but different * indexes. * * LinkGroups are useful if you want to step through all the * instances of the same kind of table, like GEO. */ DBLinkGroup GetLinkGroup(const std::string tblname); /** Adds a table type to the list of types that should *not* be loaded from the * default plane. * * @param tblname Name (type) of the table to be fetched only from user and run plane */ void AddTblNoDefaultPlane(std::string tblname); void RemoveTblNoDefaultPlane(const std::string& tblname); void SetDefaultPlaneLockStatus(bool enable); std::vector > GetTblNoDefaultPlane() { return fTblNoDefaultPlane; } /** Checks if a table is listed as to not be fetched from the default plane * * @param tblname Name (type) of the table * @return true if the table is in the list, false otherwise */ bool GetTblNoDefaultPlaneStatus(const std::string &tblname, const std::string &index = ""); /** Attempt to push a table into the database. * * Push a DBTable to the database. If the data insertion fails, * a DBInsertFailed exception is thrown. This does not keep a local * index of the table (no DBLink), so the table is not available for use * through this just, until a new run is started and BeginOfRun is called * */ void UploadTableOnline(DBTable &tbl); void SetDbTag(const std::string& tag); const std::string& GetDBTag() const {return fDbTag;} // Manually set fields on the user plane. Also for user use. // DO THIS AFTER loading files from disk, or your changes will be // stomped on later. /** Check that fields being set exist in the loaded tables (or fAllowDanger is set), * and that they have not already been accessed. */ bool IsValidField(std::string tblname, std::string index, std::string fieldname); /** Check that fields are being set with the right type * If the table is on the whitelist it can't check as it * hasn't been loaded yet */ bool IsValidFieldType(std::string tblname, std::string index, std::string fieldname, int valfieldtype); /** Set arbitrary-type field in user plane, with table index */ template void Set(std::string tblname, std::string index, std::string fieldname, T val) { std::string key = tblname + "[" + index + "]." + fieldname; Log::Assert(IsValidField(tblname, index, fieldname), "DB::Set: Unable to set DB field " + key); DBTable *t = FindOrCreateTable(tblname, index, -1); t->SetJSON(fieldname, val); // require that the type be convertable to JSON (most general) fDBSetList.push_back(key); } /** Set arbitrary-type field in user plane, no table index */ template void Set(std::string tblname, std::string fieldname, T val) { Set(tblname,"",fieldname,val); } /** Set arbitrary-type value in array in field in user plane, with table index */ template void SetValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, T val) { std::string key = tblname + "[" + index + "]." + fieldname; Log::Assert(IsValidField(tblname, index, fieldname), "DB::SetValInArray: Unable to set DB field " + key); DBTable *t = FindOrCreateTable(tblname, index, -1); DBLinkPtr ltab = DB::Get()->GetLink(tblname,index); json::Value temp = ltab->GetJSON(fieldname); Log::Assert(temp.getArraySize() >= position, "DB::SetValInArray: Cannot set entry, array too small " + key); temp[position] = val; // require that the type be convertable to JSON (most general) t->SetJSON(fieldname, temp); fDBSetList.push_back(key); } /** Set arbitrary-type value in array in field in user plane, no table index */ template void SetValInArray(std::string tblname, std::string fieldname, size_t position, T val) { SetValInArray(tblname,"",fieldname,position,val); } /** Set integer field in user plane, no table index. */ void SetI(std::string tblname, std::string fieldname, int val) { SetI(tblname, "", fieldname, val); }; /** Set integer field in user plane, with table index. */ void SetI(std::string tblname, std::string index, std::string fieldname, int val); /** Set double field in user plane, no table index. */ void SetD(std::string tblname, std::string fieldname, double val) { SetD(tblname, "", fieldname, val); }; /** Set double field in user plane, with table index. */ void SetD(std::string tblname, std::string index, std::string fieldname, double val); /** Set string field in user plane, no table index. */ void SetS(std::string tblname, std::string fieldname, std::string val) { SetS(tblname, "", fieldname, val); }; /** Set string field in user plane, with table index. */ void SetS(std::string tblname, std::string index, std::string fieldname, std::string val); /** Set boolean field in user plane, no table index. */ void SetZ(std::string tblname, std::string fieldname, bool val) { SetZ(tblname, "", fieldname, val); }; /** Set boolean field in user plane, with table index. */ void SetZ(std::string tblname, std::string index, std::string fieldname, bool val); /** Set integer array field in user plane, no table index. */ void SetIArray(std::string tblname, std::string fieldname, const std::vector &val) { SetIArray(tblname, "", fieldname, val); }; /** Set integer array field in user plane, with table index. */ void SetIArray(std::string tblname, std::string index, std::string fieldname, const std::vector &val); /** Set single value in integer array field in user plane, no table index. */ void SetIValInArray(std::string tblname, std::string fieldname, size_t position, int val) { SetIValInArray(tblname,"",fieldname,position,val); }; /** Set single value in integer array field in user plane, with table index. */ void SetIValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, int val); /** Set double array field in user plane, no table index. */ void SetDArray(std::string tblname, std::string fieldname, const std::vector &val) { SetDArray(tblname, "", fieldname, val); }; /** Set double array field in user plane, with table index. */ void SetDArray(std::string tblname, std::string index, std::string fieldname, const std::vector &val); /** Set single value in double array field in user plane, no table index. */ void SetDValInArray(std::string tblname, std::string fieldname, size_t position, double val) { SetDValInArray(tblname,"",fieldname,position,val); }; /** Set single value in integer array field in user plane, with table index. */ void SetDValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, double val); /** Set string array field in user plane, no table index. */ void SetSArray(std::string tblname, std::string fieldname, const std::vector &val) { SetSArray(tblname, "", fieldname, val); }; /** Set string array field in user plane, with table index. */ void SetSArray(std::string tblname, std::string index, std::string fieldname, const std::vector &val); /** Set single value in string array field in user plane, no table index. */ void SetSValInArray(std::string tblname, std::string fieldname, size_t position, std::string val) { SetSValInArray(tblname,"",fieldname,position,val); }; /** Set single value in string array field in user plane, with table index. */ void SetSValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, std::string val); /** Set bool array field in user plane, no table index. */ void SetZArray(std::string tblname, std::string fieldname, const std::vector &val) { SetZArray(tblname, "", fieldname, val); }; /** Set bool array field in user plane, with table index. */ void SetZArray(std::string tblname, std::string index, std::string fieldname, const std::vector &val); /** Set bool value in string array field in user plane, no table index. */ void SetZValInArray(std::string tblname, std::string fieldname, size_t position, bool val) { SetZValInArray(tblname,"",fieldname,position,val); }; /** Set bool value in string array field in user plane, with table index. */ void SetZValInArray(std::string tblname, std::string index, std::string fieldname, size_t position, bool val); /************************DBLink interface********************/ // This is the low level interface that DBLinks use. // You should not call these methods. /** Get pointer to a table in user plane: do not use unless you know what you are doing! */ DBTable *GetUserTable(const std::string tblname, const std::string index) { return FindTableUser(tblname, index); }; /** Get pointer to a table in run plane: do not use unless you know what you are doing! */ DBTable *GetRunTable(const std::string tblname, const std::string index, int runNumber, int passNumber = -1) { return FindTableRun(tblname, index, runNumber,passNumber); }; /** Get pointer to a table in default plane: do not use unless you know what you are doing! */ DBTable *GetDefaultTable(const std::string tblname, const std::string index) { return FindTableDefault(tblname, index); }; /** Remove a table link from the list of outstanding links: do not * use unless you know what you are doing! * * This method should only be used by DBLink objects to remove * themselves from the list during their destructor call. See * DB::links member for more information. */ void RemoveLink(DBLink *link); /** Number of still active links to this database. * * This is a count of the number of links which have been given out * GetLink() and GetLinkGroup(). See DB::links member for more * information about what this means. This method is really just * for debugging purposes. */ int NumLinks() { return links.size(); }; static std::vector FloatToDoubleArray(const std::vector &in); /** * Set the present run */ void SetRunID(const unsigned int aRun) {fRunID = aRun;}; /** * Return the current run */ unsigned int GetRunID() {return fRunID;} /** Set a pass for a given table entry * * @param tblname Type (name) of the table * @param index Index of the table * @param pass Pass number */ void SetPassDBEntry(const std::string &tblname,const std::string &index, int pass); /** Get the pass number that was previously set * * @param tblname Type (name of the table) * @param index Index of the table. * @return pass number */ int GetPassDBEntry(std::string tblname, std::string index = ""); /** Enable/Disable airplane mode (no default remote DB connection) **/ void SetAirplaneModeStatus(bool enable = true); /** Get current status of airplane mode **/ bool GetAirplaneModeStatus() {return fAirplaneMode;} bool AreDefaultsLoaded() { return fDefaultsLoaded;} protected: std::string decode_server(const std::string &original); std::map,int> fPassMap; /** Obtain a pointer to a RUN table. It grabs the first table that matches, disregarding the run number * @param[in] tblname table type * @param[in] index table index * * @returns Pointer to table if found, otherwise 0. */ DBTable *FindAnyTableRemote(const std::string tblname, const std::string index); /** Obtain a pointer to a table. It tries all three levels of priority. * @param[in] tblname table type * @param[in] index table index * @param[in] runNumber number of run to find the table * * @returns Pointer to table if found, otherwise 0. */ DBTable *FindTableStructure(const std::string tblname, const std::string index, int runNumber = -1); /** Obtain a pointer to a table in the user plane * * @param tblname * @param index * @return pointer to the user table, if exists, 0 if doesn't */ DBTable *FindTableUser(const std::string tblname, const std::string index); /** Obtain a pointer to a table in the default plane. * @warning This does not check whether access to the default plane is enabled or not * for this table * @param tblname * @param index * @return Pointer to the default table, if exists, otherwise 0 */ DBTable *FindTableDefault(const std::string tblname, const std::string index); /** Obtain a pointer to a table in the run plane * * @param tblname * @param index * @param runNumber * @param passNumber (default:-1, i.e., grab the highest pass available) * @return Pointer to the default table, if exists, otherwise 0 */ DBTable *FindTableRun(const std::string tblname, const std::string index, const int runNumber, int passNumber = 1); /** Obtain a pointer to a table in memory, or create it if * not found. * * @returns Pointer to existing table, or pointer to new table, * which is also added to @p tblset. */ DBTable *FindOrCreateTable(const std::string tblname, const std::string index, int runNumber); /** List of DBLinks which have been issued by GetLink() or * GetLinkGroup(). * * In order to allow a DBLink to do local caching of values, * the DB object must retain pointers to all of the previously * issued link objects. Then, if something changes, like new * tables are loaded or become valid, DB can go out and expire * the relevant cached values inside the affected DBLinks. * * In the DBLink destructor, RemoveLink() is called to remove * the link from the list before its memory is reclaimed. */ std::list links; /** URL to CouchDB server. Empty string means no server will be * used. */ std::string server; /** Cache of table names present on the CouchDB server. Check this before issuing a query. */ std::set tableNamesOnServer; /** Cache of full table keys (name, index, run) that are *not* on the server. Important to cut down on needless network traffic from future queries once a rejection has been received. */ std::set tablesNotOnServer; /** Helper class to download tables from the backend */ PgSQLBackend *fDbBackend; /** Sets containig all tables. For performance reasons keep one set per plane * We always know that the planes are prioritized in a very specific way */ DBTableSet tables_default; DBTableSet tables_user; DBTableSet tables_run; std::deque tablesFromServer; /** * Run number: Defaults to 0 (default plane) */ unsigned int fRunID; /** List of tables for which RATDB should not search the default plane **/ std::vector > fTblNoDefaultPlane; std::vector > fTblNoDefaultPlaneUser; std::vector fTblNoDefaultPlaneExceptions; /** Set of files that have already been loaded in **/ std::set fFileList; std::string fDbTag; static const std::string default_server; bool fAirplaneMode; bool fDefaultPlaneLockEnabled; bool fDefaultsLoaded; }; inline std::vector DB::FloatToDoubleArray(const std::vector &in) { std::vector out(in.size()); for (unsigned int i=0; i < in.size(); i++) out[i] = in[i]; return out; } } // namespace RAT #endif