From 6e87a48ebaf6308e58afbcc65ddfa6da9d4acc9b Mon Sep 17 00:00:00 2001 From: Tobias Kremer Date: Tue, 20 Feb 2024 15:20:49 +0100 Subject: [PATCH] Commented FDB.h and restructuring --- docs/content/architectural-introduction.rst | 2 +- docs/content/operational-introduction.rst | 8 +- .../{technical => operational}/DebugTools.rst | 2 +- .../DebugTools/dump-index.rst | 4 +- .../DebugTools/dump-toc.rst | 4 +- .../DebugTools/dump.rst | 2 +- .../DebugTools/hammer.rst | 2 +- .../DebugTools/patch.rst | 4 +- .../DebugTools/read.rst | 2 +- .../DebugTools/reconsolidate-toc.rst | 4 +- .../DebugTools/write.rst | 2 +- .../GeneralPurposeTools.rst | 4 +- .../GeneralPurposeTools/fdb-stats.rst | 8 +- .../GeneralPurposeTools/fdb.rst | 4 +- .../GeneralPurposeTools/grib2fdb.rst | 4 +- .../GeneralPurposeTools/info.rst | 4 +- .../GeneralPurposeTools/list.rst | 2 +- .../GeneralPurposeTools/purge.rst | 4 +- .../GeneralPurposeTools/schema.rst | 0 .../GeneralPurposeTools/where.rst | 2 +- .../GeneralPurposeTools/wipe.rst | 4 +- .../SpecialPurposeTools.rst | 4 +- .../SpecialPurposeTools/hide.rst | 4 +- .../SpecialPurposeTools/move.rst | 2 +- .../SpecialPurposeTools/overlay.rst | 2 +- .../SpecialPurposeTools/root.rst | 4 +- .../{technical => operational}/tools.rst | 2 +- docs/content/reference.rst | 7 +- docs/content/technical-introduction.rst | 11 +-- docs/content/technical/Api/api_python.rst | 9 +++ .../UsageExamples/check-installation.rst | 36 --------- .../UsageExamples/data-retrieval.rst | 29 ------- docs/content/technical/check-installation.rst | 1 - docs/content/technical/config-schema.rst | 77 ------------------- docs/content/technical/configuration.rst | 40 ++++++++++ docs/content/technical/genindex.rst | 2 - docs/content/technical/installation.rst | 3 - docs/content/technical/introduction.rst | 11 --- docs/content/technical/schema.rst | 2 +- docs/content/technical/usage-examples.rst | 11 --- docs/index.rst | 9 ++- src/fdb5/api/FDB.h | 20 +++++ 42 files changed, 131 insertions(+), 227 deletions(-) rename docs/content/{technical => operational}/DebugTools.rst (89%) rename docs/content/{technical => operational}/DebugTools/dump-index.rst (97%) rename docs/content/{technical => operational}/DebugTools/dump-toc.rst (99%) rename docs/content/{technical => operational}/DebugTools/dump.rst (99%) rename docs/content/{technical => operational}/DebugTools/hammer.rst (99%) rename docs/content/{technical => operational}/DebugTools/patch.rst (98%) rename docs/content/{technical => operational}/DebugTools/read.rst (99%) rename docs/content/{technical => operational}/DebugTools/reconsolidate-toc.rst (90%) rename docs/content/{technical => operational}/DebugTools/write.rst (99%) rename docs/content/{technical => operational}/GeneralPurposeTools.rst (69%) rename docs/content/{technical => operational}/GeneralPurposeTools/fdb-stats.rst (98%) rename docs/content/{technical => operational}/GeneralPurposeTools/fdb.rst (98%) rename docs/content/{technical => operational}/GeneralPurposeTools/grib2fdb.rst (99%) rename docs/content/{technical => operational}/GeneralPurposeTools/info.rst (98%) rename docs/content/{technical => operational}/GeneralPurposeTools/list.rst (99%) rename docs/content/{technical => operational}/GeneralPurposeTools/purge.rst (98%) rename docs/content/{technical => operational}/GeneralPurposeTools/schema.rst (100%) rename docs/content/{technical => operational}/GeneralPurposeTools/where.rst (99%) rename docs/content/{technical => operational}/GeneralPurposeTools/wipe.rst (99%) rename docs/content/{technical => operational}/SpecialPurposeTools.rst (69%) rename docs/content/{technical => operational}/SpecialPurposeTools/hide.rst (98%) rename docs/content/{technical => operational}/SpecialPurposeTools/move.rst (99%) rename docs/content/{technical => operational}/SpecialPurposeTools/overlay.rst (99%) rename docs/content/{technical => operational}/SpecialPurposeTools/root.rst (96%) rename docs/content/{technical => operational}/tools.rst (97%) create mode 100644 docs/content/technical/Api/api_python.rst delete mode 100644 docs/content/technical/UsageExamples/check-installation.rst delete mode 100644 docs/content/technical/UsageExamples/data-retrieval.rst delete mode 100644 docs/content/technical/config-schema.rst delete mode 100644 docs/content/technical/genindex.rst delete mode 100644 docs/content/technical/installation.rst delete mode 100644 docs/content/technical/introduction.rst delete mode 100644 docs/content/technical/usage-examples.rst diff --git a/docs/content/architectural-introduction.rst b/docs/content/architectural-introduction.rst index 142b369cb..9cb735060 100644 --- a/docs/content/architectural-introduction.rst +++ b/docs/content/architectural-introduction.rst @@ -1,7 +1,7 @@ .. _architectural-introduction-label: Architectural Introduction -########################## +~~~~~~~~~~~~~~~~~~~~~~~~~~ |Licence| diff --git a/docs/content/operational-introduction.rst b/docs/content/operational-introduction.rst index e3d72aa24..5a2e18cad 100644 --- a/docs/content/operational-introduction.rst +++ b/docs/content/operational-introduction.rst @@ -1,4 +1,10 @@ .. _operational-introduction-label: Operational Introduction -######################## +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. toctree:: + :maxdepth: 2 + + operational/tools + diff --git a/docs/content/technical/DebugTools.rst b/docs/content/operational/DebugTools.rst similarity index 89% rename from docs/content/technical/DebugTools.rst rename to docs/content/operational/DebugTools.rst index 55a1cb373..d50609cc2 100644 --- a/docs/content/technical/DebugTools.rst +++ b/docs/content/operational/DebugTools.rst @@ -1,5 +1,5 @@ Debug Tools -=========== +""""""""""" The debugging tools are found here: diff --git a/docs/content/technical/DebugTools/dump-index.rst b/docs/content/operational/DebugTools/dump-index.rst similarity index 97% rename from docs/content/technical/DebugTools/dump-index.rst rename to docs/content/operational/DebugTools/dump-index.rst index 62c25a069..ac06d87bd 100644 --- a/docs/content/technical/DebugTools/dump-index.rst +++ b/docs/content/operational/DebugTools/dump-index.rst @@ -1,5 +1,5 @@ fdb dump-index -============== +************** Dump the contents of a particular index file for debugging purposes. @@ -40,4 +40,4 @@ Dump the contents of an index file. Contents of index: Fingerprint: 0:1000:130, location: FieldRefLocation(pathid=0,offset=0) Fingerprint: 0:300:130, location: FieldRefLocation(pathid=0,offset=3280398) - ... \ No newline at end of file + ... diff --git a/docs/content/technical/DebugTools/dump-toc.rst b/docs/content/operational/DebugTools/dump-toc.rst similarity index 99% rename from docs/content/technical/DebugTools/dump-toc.rst rename to docs/content/operational/DebugTools/dump-toc.rst index 664a823a0..e5c878235 100644 --- a/docs/content/technical/DebugTools/dump-toc.rst +++ b/docs/content/operational/DebugTools/dump-toc.rst @@ -1,5 +1,5 @@ fdb dump-toc -============ +************ Description ----------- @@ -45,4 +45,4 @@ Dump the contents of a toc walking the subtocs logically. Note that subtocs that TOC_INIT 2019-06-07 14:17:01.032360, version:2, fdb: 50308, uid: , pid 31508, host: Key: {class=rd,expver=xxxx,stream=oper,date=20160907,time=1200,domain=g}, sub-toc: no TOC_INDEX 2019-06-07 14:28:31.850438, version:2, fdb: 50308, uid: , pid 644 , host: Path: an:pl.20190607.142831..2765958938625.index, offset: 0, type: BTreeIndex Prefix: an:pl, key: {type=an,levtype=pl} TOC_INDEX 2019-06-07 14:17:01.943474, version:2, fdb: 50308, uid: , pid 31508, host: Path: an:pl.20190607.141701..135325829562374.index, offset: 0, type: BTreeIndex Prefix: an:pl, key: {type=an,levtype=pl} - ... \ No newline at end of file + ... diff --git a/docs/content/technical/DebugTools/dump.rst b/docs/content/operational/DebugTools/dump.rst similarity index 99% rename from docs/content/technical/DebugTools/dump.rst rename to docs/content/operational/DebugTools/dump.rst index 7a1cfdc2e..2552692f3 100644 --- a/docs/content/technical/DebugTools/dump.rst +++ b/docs/content/operational/DebugTools/dump.rst @@ -1,5 +1,5 @@ fdb dump -======== +******** Dump the structural contents of the FDB. In particular, in the TOC formulation, enumerate the different entries in the Table of Contents (including INIT and CLEAR entries). diff --git a/docs/content/technical/DebugTools/hammer.rst b/docs/content/operational/DebugTools/hammer.rst similarity index 99% rename from docs/content/technical/DebugTools/hammer.rst rename to docs/content/operational/DebugTools/hammer.rst index 2c40a22c1..395fef42e 100644 --- a/docs/content/technical/DebugTools/hammer.rst +++ b/docs/content/operational/DebugTools/hammer.rst @@ -1,5 +1,5 @@ fdb hammer -========== +********** Description ----------- diff --git a/docs/content/technical/DebugTools/patch.rst b/docs/content/operational/DebugTools/patch.rst similarity index 98% rename from docs/content/technical/DebugTools/patch.rst rename to docs/content/operational/DebugTools/patch.rst index 4f6cdc3eb..4a4c80324 100644 --- a/docs/content/technical/DebugTools/patch.rst +++ b/docs/content/operational/DebugTools/patch.rst @@ -1,5 +1,5 @@ fdb patch -========= +********* Description ----------- @@ -53,4 +53,4 @@ Note that this is a global search through all the databases of the FDB that matc 12 fields (37.5412 Mbytes) copied to {expver=xxxz} Rates: 114.971 Mbytes per second, 36.7503 fields/s - fdb patch: 0.946881 second elapsed, 0.159061 second cpu \ No newline at end of file + fdb patch: 0.946881 second elapsed, 0.159061 second cpu diff --git a/docs/content/technical/DebugTools/read.rst b/docs/content/operational/DebugTools/read.rst similarity index 99% rename from docs/content/technical/DebugTools/read.rst rename to docs/content/operational/DebugTools/read.rst index 469041a80..cd988f662 100644 --- a/docs/content/technical/DebugTools/read.rst +++ b/docs/content/operational/DebugTools/read.rst @@ -1,5 +1,5 @@ fdb read -======== +******** Read data from the FDB and write this data into a specified target file. This may involve visiting multiple databases if required by the request. diff --git a/docs/content/technical/DebugTools/reconsolidate-toc.rst b/docs/content/operational/DebugTools/reconsolidate-toc.rst similarity index 90% rename from docs/content/technical/DebugTools/reconsolidate-toc.rst rename to docs/content/operational/DebugTools/reconsolidate-toc.rst index 9544d52f3..9021518a5 100644 --- a/docs/content/technical/DebugTools/reconsolidate-toc.rst +++ b/docs/content/operational/DebugTools/reconsolidate-toc.rst @@ -1,5 +1,5 @@ fdb reconsolidate-toc -===================== +********************* This is an advanced tool that exists to assist cleanup in a situation where databases have been poorly written. This can occur in a number of contexts such as: @@ -12,4 +12,4 @@ In general it is preferable to wipe such databases, and rerun with correct exper Usage ----- -``fdb reconsolidate-toc [database path]`` \ No newline at end of file +``fdb reconsolidate-toc [database path]`` diff --git a/docs/content/technical/DebugTools/write.rst b/docs/content/operational/DebugTools/write.rst similarity index 99% rename from docs/content/technical/DebugTools/write.rst rename to docs/content/operational/DebugTools/write.rst index 5ed793863..22ae85be3 100644 --- a/docs/content/technical/DebugTools/write.rst +++ b/docs/content/operational/DebugTools/write.rst @@ -1,5 +1,5 @@ fdb write -========= +********* Inserts data into the FDB, creating a new databases if needed. The data is copied into the FDB, and the tool reports the location where it was inserted. diff --git a/docs/content/technical/GeneralPurposeTools.rst b/docs/content/operational/GeneralPurposeTools.rst similarity index 69% rename from docs/content/technical/GeneralPurposeTools.rst rename to docs/content/operational/GeneralPurposeTools.rst index 119c48635..0534fd07b 100644 --- a/docs/content/technical/GeneralPurposeTools.rst +++ b/docs/content/operational/GeneralPurposeTools.rst @@ -1,5 +1,5 @@ General Purpose Tools -===================== +""""""""""""""""""""" The General Purpose tools are found here: @@ -7,4 +7,4 @@ The General Purpose tools are found here: :maxdepth: 1 :glob: - GeneralPurposeTools/* \ No newline at end of file + GeneralPurposeTools/* diff --git a/docs/content/technical/GeneralPurposeTools/fdb-stats.rst b/docs/content/operational/GeneralPurposeTools/fdb-stats.rst similarity index 98% rename from docs/content/technical/GeneralPurposeTools/fdb-stats.rst rename to docs/content/operational/GeneralPurposeTools/fdb-stats.rst index 3ab639af8..77948ef6c 100644 --- a/docs/content/technical/GeneralPurposeTools/fdb-stats.rst +++ b/docs/content/operational/GeneralPurposeTools/fdb-stats.rst @@ -1,5 +1,5 @@ fdb stats -========= +********* Prints information about FDB databases, aggregating the information over all the databases visited into a final summary. @@ -33,7 +33,7 @@ Example 1 You may pass a partial request (as a key) that will print information on all FDB databases that match that key. -:: +.. code-block:: bash % fdb stats class=od,expver=0001,stream=oper,date=20151001 ... @@ -67,7 +67,7 @@ Example 2 The --details flag prints a report per database that is visited, as well as the overall summary -:: +.. code-block:: bash % fdb stats class=od,expver=0001 ... @@ -98,4 +98,4 @@ The --details flag prints a report per database that is visited, as well as the ======== Number of databases : 4 - ... \ No newline at end of file + ... diff --git a/docs/content/technical/GeneralPurposeTools/fdb.rst b/docs/content/operational/GeneralPurposeTools/fdb.rst similarity index 98% rename from docs/content/technical/GeneralPurposeTools/fdb.rst rename to docs/content/operational/GeneralPurposeTools/fdb.rst index c7555e8cf..554698849 100644 --- a/docs/content/technical/GeneralPurposeTools/fdb.rst +++ b/docs/content/operational/GeneralPurposeTools/fdb.rst @@ -1,5 +1,5 @@ fdb -=== +*** Description ----------- @@ -38,4 +38,4 @@ fdb list example {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} - ... \ No newline at end of file + ... diff --git a/docs/content/technical/GeneralPurposeTools/grib2fdb.rst b/docs/content/operational/GeneralPurposeTools/grib2fdb.rst similarity index 99% rename from docs/content/technical/GeneralPurposeTools/grib2fdb.rst rename to docs/content/operational/GeneralPurposeTools/grib2fdb.rst index a4a6019c5..2995e31b4 100644 --- a/docs/content/technical/GeneralPurposeTools/grib2fdb.rst +++ b/docs/content/operational/GeneralPurposeTools/grib2fdb.rst @@ -1,5 +1,5 @@ grib2fdb5 -========= +********* Inserts data into the FDB, creating a new databases if needed. @@ -51,4 +51,4 @@ Check that the supplied keys match Processing data.grib Key {class=rd,expver=xxxx,type=an,stream=oper} FDB archive 12 fields, size 37.5412 Mbytes, in 0.086995 second (431.518 Mbytes per second) - fdb::service::archive: 0.087076 second elapsed, 0.087075 second cpu \ No newline at end of file + fdb::service::archive: 0.087076 second elapsed, 0.087075 second cpu diff --git a/docs/content/technical/GeneralPurposeTools/info.rst b/docs/content/operational/GeneralPurposeTools/info.rst similarity index 98% rename from docs/content/technical/GeneralPurposeTools/info.rst rename to docs/content/operational/GeneralPurposeTools/info.rst index 1edb3be7b..13d6efaaa 100644 --- a/docs/content/technical/GeneralPurposeTools/info.rst +++ b/docs/content/operational/GeneralPurposeTools/info.rst @@ -1,5 +1,5 @@ fdb info -======== +******** Get information about the FDB configuration and binaries @@ -61,4 +61,4 @@ Get location of current FDB configuration file in an easily parsable form: :: % fdb info --config - /testcases/fdb5/fdb5_simple.yaml \ No newline at end of file + /testcases/fdb5/fdb5_simple.yaml diff --git a/docs/content/technical/GeneralPurposeTools/list.rst b/docs/content/operational/GeneralPurposeTools/list.rst similarity index 99% rename from docs/content/technical/GeneralPurposeTools/list.rst rename to docs/content/operational/GeneralPurposeTools/list.rst index 72d764c53..2603226ca 100644 --- a/docs/content/technical/GeneralPurposeTools/list.rst +++ b/docs/content/operational/GeneralPurposeTools/list.rst @@ -1,5 +1,5 @@ fdb list -======== +******** Lists the contents of the FDB databases. In the body of the output, one line is given per field that has been archived. These (by default) present the fields that are available and will be retrievable - i.e. masked duplicates are skipped. diff --git a/docs/content/technical/GeneralPurposeTools/purge.rst b/docs/content/operational/GeneralPurposeTools/purge.rst similarity index 98% rename from docs/content/technical/GeneralPurposeTools/purge.rst rename to docs/content/operational/GeneralPurposeTools/purge.rst index 43b1fd636..85a3f83fe 100644 --- a/docs/content/technical/GeneralPurposeTools/purge.rst +++ b/docs/content/operational/GeneralPurposeTools/purge.rst @@ -1,5 +1,5 @@ fdb purge -========= +********* Purge duplicate entries from the database and remove the associated data (if the data is owned, not adopted). @@ -99,4 +99,4 @@ Additionally pass the --doit flag to delete the duplicates. ... % du -sh /data/fdb/od\:0001\:oper\:20160907\:1200\:g/ - 20M fdb_root/root/od:0001:oper:20160907:1200:g/ \ No newline at end of file + 20M fdb_root/root/od:0001:oper:20160907:1200:g/ diff --git a/docs/content/technical/GeneralPurposeTools/schema.rst b/docs/content/operational/GeneralPurposeTools/schema.rst similarity index 100% rename from docs/content/technical/GeneralPurposeTools/schema.rst rename to docs/content/operational/GeneralPurposeTools/schema.rst diff --git a/docs/content/technical/GeneralPurposeTools/where.rst b/docs/content/operational/GeneralPurposeTools/where.rst similarity index 99% rename from docs/content/technical/GeneralPurposeTools/where.rst rename to docs/content/operational/GeneralPurposeTools/where.rst index 0a91247aa..29d082ac9 100644 --- a/docs/content/technical/GeneralPurposeTools/where.rst +++ b/docs/content/operational/GeneralPurposeTools/where.rst @@ -1,5 +1,5 @@ fdb where -========= +********* Print the location of FDB5 database. diff --git a/docs/content/technical/GeneralPurposeTools/wipe.rst b/docs/content/operational/GeneralPurposeTools/wipe.rst similarity index 99% rename from docs/content/technical/GeneralPurposeTools/wipe.rst rename to docs/content/operational/GeneralPurposeTools/wipe.rst index 4b19758ad..4874264e0 100644 --- a/docs/content/technical/GeneralPurposeTools/wipe.rst +++ b/docs/content/operational/GeneralPurposeTools/wipe.rst @@ -1,5 +1,5 @@ fdb wipe -======== +******** Deletes FDB databases and the data therein contained. Uses the passed request to identify the database to delete. @@ -142,4 +142,4 @@ Use --minimum-keys with caution! Setting --minimum-keys=class is a BAD IDEA! You Unlink /data/fdb5/rd:wxyz:oper:20151004:1200:g/toc Unlink /data/fdb5/rd:wxyz:oper:20151004:1200:g/schema Unlink /data/fdb5/rd:wxyz:oper:20151004:1200:g/an:pl.20170323.164359.host.30249454665731.data - Unlink /data/fdb5/rd:wxyz:oper:20151004:1200:g/an:pl.20170323.164359.host.30249454665730.index \ No newline at end of file + Unlink /data/fdb5/rd:wxyz:oper:20151004:1200:g/an:pl.20170323.164359.host.30249454665730.index diff --git a/docs/content/technical/SpecialPurposeTools.rst b/docs/content/operational/SpecialPurposeTools.rst similarity index 69% rename from docs/content/technical/SpecialPurposeTools.rst rename to docs/content/operational/SpecialPurposeTools.rst index baadc2da5..48f47950c 100644 --- a/docs/content/technical/SpecialPurposeTools.rst +++ b/docs/content/operational/SpecialPurposeTools.rst @@ -1,5 +1,5 @@ Special Purpose Tools -===================== +""""""""""""""""""""" The Special Purpose tools are found here: @@ -7,4 +7,4 @@ The Special Purpose tools are found here: :maxdepth: 1 :glob: - SpecialPurposeTools/* \ No newline at end of file + SpecialPurposeTools/* diff --git a/docs/content/technical/SpecialPurposeTools/hide.rst b/docs/content/operational/SpecialPurposeTools/hide.rst similarity index 98% rename from docs/content/technical/SpecialPurposeTools/hide.rst rename to docs/content/operational/SpecialPurposeTools/hide.rst index 5a5aa9fda..f15268515 100644 --- a/docs/content/technical/SpecialPurposeTools/hide.rst +++ b/docs/content/operational/SpecialPurposeTools/hide.rst @@ -1,5 +1,5 @@ fdb hide -======== +******** Hide the contents of one FDB database. This masks all existing entries in the database such that they are permanently inaccessible, without destructively damaging the data or indexes. @@ -38,4 +38,4 @@ These changes can then be made permanent :: % fdb hide --doit class=rd,expver=xxxx,stream=oper,date=20160907,time=0000,domain=g - Hide contents of DB: TocDBReader(/fdb5/root/rd:xxxx:oper:20160907:0000:g) \ No newline at end of file + Hide contents of DB: TocDBReader(/fdb5/root/rd:xxxx:oper:20160907:0000:g) diff --git a/docs/content/technical/SpecialPurposeTools/move.rst b/docs/content/operational/SpecialPurposeTools/move.rst similarity index 99% rename from docs/content/technical/SpecialPurposeTools/move.rst rename to docs/content/operational/SpecialPurposeTools/move.rst index 736157a03..3f3210379 100644 --- a/docs/content/technical/SpecialPurposeTools/move.rst +++ b/docs/content/operational/SpecialPurposeTools/move.rst @@ -1,5 +1,5 @@ fdb move -======== +******** Move the content of one FDB database. This locks the source database, make it possible to create a second database in another root, duplicates all data. Source data are not automatically removed. diff --git a/docs/content/technical/SpecialPurposeTools/overlay.rst b/docs/content/operational/SpecialPurposeTools/overlay.rst similarity index 99% rename from docs/content/technical/SpecialPurposeTools/overlay.rst rename to docs/content/operational/SpecialPurposeTools/overlay.rst index 5b2af1f02..138cb8317 100644 --- a/docs/content/technical/SpecialPurposeTools/overlay.rst +++ b/docs/content/operational/SpecialPurposeTools/overlay.rst @@ -1,5 +1,5 @@ fdb overlay -=========== +*********** Make the contents of one FDB database available as though they were archived using different keys (class or expver). diff --git a/docs/content/technical/SpecialPurposeTools/root.rst b/docs/content/operational/SpecialPurposeTools/root.rst similarity index 96% rename from docs/content/technical/SpecialPurposeTools/root.rst rename to docs/content/operational/SpecialPurposeTools/root.rst index f7c03e49a..9cd78681e 100644 --- a/docs/content/technical/SpecialPurposeTools/root.rst +++ b/docs/content/operational/SpecialPurposeTools/root.rst @@ -1,5 +1,5 @@ fdb root -======== +******** Find the location on disk of the specified databases. @@ -42,4 +42,4 @@ Note that the supplied keys must be sufficient to identify the database. This su % fdb root --create class=od,expver=0001,stream=oper,date=20160907,time=1200 {class=od,expver=0001,stream=oper,date=20160907,time=1200,domain=g} (Rule[line=128]) - TocDBWriter(/data/fdb/od:0001:oper:20160907:1200:g) \ No newline at end of file + TocDBWriter(/data/fdb/od:0001:oper:20160907:1200:g) diff --git a/docs/content/technical/tools.rst b/docs/content/operational/tools.rst similarity index 97% rename from docs/content/technical/tools.rst rename to docs/content/operational/tools.rst index e391969a0..6fd9f0726 100644 --- a/docs/content/technical/tools.rst +++ b/docs/content/operational/tools.rst @@ -1,7 +1,7 @@ .. _tools-label: FDB Tools -========= +````````` FDB has a set of general purpose CLI tools for archiving and retrieving user data, some special purpose tools for administrative maintenance and for developer tools for debugging. diff --git a/docs/content/reference.rst b/docs/content/reference.rst index eb90a9fc6..993636369 100644 --- a/docs/content/reference.rst +++ b/docs/content/reference.rst @@ -5,7 +5,8 @@ Two publications, co-authored by Simon D. Smart, Tiago Quintino, Baudouin Raoult describe fdb architecture and have been presented at PASC'17 `A Scalable Object Store for Meteorological and Climate Data`_ and PASC'19 `A High-Performance Distributed Object-Store for Exascale Numerical Weather Prediction and Climate`_ In the following the two BibTeX snippets: -:: + +.. code-block:: latex @inproceedings{10.1145/3093172.3093238, author = {Smart, Simon D. and Quintino, Tiago and Raoult, Baudouin}, @@ -23,8 +24,6 @@ In the following the two BibTeX snippets: series = {PASC ’17} } - - @inproceedings{10.1145/3324989.3325726, author = {Smart, Simon D. and Quintino, Tiago and Raoult, Baudouin}, title = {A High-Performance Distributed Object-Store for Exascale Numerical Weather Prediction and Climate}, @@ -43,4 +42,4 @@ In the following the two BibTeX snippets: .. _A Scalable Object Store for Meteorological and Climate Data: https://dl.acm.org/doi/pdf/10.1145/3093172.3093238 -.. _A High-Performance Distributed Object-Store for Exascale Numerical Weather Prediction and Climate: https://dl.acm.org/doi/pdf/10.1145/3324989.3325726 \ No newline at end of file +.. _A High-Performance Distributed Object-Store for Exascale Numerical Weather Prediction and Climate: https://dl.acm.org/doi/pdf/10.1145/3324989.3325726 diff --git a/docs/content/technical-introduction.rst b/docs/content/technical-introduction.rst index 3142455dd..28ec7fb7e 100644 --- a/docs/content/technical-introduction.rst +++ b/docs/content/technical-introduction.rst @@ -1,33 +1,28 @@ .. _technical-introduction-label: Technical Introduction -###################### +~~~~~~~~~~~~~~~~~~~~~~ |Licence| FDB (Fields DataBase) is a domain-specific object store developed at ECMWF for storing, indexing and retrieving GRIB data. Each GRIB message is stored as a field and indexed trough semantic metadata (i.e. physical variables such as temperature, pressure, ...). A set of fields can be retrieved specifying a request using a specific language developed for accessing :doc:`mars` Archive -FDB exposes a C++ API as well as CLI :doc:`tools`. +FDB exposes a C++ API as well as CLI :doc:`technical/api`. .. toctree:: :maxdepth: 1 technical/requirements technical/build-and-installation + technical/check-installation technical/configuration technical/config technical/schema - technical/implementation - technical/usage-examples technical/api - technical/tools - technical/genindex .. |Licence| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg :target: https://github.com/ecmwf/fdb/blob/develop/LICENSE :alt: Apache Licence .. _mars: mars.html -.. _tools: tools.html - diff --git a/docs/content/technical/Api/api_python.rst b/docs/content/technical/Api/api_python.rst new file mode 100644 index 000000000..942aa7587 --- /dev/null +++ b/docs/content/technical/Api/api_python.rst @@ -0,0 +1,9 @@ +.. index:: Reference; Python API + :name: Python-reference + +Work on adding documentation for the FDB Python API is in progress here. + +The Python Api relies on a seperate project, see https://github.com/ecmwf/pyfdb. + +Python API +========== diff --git a/docs/content/technical/UsageExamples/check-installation.rst b/docs/content/technical/UsageExamples/check-installation.rst deleted file mode 100644 index 537982e6b..000000000 --- a/docs/content/technical/UsageExamples/check-installation.rst +++ /dev/null @@ -1,36 +0,0 @@ -Checking Installation -================================= - -To check whether the installation of the **FDB** was successful we can execute -the following command - -.. code-block:: console - - which fdb - -This should show the path to the installed FDB instance. If you receive and error -message at this point in time, make sure that you added the path FDB is located -at to you `PATH` environment variable. - -.. code-block:: console - - fdb help - -The help argument shows a list of options we have to interact with the FDB application. -The most useful for this stage would be to run ``fdb home`` to check whether the configuration -of FDB and the corresponding home directory was successful. Running this command -results in the following output: - -.. code-block:: console - - /path/to/fdb/home - -If this isn't the case, make sure you exported the ``FDB_HOME`` environment variable -as stated in the installation guide. Next we want to show information about -the FDB instance. We run - -.. code-block:: console - - fdb schema - -This should print the preconfigured schema. diff --git a/docs/content/technical/UsageExamples/data-retrieval.rst b/docs/content/technical/UsageExamples/data-retrieval.rst deleted file mode 100644 index 5f529d1c1..000000000 --- a/docs/content/technical/UsageExamples/data-retrieval.rst +++ /dev/null @@ -1,29 +0,0 @@ -Data retrieval and FDB insertion -===================================== - -This task of loading data into **FDB** should help you to familiarize -yourself with the software. - -In this scenario we want to let FDB handle a GRIB file. - -Inserting into FDB ------------------- -Writing the GRIB file to FDB can be accomplished by the following command - -:: - - fdb write /path/to/grib/file - -Which will result in the following output (or similar depending on the GRIB file) - -:: - - Processing /path/to/grib/file - FDB archive 50 messages, size 156.422 Mbytes, in 0.350135 second (446.746 Mbytes per second) - fdb::service::archive: 0.350308 second elapsed, 0.118118 second cpu - -.. **WARNING: Do not run the following command -.. in a production surrounding due it otherwise querying the entire FDB database.** -.. -.. In case you are running an FDB instance locally (and only then) you can check the -.. output of ``fdb list --all --minimum-keys=""``. diff --git a/docs/content/technical/check-installation.rst b/docs/content/technical/check-installation.rst index 841576d0c..488cf0d65 100644 --- a/docs/content/technical/check-installation.rst +++ b/docs/content/technical/check-installation.rst @@ -35,4 +35,3 @@ the FDB instance. We run This should print the set up schema. - diff --git a/docs/content/technical/config-schema.rst b/docs/content/technical/config-schema.rst deleted file mode 100644 index 42db43721..000000000 --- a/docs/content/technical/config-schema.rst +++ /dev/null @@ -1,77 +0,0 @@ -FDB Config File -=============== - -Config files define a number of parameters for the FDB. There are several different -types, which can be described by a FDB config file. - -Local -****** - -:: - - type: local - engine: toc - schema: ./schema - spaces: - - handler: Default - roots: - -path: /path/to/fdb/root - -There a number of different types such as local, remote, distributed, and select. - -Local implements the passage of data from the frontend to storage backend, talk to the FDB Store and Catalogue. -Depending on the backend, the data or metadata may not actually be local. - -Select dispatches requests to different FDB's based on the metadata associated with the Messages, and can be used to send split requests OD from RD. - -Select -******* - -:: - - type: select - fdbs: - - select: class=od - type:local - spaces: - roots: - -path: /path/to/fdb/od - -select: class=rd,expver=xx.?.? - type: local - spaces: - roots: - - path: /path/to/fdb/rd - -The remote type handles access to the remote FDB via TCP/IP. It talks to the FDB server using an asynchronous protocol. -It only handles the transition. Not the distribution of data. - -Remote -******* - -:: - - type: remote - host: fdb-minus - port: 36604 - -The distributed type implements the multi-lane access to multiple FDB's. It uses rendezvous hashing to avoid synchronisations. - -Dist -***** - -:: - - type: dist - lanes: - -type: remote - host: fdb-minus-1 - port: 36604 - -type: remote - host: fdb-minus-2 - port: 36604 - -These types can be composed together in the config file when using FDB. - -.. - _## TODO: Get this reviewed and add more information. - diff --git a/docs/content/technical/configuration.rst b/docs/content/technical/configuration.rst index 268e483ab..090651252 100644 --- a/docs/content/technical/configuration.rst +++ b/docs/content/technical/configuration.rst @@ -32,5 +32,45 @@ and is used for further storage operations. This is to guarantee data consistenc over the course of several storage procedures even if the global schema is subject to changes. +Checking Installation +--------------------------------- + +To check whether the installation of the **FDB** was successful we can execute +the following command + +.. code-block:: console + + which fdb + +This should show the path to the installed FDB instance. If you receive and error +message at this point in time, make sure that you added the path FDB is located +at to you `PATH` environment variable. + +.. code-block:: console + + fdb help + +The help argument shows a list of options we have to interact with the FDB application. +The most useful for this stage would be to run ``fdb home`` to check whether the configuration +of FDB and the corresponding home directory was successful. Running this command +results in the following output: + +.. code-block:: console + + /path/to/fdb/home + +If this isn't the case, make sure you exported the ``FDB_HOME`` environment variable +as stated in the installation guide. Next we want to show information about +the FDB instance. We run + +.. code-block:: console + + fdb schema + +This should print the set up schema. + + + + .. _Config: config-schema.html .. _Schema: config-schema.html#schema diff --git a/docs/content/technical/genindex.rst b/docs/content/technical/genindex.rst deleted file mode 100644 index 9e530fa2f..000000000 --- a/docs/content/technical/genindex.rst +++ /dev/null @@ -1,2 +0,0 @@ -Index -===== diff --git a/docs/content/technical/installation.rst b/docs/content/technical/installation.rst deleted file mode 100644 index cd7a81df9..000000000 --- a/docs/content/technical/installation.rst +++ /dev/null @@ -1,3 +0,0 @@ -Installation -============ - diff --git a/docs/content/technical/introduction.rst b/docs/content/technical/introduction.rst deleted file mode 100644 index eb8f61788..000000000 --- a/docs/content/technical/introduction.rst +++ /dev/null @@ -1,11 +0,0 @@ -Introduction -============ - -FDB exposes a C++ API as well as CLI :doc:`tools`. - -.. |Licence| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg - :target: https://github.com/ecmwf/fdb/blob/develop/LICENSE - :alt: Apache Licence - -.. _mars: mars.html -.. _tools: tools.html diff --git a/docs/content/technical/schema.rst b/docs/content/technical/schema.rst index 800ed10e0..172d9c565 100644 --- a/docs/content/technical/schema.rst +++ b/docs/content/technical/schema.rst @@ -55,7 +55,7 @@ With the rule from above, a message with the given metadata, retrieves the follo As you can see, the three levels are represented in the final key, describing the data. The individual sub-keys are: -:: +.. code-block:: bash {class=od,expver=0001,stream=oper,date=20240202,time=0000,domain=g} # Dataset Key {type=fc,levtype=pl} # Colloctation Key diff --git a/docs/content/technical/usage-examples.rst b/docs/content/technical/usage-examples.rst deleted file mode 100644 index 66286ac26..000000000 --- a/docs/content/technical/usage-examples.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. index:: Usage, Examples - -Usage Examples -============== - -.. toctree:: - :maxdepth: 1 - - UsageExamples/check-installation - UsageExamples/data-retrieval - diff --git a/docs/index.rst b/docs/index.rst index b96d82896..33cb1442b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,3 +1,4 @@ +=============================== Welcome to FDB's documentation! =============================== @@ -9,6 +10,7 @@ request using a specific language developed for accessing MARS Archive. The documentation is divided into three parts: +*************************************** :ref:`architectural-introduction-label` *************************************** @@ -16,14 +18,16 @@ The aim of this document to given an overview of the system landscape, showing how the FDB integrates into an existing setup, consisting of a data archive, using the example of `MARS `_. +*********************************** :ref:`technical-introduction-label` -*************************************** +*********************************** The aim of this part of the documentation is to give a broad and technical overview of the API of the FDB. +************************************* :ref:`operational-introduction-label` -*************************************** +************************************* This part of the documentations aims at operations and how to configure and deploy the FDB. @@ -38,6 +42,7 @@ This part of the documentations aims at operations and how to configure and depl content/operational-introduction .. raw:: html +
.. toctree:: diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index 9d49fb51d..42ee9eb42 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -118,10 +118,22 @@ class FDB { eckit::DataHandle* read(ListIterator& it, bool sorted = false); + /** @brief Retrieve data which is specified by a MARS request + * + * @param request MarsRequest which describes the data which should be retrieved + * @return DataHandle for reading the requested data from + */ eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request); ListIterator inspect(const metkit::mars::MarsRequest& request); + /** @brief List data present at the archive and which can be retrieved + * + * @param request FDBToolRequest stating which data should be queried + * @param deduplicate bool, whether the returned iterator should ignore duplicates + * @param length Size of the data to archive with the given @p key + * @return ListIterator for iterating over the set of found items + */ ListIterator list(const FDBToolRequest& request, bool deduplicate=false); DumpIterator dump(const FDBToolRequest& request, bool simple=false); @@ -129,6 +141,14 @@ class FDB { /// TODO: Is this function superfluous given the control() function? StatusIterator status(const FDBToolRequest& request); + /** @brief Wipe data from the database + * + * @param request FDBToolRequest stating which data should be queried + * @param doit flag for committing to the wipe (default is dry-run) + * @param porcelain flag for committing to the wipe (default is dry-run) (TODO: comment it) + * @param unsafeWipeAll flag for omitting all security checks and force a wipe + * @return WipeIterator for iterating over the set of wiped items + */ WipeIterator wipe(const FDBToolRequest& request, bool doit=false, bool porcelain=false, bool unsafeWipeAll=false); MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest);