Merge branch 'hotfix/5.11.29'

.github/ci-config.yml

@@ -0,0 +1,8 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+  ecmwf/metkit
+dependency_branch: develop
+parallelism_factor: 8

.github/ci-hpc-config.yml

@@ -0,0 +1,14 @@
+build:
+  modules:
+    - ninja
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/metkit@develop
+  cmake_options:
+    - -DENABLE_LUSTRE=OFF
+  parallel: 64
+  env:
+    - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+    - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions

.github/workflows/cd.yml

@@ -0,0 +1,11 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+  deploy:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2
+    secrets: inherit

.github/workflows/ci.yml

@@ -1,36 +1,76 @@
 name: ci
 
-# Controls when the workflow will run
 on:
-
-  # Trigger the workflow on all pushes, except on tag creation
+  # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-    - '**'
+      - 'master'
+      - 'develop'
     tags-ignore:
-    - '**'
+      - '**'
 
-  # Trigger the workflow on all pull requests
+  # Trigger the workflow on pull request
   pull_request: ~
 
-  # Allow workflow to be dispatched on demand
+  # Trigger the workflow manually
   workflow_dispatch: ~
 
-jobs:
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
 
-  # Calls a reusable CI workflow to build & test the current repository.
-  #   It will pull in all needed dependencies and produce a code coverage report on success.
-  ci:
-    name: ci
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/ci.yml@v1
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
     with:
+      fdb: ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
-      build_package_inputs: |
-        self_coverage: true
-        dependencies: |
-          ecmwf/ecbuild
-          ecmwf/eckit
-          MathisRosenhauer/libaec@master
-          ecmwf/eccodes
-          ecmwf/metkit
-        dependency_branch: develop
+    secrets: inherit
+
+  # Run CI of private downstream packages on self-hosted runners
+  private-downstream-ci:
+    name: private-downstream-ci
+    needs: [downstream-ci]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci
+          payload: '{"fdb": "ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      fdb: ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit
+
+  # Run CI of private downstream packages on HPC
+  private-downstream-ci-hpc:
+    name: private-downstream-ci-hpc
+    needs: [downstream-ci-hpc]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci-hpc
+          payload: '{"fdb": "ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}"}'

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

VERSION

@@ -1 +1 @@
-5.11.28
+5.11.29

docs/Doxyfile.in

@@ -864,16 +864,8 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-<<<<<<< HEAD
 INPUT                  = "../src/fdb5/api/fdb_c.h" \
                          "../src/fdb5/api/FDB.h"
-=======
-INPUT                  = "../src/odc/api/odc.h" \
-                         "../src/odc/api/Odb.h" \
-                         "../src/odc/api/ColumnType.h" \
-                         "../src/odc/api/ColumnInfo.h" \
-                         "../src/odc/api/StridedData.h"
->>>>>>> c7a78c9a (docs structure from odc)
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

docs/content/DebugTools/dump-index.rst

@@ -12,11 +12,8 @@ Usage
 Options
 -------
 
-+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| ``--config=string``                    | FDB configuration filename                                                                                                                                                                                       |
-+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-
-
+Examples
+--------
 
 Example 1
 ---------

docs/content/DebugTools/dump-toc.rst

@@ -20,6 +20,8 @@ Options
 | ``--config=string``                    | FDB configuration filename.                                                                                                                                                                                      |
 +----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 
+Examples
+--------
 Example 1
 ---------
 

docs/content/DebugTools/dump.rst

@@ -25,6 +25,8 @@ Options
 | ``--config=string``                    | FDB configuration filename.                                                                                         |
 +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 
+Examples
+--------
 
 Example 1
 ---------

docs/content/DebugTools/read.rst

@@ -35,7 +35,7 @@ Note that this MARS request must be fully expanded. For example, it may not cont
   retrieve,class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g,type=an,levtype=pl,step=0,levelist=700,param=155
   
   # this will retrieve 2 fields
-  % fdbr read myrequest foo.grib
+  % fdb read myrequest foo.grib
   retrieve,class=od,date=20151004,domain=g,expver=0001,levelist=500/700,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an
   Compress handle: 3.3e-05 second elapsed, 3.2e-05 second cpu
   Compress handle: 2e-06 second elapsed, 2e-06 second cpu

docs/content/GeneralPurposeTools/wipe.rst

@@ -37,10 +37,6 @@ Options
 +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 | ``--all``                              | (Debug and testing only) Visit all FDB databases                                                                    |
 +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
-| ``--unsafe-wipe-all``                  | Wipe all (unowned) contents of an unclean database                                                                  |
-+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
-| ``--config=string``                    | FDB configuration filename                                                                                          |
-+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 
 Example 1
 ---------

docs/content/config-schema.rst

@@ -5,25 +5,23 @@ Config files define a number of parameters for the FDB.
 
 The following is of the form local:
 ::
-  
   type: local
   engine: toc
   schema: ./schema
   spaces:
-  - hanlder: Default    
+  - handler: Default    
     roots:
     -path: /path/to/fdb/root
 
 There a number of different types such as local, remote, distributed, and select.
 
-Local implements the passage of dara from the frontend to storage backend, talk to the FDB Store and Catalogue. 
+Local implements the passage of data from the frontend to storage backend, talk to the FDB Store and Catalogue. 
 Depending on the backend, the data or metadata may not actually be local.
 
 Select dispatches requests to different FDB's based on the metadata associated with the Messages, and can be used to send split requests OD from RD.
 
 Select Type:
 ::
-  
   type: select
   fdbs:
   - select: class=od
@@ -42,7 +40,6 @@ It only handles the transition. not the distribution of data.
 
 Remote type:
 ::
-  
   type: remote
   host: fdb-minus
   port: 36604
@@ -51,7 +48,6 @@ The distributed type implements the multi-lane access to multiple FDB's. It uses
 
 Dist type:
 ::
-  
   type: dist
   lanes:
     -type: remote
@@ -68,11 +64,10 @@ These types can be composed together in the config file when using FDB.
 Schema
 ======
 
-The schema indexes structure and dala collocation policy. It is used to uniquely describe the data that is being stored and indexed. 
+The schema indexes structure and data collocation policy. It is used to uniquely describe the data that is being stored and indexed. 
 
 The schema uses global attributes that describe the underlying data with each attribute having a name and datatype, these are added at the beginning of the file.
 ::
-  
   param:     Param;
   step:      Step;
   date;      Date;
@@ -81,44 +76,39 @@ The schema uses global attributes that describe the underlying data with each at
 
 The schema then describes rules for accessing all data stored by the fdb.
 
-Each rule is described using three levels. The first level defines the attributes of te top level directory, the second level defines the attributes used to name the data files, and the third level attributes are used as index keys.
+Each rule is described using three levels. The first level defines the attributes of the top level directory, the second level defines the attributes used to name the data files, and the third level attributes are used as index keys.
 Example of a rule:
 ::
-  
   [ class=ti/s2, expver, stream, date, time, model
     [ origin, type, levtype, hdate?
       [ step, number?, levelist?, param ]]
   ]
 
 Rules can be grouped in the form:
 ::
-  
   [a1, a2, a3 ...
     b1, b2, b3... [c1, c2, c3...]]
     B1, B2, B3... [C1, C2, C3...]]
   ]
 
 A list of values can be given for an attribute.
 ::
-  
   [ ..., stream=enfo/efov, ... ]
 
 Attributes in rules can also be optional using the ? character.
 ::
-  
   [ step, levelist?, param ]
 
 Attributes can be removed using the - character.
 ::
-  
   [grid-]
 
 Rules are then matched if:
   * the attributes are present or marked optional
   * a list is provided, one of them matched
 
-.. An example schema is provided schema_.
+An example schema is provided schema_.
 
 .. _schema: schema
 
-## TODO: add more info on the schema
+## TODO: add more info on the schema

docs/index.rst

@@ -16,6 +16,6 @@ A set of fields can be retrieved specifying a request using a specific language
    content/requirements
    content/reference
    content/tools
-   content/api
    content/config-schema
    content/license
+   genindex

src/fdb5/toc/FileSpace.cc

@@ -55,7 +55,7 @@ TocPath FileSpace::filesystem(const Key& key, const eckit::PathName& db) const {
 std::vector<eckit::PathName> FileSpace::enabled(const ControlIdentifier& controlIdentifier) const {
     std::vector<eckit::PathName> result;
     for (RootVec::const_iterator i = roots_.begin(); i != roots_.end(); ++i) {
-        if (i->exists() and i->enabled(controlIdentifier)) {
+        if (i->enabled(controlIdentifier) && i->exists()) {
             result.push_back(i->path());
         }
     }
@@ -72,7 +72,7 @@ void FileSpace::all(eckit::StringSet& roots) const {
 
 void FileSpace::enabled(const ControlIdentifier& controlIdentifier, eckit::StringSet& roots) const {
     for (RootVec::const_iterator i = roots_.begin(); i != roots_.end(); ++i) {
-        if (i->exists() && i->enabled(controlIdentifier)) {
+        if (i->enabled(controlIdentifier) && i->exists()) {
             roots.insert(i->path());
         }
     }
@@ -89,7 +89,7 @@ bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& roo
 //    std::vector<const Root&> visitables = visitable();
     std::string matchList;
     for (RootVec::const_iterator i = roots_.begin(); i != roots_.end(); ++i) {
-        if (i->exists() && i->enabled(ControlIdentifier::List)) {
+        if (i->enabled(ControlIdentifier::List) && i->exists()) {
             eckit::PathName fullDB = i->path() / db;
             eckit::PathName dbToc = i->path() / db / "toc";
             if (fullDB.exists() && dbToc.exists()) {