From a6a9cde62cb9396766ec064ebdc23b0c8bc656cc Mon Sep 17 00:00:00 2001 From: Fede Raimondo Date: Mon, 25 Nov 2024 16:10:51 +0100 Subject: [PATCH 01/10] Add config manager capacitt --- junifer/utils/__init__.pyi | 2 ++ junifer/utils/config.py | 71 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 junifer/utils/config.py diff --git a/junifer/utils/__init__.pyi b/junifer/utils/__init__.pyi index b9db68f189..b014708760 100644 --- a/junifer/utils/__init__.pyi +++ b/junifer/utils/__init__.pyi @@ -1,6 +1,7 @@ __all__ = [ "make_executable", "configure_logging", + "config", "logger", "raise_error", "warn_with_log", @@ -11,5 +12,6 @@ __all__ = [ from .fs import make_executable from .logging import configure_logging, logger, raise_error, warn_with_log +from .config import config from .helpers import run_ext_cmd, deep_update from ._yaml import yaml diff --git a/junifer/utils/config.py b/junifer/utils/config.py new file mode 100644 index 0000000000..515980448f --- /dev/null +++ b/junifer/utils/config.py @@ -0,0 +1,71 @@ +"""Provide junifer global configuration.""" + +# Authors: Federico Raimondo +# Synchon Mandal +# License: AGPL +import os +from typing import Any + +from .logging import logger +from .singleton import Singleton + + +class ConfigManager(metaclass=Singleton): + """Manage configuration parameters. + + Attributes + ---------- + _config : dict + Configuration parameters. + + """ + + def __init__(self) -> None: + """Initialize the class.""" + self._config = {} + for t_var in os.environ: + if t_var.startswith("JUNIFER_"): + varname = t_var.replace("JUNIFER_", "") + varname = varname.lower() + varname.replace("_", ".") + logger.debug( + f"Setting {varname} from environment to " + f"{os.environ[t_var]}" + ) + self._config[varname] = os.environ[t_var] + + def get(self, key: str, default: Any = None) -> Any: + """Get configuration parameter. + + Parameters + ---------- + key : str + The configuration key. + + default : Any, optional + The default value to return if the key is not found (default None). + + Returns + ------- + Any + The configuration value. + + """ + return self._config.get(key, default) + + def set(self, key: str, value: Any) -> None: + """Set configuration parameter. + + Parameters + ---------- + key : str + The configuration key. + value : Any + The configuration value. + + """ + logger.debug(f"Setting {key} to {value}") + self._config[key] = value + + +config = ConfigManager() From 75334add76c709ad4935fe37620a19a2e4c50969 Mon Sep 17 00:00:00 2001 From: Fede Raimondo Date: Mon, 25 Nov 2024 21:34:32 +0100 Subject: [PATCH 02/10] fix replace + cast to bool/int/float --- junifer/utils/config.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/junifer/utils/config.py b/junifer/utils/config.py index 515980448f..637ff40f2a 100644 --- a/junifer/utils/config.py +++ b/junifer/utils/config.py @@ -27,12 +27,25 @@ def __init__(self) -> None: if t_var.startswith("JUNIFER_"): varname = t_var.replace("JUNIFER_", "") varname = varname.lower() - varname.replace("_", ".") + varname = varname.replace("_", ".") + var_value = os.environ[t_var] + if var_value.lower() == "true": + var_value = True + elif var_value.lower() == "false": + var_value = False + else: + try: + var_value = int(var_value) + except ValueError: + try: + var_value = float(var_value) + except ValueError: + pass logger.debug( f"Setting {varname} from environment to " - f"{os.environ[t_var]}" + f"{var_value} (type: {type(var_value)})" ) - self._config[varname] = os.environ[t_var] + self._config[varname] = var_value def get(self, key: str, default: Any = None) -> Any: """Get configuration parameter. From 0a7f12257fd238354ca892c8a0176cdc482efea6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 10:37:57 +0100 Subject: [PATCH 03/10] chore: reorder __all__ using ruff --- junifer/api/decorators.py | 2 +- junifer/api/functions.py | 2 +- junifer/cli/cli.py | 16 ++++++++-------- junifer/cli/parser.py | 2 +- junifer/data/_dispatch.py | 2 +- junifer/data/masks/_masks.py | 2 +- junifer/data/template_spaces.py | 2 +- junifer/stats.py | 2 +- junifer/storage/utils.py | 4 ++-- junifer/testing/datagrabbers.py | 2 +- junifer/typing/_typing.py | 12 ++++++------ junifer/utils/helpers.py | 2 +- junifer/utils/logging.py | 2 +- junifer/utils/singleton.py | 2 +- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/junifer/api/decorators.py b/junifer/api/decorators.py index b063dc24a6..bf39c6c5c8 100644 --- a/junifer/api/decorators.py +++ b/junifer/api/decorators.py @@ -13,8 +13,8 @@ __all__ = [ "register_datagrabber", "register_datareader", - "register_preprocessor", "register_marker", + "register_preprocessor", "register_storage", ] diff --git a/junifer/api/functions.py b/junifer/api/functions.py index a67a22b621..d721c9dff5 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -24,7 +24,7 @@ from ..utils import logger, raise_error, yaml -__all__ = ["run", "collect", "queue", "reset", "list_elements"] +__all__ = ["collect", "list_elements", "queue", "reset", "run"] def _get_datagrabber(datagrabber_config: dict) -> DataGrabberLike: diff --git a/junifer/cli/cli.py b/junifer/cli/cli.py index da0448c9e4..2ac97b3e18 100644 --- a/junifer/cli/cli.py +++ b/junifer/cli/cli.py @@ -29,19 +29,19 @@ __all__ = [ + "afni_docker", + "ants_docker", "cli", - "run", "collect", + "freesurfer_docker", + "fsl_docker", + "list_elements", "queue", - "wtf", - "selftest", "reset", - "list_elements", + "run", + "selftest", "setup", - "afni_docker", - "fsl_docker", - "ants_docker", - "freesurfer_docker", + "wtf", ] diff --git a/junifer/cli/parser.py b/junifer/cli/parser.py index 7d17c92461..99577c1021 100644 --- a/junifer/cli/parser.py +++ b/junifer/cli/parser.py @@ -15,7 +15,7 @@ from ..utils import logger, raise_error, warn_with_log, yaml -__all__ = ["parse_yaml", "parse_elements"] +__all__ = ["parse_elements", "parse_yaml"] def parse_yaml(filepath: Union[str, Path]) -> dict: # noqa: C901 diff --git a/junifer/data/_dispatch.py b/junifer/data/_dispatch.py index b504fbe6a1..c8be8cdb22 100644 --- a/junifer/data/_dispatch.py +++ b/junifer/data/_dispatch.py @@ -25,11 +25,11 @@ __all__ = [ + "deregister_data", "get_data", "list_data", "load_data", "register_data", - "deregister_data", ] diff --git a/junifer/data/masks/_masks.py b/junifer/data/masks/_masks.py index e7cd2bda01..779aa3eabf 100644 --- a/junifer/data/masks/_masks.py +++ b/junifer/data/masks/_masks.py @@ -35,7 +35,7 @@ from nibabel.nifti1 import Nifti1Image -__all__ = ["compute_brain_mask", "MaskRegistry"] +__all__ = ["MaskRegistry", "compute_brain_mask"] # Path to the masks diff --git a/junifer/data/template_spaces.py b/junifer/data/template_spaces.py index 76475b49a4..1ec253580d 100644 --- a/junifer/data/template_spaces.py +++ b/junifer/data/template_spaces.py @@ -16,7 +16,7 @@ from .utils import closest_resolution -__all__ = ["get_xfm", "get_template"] +__all__ = ["get_template", "get_xfm"] def get_xfm( diff --git a/junifer/stats.py b/junifer/stats.py index 63678efef9..61a35bdca0 100644 --- a/junifer/stats.py +++ b/junifer/stats.py @@ -13,7 +13,7 @@ from .utils import logger, raise_error -__all__ = ["get_aggfunc_by_name", "count", "winsorized_mean", "select"] +__all__ = ["count", "get_aggfunc_by_name", "select", "winsorized_mean"] def get_aggfunc_by_name( diff --git a/junifer/storage/utils.py b/junifer/storage/utils.py index 21615e7405..1d4bb7fe4d 100644 --- a/junifer/storage/utils.py +++ b/junifer/storage/utils.py @@ -15,11 +15,11 @@ __all__ = [ + "element_to_prefix", "get_dependency_version", + "matrix_to_vector", "process_meta", - "element_to_prefix", "store_matrix_checks", - "matrix_to_vector", ] diff --git a/junifer/testing/datagrabbers.py b/junifer/testing/datagrabbers.py index e305937a1c..63703a9abc 100644 --- a/junifer/testing/datagrabbers.py +++ b/junifer/testing/datagrabbers.py @@ -15,8 +15,8 @@ __all__ = [ "OasisVBMTestingDataGrabber", - "SPMAuditoryTestingDataGrabber", "PartlyCloudyTestingDataGrabber", + "SPMAuditoryTestingDataGrabber", ] diff --git a/junifer/typing/_typing.py b/junifer/typing/_typing.py index b7be284156..9e2cd47ca5 100644 --- a/junifer/typing/_typing.py +++ b/junifer/typing/_typing.py @@ -19,16 +19,16 @@ __all__ = [ + "ConditionalDependencies", "DataGrabberLike", - "PreprocessorLike", - "MarkerLike", - "StorageLike", - "PipelineComponent", + "DataGrabberPatterns", "Dependencies", - "ConditionalDependencies", "ExternalDependencies", "MarkerInOutMappings", - "DataGrabberPatterns", + "MarkerLike", + "PipelineComponent", + "PreprocessorLike", + "StorageLike", ] diff --git a/junifer/utils/helpers.py b/junifer/utils/helpers.py index 229b750420..f4f67dbe0e 100644 --- a/junifer/utils/helpers.py +++ b/junifer/utils/helpers.py @@ -10,7 +10,7 @@ from .logging import logger, raise_error -__all__ = ["run_ext_cmd", "deep_update"] +__all__ = ["deep_update", "run_ext_cmd"] def run_ext_cmd(name: str, cmd: list[str]) -> None: diff --git a/junifer/utils/logging.py b/junifer/utils/logging.py index 0fb26bdbe8..93ff8618fa 100644 --- a/junifer/utils/logging.py +++ b/junifer/utils/logging.py @@ -24,9 +24,9 @@ __all__ = [ "WrapStdOut", + "configure_logging", "get_versions", "log_versions", - "configure_logging", "raise_error", "warn_with_log", ] diff --git a/junifer/utils/singleton.py b/junifer/utils/singleton.py index 1503fc9c78..e6e31baa2b 100644 --- a/junifer/utils/singleton.py +++ b/junifer/utils/singleton.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar -__all__ = ["Singleton", "ABCSingleton"] +__all__ = ["ABCSingleton", "Singleton"] class Singleton(type): From 04b3b154a2b56677f5d71e101f9f589cfbd13482 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 10:52:44 +0100 Subject: [PATCH 04/10] style: add commentary, __all__; chore: clean up whitespace --- junifer/utils/config.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/junifer/utils/config.py b/junifer/utils/config.py index 637ff40f2a..b515fa9173 100644 --- a/junifer/utils/config.py +++ b/junifer/utils/config.py @@ -3,6 +3,7 @@ # Authors: Federico Raimondo # Synchon Mandal # License: AGPL + import os from typing import Any @@ -10,6 +11,9 @@ from .singleton import Singleton +__all__ = ["ConfigManager", "config"] + + class ConfigManager(metaclass=Singleton): """Manage configuration parameters. @@ -54,13 +58,12 @@ def get(self, key: str, default: Any = None) -> Any: ---------- key : str The configuration key. - - default : Any, optional + default : any, optional The default value to return if the key is not found (default None). Returns ------- - Any + any The configuration value. """ @@ -73,7 +76,7 @@ def set(self, key: str, value: Any) -> None: ---------- key : str The configuration key. - value : Any + value : any The configuration value. """ @@ -81,4 +84,5 @@ def set(self, key: str, value: Any) -> None: self._config[key] = value +# Initialize here to access from anywhere config = ConfigManager() From 380eb567b7a2cfbca54eb31e763c1aa17c222017 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 10:53:34 +0100 Subject: [PATCH 05/10] update: add tests for ConfigManager --- junifer/utils/tests/test_config.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 junifer/utils/tests/test_config.py diff --git a/junifer/utils/tests/test_config.py b/junifer/utils/tests/test_config.py new file mode 100644 index 0000000000..43c3bdf77b --- /dev/null +++ b/junifer/utils/tests/test_config.py @@ -0,0 +1,27 @@ +"""Provide tests for ConfigManager.""" + +# Authors: Federico Raimondo +# Synchon Mandal +# License: AGPL + +from junifer.utils import config +from junifer.utils.config import ConfigManager + + +def test_config_manager_singleton() -> None: + """Test that ConfigManager is a singleton.""" + config_mgr_1 = ConfigManager() + config_mgr_2 = ConfigManager() + assert id(config_mgr_1) == id(config_mgr_2) + + +def test_config_manager_get_set() -> None: + """Test config getting and setting for ConfigManager.""" + # Get non-existing with default + assert "mystery_machine" == config.get( + key="scooby", default="mystery_machine" + ) + # Set + config.set(key="scooby", value="doo") + # Get existing + assert "doo" == config.get("scooby") From 6396868803797a1eb44621be940a6f992cb5037d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 11:06:44 +0100 Subject: [PATCH 06/10] chore: import ConfigManager to docs ref --- junifer/utils/__init__.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/junifer/utils/__init__.pyi b/junifer/utils/__init__.pyi index b014708760..afcca40b31 100644 --- a/junifer/utils/__init__.pyi +++ b/junifer/utils/__init__.pyi @@ -8,10 +8,11 @@ __all__ = [ "run_ext_cmd", "deep_update", "yaml", + "ConfigManager", ] from .fs import make_executable from .logging import configure_logging, logger, raise_error, warn_with_log -from .config import config +from .config import config, ConfigManager from .helpers import run_ext_cmd, deep_update from ._yaml import yaml From 797b427c6e428e78ec20f174c2d100e6933cb4e6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 14:41:55 +0100 Subject: [PATCH 07/10] chore: rename config.py to _config.py to avoid possible name mangle conflicts; update: add delete method to ConfigManager --- junifer/utils/__init__.pyi | 2 +- junifer/utils/{config.py => _config.py} | 56 +++++++++++++++++-------- junifer/utils/tests/test_config.py | 2 +- 3 files changed, 41 insertions(+), 19 deletions(-) rename junifer/utils/{config.py => _config.py} (55%) diff --git a/junifer/utils/__init__.pyi b/junifer/utils/__init__.pyi index afcca40b31..c840252d3b 100644 --- a/junifer/utils/__init__.pyi +++ b/junifer/utils/__init__.pyi @@ -13,6 +13,6 @@ __all__ = [ from .fs import make_executable from .logging import configure_logging, logger, raise_error, warn_with_log -from .config import config, ConfigManager +from ._config import config, ConfigManager from .helpers import run_ext_cmd, deep_update from ._yaml import yaml diff --git a/junifer/utils/config.py b/junifer/utils/_config.py similarity index 55% rename from junifer/utils/config.py rename to junifer/utils/_config.py index b515fa9173..b2b8e4643c 100644 --- a/junifer/utils/config.py +++ b/junifer/utils/_config.py @@ -5,8 +5,9 @@ # License: AGPL import os -from typing import Any +from typing import Optional +from ..typing import ConfigVal from .logging import logger from .singleton import Singleton @@ -27,16 +28,21 @@ class ConfigManager(metaclass=Singleton): def __init__(self) -> None: """Initialize the class.""" self._config = {} + # Initial setup from process env + self._reload() + + def _reload(self) -> None: + """Reload env vars.""" for t_var in os.environ: if t_var.startswith("JUNIFER_"): - varname = t_var.replace("JUNIFER_", "") - varname = varname.lower() - varname = varname.replace("_", ".") + # Set correct type var_value = os.environ[t_var] + # bool if var_value.lower() == "true": var_value = True elif var_value.lower() == "false": var_value = False + # numeric else: try: var_value = int(var_value) @@ -45,43 +51,59 @@ def __init__(self) -> None: var_value = float(var_value) except ValueError: pass + # Set value + var_name = ( + t_var.replace("JUNIFER_", "").lower().replace("_", ".") + ) logger.debug( - f"Setting {varname} from environment to " - f"{var_value} (type: {type(var_value)})" + f"Setting `{var_name}` from environment to " + f"`{var_value}` (type: {type(var_value)})" ) - self._config[varname] = var_value + self._config[var_name] = var_value - def get(self, key: str, default: Any = None) -> Any: + def get(self, key: str, default: Optional[ConfigVal] = None) -> ConfigVal: """Get configuration parameter. Parameters ---------- key : str - The configuration key. - default : any, optional + The configuration key to get. + default : bool or int or float or None, optional The default value to return if the key is not found (default None). Returns ------- - any + bool or int or float The configuration value. """ return self._config.get(key, default) - def set(self, key: str, value: Any) -> None: + def set(self, key: str, val: ConfigVal) -> None: """Set configuration parameter. Parameters ---------- key : str - The configuration key. - value : any - The configuration value. + The configuration key to set. + val : bool or int or float + The value to set ``key`` to. + + """ + logger.debug(f"Setting `{key}` to `{val}` (type: {type(val)})") + self._config[key] = val + + def delete(self, key: str) -> None: + """Delete configuration parameter. + + Parameters + ---------- + key : str + The configuration key to delete. """ - logger.debug(f"Setting {key} to {value}") - self._config[key] = value + logger.debug(f"Deleting `{key}` from config") + _ = self._config.pop(key) # Initialize here to access from anywhere diff --git a/junifer/utils/tests/test_config.py b/junifer/utils/tests/test_config.py index 43c3bdf77b..6ab3b26be4 100644 --- a/junifer/utils/tests/test_config.py +++ b/junifer/utils/tests/test_config.py @@ -5,7 +5,7 @@ # License: AGPL from junifer.utils import config -from junifer.utils.config import ConfigManager +from junifer.utils._config import ConfigManager def test_config_manager_singleton() -> None: From 56b36095db4696445cdf0c06c48fbc519e0a0e45 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 14:43:02 +0100 Subject: [PATCH 08/10] chore: add ConfigVal type hint --- junifer/typing/__init__.pyi | 2 ++ junifer/typing/_typing.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/junifer/typing/__init__.pyi b/junifer/typing/__init__.pyi index 43ab84c024..bfe026fda6 100644 --- a/junifer/typing/__init__.pyi +++ b/junifer/typing/__init__.pyi @@ -9,6 +9,7 @@ __all__ = [ "ExternalDependencies", "MarkerInOutMappings", "DataGrabberPatterns", + "ConfigVal", ] from ._typing import ( @@ -22,4 +23,5 @@ from ._typing import ( ExternalDependencies, MarkerInOutMappings, DataGrabberPatterns, + ConfigVal, ) diff --git a/junifer/typing/_typing.py b/junifer/typing/_typing.py index 9e2cd47ca5..d7275b597f 100644 --- a/junifer/typing/_typing.py +++ b/junifer/typing/_typing.py @@ -20,6 +20,7 @@ __all__ = [ "ConditionalDependencies", + "ConfigVal", "DataGrabberLike", "DataGrabberPatterns", "Dependencies", @@ -60,3 +61,4 @@ DataGrabberPatterns = dict[ str, Union[dict[str, str], Sequence[dict[str, str]]] ] +ConfigVal = Union[bool, int, float] From 4388cb0071d8a6c7bab5f6547aaf25f995acbda6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 14:43:20 +0100 Subject: [PATCH 09/10] update: improve tests for ConfigManager --- junifer/utils/tests/test_config.py | 46 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/junifer/utils/tests/test_config.py b/junifer/utils/tests/test_config.py index 6ab3b26be4..209acfa648 100644 --- a/junifer/utils/tests/test_config.py +++ b/junifer/utils/tests/test_config.py @@ -4,6 +4,11 @@ # Synchon Mandal # License: AGPL +import os + +import pytest + +from junifer.typing import ConfigVal from junifer.utils import config from junifer.utils._config import ConfigManager @@ -15,13 +20,40 @@ def test_config_manager_singleton() -> None: assert id(config_mgr_1) == id(config_mgr_2) -def test_config_manager_get_set() -> None: - """Test config getting and setting for ConfigManager.""" +def test_config_manager() -> None: + """Test config operations for ConfigManager.""" # Get non-existing with default - assert "mystery_machine" == config.get( - key="scooby", default="mystery_machine" - ) + assert config.get(key="scooby") is None # Set - config.set(key="scooby", value="doo") + config.set(key="scooby", val=True) # Get existing - assert "doo" == config.get("scooby") + assert config.get("scooby") + # Delete + config.delete("scooby") + # Get non-existing with default + assert config.get(key="scooby") is None + + +@pytest.mark.parametrize( + "val, expected_val", + [("TRUE", True), ("FALSE", False), ("1", 1), ("0.0", 0.0)], +) +def test_config_manager_env_reload(val: str, expected_val: ConfigVal) -> None: + """Test config parsing from env reload. + + Parameters + ---------- + val : str + The parametrized values. + expected_val : bool or int or float + The parametrized expected value. + + """ + # Set env var + os.environ["JUNIFER_TESTME"] = val + # Check + config._reload() + assert config.get("testme") == expected_val + # Cleanup + del os.environ["JUNIFER_TESTME"] + config.delete("testme") From 8b7b59978da001db953781791b10588d058a4df5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 26 Nov 2024 15:49:07 +0100 Subject: [PATCH 10/10] chore: add changelog 401.feature --- docs/changes/newsfragments/401.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/401.feature diff --git a/docs/changes/newsfragments/401.feature b/docs/changes/newsfragments/401.feature new file mode 100644 index 0000000000..24e89567ee --- /dev/null +++ b/docs/changes/newsfragments/401.feature @@ -0,0 +1 @@ +Introduce :class:`junifer.utils.ConfigManager` singleton class to manage global configuration by `Fede Raimondo`_