├── kloch ├── launchers │ ├── _platform.py │ ├── base │ │ ├── __init__.py │ │ └── _dataclass.py │ ├── python │ │ ├── __init__.py │ │ ├── _dataclass.py │ │ └── _serialized.py │ ├── system │ │ ├── __init__.py │ │ ├── _dataclass.py │ │ └── _serialized.py │ ├── __init__.py │ ├── _get.py │ ├── _serialized.py │ ├── _context.py │ └── _plugins.py ├── __main__.py ├── filesyntax │ ├── __init__.py │ ├── _profile.py │ └── _io.py ├── _utils.py ├── __init__.py ├── constants.py ├── session.py ├── config.py └── _dictmerge.py ├── tests ├── data │ ├── fake-profile.yml │ ├── not-a-profile.txt │ ├── plugins-rerr │ │ └── kloch_rerr.py │ ├── config-molg.yml │ ├── test-script-a.py │ ├── e2e-1 │ │ ├── profile.prod.yml │ │ ├── config.yml │ │ ├── profile.prodml.yml │ │ └── profile.studio.yml │ ├── profile.lxm.yml │ ├── profile.echoes-beta.yml │ ├── profile.echoes-tmp.yml │ ├── config-blaj.yml │ ├── profile.system-expand.yml │ ├── config-fuel.yml │ ├── profile.nolauncher.yml │ ├── profile.echoes.yml │ ├── profile.system-test.yml │ ├── profile.priorities-fail.yml │ ├── profile.priorities-success.yml │ ├── profile-old-version.yml │ ├── profile.system-test-asstr.yml │ ├── profile.mult-launchers.yml │ ├── profile.knots.yml │ ├── plugins-behr │ │ └── kloch_behr │ │ │ └── __init__.py │ └── plugins-tyfa │ │ └── kloch_tyfa.py ├── pytest.ini ├── conftest.py ├── test_constants.py ├── test_utils.py ├── test_session.py ├── test_launchers_context.py ├── test_launchers_plugins.py ├── test_profile.py ├── test_launchers_get.py ├── test_launchers_python.py ├── test_config.py ├── test_launchers_base.py ├── test_filesyntax_io.py ├── test_e2e.py ├── test_launchers_serialized.py └── test_dictmerge.py ├── doc ├── source │ ├── changelog.md │ ├── .pyproject.toml │ ├── developer.md │ ├── _static │ │ ├── fonts │ │ │ └── Dosis-VariableFont_wght.ttf │ │ ├── extra.css │ │ ├── logo-color.svg │ │ ├── logotype.svg │ │ └── banner.svg │ ├── public-api │ │ ├── session.rst │ │ ├── config.rst │ │ ├── constants.rst │ │ ├── profile.rst │ │ ├── cli.rst │ │ ├── index.rst │ │ ├── utils.rst │ │ ├── io.rst │ │ └── launchers.rst │ ├── _injected │ │ ├── exec-config-envvar.py │ │ ├── demo-usage-basics │ │ │ ├── profile.yml │ │ │ └── exec-list.py │ │ ├── demo-usage-tokens │ │ │ ├── profile-a.yml │ │ │ ├── profile-b.yml │ │ │ ├── profile-c.yml │ │ │ └── exec-merge.py │ │ ├── demo-fileformat │ │ │ ├── profile.yml │ │ │ ├── profile-beta.yml │ │ │ └── exec-merge.py │ │ ├── demo-usage-standard │ │ │ ├── profile-a.yml │ │ │ ├── profile-b.yml │ │ │ └── exec-merge.py │ │ ├── exec-config-yaml.py │ │ ├── demo-usage-.base │ │ │ ├── profile-b.yml │ │ │ ├── profile-a.yml │ │ │ └── exec-merge.py │ │ ├── exec-fileformat-token-append.py │ │ ├── tokens.rst │ │ ├── exec-public-api.py │ │ ├── exec-fileformat-token-context.py │ │ ├── snip-launcher-plugin-basic.py │ │ ├── exec-config-autodoc.py │ │ └── exec-launchers-doc.py │ ├── install.rst │ ├── cli.rst │ ├── config.rst │ ├── launchers.rst │ ├── conf.py │ ├── index.rst │ ├── launcher-plugins.rst │ └── _extensions │ │ └── execinject.py ├── serve-doc.py └── build-doc.py ├── LICENSE.txt ├── .gitignore ├── .github └── workflows │ ├── pypi.yml │ ├── doc.yml │ ├── test.yml │ └── pypi-test.yml ├── CONTRIBUTING.md ├── README.md ├── pyproject.toml ├── TODO.md └── CHANGELOG.md /kloch/launchers/_platform.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data/fake-profile.yml: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /doc/source/changelog.md: -------------------------------------------------------------------------------- 1 | ../../CHANGELOG.md -------------------------------------------------------------------------------- /doc/source/.pyproject.toml: -------------------------------------------------------------------------------- 1 | ../../pyproject.toml -------------------------------------------------------------------------------- /doc/source/developer.md: -------------------------------------------------------------------------------- 1 | ../../CONTRIBUTING.md -------------------------------------------------------------------------------- /tests/data/not-a-profile.txt: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: notaprofile 3 | version: 0.2.0 4 | launchers: {} -------------------------------------------------------------------------------- /kloch/__main__.py: -------------------------------------------------------------------------------- 1 | if __name__ == "__main__": 2 | from kloch.cli import run_cli 3 | run_cli() # pragma: no cover 4 | -------------------------------------------------------------------------------- /tests/data/plugins-rerr/kloch_rerr.py: -------------------------------------------------------------------------------- 1 | SOME_DUMBASS_CONSTANT = True 2 | 3 | 4 | class ClearlyNotALauncher: 5 | useless = True 6 | -------------------------------------------------------------------------------- /tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | log_cli = True 3 | log_level = DEBUG 4 | log_cli_format = %(levelname)-7s | %(asctime)s [%(name)s] %(message)s -------------------------------------------------------------------------------- /tests/data/config-molg.yml: -------------------------------------------------------------------------------- 1 | NON_VALID_KEY: "oops" 2 | 3 | cli_logging_default_level: "WARNING" 4 | cli_logging_format: "{levelname: <7}: {message}" 5 | -------------------------------------------------------------------------------- /doc/source/_static/fonts/Dosis-VariableFont_wght.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/knotsanimation/kloch/HEAD/doc/source/_static/fonts/Dosis-VariableFont_wght.ttf -------------------------------------------------------------------------------- /kloch/launchers/base/__init__.py: -------------------------------------------------------------------------------- 1 | from ._dataclass import BaseLauncher 2 | from ._serialized import BaseLauncherFields 3 | from ._serialized import BaseLauncherSerialized 4 | -------------------------------------------------------------------------------- /kloch/launchers/python/__init__.py: -------------------------------------------------------------------------------- 1 | from ._dataclass import PythonLauncher 2 | from ._serialized import PythonLauncherFields 3 | from ._serialized import PythonLauncherSerialized 4 | -------------------------------------------------------------------------------- /kloch/launchers/system/__init__.py: -------------------------------------------------------------------------------- 1 | from ._dataclass import SystemLauncher 2 | from ._serialized import SystemLauncherFields 3 | from ._serialized import SystemLauncherSerialized 4 | -------------------------------------------------------------------------------- /tests/data/test-script-a.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import kloch 4 | 5 | if __name__ == "__main__": 6 | print(f"{kloch.__name__} test script working") 7 | print(sys.argv) 8 | -------------------------------------------------------------------------------- /tests/data/e2e-1/profile.prod.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: prod 3 | version: 0.1.0 4 | inherit: studio 5 | launchers: 6 | .base: 7 | environ: 8 | PROD_NAME: "myprod" 9 | -------------------------------------------------------------------------------- /tests/data/profile.lxm.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: lxm 3 | version: 0.1.0 4 | launchers: 5 | +=.python: 6 | python_file: test-script-a.py 7 | environ: 8 | LXMCUSTOM: 1 -------------------------------------------------------------------------------- /tests/data/profile.echoes-beta.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: knots:echoes:beta 3 | version: 0.1.0 4 | launchers: 5 | +=rezenv: 6 | +=requires: 7 | maya: 2023 8 | houdini: 20 -------------------------------------------------------------------------------- /doc/source/public-api/session.rst: -------------------------------------------------------------------------------- 1 | Session 2 | ======= 3 | 4 | .. code-block:: python 5 | 6 | import kloch.session 7 | 8 | .. automodule:: kloch.session 9 | :members: 10 | :undoc-members: 11 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | THISDIR = Path(__file__).parent 6 | 7 | 8 | @pytest.fixture() 9 | def data_dir() -> Path: 10 | return THISDIR / "data" 11 | -------------------------------------------------------------------------------- /tests/data/profile.echoes-tmp.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: knots:echoes:tmp 3 | version: 0.2.0 4 | inherit: knots:echoes 5 | launchers: 6 | -=params: [ ] 7 | requires: 8 | tmppackage: 2+ -------------------------------------------------------------------------------- /doc/source/public-api/config.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | .. code-block:: python 5 | 6 | import kloch 7 | import kloch.config 8 | 9 | .. automodule:: kloch.config 10 | :members: 11 | -------------------------------------------------------------------------------- /doc/source/public-api/constants.rst: -------------------------------------------------------------------------------- 1 | Constants 2 | ========= 3 | 4 | .. code-block:: python 5 | 6 | import kloch.constants 7 | 8 | .. automodule:: kloch.constants 9 | :members: 10 | :undoc-members: 11 | -------------------------------------------------------------------------------- /tests/data/config-blaj.yml: -------------------------------------------------------------------------------- 1 | cli_logging_default_level: "WARNING" 2 | cli_logging_format: "{levelname: <7}: {message}" 3 | cli_logging_paths: 4 | - ./tmp/kloch.log 5 | - ./tmp/kloch2.log 6 | cli_session_dir: ./.session/ -------------------------------------------------------------------------------- /tests/data/profile.system-expand.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: system-expand 3 | version: 0.1.0 4 | launchers: 5 | .system: 6 | command: 7 | - python 8 | - -V 9 | expand_first_arg: true -------------------------------------------------------------------------------- /doc/source/_injected/exec-config-envvar.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | import kloch 4 | 5 | fields = dataclasses.fields(kloch.config.KlochConfig) 6 | 7 | for field in fields: 8 | print(f"{field.metadata['environ']}") 9 | -------------------------------------------------------------------------------- /tests/data/e2e-1/config.yml: -------------------------------------------------------------------------------- 1 | cli_logging_paths: [ "$KLOCHTEST_LOG_PATH" ] 2 | cli_session_dir: "$KLOCHTEST_SESSION_PATH" 3 | cli_session_dir_lifetime: 240.0 4 | launcher_plugins: 5 | - "kloch_behr" 6 | profile_roots: 7 | - . -------------------------------------------------------------------------------- /tests/data/config-fuel.yml: -------------------------------------------------------------------------------- 1 | cli_logging_default_level: "WARNING" 2 | cli_logging_format: "{levelname: <7}: {message}" 3 | cli_logging_paths: 4 | - $FOOBAR/kloch.log 5 | - $$FOOBAR/tmp/kloch2.log 6 | cli_session_dir: $THIRDIR/.session/ -------------------------------------------------------------------------------- /doc/source/public-api/profile.rst: -------------------------------------------------------------------------------- 1 | Profile 2 | ======= 3 | 4 | .. code-block:: python 5 | 6 | import kloch 7 | import kloch.filesyntax 8 | 9 | .. automodule:: kloch.filesyntax 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /tests/data/profile.nolauncher.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: nolauncher 3 | version: 0.1.0 4 | launchers: 5 | .base: 6 | environ: 7 | TRANS_RIGHT_ARE_HUMAN_RIGHTS: why is there even the question in the first place -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-basics/profile.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: myMovie 3 | version: 0.1.0 4 | launchers: 5 | rezenv: 6 | requires: 7 | houdini: 20.2 8 | maya: 2023 9 | nuke: 15 10 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-tokens/profile-a.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: myMovie:beta 3 | version: 0.1.0 4 | launchers: 5 | rezenv: 6 | requires: 7 | maya: 2023 8 | houdini: 20.2 9 | debugTools: 2.1+ 10 | 11 | -------------------------------------------------------------------------------- /tests/test_constants.py: -------------------------------------------------------------------------------- 1 | import kloch 2 | 3 | 4 | def test__Environ(): 5 | envvars = kloch.Environ.list_all() 6 | assert isinstance(envvars, list) 7 | assert kloch.Environ.CONFIG_PATH in envvars 8 | assert all([isinstance(env, str) for env in envvars]) 9 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-basics/exec-list.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import kloch 4 | 5 | THISDIR = Path(".", "source", "_injected", "demo-usage-basics").absolute() 6 | 7 | cli = kloch.get_cli(["list", "--profile_roots", str(THISDIR)]) 8 | cli.execute() 9 | -------------------------------------------------------------------------------- /doc/source/public-api/cli.rst: -------------------------------------------------------------------------------- 1 | Command Line Interface 2 | ====================== 3 | 4 | .. code-block:: python 5 | 6 | import kloch 7 | import kloch.cli 8 | 9 | .. autofunction:: kloch.get_cli 10 | 11 | .. autofunction:: kloch.run_cli 12 | 13 | .. autoclass:: kloch.cli.BaseParser 14 | :members: 15 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-fileformat/profile.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: knots:echoes 3 | version: 0.2.0 4 | inherit: knots:echoes:beta 5 | launchers: 6 | .base: 7 | environ: 8 | PROD_STATUS: prod 9 | rezenv: 10 | requires: 11 | houdini: 20.2 12 | -=devUtils: _ 13 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | This software intentionally doesn't define a License. 2 | 3 | Additional information can be acquired through 4 | https://en.wikipedia.org/wiki/License-free_software 5 | but whose content doesn't constitute licensing terms in any way. 6 | 7 | Please contact for licensing inquiries. 8 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-tokens/profile-b.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: myMovie 3 | inherit: myMovie:beta 4 | version: 0.1.0 5 | launchers: 6 | config: 7 | package_filter: 8 | - excludes: 9 | - after(1714574770) 10 | rezenv: 11 | requires: 12 | -=debugTools: x 13 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-standard/profile-a.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: myStudio 3 | version: 0.1.0 4 | launchers: 5 | rezenv: 6 | config: 7 | default_shell: powershell 8 | requires: 9 | studio_util: 1+ 10 | environ: 11 | TOOLS_PATH: 12 | - /d/pipeline/studio/tools 13 | -------------------------------------------------------------------------------- /doc/source/public-api/index.rst: -------------------------------------------------------------------------------- 1 | Public API 2 | ========== 3 | 4 | .. exec_code:: 5 | :hide_code: 6 | :filename: _injected/exec-public-api.py 7 | :language_output: python 8 | 9 | 10 | .. toctree:: 11 | 12 | config 13 | constants 14 | cli 15 | io 16 | profile 17 | launchers 18 | session 19 | utils 20 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-standard/profile-b.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: myMovie 3 | inherit: myStudio 4 | version: 0.1.0 5 | launchers: 6 | rezenv: 7 | requires: 8 | maya: 2023 9 | houdini: 20.2 10 | environ: 11 | PROD_NAME: myMovie 12 | TOOLS_PATH: 13 | - /d/pipeline/myMovie/tools -------------------------------------------------------------------------------- /doc/source/_injected/exec-config-yaml.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | import yaml 4 | import kloch 5 | 6 | config = kloch.get_config() 7 | asdict = dataclasses.asdict(config) 8 | asyaml = yaml.dump(asdict).split("\n") 9 | print(f".. code-block:: yaml\n :caption: kloch-config.yml\n\n") 10 | for line in asyaml: 11 | print(f" {line}") 12 | -------------------------------------------------------------------------------- /tests/data/profile.echoes.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: knots:echoes 3 | version: 0.2.0 4 | inherit: knots:echoes:beta 5 | launchers: 6 | +=rezenv: 7 | +=config: 8 | +=package_filter: 9 | - excludes: 10 | - after(1714574770) 11 | +=params: 12 | - --verbose 13 | +=requires: 14 | maya: 2023 -------------------------------------------------------------------------------- /tests/data/profile.system-test.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: system-test 3 | version: 0.1.0 4 | launchers: 5 | +=.system: 6 | environ: 7 | LXMCUSTOM: "表ポあA鷗ŒéＢ逍Üߪąñ丂㐀𠀀" 8 | HEH: "(╯°□°）╯︵ ┻━┻)" 9 | command: 10 | - paint.exe 11 | - new 12 | subprocess_kwargs: 13 | shell: True 14 | timeout: 2 -------------------------------------------------------------------------------- /tests/data/e2e-1/profile.prodml.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: prodml 3 | version: 0.1.0 4 | inherit: studio 5 | launchers: 6 | .base: 7 | environ: 8 | PROD_NAME: "myprod" 9 | .system: 10 | command: 11 | - echo 12 | - hello 13 | - from 14 | .system@os=windows: 15 | subprocess_kwargs: 16 | shell: True -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-.base/profile-b.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: diagnose 3 | inherit: myStudio 4 | version: 0.1.0 5 | launchers: 6 | .base: 7 | environ: 8 | LOG_LEVEL: DEBUG 9 | .system: 10 | command: 11 | - diagnose 12 | .python: 13 | cwd: $STUDIO_COMMON_TOOLS_PATH 14 | python_file: ./diagnose.py 15 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-tokens/profile-c.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: greetings 3 | version: 0.1.0 4 | launchers: 5 | .base: 6 | environ: 7 | BASE_GREETING: "hello" 8 | .system@os=windows: 9 | command: powershell -c "echo $Env:BASE_GREETING there 👋" 10 | .system@os=linux: 11 | command: sh -c "echo $BASE_GREETING there 👋" -------------------------------------------------------------------------------- /tests/data/profile.priorities-fail.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: priorities-fail 3 | version: 0.1.0 4 | launchers: 5 | .system: 6 | priority: 5 7 | command: 8 | - echo 9 | - success_prio_test 10 | .system@os=windows: 11 | subprocess_kwargs: 12 | shell: true 13 | .python: 14 | priority: 5 15 | python_file: test-script-a.py -------------------------------------------------------------------------------- /tests/data/profile.priorities-success.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: priorities-success 3 | version: 0.1.0 4 | launchers: 5 | .system: 6 | priority: 5 7 | command: 8 | - echo 9 | - success_prio_test 10 | .system@os=windows: 11 | subprocess_kwargs: 12 | shell: true 13 | .python: 14 | priority: 3 15 | python_file: test-script-a.py -------------------------------------------------------------------------------- /tests/data/profile-old-version.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:1 2 | identifier: version1 3 | version: 0.2.0 4 | managers: 5 | +=rezenv: 6 | +=config: 7 | +=package_filter: 8 | - excludes: 9 | - after(1714574770) 10 | +=params: 11 | - --verbose 12 | +=requires: 13 | houdini: 20.2 14 | -=devUtils: _ 15 | +=environ: 16 | PROD_STATUS: prod -------------------------------------------------------------------------------- /tests/data/profile.system-test-asstr.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: system-test-asstr 3 | version: 0.1.0 4 | launchers: 5 | +=.system: 6 | environ: 7 | LXMCUSTOM: "表ポあA鷗ŒéＢ逍Üߪąñ丂㐀𠀀" 8 | HEH: "(╯°□°）╯︵ ┻━┻)" 9 | command: 10 | - paint.exe 11 | - new 12 | command_as_str: true 13 | subprocess_kwargs: 14 | shell: True 15 | timeout: 2 -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-.base/profile-a.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: myStudio 3 | version: 0.1.0 4 | launchers: 5 | .base: 6 | environ: 7 | STUDIO_PIPELINE_PATH: /d/pipeline 8 | STUDIO_LOCAL_PREFS_PATH: ~/studio 9 | STUDIO_COMMON_TOOLS_PATH: $STUDIO_PIPELINE_PATH/tools/bin 10 | PATH: 11 | - $PATH 12 | - $STUDIO_COMMON_TOOLS_PATH 13 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-tokens/exec-merge.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import kloch 4 | 5 | THISDIR = Path(".", "source", "_injected", "demo-usage-tokens").absolute() 6 | 7 | profile_path = THISDIR / "profile-b.yml" 8 | profile = kloch.read_profile_from_file(profile_path, profile_locations=[THISDIR]) 9 | profile = profile.get_merged_profile() 10 | print(kloch.serialize_profile(profile)) 11 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-standard/exec-merge.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import kloch 4 | 5 | THISDIR = Path(".", "source", "_injected", "demo-usage-standard").absolute() 6 | 7 | profile_path = THISDIR / "profile-b.yml" 8 | profile = kloch.read_profile_from_file(profile_path, profile_locations=[THISDIR]) 9 | profile = profile.get_merged_profile() 10 | print(kloch.serialize_profile(profile)) 11 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-fileformat/profile-beta.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: knots:echoes:beta 3 | version: 0.3.2 4 | launchers: 5 | .base: 6 | environ: 7 | PROD_STATUS: beta 8 | PROD_NAME: echoes 9 | rezenv: 10 | requires: 11 | houdini: 20.1 12 | maya: 2023 13 | devUtils: 1+ 14 | rezenv@os=windows: 15 | config: 16 | default_shell: "powershell" 17 | -------------------------------------------------------------------------------- /tests/data/profile.mult-launchers.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: mult-launchers 3 | version: 0.1.0 4 | launchers: 5 | .base: 6 | environ: 7 | AYYYYY: "⭐" 8 | .python: 9 | python_file: test-script-a.py 10 | environ: 11 | LXMCUSTOM: 1 12 | .system: 13 | environ: 14 | LXMCUSTOM: "表ポあA鷗ŒéＢ逍Üߪąñ丂㐀𠀀" 15 | HEH: "(╯°□°）╯︵ ┻━┻)" 16 | command: 17 | - paint.exe 18 | - new -------------------------------------------------------------------------------- /doc/source/public-api/utils.rst: -------------------------------------------------------------------------------- 1 | Low-level Utilities 2 | =================== 3 | 4 | .. code-block:: python 5 | 6 | import kloch 7 | 8 | .. autoclass:: kloch.MergeableDict 9 | :members: 10 | :show-inheritance: 11 | :undoc-members: 12 | :special-members: __add__ 13 | 14 | .. autoclass:: kloch.MergeRule 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | 19 | .. autofunction:: kloch.refacto_dict 20 | 21 | .. autofunction:: kloch.deepmerge_dicts 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Unit test / coverage reports 7 | htmlcov/ 8 | .tox/ 9 | .nox/ 10 | .coverage 11 | .coverage.* 12 | .cache 13 | nosetests.xml 14 | coverage.xml 15 | *.cover 16 | .hypothesis/ 17 | .pytest_cache/ 18 | 19 | # PyCharm 20 | .idea/ 21 | 22 | # logging files 23 | *.log 24 | 25 | # documentation 26 | doc/build/ 27 | 28 | # pyinstaller 29 | scripts/.pyinstaller/ 30 | scripts/.nuitka/ 31 | scripts/build/ -------------------------------------------------------------------------------- /tests/data/e2e-1/profile.studio.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: studio 3 | version: 0.1.0 4 | launchers: 5 | +=.base: 6 | +=environ: 7 | STUD_SKYNET_PATH: "N:" 8 | STUD_LOCAL_INSTALL_PATH: $LOCALAPPDATA\studio 9 | STUD_LOCAL_REZ_INSTALL_PATH: $STUD_LOCAL_INSTALL_PATH\rez 10 | STUD_LOCAL_REZ_SCRIPTS_PATH: $STUD_LOCAL_REZ_INSTALL_PATH\Scripts\rez 11 | +=behr: 12 | +=environ: 13 | PATH: 14 | - $PATH 15 | - $STUD_LOCAL_REZ_SCRIPTS_PATH 16 | -------------------------------------------------------------------------------- /doc/source/public-api/io.rst: -------------------------------------------------------------------------------- 1 | Input/Output 2 | ============ 3 | 4 | .. code-block:: python 5 | 6 | import kloch 7 | import kloch.filesyntax 8 | 9 | .. autofunction:: kloch.get_all_profile_file_paths 10 | .. autofunction:: kloch.get_profile_file_path 11 | .. autofunction:: kloch.serialize_profile 12 | .. autofunction:: kloch.write_profile_to_file 13 | .. autofunction:: kloch.read_profile_from_file 14 | .. autofunction:: kloch.read_profile_from_id 15 | 16 | .. autofunction:: kloch.filesyntax.is_file_environment_profile 17 | -------------------------------------------------------------------------------- /tests/data/profile.knots.yml: -------------------------------------------------------------------------------- 1 | __magic__: kloch_profile:4 2 | identifier: knots 3 | version: 0.1.0 4 | launchers: 5 | +=.base: 6 | +=environ: 7 | KNOTS_SKYNET_PATH: "N:" 8 | KNOTS_LOCAL_INSTALL_PATH: $LOCALAPPDATA\knots 9 | KNOTS_LOCAL_REZ_INSTALL_PATH: $KNOTS_LOCAL_INSTALL_PATH\rez 10 | KNOTS_LOCAL_REZ_SCRIPTS_PATH: $KNOTS_LOCAL_REZ_INSTALL_PATH\Scripts\rez 11 | +=rezenv: 12 | +=requires: [] 13 | +=environ: 14 | PATH: 15 | - $PATH 16 | - $KNOTS_LOCAL_REZ_SCRIPTS_PATH 17 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-fileformat/exec-merge.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import sys 3 | import subprocess 4 | 5 | THISDIR = Path(".", "source", "_injected", "demo-fileformat").absolute() 6 | 7 | result = subprocess.run( 8 | [ 9 | sys.executable, 10 | "-m", 11 | "kloch", 12 | "resolve", 13 | "knots:echoes", 14 | "--profile_roots", 15 | str(THISDIR), 16 | "--skip-context-filtering", 17 | ], 18 | capture_output=True, 19 | text=True, 20 | ) 21 | print(result.stdout) 22 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import kloch._utils 4 | 5 | LOGGER = logging.getLogger(__name__) 6 | 7 | 8 | def test__expand_envvars(): 9 | src_str = "${PATH}/foobar" 10 | result = kloch._utils.expand_envvars(src_str) 11 | assert result != src_str 12 | 13 | src_str = "foo/$${PATH}/foobar" 14 | result = kloch._utils.expand_envvars(src_str) 15 | assert result == "foo/${PATH}/foobar" 16 | 17 | src_str = "foo/tmp##${PATH}/foobar" 18 | result = kloch._utils.expand_envvars(src_str) 19 | assert result.startswith("foo/tmp##") 20 | -------------------------------------------------------------------------------- /doc/source/_injected/demo-usage-.base/exec-merge.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import yaml 4 | 5 | import kloch 6 | 7 | THISDIR = Path(".", "source", "_injected", "demo-usage-.base").absolute() 8 | 9 | profile_path = THISDIR / "profile-b.yml" 10 | profile = kloch.read_profile_from_file(profile_path, profile_locations=[THISDIR]) 11 | profile = profile.get_merged_profile() 12 | launcher_classes = kloch.launchers.get_available_launchers_serialized_classes() 13 | launcher_list = profile.launchers.to_serialized_list(launcher_classes) 14 | launcher_list = launcher_list.with_base_merged() 15 | print(yaml.dump(launcher_list.to_dict(), sort_keys=False)) 16 | -------------------------------------------------------------------------------- /doc/serve-doc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import subprocess 4 | import sys 5 | from pathlib import Path 6 | 7 | THISDIR = Path(__file__).parent 8 | 9 | SRCDIR = THISDIR / "source" 10 | BUILDIR = THISDIR / "build" 11 | 12 | shutil.rmtree(BUILDIR, ignore_errors=True) 13 | 14 | COMMAND = [ 15 | "sphinx-autobuild", 16 | str(SRCDIR), 17 | str(BUILDIR), 18 | "--watch", 19 | str( 20 | THISDIR.parent / "kloch", 21 | ), 22 | ] 23 | COMMAND += sys.argv[1:] 24 | 25 | ENVIRON = dict(os.environ) 26 | ENVIRON["PYTHONPATH"] = f"{ENVIRON['PYTHONPATH']}{os.pathsep}{THISDIR.parent}" 27 | 28 | subprocess.check_call(COMMAND, cwd=THISDIR, env=ENVIRON) 29 | -------------------------------------------------------------------------------- /doc/source/_injected/exec-fileformat-token-append.py: -------------------------------------------------------------------------------- 1 | import json 2 | from kloch import MergeableDict 3 | 4 | base = MergeableDict( 5 | { 6 | "rezenv": { 7 | "+=config": {"quiet": True}, 8 | "requires": {"houdini": "20", "devUtils": "2.1"}, 9 | "environ": {"STATUS": "wip"}, 10 | "roots": ["/d/packages"], 11 | } 12 | } 13 | ) 14 | top = MergeableDict( 15 | { 16 | "rezenv": { 17 | "+=config": {"quiet": False, "debug": True}, 18 | "requires": {"maya": "2023", "-=devUtils": "", "!=houdini": "19"}, 19 | "==environ": {"PROD": "test"}, 20 | "roots": ["/d/prods"], 21 | } 22 | } 23 | ) 24 | print(json.dumps(base + top, indent=4)) 25 | -------------------------------------------------------------------------------- /doc/build-doc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import subprocess 4 | import sys 5 | from pathlib import Path 6 | 7 | THISDIR = Path(__file__).parent 8 | 9 | SRCDIR = THISDIR / "source" 10 | BUILDIR = THISDIR / "build" 11 | 12 | shutil.rmtree(BUILDIR, ignore_errors=True) 13 | 14 | COMMAND = [ 15 | "sphinx-build", 16 | "-M", 17 | "html", 18 | str(SRCDIR), 19 | str(BUILDIR), 20 | ] 21 | COMMAND += sys.argv[1:] 22 | 23 | ENVIRON = dict(os.environ) 24 | PYTHONPATH = f"{ENVIRON.get('PYTHONPATH', '')}{os.pathsep}{THISDIR.parent}" 25 | PYTHONPATH = PYTHONPATH.lstrip(os.pathsep) 26 | ENVIRON["PYTHONPATH"] = PYTHONPATH 27 | 28 | subprocess.check_call(COMMAND, cwd=THISDIR, env=ENVIRON) 29 | print(f"documentation generated in '{BUILDIR}'") 30 | url = "file:///" + str(Path(BUILDIR / "html" / "index.html").as_posix()) 31 | print(f"html at '{url}'") 32 | -------------------------------------------------------------------------------- /doc/source/_static/extra.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: "Dosis"; 3 | src: url("./fonts/Dosis-VariableFont_wght.ttf"); 4 | } 5 | 6 | 7 | img.sidebar-logo { 8 | width: 64px; 9 | max-width: unset; 10 | margin: 5px; 11 | } 12 | 13 | h1 { 14 | font-size: 4em; 15 | } 16 | 17 | /*sidebar site title*/ 18 | span.sidebar-brand-text { 19 | font-family: var(--font-stack--headings); 20 | font-weight: 700; 21 | } 22 | /*sidebar page listing*/ 23 | div.sidebar-tree .reference:hover { 24 | /*we are sure it's always white no matter the theme*/ 25 | color: #FEFEFE; 26 | } 27 | 28 | /* custom vertical column split system*/ 29 | div.columns { 30 | display: grid; 31 | gap: 2%; 32 | grid-auto-flow: column; 33 | grid-auto-columns: minmax(0, 1fr); 34 | } 35 | 36 | /* improve tables look */ 37 | table.docutils { 38 | width: 100%; 39 | } 40 | table.docutils td, table.docutils th { 41 | text-align: left; 42 | } 43 | -------------------------------------------------------------------------------- /kloch/filesyntax/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serialization and unserialization of environment profile to disk. 3 | """ 4 | 5 | __all__ = [ 6 | "EnvironmentProfile", 7 | "ProfileInheritanceError", 8 | "ProfileAPIVersionError", 9 | "ProfileIdentifierError", 10 | "is_file_environment_profile", 11 | "get_profile_file_path", 12 | "get_all_profile_file_paths", 13 | "read_profile_from_file", 14 | "read_profile_from_id", 15 | "serialize_profile", 16 | "write_profile_to_file", 17 | ] 18 | 19 | from ._profile import EnvironmentProfile 20 | from ._io import ProfileInheritanceError 21 | from ._io import ProfileAPIVersionError 22 | from ._io import ProfileIdentifierError 23 | from ._io import is_file_environment_profile 24 | from ._io import get_profile_file_path 25 | from ._io import get_all_profile_file_paths 26 | from ._io import read_profile_from_file 27 | from ._io import read_profile_from_id 28 | from ._io import serialize_profile 29 | from ._io import write_profile_to_file 30 | -------------------------------------------------------------------------------- /doc/source/_static/logo-color.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python distribution to PyPI 2 | # based on https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ 3 | 4 | on: 5 | release: 6 | types: 7 | - published 8 | 9 | jobs: 10 | publish-to-pypi: 11 | name: Build and publish distribution to PyPI 12 | runs-on: ubuntu-latest 13 | 14 | permissions: 15 | id-token: write # IMPORTANT: mandatory for trusted publishing 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Install Poetry 20 | # poetry 1.8.4 21 | run: pipx install git+https://github.com/python-poetry/poetry.git@6a071c181ad82ec3a5d4ecd54d50d08a75470e78 22 | - name: Build with Poetry 23 | run: poetry build 24 | # Poetry publish doesn't work with trusted publisher, use official action 25 | # https://github.com/python-poetry/poetry/issues/7940 26 | - name: Publish to PyPI 27 | uses: pypa/gh-action-pypi-publish@release/v1.10 28 | with: 29 | packages-dir: dist/ 30 | -------------------------------------------------------------------------------- /kloch/_utils.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import os 3 | 4 | 5 | @contextlib.contextmanager 6 | def patch_environ(**environ): 7 | """ 8 | Temporarily change ``os.environ`` and restore it once finished. 9 | 10 | Usage:: 11 | 12 | with patch_environ(): 13 | # the environ is left untouched by default 14 | os.environ.clear() 15 | os.environ["PATH"] = "/foo" 16 | 17 | """ 18 | old_environ = dict(os.environ) 19 | os.environ.update(environ) 20 | try: 21 | yield 22 | finally: 23 | os.environ.clear() 24 | os.environ.update(old_environ) 25 | 26 | 27 | def expand_envvars(src_str: str) -> str: 28 | """ 29 | Resolve environment variable pattern in the given string. 30 | 31 | Using ``os.path.expandvars`` but allow escaping using ``$$`` 32 | """ 33 | # temporary remove escape character 34 | new_str = src_str.replace("$$", "##tmp##") 35 | # environment variable expansion 36 | new_str = os.path.expandvars(new_str) 37 | # restore escaped character 38 | new_str = new_str.replace("##tmp##", "$") 39 | return new_str 40 | -------------------------------------------------------------------------------- /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: doc 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | name: Build Documentation 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Install Poetry 15 | run: pipx install poetry 16 | - name: Install Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.9' 20 | cache: 'poetry' 21 | - run: poetry install --extras "doc" 22 | - run: poetry run python ./doc/build-doc.py 23 | - name: Upload artifact 24 | uses: actions/upload-pages-artifact@v3 25 | with: 26 | path: './doc/build/html' 27 | deploy: 28 | environment: 29 | name: github-pages 30 | url: ${{ steps.deployment.outputs.page_url }} 31 | runs-on: ubuntu-latest 32 | needs: build 33 | permissions: 34 | pages: write # to deploy to Pages 35 | id-token: write # to verify the deployment originates from an appropriate source 36 | steps: 37 | - name: Deploy to GitHub Pages 38 | id: deployment 39 | uses: actions/deploy-pages@v4 40 | -------------------------------------------------------------------------------- /doc/source/_injected/tokens.rst: -------------------------------------------------------------------------------- 1 | +--------------------+-----------------------------------------------------------------+ 2 | | token | description | 3 | +====================+=================================================================+ 4 | | ``+=`` | indicate the key's value should be appended to the base's value | 5 | +--------------------+-----------------------------------------------------------------+ 6 | | ``-=`` | indicate the base's key should be removed if it exist | 7 | +--------------------+-----------------------------------------------------------------+ 8 | | ``==`` | indicate the base's value should be overriden | 9 | +--------------------+-----------------------------------------------------------------+ 10 | | ``!=`` | add the key and value only if the key doesn't exist in base | 11 | +--------------------+-----------------------------------------------------------------+ 12 | | `unset` | no token is similar to += (append) | 13 | +--------------------+-----------------------------------------------------------------+ -------------------------------------------------------------------------------- /tests/test_session.py: -------------------------------------------------------------------------------- 1 | import time 2 | from pathlib import Path 3 | 4 | import kloch.session 5 | 6 | 7 | def test__SessionDir(tmp_path: Path): 8 | session = kloch.session.SessionDirectory.initialize(tmp_path) 9 | assert session.path.exists() 10 | assert session.path is not tmp_path 11 | assert session.meta_session_path.exists() 12 | assert not session.profile_path.exists() 13 | 14 | 15 | def test__clean_outdated_session_dirs(tmp_path: Path): 16 | session1 = kloch.session.SessionDirectory.initialize(tmp_path) 17 | time.sleep(0.05) 18 | session2 = kloch.session.SessionDirectory.initialize(tmp_path) 19 | time.sleep(0.05) 20 | session3 = kloch.session.SessionDirectory.initialize(tmp_path) 21 | time.sleep(0.05) 22 | session4 = kloch.session.SessionDirectory.initialize(tmp_path) 23 | time.sleep(0.05) 24 | 25 | Path(tmp_path / "nothingtoseehere.randomfile").write_text("eat the rich") 26 | 27 | lifetime = 0.15 / 3600 # to hours 28 | cleaned = kloch.session.clean_outdated_session_dirs(tmp_path, lifetime=lifetime) 29 | assert len(cleaned) == 2 30 | assert session4.path.exists() 31 | assert session3.path.exists() 32 | assert not session2.path.exists() 33 | assert not session1.path.exists() 34 | -------------------------------------------------------------------------------- /tests/data/plugins-behr/kloch_behr/__init__.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from pathlib import Path 3 | from typing import List 4 | from typing import Optional 5 | 6 | from kloch.launchers import BaseLauncher 7 | from kloch.launchers import BaseLauncherSerialized 8 | from kloch.launchers import BaseLauncherFields 9 | 10 | 11 | @dataclasses.dataclass 12 | class BehrLauncher(BaseLauncher): 13 | 14 | name = "behr" 15 | 16 | def execute( 17 | self, 18 | tmpdir: Path, 19 | command: Optional[List[str]] = None, 20 | ): # pragma: no cover 21 | pass 22 | 23 | 24 | @dataclasses.dataclass(frozen=True) 25 | class BehrLauncherFields(BaseLauncherFields): 26 | pass 27 | 28 | 29 | class BehrLauncherSerialized(BaseLauncherSerialized): 30 | source = BehrLauncher 31 | 32 | identifier = BehrLauncher.name 33 | 34 | fields = BehrLauncherFields 35 | 36 | summary = "Whatever" 37 | description = "Follow me on Mastodon ! https://mastodon.gamedev.place/@liamcollod" 38 | 39 | def validate(self): # pragma: no cover 40 | super().validate() 41 | 42 | # we override for type-hint 43 | def unserialize(self) -> BehrLauncher: # pragma: no cover 44 | # noinspection PyTypeChecker 45 | return super().unserialize() 46 | -------------------------------------------------------------------------------- /tests/data/plugins-tyfa/kloch_tyfa.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from pathlib import Path 3 | from typing import List 4 | from typing import Optional 5 | 6 | from kloch.launchers import BaseLauncher 7 | from kloch.launchers import BaseLauncherSerialized 8 | from kloch.launchers import BaseLauncherFields 9 | 10 | # intentional import 11 | from kloch.launchers import PythonLauncher 12 | 13 | 14 | @dataclasses.dataclass 15 | class TyfaLauncher(BaseLauncher): 16 | 17 | name = "tyfa" 18 | 19 | def execute( 20 | self, 21 | tmpdir: Path, 22 | command: Optional[List[str]] = None, 23 | ): # pragma: no cover 24 | pass 25 | 26 | 27 | @dataclasses.dataclass(frozen=True) 28 | class TyfaLauncherFields(BaseLauncherFields): 29 | pass 30 | 31 | 32 | class TyfaLauncherSerialized(BaseLauncherSerialized): 33 | source = TyfaLauncher 34 | 35 | identifier = TyfaLauncher.name 36 | 37 | fields = TyfaLauncherFields 38 | 39 | summary = "Whatever" 40 | description = "Follow me on GitHub ! https://github.com/MrLixm" 41 | 42 | def validate(self): # pragma: no cover 43 | super().validate() 44 | 45 | # we override for type-hint 46 | def unserialize(self) -> TyfaLauncher: # pragma: no cover 47 | # noinspection PyTypeChecker 48 | return super().unserialize() 49 | -------------------------------------------------------------------------------- /doc/source/_injected/exec-public-api.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import types 3 | 4 | import kloch 5 | 6 | 7 | output = { 8 | "functions": [], 9 | "classes": [], 10 | "modules": [], 11 | "other": [], 12 | } 13 | 14 | print("import kloch") 15 | for obj_name in sorted(kloch.__all__): 16 | if not hasattr(kloch, obj_name): 17 | output["other"].append((obj_name, "")) 18 | 19 | obj = getattr(kloch, obj_name) 20 | doc = "" 21 | if hasattr(obj, "__doc__") and obj.__doc__: 22 | doc = obj.__doc__.lstrip(" ").lstrip("\n") 23 | doc = doc.split("\n") 24 | doc = doc[0].lstrip(" ") 25 | 26 | if isinstance(obj, types.FunctionType): 27 | obj_name = obj_name + "(...)" 28 | output["functions"].append((obj_name, doc)) 29 | elif inspect.isclass(obj): 30 | obj_name = obj_name + "(...)" 31 | output["classes"].append((obj_name, doc)) 32 | elif inspect.ismodule(obj): 33 | output["modules"].append((obj_name, doc)) 34 | else: 35 | output["other"].append((obj_name, doc)) 36 | 37 | for category, content_list in output.items(): 38 | print(f'\n""" {category} """') 39 | for content in content_list: 40 | print(f"kloch.{content[0]}") 41 | doc = content[1] 42 | if doc: 43 | print(f"# {doc}") 44 | print("") 45 | -------------------------------------------------------------------------------- /kloch/launchers/python/_dataclass.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import logging 3 | import subprocess 4 | import sys 5 | from pathlib import Path 6 | from typing import List 7 | from typing import Optional 8 | 9 | from kloch.launchers import BaseLauncher 10 | 11 | 12 | LOGGER = logging.getLogger(__name__) 13 | 14 | 15 | @dataclasses.dataclass 16 | class PythonLauncher(BaseLauncher): 17 | """ 18 | A launcher that execute the given python file with kloch's own interpreter. 19 | """ 20 | 21 | python_file: str = None 22 | """ 23 | Filesystem path to an existing python file. 24 | """ 25 | 26 | required_fields = ["python_file"] 27 | 28 | name = ".python" 29 | 30 | def execute(self, tmpdir: Path, command: Optional[List[str]] = None): 31 | """ 32 | Just call ``subprocess.run`` with ``sys.executable`` + the file path 33 | """ 34 | # XXX: if packaged with nuitka, sys.executable is the built executable path, 35 | # but we support this at the CLI level 36 | _command = [sys.executable, self.python_file] 37 | _command += self.command + (command or []) 38 | 39 | LOGGER.debug(f"subprocess.run({_command}, env={self.environ}, cwd={self.cwd})") 40 | result = subprocess.run(_command, env=self.environ, cwd=self.cwd) 41 | 42 | return result.returncode 43 | -------------------------------------------------------------------------------- /kloch/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "config", 3 | "deepmerge_dicts", 4 | "EnvironmentProfile", 5 | "Environ", 6 | "filesyntax", 7 | "get_all_profile_file_paths", 8 | "get_profile_file_path", 9 | "get_cli", 10 | "get_config", 11 | "KlochConfig", 12 | "launchers", 13 | "MergeableDict", 14 | "MergeRule", 15 | "read_profile_from_file", 16 | "read_profile_from_id", 17 | "refacto_dict", 18 | "run_cli", 19 | "serialize_profile", 20 | "write_profile_to_file", 21 | ] 22 | from .constants import Environ 23 | from ._dictmerge import MergeableDict 24 | from ._dictmerge import MergeRule 25 | from ._dictmerge import refacto_dict 26 | from ._dictmerge import deepmerge_dicts 27 | from .config import KlochConfig 28 | from .config import get_config 29 | from . import config 30 | from . import launchers 31 | from . import filesyntax 32 | from .filesyntax import EnvironmentProfile 33 | from .filesyntax import serialize_profile 34 | from .filesyntax import read_profile_from_file 35 | from .filesyntax import read_profile_from_id 36 | from .filesyntax import write_profile_to_file 37 | from .filesyntax import get_profile_file_path 38 | from .filesyntax import get_all_profile_file_paths 39 | from .cli import get_cli 40 | from .cli import run_cli 41 | 42 | # keep in sync with pyproject.toml 43 | __version__ = "0.13.1" 44 | -------------------------------------------------------------------------------- /kloch/launchers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Entities that serialize a function call made to execute a software. 3 | """ 4 | 5 | from ._context import LauncherContext 6 | from ._context import LauncherPlatform 7 | 8 | from .base import BaseLauncher 9 | from .base import BaseLauncherSerialized 10 | from .base import BaseLauncherFields 11 | 12 | 13 | from .system import SystemLauncher 14 | from .system import SystemLauncherSerialized 15 | from .python import PythonLauncher 16 | from .python import PythonLauncherSerialized 17 | 18 | from ._plugins import check_launcher_plugins 19 | from ._plugins import LoadedPluginsLaunchers 20 | from ._plugins import load_plugin_launchers 21 | 22 | from ._get import get_available_launchers_classes 23 | from ._get import get_available_launchers_serialized_classes 24 | from ._get import is_launcher_plugin 25 | 26 | from ._serialized import LauncherSerializedDict 27 | from ._serialized import LauncherSerializedList 28 | 29 | _BUILTINS_LAUNCHERS = [ 30 | BaseLauncher, 31 | SystemLauncher, 32 | PythonLauncher, 33 | ] 34 | """ 35 | List of launchers class implementation, including the base one. 36 | """ 37 | 38 | _BUILTINS_LAUNCHERS_SERIALIZED = [ 39 | BaseLauncherSerialized, 40 | SystemLauncherSerialized, 41 | PythonLauncherSerialized, 42 | ] 43 | """ 44 | List of serialized launchers class implementation, including the base one. 45 | """ 46 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - 'doc/**' 7 | - 'README.md' 8 | - 'CHANGELOG.md' 9 | - 'TODO.md' 10 | push: 11 | branches: 12 | - main 13 | 14 | concurrency: 15 | group: ${{ github.workflow }}-${{ github.ref }} 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | tests: 20 | name: unittests python-${{ matrix.python-version }} ${{ matrix.os }} 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | python-version: 25 | - "3.7" 26 | - "3.11" 27 | os: 28 | # not latest because this is last version that supports python-3.7 29 | - ubuntu-22.04 30 | - windows-latest 31 | runs-on: ${{ matrix.os }} 32 | steps: 33 | - uses: actions/checkout@v4 34 | - name: Install Poetry 35 | run: pipx install git+https://github.com/python-poetry/poetry.git@5bab98c9500f1050c6bb6adfb55580a23173f18d 36 | - name: Install Python ${{ matrix.python-version }} 37 | id: setup-python 38 | uses: actions/setup-python@v5 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | update-environment: false 42 | - run: poetry env use ${{ steps.setup-python.outputs.python-path }} 43 | - run: poetry install --extras "tests" --no-interaction --no-cache -vvv 44 | - run: poetry run pytest ./tests 45 | -------------------------------------------------------------------------------- /doc/source/install.rst: -------------------------------------------------------------------------------- 1 | Install 2 | ======= 3 | 4 | `kloch` is a python-based tool with ``PyYAML`` as only dependency. 5 | 6 | It's available on `PyPI `_ and can be installed 7 | with: 8 | 9 | .. code-block:: shell 10 | 11 | pip install kloch 12 | 13 | If you use poetry you can also include it in your project as a git dependency 14 | 15 | .. code-block:: shell 16 | 17 | poetry add "git+https://github.com/knotsanimation/kloch.git" 18 | 19 | the manual way 20 | -------------- 21 | 22 | If you don't like all that mess that is the python packaging ecosystem nothing 23 | prevent you to go with a more low-level approach. 24 | 25 | You can copy the ``kloch`` directory (the one with an ``__init__.py`` inside) 26 | at any location that is registred in your PYTHONPATH env var. 27 | 28 | You just need to ensure PyYAML is also downloaded and accessible in your PYTHONPATH. 29 | 30 | Assuming ``python`` is available you coudl use the following recipe: 31 | 32 | .. code-block:: shell 33 | :caption: shell script 34 | 35 | install_dir="/d/demo" 36 | mkdir $install_dir 37 | 38 | cd ./tmp 39 | git clone https://github.com/knotsanimation/kloch.git 40 | cp --recursive ./kloch/kloch "$install_dir/kloch" 41 | # we should now have /d/demo/kloch/__init__.py 42 | 43 | python -m pip install PyYAML --target "$install_dir" 44 | 45 | export PYTHONPATH="$install_dir" 46 | python -m kloch --help 47 | 48 | -------------------------------------------------------------------------------- /kloch/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Variables whose values doesn't change during application runtime. 3 | """ 4 | 5 | from typing import List 6 | 7 | _KLOCH_CONFIG_PREFIX = "KLOCH_CONFIG" 8 | 9 | 10 | class Environ: 11 | """ 12 | Environment variables used across the application. 13 | """ 14 | 15 | CONFIG_PATH = f"{_KLOCH_CONFIG_PREFIX}_PATH" 16 | """ 17 | Environment variable that must specify a file path to an existing configuration file. 18 | """ 19 | 20 | CONFIG_LAUNCHER_PLUGINS = f"{_KLOCH_CONFIG_PREFIX}_launcher_plugins".upper() 21 | 22 | CONFIG_CLI_LOGGING_PATHS = f"{_KLOCH_CONFIG_PREFIX}_cli_logging_paths".upper() 23 | 24 | CONFIG_CLI_LOGGING_FORMAT = f"{_KLOCH_CONFIG_PREFIX}_cli_logging_format".upper() 25 | 26 | CONFIG_CLI_LOGGING_DEFAULT_LEVEL = ( 27 | f"{_KLOCH_CONFIG_PREFIX}_cli_logging_default_level".upper() 28 | ) 29 | 30 | CONFIG_CLI_SESSION_PATH = f"{_KLOCH_CONFIG_PREFIX}_cli_session_path".upper() 31 | 32 | CONFIG_CLI_SESSION_LIFETIME = f"{_KLOCH_CONFIG_PREFIX}_cli_lifetime".upper() 33 | 34 | CONFIG_PROFILE_ROOTS = f"{_KLOCH_CONFIG_PREFIX}_profile_roots".upper() 35 | 36 | @classmethod 37 | def list_all(cls) -> List[str]: 38 | """ 39 | Return all environment variables supported by the application. 40 | """ 41 | return [ 42 | obj 43 | for obj_name, obj in vars(cls).items() 44 | if not obj_name.startswith("__") 45 | and isinstance(obj, str) 46 | and obj.startswith("KLOCH") 47 | ] 48 | -------------------------------------------------------------------------------- /doc/source/cli.rst: -------------------------------------------------------------------------------- 1 | CLI 2 | === 3 | 4 | Command Line Interface documentation. 5 | 6 | Usage 7 | ----- 8 | 9 | Assuming kloch and its dependencies are installed: 10 | 11 | .. code-block:: shell 12 | 13 | python -m kloch --help 14 | 15 | The CLI can also be started from a python script: 16 | 17 | .. code-block:: python 18 | 19 | import kloch 20 | kloch.run_cli(["--help"]) 21 | 22 | # you can also use get_cli to have more control 23 | # main difference is logging not being configuring 24 | cli = kloch.get_cli(["..."]) 25 | cli.execute() 26 | 27 | Commands 28 | -------- 29 | 30 | ``--help`` 31 | __________ 32 | 33 | .. exec_code:: 34 | 35 | import kloch 36 | kloch.get_cli(["--help"]) 37 | 38 | run 39 | ___ 40 | 41 | .. exec_code:: 42 | 43 | import kloch 44 | kloch.get_cli(["run", "--help"]) 45 | 46 | Be aware that the `run` command need to create files on the system. The 47 | location is determined by the :option:`cli_session_dir ` 48 | option. Those locations can be cleared automaticaly based on a 49 | :option:`lifetime option `. 50 | 51 | list 52 | ____ 53 | 54 | .. exec_code:: 55 | 56 | import kloch 57 | kloch.get_cli(["list", "--help"]) 58 | 59 | resolve 60 | _______ 61 | 62 | .. exec_code:: 63 | 64 | import kloch 65 | kloch.get_cli(["resolve", "--help"]) 66 | 67 | python 68 | ______ 69 | 70 | .. exec_code:: 71 | 72 | import kloch 73 | kloch.get_cli(["python", "--help"]) 74 | 75 | plugins 76 | _______ 77 | 78 | .. exec_code:: 79 | 80 | import kloch 81 | kloch.get_cli(["plugins", "--help"]) 82 | -------------------------------------------------------------------------------- /doc/source/config.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | 5 | Tweak the behavior of the kloch runtime using a configuration system. 6 | 7 | Configuration are serialized on disk using yaml file with a simple flat key/value structure: 8 | 9 | .. exec-inject:: 10 | :filename: _injected/exec-config-yaml.py 11 | 12 | You specify which configuration file to use with the environment variable: 13 | 14 | .. exec_code:: 15 | :hide_code: 16 | 17 | import kloch 18 | print(kloch.Environ.CONFIG_PATH) 19 | 20 | Each config key can also be set using an individual environment variable: 21 | 22 | .. exec_code:: 23 | :hide_code: 24 | :filename: _injected/exec-config-envvar.py 25 | 26 | Be aware that specifying a config key in an environment variable will 27 | override any value specified in the config file. 28 | 29 | 30 | Content 31 | ------- 32 | 33 | .. admonition:: Relative Paths 34 | :class: tip 35 | 36 | All config **file** key that expect a ``Path`` and whose value is a relative path 37 | are turned absolute to the config parent directory. 38 | 39 | Example: config is read from ``C:/configs/kloch.yml``, 40 | then ``cli_session_dir: ../sessions/`` will produce ``C:/sessions/`` 41 | 42 | .. admonition:: Environment Variables in Paths 43 | :class: tip 44 | 45 | All config **file** key that expect a ``Path`` can have environment variable that 46 | will be resolved. Environment variable are specified like ``$MYVAR/mydir``. 47 | You can escape the resolving by doubling the ``$`` like ``$$myvar/mydir``. 48 | 49 | .. program:: config 50 | 51 | .. exec-inject:: 52 | :filename: _injected/exec-config-autodoc.py -------------------------------------------------------------------------------- /doc/source/launchers.rst: -------------------------------------------------------------------------------- 1 | Launchers 2 | ========= 3 | 4 | A launcher is a fancy python function, but implemented using 5 | :abbr:`OOP (Object Oriented Programming)`. 6 | 7 | You can break it in 4 components: 8 | 9 | - some user configurable options. 10 | - a execution block that will use the user provided options. 11 | - how it can be serialized in the yaml syntax. 12 | - an unique identifier to refers to it 13 | 14 | Kloch comes with a bunch of built-in launchers: 15 | 16 | .. exec-inject:: 17 | 18 | import kloch.launchers 19 | 20 | launchers = kloch.launchers.get_available_launchers_serialized_classes() 21 | txt = "\n- ".join([""] + [f"``{launcher.identifier}`` : {launcher.summary}" for launcher in launchers]) 22 | print(txt) 23 | 24 | But you can implement as much as you want using the 25 | :doc:`launcher-plugins` system. 26 | 27 | The ``.base`` launch is particular because all launchers inherit from it. Which 28 | mean all launcher, even custom will at least have the options of the ``.base`` 29 | launcher. But it's also handled differently when specified in a profile file 30 | because its option values will be used as base to merge all the other launchers 31 | values over it (example in :ref:`usage:.base inheritance`). 32 | 33 | 34 | builtin launchers 35 | ----------------- 36 | 37 | The following describe how to configure the builtin launchers in the 38 | :doc:`file-format`. 39 | 40 | .. exec-inject:: 41 | :filename: _injected/exec-launchers-doc.py 42 | 43 | ---- 44 | 45 | **References** 46 | 47 | .. [1] https://docs.python.org/3/library/os.path.html#os.path.expandvars 48 | .. [2] ``:`` on UNIX, ``;`` on Windows 49 | .. [4] with https://docs.python.org/3.9/library/pathlib.html#pathlib.Path.resolve 50 | -------------------------------------------------------------------------------- /doc/source/_injected/exec-fileformat-token-context.py: -------------------------------------------------------------------------------- 1 | from kloch.launchers._context import _FIELDS_MAPPING 2 | 3 | RowType = tuple[str, str, str] 4 | 5 | 6 | def enforce_col_length( 7 | row: tuple[str, str, str], 8 | lengths: list[int], 9 | ) -> RowType: 10 | # noinspection PyTypeChecker 11 | return tuple( 12 | col.ljust(lengths[index], col[-1] if col else " ") 13 | for index, col in enumerate(row) 14 | ) 15 | 16 | 17 | def unify_columns(table: list[RowType]) -> list[str]: 18 | """ 19 | Add the columns separator to return a row string. 20 | """ 21 | new_table = [] 22 | for row in table: 23 | sep = "+" if row[-1][-1] in ["-", "="] else "|" 24 | row = sep + sep.join(row) + sep 25 | new_table.append(row) 26 | return new_table 27 | 28 | 29 | def generate_table(): 30 | fields_table: list[RowType] = [] 31 | for prop_name, field in _FIELDS_MAPPING.items(): 32 | field_doc = field.metadata["doc"] 33 | fields_table += [ 34 | ( 35 | f" ``{prop_name}`` ", 36 | f" {field_doc['value']} ", 37 | f" {field_doc['description']} ", 38 | ), 39 | ("-", "-", "-"), 40 | ] 41 | 42 | header_table = [ 43 | ("-", "-", "-"), 44 | (" property name ", " property value ", " description "), 45 | ("=", "=", "="), 46 | ] 47 | table = header_table + fields_table 48 | 49 | col_lens = [max([len(row[index]) for row in table]) for index in range(3)] 50 | 51 | table = [enforce_col_length(row, col_lens) for row in table] 52 | table = unify_columns(table) 53 | 54 | return "\n".join(table) 55 | 56 | 57 | print(generate_table()) 58 | -------------------------------------------------------------------------------- /doc/source/public-api/launchers.rst: -------------------------------------------------------------------------------- 1 | Launchers 2 | ========= 3 | 4 | .. code-block:: python 5 | 6 | import kloch.launchers 7 | 8 | Contexts 9 | -------- 10 | 11 | .. autoclass:: kloch.launchers.LauncherContext 12 | :members: 13 | :undoc-members: 14 | 15 | .. autoclass:: kloch.launchers.LauncherPlatform 16 | :members: 17 | :undoc-members: 18 | 19 | 20 | Getters 21 | ------- 22 | 23 | .. autofunction:: kloch.launchers.get_available_launchers_classes 24 | 25 | .. autofunction:: kloch.launchers.get_available_launchers_serialized_classes 26 | 27 | 28 | Plugins 29 | ------- 30 | 31 | .. autoclass:: kloch.launchers.LoadedPluginsLaunchers 32 | 33 | .. autofunction:: kloch.launchers.check_launcher_plugins 34 | 35 | .. autofunction:: kloch.launchers.is_launcher_plugin 36 | 37 | 38 | Serialized 39 | ---------- 40 | 41 | .. autoclass:: kloch.launchers.LauncherSerializedDict 42 | :members: 43 | :show-inheritance: 44 | 45 | .. autoclass:: kloch.launchers.LauncherSerializedList 46 | :members: 47 | :show-inheritance: 48 | 49 | 50 | BaseLauncher 51 | ------------ 52 | 53 | .. autoclass:: kloch.launchers.BaseLauncher 54 | :members: 55 | :inherited-members: 56 | :show-inheritance: 57 | 58 | .. autoclass:: kloch.launchers.BaseLauncherSerialized 59 | :members: 60 | :show-inheritance: 61 | 62 | .. autoclass:: kloch.launchers.BaseLauncherFields 63 | :members: 64 | 65 | 66 | BaseLauncher Subclasses 67 | ----------------------- 68 | 69 | .. autoclass:: kloch.launchers.SystemLauncher 70 | :members: 71 | :inherited-members: 72 | :show-inheritance: 73 | 74 | .. autoclass:: kloch.launchers.SystemLauncherSerialized 75 | :members: 76 | :show-inheritance: 77 | 78 | .. autoclass:: kloch.launchers.PythonLauncher 79 | :members: 80 | :inherited-members: 81 | :show-inheritance: 82 | 83 | .. autoclass:: kloch.launchers.PythonLauncherSerialized 84 | :members: 85 | :show-inheritance: 86 | -------------------------------------------------------------------------------- /kloch/launchers/system/_dataclass.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import logging 3 | import shutil 4 | import subprocess 5 | from pathlib import Path 6 | from typing import List 7 | from typing import Optional 8 | 9 | from kloch.launchers import BaseLauncher 10 | 11 | 12 | LOGGER = logging.getLogger(__name__) 13 | 14 | 15 | @dataclasses.dataclass 16 | class SystemLauncher(BaseLauncher): 17 | """ 18 | A minimal launcher that just start a subprocess with the given command. 19 | """ 20 | 21 | name = ".system" 22 | 23 | command_as_str: bool = False 24 | 25 | subprocess_kwargs: dict = dataclasses.field(default_factory=dict) 26 | """ 27 | Mapping of kwargs to pass to python's 'subprocess.run' call. 28 | """ 29 | 30 | expand_first_arg: bool = False 31 | 32 | def execute(self, tmpdir: Path, command: Optional[List[str]] = None): 33 | """ 34 | Just call subprocess.run. 35 | """ 36 | _command = self.command + (command or []) 37 | 38 | if self.expand_first_arg: 39 | toexpand = _command.pop(0) 40 | env_path = self.environ.get("PATH") 41 | expanded = shutil.which(toexpand, path=env_path) 42 | if not expanded: 43 | raise FileNotFoundError( 44 | f"Could not expand the '{toexpand}' argument; " 45 | f"searched PATH variable '{env_path}'" 46 | ) 47 | _command.insert(0, expanded) 48 | 49 | if self.command_as_str: 50 | _command = subprocess.list2cmdline(_command) 51 | 52 | LOGGER.debug( 53 | f"subprocess.run({_command}, env={self.environ}, cwd={self.cwd}, **{self.subprocess_kwargs})" 54 | ) 55 | result = subprocess.run( 56 | _command, 57 | env=self.environ, 58 | cwd=self.cwd, 59 | **self.subprocess_kwargs, 60 | ) 61 | 62 | return result.returncode 63 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributing guidelines and workflow for developers. 4 | 5 | ## pre-requisites 6 | 7 | - `git` is available on your system 8 | - `poetry` is available on your system 9 | 10 | ## getting started 11 | 12 | ```shell 13 | cd /some/place/to/develop 14 | git clone https://github.com/knotsanimation/kloch.git 15 | cd kloch 16 | # start the python venv 17 | poetry shell 18 | poetry install --all-extras 19 | # create and checkout new branch, DON'T work on main ! 20 | git checkout -b 21 | ``` 22 | 23 | ## running tests 24 | 25 | ```shell 26 | # useless if you used `install --all-extras` 27 | poetry install --extras tests 28 | python -m pytest ./tests -s 29 | ``` 30 | 31 | ## building documentation 32 | 33 | build from scratch once: 34 | 35 | ```shell 36 | # useless if you used `install --all-extras` 37 | poetry install --extras doc 38 | python .doc/build-doc.py 39 | ``` 40 | 41 | build from scratch but start live changes detection: 42 | 43 | ```shell 44 | # useless if you used `install --all-extras` 45 | poetry install --extras doc 46 | python .doc/serve-doc.py 47 | ``` 48 | 49 | ## publishing 50 | 51 | `kloch` is published on [PyPI](https://pypi.org/project/kloch/). 52 | This is an automatic process performed by GitHub actions `pypi.yml` 53 | (in `.github/workflows`). 54 | This action is triggered when a new GitHub release is created. 55 | 56 | For validation `kloch` is also published 57 | to [TestPyPI](https://test.pypi.org/project/kloch/). 58 | This is also an automatic process performed by GitHub actions `pypi-test.yml` 59 | (in `.github/workflows`). 60 | This action is triggered when a PR is set "ready-for-review" or when a commit 61 | is pushed to the `main` branch. 62 | 63 | Note for the TestPyPI publish: 64 | - TestPyPI is "temporary", repo can be deleted at any time and would need 65 | to be recreated. 66 | - authentification is done via an API token which may expire at some point and 67 | would need to be recreated (and added back as secret to this GitHub repo). 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kloch 2 | 3 | [![documentation badge](https://img.shields.io/badge/documentation-blue?style=flat&logo=readthedocs&logoColor=white)](https://knotsanimation.github.io/kloch/) 4 | ![Made with Python](https://img.shields.io/badge/Python->=3.7-blue?logo=python&logoColor=white) 5 | 6 | 7 | > [!WARNING] 8 | > Despite being public, this repository is still in development stage and 9 | > have not been tested extensively yet. 10 | 11 | ![banner with logo and logotype](./doc/source/_static/banner.svg) 12 | 13 | ``kloch`` _/klˈoʃ/_ is a configuration system for launching software. 14 | Configurations are yaml files referred as "environment profile" which specify 15 | the parameters for one or multiple pre-defined _launchers_. 16 | 17 | _Launchers_ are internally defined python objects that specify how to execute 18 | a combinations of options and (optional) command. 19 | 20 | In a very abstract way, `kloch` is a system that: 21 | - serialize the arguments passed to a pre-defined function as yaml files. 22 | - execute that function by unserializing the parameters provided at runtime. 23 | 24 | `kloch` was initially designed as the environment manager layer when used with 25 | the [rez](https://rez.readthedocs.io) package manager. 26 | 27 | ## features 28 | 29 | - offer a CLI and a public python API 30 | - custom config format for environment profiles: 31 | - inheritance 32 | - inheritance merging rules with token system 33 | - arbitrary profile locations with flexible configuration 34 | - plugin system for launchers 35 | 36 | ## programming distinctions 37 | 38 | - robust logging. 39 | - good amount of unittesting 40 | - good documentation with a lot of doc created at code level 41 | - python 3.7+ support (EOL was in June 2023). 42 | - PyYAML as only mandatory external python dependency. 43 | - flexible "meta" configuration from environment variable or config file. 44 | - clear public and private API 45 | 46 | ## quick-start 47 | 48 | ```shell 49 | pip install kloch 50 | kloch --help 51 | # or: 52 | python -m kloch --help 53 | ``` 54 | 55 | 56 | ## documentation 57 | 58 | Check the detailed documentation at https://knotsanimation.github.io/kloch/ 59 | -------------------------------------------------------------------------------- /tests/test_launchers_context.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | from kloch.launchers._context import LauncherContext 4 | from kloch.launchers._context import LauncherPlatform 5 | from kloch.launchers._context import unserialize_context_expression 6 | from kloch.launchers._context import resolve_context_expression 7 | 8 | 9 | def test__ProfileContext(): 10 | 11 | ctx1 = LauncherContext(platform=LauncherPlatform.linux, user="unittest") 12 | ctx2 = LauncherContext(platform=LauncherPlatform.linux, user="unittest") 13 | assert ctx1 == ctx2 14 | 15 | ctx3 = LauncherContext(platform=LauncherPlatform.linux) 16 | assert ctx1 == ctx3 17 | 18 | ctx4 = LauncherContext(platform=LauncherPlatform.windows) 19 | assert ctx1 != ctx4 20 | 21 | assert LauncherContext.create_from_system() == LauncherContext.create_from_system() 22 | 23 | 24 | def test__ProfileContext__devmistake(): 25 | for field in dataclasses.fields(LauncherContext): 26 | assert "unserialize" in field.metadata 27 | assert "serialized_name" in field.metadata 28 | assert "doc" in field.metadata 29 | assert "value" in field.metadata["doc"] 30 | assert "description" in field.metadata["doc"] 31 | 32 | 33 | def test__unserialize_context_expression(): 34 | source = "he!$69@os=windows" 35 | result = unserialize_context_expression(source) 36 | assert result.platform == LauncherPlatform.windows 37 | assert result.user is None 38 | 39 | source = "he!$69@os=windows@user=babos@@mik" 40 | result = unserialize_context_expression(source) 41 | assert result.platform == LauncherPlatform.windows 42 | assert result.user == "babos@mik" 43 | 44 | source = "wow@@gmail.com" 45 | result = unserialize_context_expression(source) 46 | assert result.platform is None 47 | assert result.user is None 48 | 49 | 50 | def test__resolve_context_expression(): 51 | source = "he!$69@os=windows" 52 | result = resolve_context_expression(source) 53 | expected = "he!$69" 54 | assert result == expected 55 | 56 | source = "he!$69@os=windows@user=babos@@mik" 57 | result = resolve_context_expression(source) 58 | expected = "he!$69" 59 | assert result == expected 60 | 61 | source = "wow@@gmail.com" 62 | result = resolve_context_expression(source) 63 | expected = "wow@gmail.com" 64 | assert result == expected 65 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "kloch" 3 | # keep in sync with __init__.py 4 | version = "0.13.1" 5 | description = "Environment Manager CLI wrapping package managers calls as serialized config file" 6 | authors = ["Liam Collod "] 7 | readme = "README.md" 8 | license = "LICENSE.txt" 9 | documentation = "https://knotsanimation.github.io/kloch/" 10 | keywords = ["environment", "kloch", "packaging", "managing"] 11 | classifiers = [ 12 | "Development Status :: 4 - Beta", 13 | "Intended Audience :: Developers", 14 | "Programming Language :: Python :: 3", 15 | "Programming Language :: Python :: 3.7", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Programming Language :: Python :: 3.10", 19 | "Programming Language :: Python :: 3.11", 20 | ] 21 | [tool.poetry.urls] 22 | # https://peps.python.org/pep-0753/ 23 | "homepage" = "https://github.com/knotsanimation/kloch" 24 | "issues" = "https://github.com/knotsanimation/kloch/issues" 25 | "github" = "https://github.com/knotsanimation/kloch" 26 | "releasenotes" = "https://knotsanimation.github.io/kloch/changelog.html" 27 | 28 | [tool.poetry.dependencies] 29 | python = ">=3.7" 30 | PyYAML = "^6.0.1" 31 | 32 | # // extras 33 | black = { version = "^24.4.2", python = ">=3.9", optional = true } 34 | pytest = { version = "7.3.2", python = ">=3.7", optional = true } 35 | # XXX: issue with CI on unix machine 36 | typing-extensions = { version = "<4.8" } 37 | Sphinx = { version = "^7.3.7", python = ">=3.9", optional = true } 38 | sphinx-autobuild = { version = "2024.2.4", python = ">=3.9", optional = true } 39 | sphinx-exec-code = { version = "0.12", python = ">=3.9", optional = true } 40 | sphinx-copybutton = { version = "0.5", python = ">=3.9", optional = true } 41 | furo = { version = "^2024.4.27", python = ">=3.9", optional = true } 42 | myst_parser = { version = "^3", python = ">=3.9,<3.13", optional = true } 43 | 44 | [tool.poetry.extras] 45 | dev = [ 46 | "black", 47 | ] 48 | tests = [ 49 | "pytest", 50 | "typing-extensions", 51 | ] 52 | doc = [ 53 | "Sphinx", 54 | "myst_parser", 55 | "sphinx-autobuild", 56 | "sphinx-exec-code", 57 | "sphinx-copybutton", 58 | "furo", 59 | ] 60 | 61 | [tool.poetry.scripts] 62 | kloch = 'kloch.cli:run_cli' 63 | 64 | [build-system] 65 | requires = ["poetry-core"] 66 | build-backend = "poetry.core.masonry.api" 67 | -------------------------------------------------------------------------------- /doc/source/_static/logotype.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /kloch/launchers/_get.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from typing import Optional 3 | from typing import Type 4 | from typing import TypeVar 5 | from typing import Union 6 | 7 | import kloch.launchers 8 | from kloch.launchers import BaseLauncher 9 | from kloch.launchers import BaseLauncherSerialized 10 | from ._plugins import LoadedPluginsLaunchers 11 | 12 | 13 | T = TypeVar("T") 14 | 15 | 16 | def _collect_launchers( 17 | natives: List[Type[T]], 18 | plugins: Optional[LoadedPluginsLaunchers[Type[T]]] = None, 19 | ) -> List[Type[T]]: 20 | """ 21 | Combine given launchers to flat list of unique objects. 22 | """ 23 | plugins = plugins.launchers if plugins else [] 24 | launchers = natives + plugins 25 | # remove duplicates but preserve order 26 | return [ 27 | launcher 28 | for index, launcher in enumerate(launchers) 29 | if launcher not in launchers[:index] 30 | ] 31 | 32 | 33 | def get_available_launchers_classes( 34 | plugins: Optional[LoadedPluginsLaunchers[Type[BaseLauncher]]] = None, 35 | ) -> List[Type[BaseLauncher]]: 36 | """ 37 | Collect all the launchers classes available. 38 | 39 | This is simply the given plugin launchers + builtin launchers. 40 | 41 | Args: 42 | plugins: collection of launcher loaded from plugins. 43 | """ 44 | # noinspection PyProtectedMember 45 | return _collect_launchers( 46 | natives=kloch.launchers._BUILTINS_LAUNCHERS.copy(), 47 | plugins=plugins, 48 | ) 49 | 50 | 51 | def get_available_launchers_serialized_classes( 52 | plugins: Optional[LoadedPluginsLaunchers[Type[BaseLauncherSerialized]]] = None, 53 | ) -> List[Type[BaseLauncherSerialized]]: 54 | """ 55 | Collect all the serialized launchers classes available. 56 | 57 | This is simply the given plugin launchers + builtin launchers. 58 | 59 | Args: 60 | plugins: collection of launcher loaded from plugins 61 | """ 62 | # noinspection PyProtectedMember 63 | return _collect_launchers( 64 | natives=kloch.launchers._BUILTINS_LAUNCHERS_SERIALIZED.copy(), 65 | plugins=plugins, 66 | ) 67 | 68 | 69 | # noinspection PyProtectedMember 70 | def is_launcher_plugin( 71 | launcher: Union[Type[BaseLauncher], Type[BaseLauncherSerialized]] 72 | ) -> bool: 73 | """ 74 | Return True if the given launcher is an external plugin else False if builtin. 75 | """ 76 | if launcher in kloch.launchers._BUILTINS_LAUNCHERS: 77 | return False 78 | return launcher not in kloch.launchers._BUILTINS_LAUNCHERS_SERIALIZED 79 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | Collection of various ideas to improve kloch. 4 | 5 | | icon | priority | 6 | |-------|----------| 7 | | 🔴 | high | 8 | | 🟠 | medium | 9 | | 🟡 | normal | 10 | | ⚪ | unsure | 11 | 12 | 13 | ## features 14 | 15 | - [ ] 🔴 allow to create "on-the-fly" profile directly from the command line 16 | - [ ] 🟠 allow multiples arg in `inherit` 17 | - [ ] 🟠 do something with the version attribute ? allow duplicate identifier in path, but with different version ? 18 | - [ ] 🟡 private profiles ? with a dot prefix signifying they can only be inherited and not used directly ? 19 | - [ ] ⚪ set cwd for relative paths 20 | - [ ] ⚪ token replacement system ? 21 | - [ ] PROFILE_DIR token 22 | - [ ] environment variable resolving ? 23 | - [ ] 🟡 allow `python_file` to be an url to a python file to download 24 | - [x] ~~🟠 introduce operating system functions ? like maybe tokens or if conditions ?~~ 25 | - [x] ~~🔴 allow to specify an absolute path to a profile instead of identifier~~ 26 | - [x] ~~log on disk ?~~ 27 | - [x] ~~validation of keys/value on profile read~~ 28 | - [x] ~~add a "python/pip/poetry" package manager ?~~ (implemented as launcher with kiche) 29 | - [x] ~~use `+-` to append command~~ (rejected) 30 | - [x] ~~new merge rule `!=` create if doesn't exist (for dict keys)~~ 31 | 32 | ## refacto 33 | 34 | - [ ] ⚪ ensure environment can be reproducible 35 | - always store them as an intermediate resolved file before execution ? 36 | - remove all resolving from launcher and perform all of this upstream ? 37 | - store intermediate resolved file at user-defined location OR tmp location ? 38 | Including also the tmp dir passed to launcher.execute() ? 39 | - define unique hash for dir ? like `timestamp-machine-user-uuid` 40 | - define config param to auto clear archived entries after X time 41 | - perhaps its too complicated to achieve, so achieve a middle ground, where 42 | you resolve the requirements formulated in the profile and store them on disk, 43 | (code at Serialized level) then use that file to launch. 44 | - [ ] ⚪ abstract `subprocess.run` in BaseLauncher and offer a `prepare_execution` 45 | abstractmethod instead. 46 | - [x] ~~naming of thing ? package manager could just be "launchers"~~ 47 | - [x] ~~internalise PyYaml dependenciyes (add it to vendor module)~~ (impossible) 48 | - [x] ~~change default merge rule to be append and add token to specify explicit override~~ 49 | - `-=` remove 50 | - `==` override 51 | - `+=` append (default) 52 | - [x] ~~rename profile `base` key to `inherit` (limit similarities with `.base`)~~ 53 | 54 | ## chore 55 | 56 | - [x] ~~doc: explain how to add a new package manager support~~ 57 | - [x] ~~doc: replace index h1 with svg logotype~~ -------------------------------------------------------------------------------- /kloch/launchers/python/_serialized.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import logging 3 | import os 4 | from pathlib import Path 5 | from typing import Dict 6 | from typing import List 7 | 8 | from kloch.launchers import BaseLauncherSerialized 9 | from kloch.launchers import BaseLauncherFields 10 | from ._dataclass import PythonLauncher 11 | 12 | LOGGER = logging.getLogger(__name__) 13 | 14 | 15 | # noinspection PyTypeChecker 16 | @dataclasses.dataclass(frozen=True) 17 | class PythonLauncherFields(BaseLauncherFields): 18 | python_file: str = dataclasses.field( 19 | default="python_file", 20 | metadata={ 21 | "description": ( 22 | "Filesystem path to an existing python file.\n" 23 | "The path will have environment variables expanded with ``os.expandvars`` [1]_.\n" 24 | "The path is turned absolute and normalized. [4]_\n" 25 | ), 26 | "required": True, 27 | }, 28 | ) 29 | # we override just for the metadata attribute 30 | # noinspection PyDataclass 31 | command: List[str] = dataclasses.field( 32 | default=BaseLauncherFields.command, 33 | metadata={ 34 | "description": "Arbitrary list of command line arguments passed to the python file.", 35 | "required": False, 36 | }, 37 | ) 38 | 39 | 40 | class PythonLauncherSerialized(BaseLauncherSerialized): 41 | source = PythonLauncher 42 | 43 | identifier = PythonLauncher.name 44 | 45 | fields = PythonLauncherFields 46 | 47 | summary = ( 48 | "A launcher that execute the given python file with kloch's own interpreter." 49 | ) 50 | description = summary + ( 51 | "\n\n" 52 | "Execute the given python file with the python interpreter used to run kloch.\n" 53 | "\n" 54 | "Any command will be basse as command line arguments to the script." 55 | ) 56 | 57 | def validate(self): 58 | super().validate() 59 | python_file = self.fields.python_file 60 | assert python_file in self, f"'{python_file}': missing or empty attribute." 61 | assert isinstance(self[python_file], str), f"'{python_file}': must be a str." 62 | 63 | def resolved(self) -> Dict: 64 | resolved = super().resolved() 65 | python_file = self.fields.python_file 66 | 67 | old_python_file = self[python_file] 68 | new_python_file = Path(os.path.expandvars(old_python_file)).absolute().resolve() 69 | resolved[python_file] = str(new_python_file) 70 | 71 | return resolved 72 | 73 | # we override for type-hint 74 | def unserialize(self) -> PythonLauncher: 75 | # noinspection PyTypeChecker 76 | return super().unserialize() 77 | -------------------------------------------------------------------------------- /kloch/launchers/system/_serialized.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import logging 3 | 4 | from kloch.launchers import BaseLauncherSerialized 5 | from kloch.launchers import BaseLauncherFields 6 | from ._dataclass import SystemLauncher 7 | 8 | LOGGER = logging.getLogger(__name__) 9 | 10 | 11 | @dataclasses.dataclass(frozen=True) 12 | class SystemLauncherFields(BaseLauncherFields): 13 | 14 | command_as_str: bool = dataclasses.field( 15 | default="command_as_str", 16 | metadata={ 17 | "description": ( 18 | "If True a str is passed to ``subprocess.run``, else a list is passed. " 19 | "This can be useful in the context of setting ``shell=True`` or not on UNIX platforms." 20 | ), 21 | "required": False, 22 | }, 23 | ) 24 | 25 | subprocess_kwargs: dict = dataclasses.field( 26 | default="subprocess_kwargs", 27 | metadata={ 28 | "description": ( 29 | "Mapping of kwargs to pass to the internal ``subprocess.run`` call." 30 | ), 31 | "required": False, 32 | }, 33 | ) 34 | 35 | expand_first_arg: bool = dataclasses.field( 36 | default="expand_first_arg", 37 | metadata={ 38 | "description": ( 39 | "If True the first argument of the passed command will be expanded " 40 | "using ``shutil.which`` to find its executable file on disk. Will " 41 | "raise if no path is found.\n\n" 42 | "Useful for avoiding using ``shell=True`` in ``subprocess_kwargs``." 43 | ), 44 | "required": False, 45 | }, 46 | ) 47 | 48 | 49 | class SystemLauncherSerialized(BaseLauncherSerialized): 50 | source = SystemLauncher 51 | 52 | identifier = SystemLauncher.name 53 | 54 | fields = SystemLauncherFields 55 | 56 | summary = ( 57 | "A simple launcher executing the given command in the default system console." 58 | ) 59 | description = summary + ( 60 | "\n\n" 61 | "The launcher will just set the given environment variables for the session," 62 | "execute the command, then exit. Which make it useless without a command specified." 63 | "\n\n" 64 | "The launcher use ``subprocess.run`` to execute the command." 65 | ) 66 | 67 | def validate(self): 68 | subprocesskw = self.fields.subprocess_kwargs 69 | if subprocesskw in self: 70 | assert isinstance( 71 | self[subprocesskw], dict 72 | ), f"'{subprocesskw}': must be a dict." 73 | super().validate() 74 | 75 | # we override for type-hint 76 | def unserialize(self) -> SystemLauncher: 77 | # noinspection PyTypeChecker 78 | return super().unserialize() 79 | -------------------------------------------------------------------------------- /doc/source/_injected/snip-launcher-plugin-basic.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import http.client 3 | import subprocess 4 | from pathlib import Path 5 | from typing import List 6 | from typing import Optional 7 | 8 | from kloch.launchers import BaseLauncher 9 | from kloch.launchers import BaseLauncherSerialized 10 | from kloch.launchers import BaseLauncherFields 11 | 12 | 13 | @dataclasses.dataclass 14 | class GitCloneLauncher(BaseLauncher): 15 | 16 | name = "git-clone" 17 | 18 | # all field must have a default value, but we make it 19 | # required using the bellow `required_fields` class attribute 20 | remote_url: str = "" 21 | 22 | required_fields = ["remote_url"] 23 | 24 | def execute(self, tmpdir: Path, command: Optional[List[str]] = None): 25 | # we consider `command` are just extra args to git clone command 26 | _command = ["git", "clone", self.remote_url] + command 27 | subprocess.run(_command, env=self.environ, cwd=self.cwd) 28 | 29 | 30 | @dataclasses.dataclass(frozen=True) 31 | class GitCloneLauncherFields(BaseLauncherFields): 32 | 33 | # field name must be the same as in the BaseLauncher subclass above 34 | remote_url: str = dataclasses.field( 35 | # this is the expected key name in the serialized representation 36 | default="remote-url", 37 | # this if for automated documentation generation 38 | metadata={ 39 | "description": "An URL to a valid remote git repository.", 40 | "required": True, 41 | }, 42 | ) 43 | 44 | 45 | def does_url_exists(url: str) -> bool: 46 | connection = http.client.HTTPConnection(url) 47 | connection.request("HEAD", "") 48 | return connection.getresponse().status < 400 49 | 50 | 51 | class GitCloneLauncherSerialized(BaseLauncherSerialized): 52 | # the class it serialize 53 | source = GitCloneLauncher 54 | 55 | # we can pick a different name but we keep it similar for simplicity 56 | identifier = GitCloneLauncher.name 57 | 58 | fields = GitCloneLauncherFields 59 | 60 | # short one line description of the launcher 61 | summary = "Just clone a repository from a git remote." 62 | 63 | # full documentation of an arbitrary length for the launcher 64 | description = "From a git remote repository url, git clone it in the current working directory." 65 | 66 | def validate(self): 67 | super().validate() 68 | remote_url = self.fields.remote_url 69 | assert remote_url in self, f"'{remote_url}': missing or empty attribute." 70 | assert does_url_exists( 71 | self[remote_url] 72 | ), f"'{remote_url}': url provided doesn't exists: {self[remote_url]}." 73 | 74 | # we override for type-hint 75 | def unserialize(self) -> GitCloneLauncher: 76 | # noinspection PyTypeChecker 77 | return super().unserialize() 78 | -------------------------------------------------------------------------------- /tests/test_launchers_plugins.py: -------------------------------------------------------------------------------- 1 | from kloch.launchers import BaseLauncher 2 | from kloch.launchers import BaseLauncherSerialized 3 | import kloch.launchers._plugins 4 | 5 | 6 | def test__load_plugin_launchers(data_dir, monkeypatch): 7 | 8 | plugin_path = data_dir / "plugins-behr" 9 | monkeypatch.syspath_prepend(plugin_path) 10 | 11 | loaded = kloch.launchers._plugins.load_plugin_launchers( 12 | module_names=["kloch_behr"], 13 | subclass_type=kloch.launchers.BaseLauncher, 14 | ) 15 | assert loaded.given == ["kloch_behr"] 16 | assert not loaded.missed 17 | assert len(loaded.launchers) == 1 18 | assert issubclass(loaded.launchers[0], kloch.launchers.BaseLauncher) 19 | assert loaded.launchers[0].name == "behr" 20 | 21 | loaded = kloch.launchers._plugins.load_plugin_launchers( 22 | module_names=["kloch_behr"], 23 | subclass_type=kloch.launchers.BaseLauncherSerialized, 24 | ) 25 | assert not loaded.missed 26 | assert len(loaded.launchers) == 1 27 | assert issubclass(loaded.launchers[0], kloch.launchers.BaseLauncherSerialized) 28 | assert loaded.launchers[0].identifier == "behr" 29 | 30 | plugin_path = data_dir / "plugins-tyfa" 31 | monkeypatch.syspath_prepend(plugin_path) 32 | 33 | loaded = kloch.launchers._plugins.load_plugin_launchers( 34 | module_names=["kloch_tyfa"], 35 | subclass_type=kloch.launchers.BaseLauncher, 36 | ) 37 | # XXX: tyfa import PythonLauncher so it is discovered 38 | assert len(loaded.launchers) == 2 39 | 40 | 41 | def test__load_plugin_launchers__missing(data_dir, monkeypatch): 42 | 43 | plugin_path = data_dir / "plugins-rerr" 44 | monkeypatch.syspath_prepend(plugin_path) 45 | 46 | loaded = kloch.launchers._plugins.load_plugin_launchers( 47 | module_names=["kloch_rerr"], 48 | subclass_type=kloch.launchers.BaseLauncher, 49 | ) 50 | assert len(loaded.missed) == 1 51 | assert "kloch_rerr" in loaded.missed 52 | assert not loaded.launchers 53 | 54 | 55 | def test__check_launcher_plugins(data_dir, monkeypatch): 56 | plugin_path = data_dir / "plugins-behr" 57 | monkeypatch.syspath_prepend(str(plugin_path)) 58 | plugin_path = data_dir / "plugins-rerr" 59 | monkeypatch.syspath_prepend(str(plugin_path)) 60 | 61 | plugins = kloch.launchers.load_plugin_launchers( 62 | ["kloch_behr"], BaseLauncherSerialized 63 | ) 64 | errors = kloch.launchers._plugins.check_launcher_plugins(plugins) 65 | 66 | assert not errors 67 | 68 | plugins = kloch.launchers.load_plugin_launchers( 69 | ["kloch_rerr"], BaseLauncherSerialized 70 | ) 71 | errors = kloch.launchers._plugins.check_launcher_plugins(plugins) 72 | error1 = errors[0] 73 | assert isinstance(error1, kloch.launchers._plugins.PluginModuleError) 74 | assert BaseLauncher.__name__ in str(error1) 75 | -------------------------------------------------------------------------------- /tests/test_profile.py: -------------------------------------------------------------------------------- 1 | from kloch import EnvironmentProfile 2 | from kloch.launchers import LauncherSerializedDict 3 | 4 | 5 | def test__EnvironmentProfile__merging(): 6 | profile1 = EnvironmentProfile( 7 | identifier="knots", 8 | version="0.1.0", 9 | inherit=None, 10 | launchers=LauncherSerializedDict( 11 | { 12 | "rezenv": { 13 | "config": {"exclude": "whatever"}, 14 | "==requires": { 15 | "echoes": "2", 16 | "maya": "2023", 17 | }, 18 | "+=tests": { 19 | "+=foo": [1, 2, 3], 20 | "deeper!": {"as deep": [1, 2]}, 21 | }, 22 | }, 23 | "testenv": {"command": "echo $cwd"}, 24 | } 25 | ), 26 | ) 27 | profile2 = EnvironmentProfile( 28 | identifier="knots:echoes", 29 | version="0.1.0", 30 | inherit=profile1, 31 | launchers=LauncherSerializedDict( 32 | { 33 | "+=rezenv": { 34 | "==config": {"include": "yes"}, 35 | "requires": { 36 | "-=maya": "_", 37 | "-=notAdded": "_", 38 | "added": "1.2", 39 | }, 40 | "+=tests": { 41 | "foo": [4, 5, 6], 42 | "+=new-echoes-key": {"working": True}, 43 | "==deeper!": {"+=as deep": [0, 0]}, 44 | }, 45 | } 46 | } 47 | ), 48 | ) 49 | result = profile2.get_merged_profile().launchers 50 | expected = { 51 | "+=rezenv": { 52 | "==config": {"include": "yes"}, 53 | "requires": { 54 | "echoes": "2", 55 | "added": "1.2", 56 | }, 57 | "+=tests": { 58 | "foo": [1, 2, 3, 4, 5, 6], 59 | "+=new-echoes-key": {"working": True}, 60 | "==deeper!": {"+=as deep": [0, 0]}, 61 | }, 62 | }, 63 | "testenv": {"command": "echo $cwd"}, 64 | } 65 | assert result == expected 66 | 67 | result = profile2.get_merged_profile().launchers.resolved() 68 | expected = { 69 | "rezenv": { 70 | "config": {"include": "yes"}, 71 | "requires": { 72 | "echoes": "2", 73 | "added": "1.2", 74 | }, 75 | "tests": { 76 | "foo": [1, 2, 3, 4, 5, 6], 77 | "new-echoes-key": {"working": True}, 78 | "deeper!": {"as deep": [0, 0]}, 79 | }, 80 | }, 81 | "testenv": {"command": "echo $cwd"}, 82 | } 83 | assert result == expected 84 | -------------------------------------------------------------------------------- /tests/test_launchers_get.py: -------------------------------------------------------------------------------- 1 | import kloch 2 | import kloch.launchers 3 | from kloch.launchers import get_available_launchers_classes 4 | from kloch.launchers import is_launcher_plugin 5 | from kloch.launchers._plugins import _check_launcher_serialized_implementation 6 | from kloch.launchers._plugins import _check_launcher_implementation 7 | from kloch.launchers import get_available_launchers_serialized_classes 8 | 9 | 10 | def test__get_available_launchers_classes(data_dir, monkeypatch): 11 | plugin_path = data_dir / "plugins-behr" 12 | monkeypatch.syspath_prepend(plugin_path) 13 | plugin_path = data_dir / "plugins-tyfa" 14 | monkeypatch.syspath_prepend(plugin_path) 15 | launchers = get_available_launchers_classes() 16 | assert len(launchers) == len(kloch.launchers._BUILTINS_LAUNCHERS) 17 | 18 | loaded = kloch.launchers.load_plugin_launchers( 19 | ["kloch_behr", "kloch_tyfa"], 20 | kloch.launchers.BaseLauncher, 21 | ) 22 | launchers = get_available_launchers_classes(plugins=loaded) 23 | assert len(launchers) == len(kloch.launchers._BUILTINS_LAUNCHERS) + 2 24 | 25 | 26 | # ensure the native subclasses have properly overridden class attributes 27 | # (this is because there is no way of having abstract attribute with abc module) 28 | def test__launchers__implementation(): 29 | for launcher in get_available_launchers_classes(): 30 | _check_launcher_implementation(launcher) 31 | 32 | 33 | # ensure developer didn't missed to add documentation before adding a new subclass 34 | def test__launchers__serialized_implementation(): 35 | # ensure each launcher have a serialized class 36 | assert len(get_available_launchers_classes()) == len( 37 | get_available_launchers_serialized_classes() 38 | ) 39 | for serialized in get_available_launchers_serialized_classes(): 40 | _check_launcher_serialized_implementation(serialized) 41 | 42 | 43 | def test__is_launcher_plugin(data_dir, monkeypatch): 44 | plugin_path = data_dir / "plugins-behr" 45 | monkeypatch.syspath_prepend(plugin_path) 46 | plugin_path = data_dir / "plugins-tyfa" 47 | monkeypatch.syspath_prepend(plugin_path) 48 | 49 | loaded = kloch.launchers.load_plugin_launchers( 50 | ["kloch_behr", "kloch_tyfa"], 51 | kloch.launchers.BaseLauncher, 52 | ) 53 | results = [is_launcher_plugin(launcher) for launcher in loaded.launchers] 54 | # XXX: kloch_tyfa will return the native PythonLauncher 55 | assert not all(results), loaded.launchers 56 | assert results == [True, False, True] 57 | 58 | loaded = kloch.launchers.load_plugin_launchers( 59 | ["kloch_behr", "kloch_tyfa"], 60 | kloch.launchers.BaseLauncherSerialized, 61 | ) 62 | results = [is_launcher_plugin(launcher) for launcher in loaded.launchers] 63 | assert all(results), loaded.launchers 64 | assert results == [True, True] 65 | -------------------------------------------------------------------------------- /kloch/filesyntax/_profile.py: -------------------------------------------------------------------------------- 1 | """ 2 | We define a simple config system that describe how to build a software environment using 3 | different "software launchers". 4 | 5 | The config system can handle the merging of 2 configs structure. 6 | """ 7 | 8 | import copy 9 | import dataclasses 10 | from typing import Any 11 | from typing import Dict 12 | from typing import Optional 13 | 14 | from kloch.launchers import LauncherSerializedDict 15 | 16 | 17 | @dataclasses.dataclass 18 | class EnvironmentProfile: 19 | """ 20 | A profile is a collection of parameters required to start a pre-defined launcher. 21 | 22 | This can be seen as the context/environment necessary to run a launcher thus its 23 | full name 'Environment Profile' that we abbreviate to profile for convenience. 24 | 25 | Profiles can inherit each other by specifying a `inherit` attribute. The inheritance 26 | only merge the content of the ``launchers`` attribute between 2 profiles. 27 | """ 28 | 29 | identifier: str 30 | version: str 31 | inherit: Optional["EnvironmentProfile"] 32 | launchers: LauncherSerializedDict 33 | 34 | @classmethod 35 | def from_dict(cls, serialized: Dict) -> "EnvironmentProfile": 36 | """ 37 | Generate a profile instance from a serialized dict object. 38 | 39 | No type checking is performed and the user is reponsible for the correct 40 | type being stored in the dict. 41 | """ 42 | identifier: str = serialized["identifier"] 43 | version: str = serialized["version"] 44 | inherit: Optional["EnvironmentProfile"] = serialized.get("inherit", None) 45 | launchers: LauncherSerializedDict = serialized["launchers"] 46 | 47 | return EnvironmentProfile( 48 | identifier=identifier, 49 | version=version, 50 | inherit=inherit, 51 | launchers=launchers, 52 | ) 53 | 54 | def to_dict(self) -> Dict[str, Any]: 55 | """ 56 | Convert a profile instance to a serialized dict object. 57 | """ 58 | serialized = { 59 | "identifier": self.identifier, 60 | "version": self.version, 61 | } 62 | if self.inherit: 63 | serialized["inherit"] = self.inherit 64 | 65 | serialized["launchers"] = copy.deepcopy(self.launchers) 66 | return serialized 67 | 68 | def get_merged_profile(self): 69 | """ 70 | Resolve the inheritance the profile might have over another profile. 71 | 72 | Returns: 73 | a new instance. 74 | """ 75 | launchers = self.launchers 76 | if self.inherit: 77 | launchers = self.inherit.get_merged_profile().launchers + launchers 78 | 79 | return EnvironmentProfile( 80 | identifier=self.identifier, 81 | version=self.version, 82 | inherit=None, 83 | launchers=launchers, 84 | ) 85 | -------------------------------------------------------------------------------- /tests/test_launchers_python.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | import pytest 5 | 6 | import kloch 7 | from kloch.launchers import BaseLauncherSerialized 8 | from kloch.launchers import PythonLauncherSerialized 9 | from kloch.launchers import PythonLauncher 10 | 11 | 12 | def test__PythonLauncher(tmp_path, data_dir, capfd): 13 | script_path = data_dir / "test-script-a.py" 14 | launcher = PythonLauncher( 15 | command=["first arg"], 16 | python_file=str(script_path), 17 | environ=os.environ.copy(), 18 | ) 19 | expected_argv = [str(script_path), "first arg", "second arg"] 20 | 21 | launcher.execute(command=["second arg"], tmpdir=tmp_path) 22 | result = capfd.readouterr() 23 | assert f"{kloch.__name__} test script working" in result.out 24 | result_out = result.out.rstrip("\n").rstrip("\r") 25 | assert result_out.endswith(f"{str(expected_argv)}") 26 | 27 | 28 | def test__PythonLauncherSerialized__add(): 29 | # we test BaseLauncherSerialized.__add__ 30 | 31 | instance1 = BaseLauncherSerialized( 32 | { 33 | "command": ["first arg"], 34 | } 35 | ) 36 | instance2 = PythonLauncherSerialized( 37 | { 38 | "+=command": ["second arg", "third arg"], 39 | } 40 | ) 41 | merged = instance1 + instance2 42 | assert isinstance(merged, PythonLauncherSerialized) 43 | assert merged == { 44 | "+=command": ["first arg", "second arg", "third arg"], 45 | } 46 | 47 | 48 | def test__PythonLauncherSerialized(data_dir, monkeypatch): 49 | src_dict = { 50 | "command": ["first arg", "second arg"], 51 | } 52 | instance = PythonLauncherSerialized(src_dict) 53 | with pytest.raises(AssertionError): 54 | instance.validate() 55 | 56 | script_path = data_dir / "test-script-a.py" 57 | 58 | src_dict = { 59 | "python_file": script_path, 60 | } 61 | instance = PythonLauncherSerialized(src_dict) 62 | with pytest.raises(AssertionError): 63 | instance.validate() 64 | 65 | src_dict = { 66 | "python_file": str(script_path), 67 | } 68 | instance = PythonLauncherSerialized(src_dict) 69 | instance.validate() 70 | resolved = instance.resolved() 71 | assert resolved["python_file"] == str(script_path) 72 | 73 | monkeypatch.setenv("XXUNITTEST", str(data_dir)) 74 | 75 | src_dict = { 76 | "python_file": "${XXUNITTEST}/test-script-a.py", 77 | } 78 | instance = PythonLauncherSerialized(src_dict) 79 | instance.validate() 80 | resolved = instance.resolved() 81 | assert resolved["python_file"] == str(script_path) 82 | 83 | launcher = instance.unserialize() 84 | assert isinstance(launcher, PythonLauncher) 85 | assert launcher.python_file == str(script_path) 86 | 87 | 88 | def test__PythonLauncherSerialized__fields(): 89 | base_fields = BaseLauncherSerialized.fields.iterate() 90 | python_fields = PythonLauncherSerialized.fields.iterate() 91 | assert len(python_fields) == len(base_fields) + 1 92 | -------------------------------------------------------------------------------- /doc/source/_injected/exec-config-autodoc.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Type 3 | 4 | import kloch 5 | 6 | RowType = tuple[str, str, str] 7 | 8 | 9 | def enforce_col_length( 10 | row: tuple[str, str, str], 11 | lengths: list[int], 12 | ) -> RowType: 13 | # noinspection PyTypeChecker 14 | return tuple( 15 | col.ljust(lengths[index], col[-1] if col else " ") 16 | for index, col in enumerate(row) 17 | ) 18 | 19 | 20 | def unify_columns(table: list[RowType]) -> list[str]: 21 | """ 22 | Add the columns separator to return a row string. 23 | """ 24 | new_table = [] 25 | for row in table: 26 | sep = "+" if row[-1][-1] in ["-", "="] else "|" 27 | row = sep + sep.join(row) + sep 28 | new_table.append(row) 29 | return new_table 30 | 31 | 32 | def create_field_table(field: dataclasses.Field) -> list[RowType]: 33 | field_name = field.name 34 | field_doc = field.metadata["documentation"] 35 | 36 | doc = field_doc.split("\n") 37 | 38 | # typing.Annotated 39 | if hasattr(field.type, "__metadata__"): 40 | ftype = field.type.__origin__ 41 | else: 42 | ftype = field.type 43 | 44 | ftype = str(ftype).replace("typing.", "") 45 | 46 | default_value = repr( 47 | field.default_factory() 48 | if field.default is dataclasses.MISSING 49 | else field.default 50 | ) 51 | 52 | env_var = field.metadata["environ"] 53 | 54 | rows = [ 55 | (f" .. option:: {field_name} ", " **type** ", f" `{ftype}` "), 56 | ("-", "-", "-"), 57 | ("", " **default** ", f" ``{default_value}`` "), 58 | ("", "-", "-"), 59 | ("", " **environment variable** ", f" ``{env_var}`` "), 60 | ("", "-", "-"), 61 | ("", " **description** ", f" {doc.pop(0)} "), 62 | ] 63 | 64 | for doc_line in doc: 65 | rows += [("", "", f" {doc_line} ")] 66 | 67 | rows += [("-", "-", "-")] 68 | return rows 69 | 70 | 71 | def replace_character(src_str: str, character: str, substitution: str) -> str: 72 | index = src_str.rfind(character, 0, -2) 73 | return src_str[:index] + substitution + src_str[index + 1 :] 74 | 75 | 76 | def generate_table(): 77 | fields = dataclasses.fields(kloch.KlochConfig) 78 | 79 | fields_table: list[RowType] = [] 80 | for field in fields: 81 | fields_table += create_field_table(field=field) 82 | 83 | header_table = [ 84 | ("-", "-", "-"), 85 | (" key name ", "", ""), 86 | ("=", "=", "="), 87 | ] 88 | table = header_table + fields_table 89 | 90 | col_lens = [max([len(row[index]) for row in table]) for index in range(3)] 91 | 92 | table = [enforce_col_length(row, col_lens) for row in table] 93 | table = unify_columns(table) 94 | 95 | table[0] = replace_character(table[0], "+", "-") 96 | table[1] = replace_character(table[1], "|", " ") 97 | 98 | return "\n".join(table) 99 | 100 | 101 | print(generate_table()) 102 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import logging 3 | from pathlib import Path 4 | 5 | import pytest 6 | 7 | import kloch.config 8 | 9 | 10 | def test__KlochConfig(): 11 | # ensure it doesn't raise errors 12 | kloch.config.KlochConfig() 13 | 14 | config = kloch.config.KlochConfig(cli_logging_default_level="DEBUG") 15 | assert config.cli_logging_default_level == "DEBUG" 16 | config = kloch.config.KlochConfig(cli_logging_default_level=logging.DEBUG) 17 | assert config.cli_logging_default_level == logging.DEBUG 18 | 19 | field = kloch.config.KlochConfig.get_field("cli_logging_default_level") 20 | assert field.name == "cli_logging_default_level" 21 | 22 | 23 | def test__KlochConfig__from_environment(monkeypatch, data_dir): 24 | config = kloch.config.KlochConfig.from_environment() 25 | assert config == kloch.config.KlochConfig() 26 | 27 | config_path = data_dir / "config-blaj.yml" 28 | monkeypatch.setenv(kloch.Environ.CONFIG_PATH, str(config_path)) 29 | config = kloch.config.KlochConfig.from_environment() 30 | assert config.cli_logging_default_level == "WARNING" 31 | assert config.cli_logging_format == "{levelname: <7}: {message}" 32 | assert len(config.cli_logging_paths) == 2 33 | assert isinstance(config.cli_logging_paths[0], Path) 34 | assert isinstance(config.cli_session_dir, Path) 35 | assert str(config.cli_session_dir) == str(config_path.parent / ".session") 36 | assert str(config.cli_logging_paths[0]) == str( 37 | config_path.parent / "tmp" / "kloch.log" 38 | ) 39 | 40 | monkeypatch.setenv("KLOCH_CONFIG_CLI_LOGGING_DEFAULT_LEVEL", "ERROR") 41 | config = kloch.config.KlochConfig.from_environment() 42 | assert config.cli_logging_default_level == "ERROR" 43 | assert config.cli_logging_format == "{levelname: <7}: {message}" 44 | 45 | 46 | def test__KlochConfig__from_file__expandvar(monkeypatch, data_dir, tmp_path: Path): 47 | monkeypatch.setenv("FOOBAR", str(tmp_path)) 48 | monkeypatch.setenv("THIRDIR", str(tmp_path / "third")) 49 | 50 | config_path = data_dir / "config-fuel.yml" 51 | config = kloch.config.KlochConfig.from_file(file_path=config_path) 52 | assert len(config.cli_logging_paths) == 2 53 | assert config.cli_logging_paths[0] == tmp_path / "kloch.log" 54 | assert config.cli_logging_paths[1] == config_path.parent / Path( 55 | "$FOOBAR/tmp/kloch2.log" 56 | ) 57 | assert config.cli_session_dir == Path(tmp_path / "third" / ".session") 58 | 59 | 60 | def test__KlochConfig__from_file__error(data_dir): 61 | config_path = data_dir / "config-molg.yml" 62 | with pytest.raises(TypeError) as error: 63 | config = kloch.config.KlochConfig.from_file(file_path=config_path) 64 | assert "NON_VALID_KEY" in str(error.value) 65 | 66 | 67 | def test__KlochConfig__documentation(): 68 | for field in dataclasses.fields(kloch.config.KlochConfig): 69 | assert field.metadata.get("documentation") 70 | config_cast = field.metadata.get("config_cast") 71 | assert config_cast 72 | # check the caster accept 2 arguments 73 | try: 74 | config_cast({}, Path()) 75 | except TypeError: 76 | pass 77 | assert field.metadata.get("environ") 78 | assert field.metadata.get("environ_cast") 79 | -------------------------------------------------------------------------------- /.github/workflows/pypi-test.yml: -------------------------------------------------------------------------------- 1 | name: testpypi publish 2 | # based on https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ 3 | 4 | on: 5 | pull_request: 6 | types: 7 | - opened 8 | - reopened 9 | - synchronize # (when you commit/push to) 10 | - ready_for_review 11 | paths-ignore: 12 | - 'doc/**' 13 | - 'tests/**' 14 | - 'CHANGELOG.md' 15 | - 'TODO.md' 16 | push: 17 | branches: 18 | # this allows to test if everything looks okay before doing a 19 | # git release (which will trigger the definitive pypi publish). 20 | - main 21 | 22 | jobs: 23 | publish-to-testpypi: 24 | name: Build and publish distribution to TestPyPI 25 | runs-on: ubuntu-latest 26 | 27 | # only runs on PR that are ready-for-review 28 | if: ${{ !github.event.pull_request.draft }} 29 | 30 | outputs: 31 | published_version: ${{steps.getversion.outputs.version}} 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | - name: Install Poetry 36 | # poetry 1.8.4 37 | run: pipx install git+https://github.com/python-poetry/poetry.git@6a071c181ad82ec3a5d4ecd54d50d08a75470e78 38 | # XXX: TestPyPI expect an unique version for each deploy 39 | # so we must create an unique version for each action run. 40 | # Note that this doesn't change the version in 'kloch/__init__.py' 41 | - name: Update version to be unique per run 42 | run: poetry version $(poetry version --short).dev${{ github.run_id }}${{ github.run_number }}${{ github.run_attempt }} 43 | - id: getversion 44 | run: echo "version=$(poetry version --short)" >> "$GITHUB_OUTPUT" 45 | - name: Configure poetry for TestPyPI 46 | run: poetry config repositories.test-pypi https://test.pypi.org/legacy/ 47 | - name: Poetry build and publish to TestPyPI 48 | run: poetry publish --build -u __token__ -p ${{ secrets.TESTPYPI_TOKEN }} --repository test-pypi 49 | # this is only stored for debugging purposes 50 | - name: Store the distribution packages 51 | uses: actions/upload-artifact@v4 52 | with: 53 | name: python-package-distributions 54 | path: dist/ 55 | 56 | test-pip-install: 57 | name: Pip install the package from TestPyPI 58 | # not latest because this is last version that supports python-3.7 59 | runs-on: ubuntu-22.04 60 | needs: publish-to-testpypi 61 | 62 | # only run on main because it's slow due to the 'sleep' we have to use 63 | if: ${{ github.ref == 'refs/heads/main' }} 64 | 65 | steps: 66 | # this is a hack to let PyPi refresh for the upload that just happened 67 | - name: Sleep for 1min30 68 | run: sleep 90s 69 | shell: bash 70 | # needed for pytest at the end 71 | - uses: actions/checkout@v4 72 | - uses: actions/setup-python@v5 73 | with: 74 | python-version: 3.7 75 | - name: Create venv 76 | run: python -m venv .testvenv 77 | - name: Activate venv 78 | run: source .testvenv/bin/activate 79 | - run: pip install -i https://test.pypi.org/simple/ kloch[tests]==${{needs.publish-to-testpypi.outputs.published_version}} --extra-index-url https://pypi.org/simple 80 | - name: test installed package 81 | run: python -m pytest ./tests -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | import sys 6 | 7 | import kloch 8 | from pathlib import Path 9 | 10 | # -- Project information ----------------------------------------------------- 11 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 12 | 13 | project = "kloch" 14 | copyright = "2024, Knots Animation" 15 | author = "Knots Animation" 16 | version = kloch.__version__ 17 | release = version 18 | 19 | # -- General configuration --------------------------------------------------- 20 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 21 | 22 | sys.path.append(str(Path("_extensions").resolve())) 23 | 24 | extensions = [ 25 | "myst_parser", 26 | "sphinx.ext.autodoc", 27 | "sphinx.ext.autosummary", 28 | "sphinx.ext.napoleon", 29 | "sphinx.ext.autosectionlabel", 30 | "sphinx_exec_code", 31 | "sphinx_copybutton", 32 | "execinject", 33 | ] 34 | 35 | templates_path = ["_templates"] 36 | exclude_patterns = [] 37 | 38 | autosectionlabel_prefix_document = True 39 | 40 | suppress_warnings = [ 41 | "autosectionlabel", 42 | ] 43 | 44 | # -- Options for HTML output ------------------------------------------------- 45 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 46 | 47 | html_theme = "furo" 48 | html_static_path = ["_static"] 49 | html_css_files = [ 50 | "extra.css", 51 | ] 52 | 53 | html_logo = "_static/logo-color.svg" 54 | 55 | html_theme_options = { 56 | "light_css_variables": { 57 | "color-highlight-on-target": "#D1DBFF", 58 | "color-sidebar-item-background--hover": "#002bc5", 59 | "font-stack--headings": "Dosis, var(--font-stack)", 60 | }, 61 | "dark_css_variables": { 62 | "color-brand-primary": "#3953FF", 63 | "color-brand-content": "#3953FF", 64 | "color-brand-visited": "#6D92C5", 65 | "color-highlight-on-target": "#002bc5", 66 | "color-sidebar-item-background--hover": "#002bc5", 67 | "color-background-primary": "linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(28,27,30,1) 50%)", 68 | "color-sidebar-background": "transparent", 69 | "color-toc-background": "transparent", 70 | "color-sidebar-search-background": "#090909", 71 | "font-stack--headings": "Dosis, var(--font-stack)", 72 | }, 73 | "footer_icons": [ 74 | { 75 | "name": "GitHub", 76 | "url": "https://github.com/knotsanimation/kloch", 77 | "html": """ 78 | 79 | 80 | 81 | """, 82 | "class": "", 83 | }, 84 | ], 85 | } 86 | -------------------------------------------------------------------------------- /doc/source/_static/banner.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | kloch 2 | ===== 3 | 4 | .. warning:: 5 | 6 | Despite being public, this repository is still in development stage and 7 | have not been tested extensively yet. 8 | 9 | ``kloch`` `/klˈoʃ/` [1]_ [2]_, is a configuration system for launching software. 10 | 11 | Configurations are `yaml `_ files referred 12 | as `environment profile` which specify the parameters for one or 13 | multiple pre-defined launchers. 14 | 15 | .. container:: columns 16 | 17 | .. container:: column-left 18 | 19 | .. literalinclude:: _injected/demo-fileformat/profile-beta.yml 20 | :language: yaml 21 | :caption: ./profiles/beta.yml 22 | 23 | .. container:: column-right 24 | 25 | .. literalinclude:: _injected/demo-fileformat/profile.yml 26 | :language: yaml 27 | :caption: ./profiles/prod.yml 28 | 29 | 30 | `Launchers` are internally-defined python objects that specify how to execute 31 | a combinations of options and (optional) command. 32 | 33 | To use the profile, one must call the :abbr:`CLI (Command Line Interface)` tool. 34 | The profile example shared above can be launched using: 35 | 36 | .. code-block:: shell 37 | 38 | python -m kloch run knots:echoes 39 | 40 | Design 41 | ------ 42 | 43 | `kloch` was initially designed as the environment manager layer when used with 44 | the `rez `_ package manager. 45 | 46 | In a very abstract way, `kloch` is a system that: 47 | 48 | - 49 | :abbr:`serialize (translating a data structure or object state into a 50 | format that can be stored on disk)` the arguments passed to a pre-defined 51 | function as yaml files. 52 | 53 | .. code-block:: shell 54 | 55 | myLauncher(optionA="paint.exe", optionB={"PATH": "/foo"}) 56 | 57 | becomes: 58 | 59 | .. code-block:: yaml 60 | :caption: myProfile.yml 61 | 62 | myLauncher: 63 | optionA: paint.exe 64 | optionB: 65 | PATH: /foo 66 | 67 | - 68 | execute that function by unserializing the parameters provided at runtime. 69 | 70 | .. code-block:: python 71 | :caption: pseudo-code in python 72 | 73 | profile = read_profile("myProfile.yml") 74 | for launcher_name, launcher_config in profile.items(): 75 | launcher = get_launcher(launcher_name) 76 | launcher(**launcher_config) 77 | 78 | 79 | Features 80 | -------- 81 | 82 | - offer a CLI and a public python API 83 | - custom config format for environment profiles: 84 | - inheritance 85 | - inheritance merging rules with token system 86 | - arbitrary profile locations with flexible configuration 87 | - plugin system for launchers 88 | 89 | 90 | Programming distinctions 91 | ------------------------ 92 | 93 | - robust logging. 94 | - good amount of unittesting 95 | - good documentation with a lot of doc created at code level 96 | - python 3.7+ support (EOL was in June 2023). 97 | - PyYAML as only mandatory external python dependency. 98 | - flexible "meta" configuration from environment variable or config file. 99 | - clear public and private API 100 | 101 | 102 | 103 | Contents 104 | -------- 105 | 106 | .. toctree:: 107 | :maxdepth: 2 108 | :caption: user 109 | 110 | Home 111 | install 112 | usage 113 | cli 114 | launchers 115 | file-format 116 | config 117 | 118 | .. toctree:: 119 | :maxdepth: 2 120 | :caption: developer 121 | 122 | public-api/index 123 | launcher-plugins 124 | developer 125 | changelog 126 | GitHub 127 | 128 | ---- 129 | 130 | **References** 131 | 132 | .. [1] https://www.internationalphoneticalphabet.org/ipa-sounds/ipa-chart-with-sounds/#ipachartstart 133 | .. [2] https://unalengua.com/ipa-translate?hl=en&ttsLocale=fr-FR&voiceId=Celine&sl=fr&text=clauche&ttsMode=word&speed=2 -------------------------------------------------------------------------------- /doc/source/_injected/exec-launchers-doc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Build an automatic rst documentation including a table from a dataclass object. 3 | 4 | All the information is extracted from the code of the dataclass. 5 | """ 6 | 7 | import dataclasses 8 | from typing import Type 9 | 10 | import kloch.launchers 11 | 12 | 13 | RowType = tuple[str, str, str] 14 | 15 | 16 | def enforce_col_length( 17 | row: tuple[str, str, str], 18 | lengths: list[int], 19 | ) -> RowType: 20 | # noinspection PyTypeChecker 21 | return tuple( 22 | col.ljust(lengths[index], col[-1] if col else " ") 23 | for index, col in enumerate(row) 24 | ) 25 | 26 | 27 | def unify_columns(table: list[RowType]) -> list[str]: 28 | """ 29 | Add the columns separator to return a row string. 30 | """ 31 | new_table = [] 32 | for row in table: 33 | sep = "+" if row[-1][-1] in ["-", "="] else "|" 34 | row = sep + sep.join(row) + sep 35 | new_table.append(row) 36 | return new_table 37 | 38 | 39 | def create_field_table( 40 | field: dataclasses.Field, 41 | required: bool, 42 | launcher: Type[kloch.launchers.BaseLauncherSerialized], 43 | ) -> list[RowType]: 44 | field_name = field.default 45 | field_doc = field.metadata["description"] 46 | required = "yes" if required else "no" 47 | 48 | doc = field_doc.split("\n") 49 | 50 | # typing.Annotated 51 | if hasattr(field.type, "__metadata__"): 52 | ftype = field.type.__origin__ 53 | else: 54 | ftype = field.type 55 | 56 | ftype = str(ftype).replace("typing.", "") 57 | 58 | rows = [ 59 | (f" .. _format-{launcher.identifier}-{field_name}: ", "", ""), 60 | (f"", "", ""), 61 | (f" ``{field_name}`` ", " **required** ", f" {required} "), 62 | ("-", "-", "-"), 63 | ("", " **type** ", f" `{ftype}` "), 64 | ("", "-", "-"), 65 | ("", " **description** ", f" {doc.pop(0)} "), 66 | ] 67 | 68 | for doc_line in doc: 69 | rows += [("", "", f" {doc_line} ")] 70 | 71 | rows += [("-", "-", "-")] 72 | return rows 73 | 74 | 75 | def replace_character(src_str: str, character: str, substitution: str) -> str: 76 | index = src_str.rfind(character, 0, -2) 77 | return src_str[:index] + substitution + src_str[index + 1 :] 78 | 79 | 80 | def document_launcher(launcher: Type[kloch.launchers.BaseLauncherSerialized]) -> str: 81 | lines = [] 82 | lines += [launcher.identifier, "_" * len(launcher.identifier)] 83 | lines += [""] + [launcher.description] + [""] 84 | 85 | fields_table: list[RowType] = [] 86 | for field in launcher.fields.iterate(): 87 | required = field.metadata["required"] 88 | fields_table += create_field_table( 89 | field=field, 90 | required=required, 91 | launcher=launcher, 92 | ) 93 | 94 | header_table = [ 95 | ("-", "-", "-"), 96 | (" ➡parent ", f" :launchers:{launcher.identifier} ", ""), 97 | ("-", "-", "-"), 98 | (" ⬇key ", "", ""), 99 | ("=", "=", "="), 100 | ] 101 | table = header_table + fields_table 102 | 103 | col_lens = [max([len(row[index]) for row in table]) for index in range(3)] 104 | 105 | table = [enforce_col_length(row, col_lens) for row in table] 106 | table = unify_columns(table) 107 | 108 | table[0] = replace_character(table[0], "+", "-") 109 | table[1] = replace_character(table[1], "|", " ") 110 | table[2] = replace_character(table[2], "+", "-") 111 | table[3] = replace_character(table[3], "|", " ") 112 | 113 | lines += table 114 | lines += [""] 115 | 116 | return "\n".join(lines) 117 | 118 | 119 | def main(): 120 | for launcher in kloch.launchers.get_available_launchers_serialized_classes(): 121 | print(document_launcher(launcher)) 122 | 123 | 124 | main() 125 | -------------------------------------------------------------------------------- /kloch/session.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import shutil 3 | import socket 4 | import time 5 | import uuid 6 | from pathlib import Path 7 | from typing import List 8 | 9 | LOGGER = logging.getLogger(__name__) 10 | 11 | 12 | class SessionDirectory: 13 | """ 14 | Manage the filesystem directory to store data during a kloch launcher session. 15 | 16 | Most of the data stored is arbitrary and generated depending on the launcher. 17 | """ 18 | 19 | def __init__(self, path: Path): 20 | self.path: Path = path 21 | """ 22 | Filesystem path to an existing directory. 23 | """ 24 | 25 | self.meta_session_path = self.path / "kloch.session" 26 | """ 27 | A file indicating the directory is a session and which contains 28 | the timestamp at which the session was created. 29 | """ 30 | 31 | self.profile_path = self.path / "profile.yml" 32 | """ 33 | A file which contain the merged profile used to start the session. 34 | """ 35 | 36 | @property 37 | def identifier(self) -> str: 38 | return self.path.name 39 | 40 | @property 41 | def timestamp(self) -> float: 42 | return float(self.meta_session_path.read_text()) 43 | 44 | @classmethod 45 | def initialize(cls, root: Path) -> "SessionDirectory": 46 | """ 47 | Generate a new unique session directory on the filesystem. 48 | 49 | This function should be safe to be executed from different thread on the same machine 50 | at the same time. 51 | """ 52 | if not root.exists(): 53 | LOGGER.debug(f"mkdir('{root}')") 54 | root.mkdir() 55 | 56 | timestamp = time.time() 57 | # XXX: we generate uuid to prevent collision but we shorten it as we 58 | # are already pretty safe with the timestamp. 59 | uuid_ = uuid.uuid4().hex[:8] 60 | identifier: str = f"{timestamp}-{socket.gethostname()}-{uuid_}" 61 | 62 | path = root / identifier 63 | LOGGER.debug(f"mkdir('{path}')") 64 | path.mkdir() 65 | 66 | instance = cls(path) 67 | 68 | LOGGER.debug(f"touch('{instance.meta_session_path}')") 69 | instance.meta_session_path.write_text(str(timestamp)) 70 | return instance 71 | 72 | 73 | def get_session_dirs(root: Path) -> List[SessionDirectory]: 74 | """ 75 | Get all the session directories found in the given root directory. 76 | """ 77 | return [ 78 | SessionDirectory(path) 79 | for path in root.glob("*") 80 | if path.is_dir() and SessionDirectory(path).meta_session_path.exists() 81 | ] 82 | 83 | 84 | def clean_outdated_session_dirs(root: Path, lifetime: float) -> List[Path]: 85 | """ 86 | Iterate through all existing session directories and delete the one which have been created longer than the given lifetime. 87 | 88 | This function should handle file or dirs of the given root which are not sessions. 89 | 90 | Args: 91 | root: filesystem path to an existing directory. 92 | lifetime: maximum lifetime in hours of a session directory 93 | 94 | Returns: 95 | list of directories filesystem path that have been removed 96 | """ 97 | # convert hours to seconds 98 | lifetime = lifetime * 3600 99 | current_time = time.time() 100 | minimal_lifetime = current_time - lifetime 101 | 102 | removed = [] 103 | 104 | for session in get_session_dirs(root=root): 105 | 106 | if session.timestamp > minimal_lifetime: 107 | continue 108 | 109 | try: 110 | shutil.rmtree(session.path) 111 | except Exception as error: 112 | LOGGER.exception( 113 | f"failed to remove outdated session dir '{session.path}': {error}" 114 | ) 115 | continue 116 | 117 | removed.append(session.path) 118 | 119 | return removed 120 | -------------------------------------------------------------------------------- /doc/source/launcher-plugins.rst: -------------------------------------------------------------------------------- 1 | Launcher Plugins 2 | ================ 3 | 4 | Kloch comes with a basic plugin system that allow you to add new launchers. 5 | The workflow can be simply described as: 6 | 7 | - create a new python module at an arbitrary location 8 | - subclass :any:`BaseLauncher` and :any:`BaseLauncherSerialized` inside 9 | - make sure its parent location is registred in the PYTHONPATH so they can be imported 10 | - append the module name to the ``launcher_plugins`` configuration key 11 | 12 | Which could be illustrated by the following bash commands: 13 | 14 | .. code-block:: shell 15 | 16 | cd /d/dev/my-kloch-plugins/ 17 | touch kloch_mylauncher.py 18 | # edit its content ^ 19 | export PYTHONPATH=$PYTHONPATH:/d/dev/my-kloch-plugins/ 20 | export KLOCH_CONFIG_LAUNCHER_PLUGINS=kloch_mylauncher 21 | # check if we succesfully registred our plugin 22 | kloch plugins 23 | 24 | 25 | As example you can check: 26 | 27 | - ``{root}/tests/data/plugins-behr`` 28 | - ``{root}/tests/data/plugins-tyfa`` 29 | - https://github.com/knotsanimation/kloch-launcher-rezenv 30 | 31 | 32 | Creating the module 33 | ------------------- 34 | 35 | A plugin for kloch is a regular python module that will be imported and parsed 36 | to find specific object. 37 | 38 | So you can create a regular python file at any location, or if you prefer 39 | you can also ship your plugin as a python package by creating a directory 40 | and an ``__init__.py`` file. 41 | 42 | 43 | .. tip:: 44 | 45 | As kloch plugins need to be in the PYTHONPATH, every python code will be 46 | able to import them so make sure to pick a name that will not conflict 47 | with other modules. Prefixing them with ``kloch_`` is recommended. 48 | 49 | 50 | Creating the subclasses 51 | ----------------------- 52 | 53 | When creating the a new launcher you start by creating a ``dataclass`` object 54 | that declare what are the user-configurable options and how are they "launched". 55 | 56 | This is done by subclassing :any:`BaseLauncher`. 57 | 58 | Next you need to declare how that dataclass must be serialized, by creating 59 | a subclass of :any:`BaseLauncherSerialized`. This act as a high-level 60 | controller for serialization. 61 | 62 | For the granular control over how each field of the dataclass is serialized you 63 | must created another dataclass subclass, but of :any:`BaseLauncherFields`. 64 | Which will just miror the ``BaseLauncher`` field structure, but where each of 65 | its field provide more metadata to unserialize the field. 66 | 67 | Here is an example which subclass both of those class, in which we create 68 | a launcher for "git cloning": 69 | 70 | .. literalinclude:: _injected/snip-launcher-plugin-basic.py 71 | :language: python 72 | :caption: kloch_gitclone.py 73 | 74 | .. tip:: 75 | 76 | When implementing a field for the user to have control on the launcher, 77 | it's best to implement it as ``dict`` type over ``list`` because it 78 | allow users to override one item in particular using the token system. 79 | 80 | Registering 81 | ----------- 82 | 83 | Then add the ``kloch_gitclone.py`` parent location in your PYTHONPATH 84 | so the plugin system can do an ``import kloch_gitclone``. 85 | 86 | .. tip:: 87 | 88 | PYTHONPATH is the standard mechanism by python to discover and import modules 89 | but nothing prevent you to use other tools or methods. 90 | 91 | You could for example create a ``pyproject.toml`` which declare ``kloch`` 92 | and your plugin as dependency and let a tool like `uv `_ 93 | or `poetry `_ create the venv. 94 | 95 | As long as it can be ``import my_plugin_module_name`` it will work ! 96 | 97 | The last step is to add the module name in the list of launcher plugins to use. 98 | You do this by modifying the kloch configuration, which is explained in :doc:`config` 99 | 100 | A quick way to do it is just to set the corresponding environment variable before 101 | starting `kloch`. 102 | 103 | .. code-block:: shell 104 | 105 | export KLOCH_CONFIG_LAUNCHER_PLUGINS=kloch_gitclone 106 | 107 | You can check your plugin is registred by calling: 108 | 109 | .. code-block:: shell 110 | 111 | kloch plugins -------------------------------------------------------------------------------- /kloch/launchers/base/_dataclass.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import dataclasses 3 | from pathlib import Path 4 | from typing import Any 5 | from typing import ClassVar 6 | from typing import Dict 7 | from typing import List 8 | from typing import Optional 9 | 10 | 11 | @dataclasses.dataclass 12 | class BaseLauncher: 13 | """ 14 | An "abstract" dataclass that describe how to start a software environment session. 15 | """ 16 | 17 | # XXX: all fields defined MUST specify a default value (else inheritance issues) 18 | # instead add them to the `required_fields` class variable. 19 | 20 | environ: Dict[str, str] = dataclasses.field(default_factory=dict) 21 | """ 22 | Mapping of environment variables to set when starting the environment. 23 | 24 | The developer is reponsible of honoring the field usage in its launcher implementation. 25 | """ 26 | 27 | command: List[str] = dataclasses.field(default_factory=list) 28 | """ 29 | Arbitrary list of command line arguments to call at the end of the launcher execution. 30 | 31 | The developer is reponsible of honoring the field usage in its launcher implementation. 32 | """ 33 | 34 | cwd: Optional[str] = None 35 | """ 36 | Current working directory. 37 | 38 | The developer is reponsible of honoring the field usage in its launcher implementation. 39 | """ 40 | 41 | priority: int = 0 42 | """ 43 | How much you should privilege this launcher to be used over other launchers. 44 | """ 45 | 46 | required_fields: ClassVar[List[str]] = [] 47 | """ 48 | List of dataclass field that are required to have a non-None value when instancing. 49 | 50 | Note that your subclass must have the default field value set to None for this to work. 51 | 52 | Example:: 53 | 54 | @dataclasses.dataclass 55 | class DemoLauncher(BaseLauncher): 56 | # override the BaseLauncher.environ field to make it required 57 | environ: Dict[str, str] = None 58 | 59 | required_fields = ["environ"] 60 | 61 | """ 62 | 63 | name: ClassVar[str] = ".base" 64 | """ 65 | A unique name among all subclasses. 66 | """ 67 | 68 | def __str__(self) -> str: 69 | fields: List[str] = [] 70 | for field in dataclasses.fields(self): 71 | name = field.name 72 | value = getattr(self, name) 73 | if name == "environ": 74 | asstr = "{...}" if value else "{}" 75 | elif name == "command": 76 | asstr = "[...]" if value else "[]" 77 | else: 78 | asstr = str(value) 79 | 80 | fields += [f"{name}={asstr}"] 81 | 82 | fieldsstr = ", ".join(fields) 83 | return f"<{self.__class__.__name__} {fieldsstr}>" 84 | 85 | def __post_init__(self): 86 | for field in dataclasses.fields(self): 87 | if field.name in self.required_fields and getattr(self, field.name) is None: 88 | raise ValueError( 89 | f"Missing required field '{field.name}' for instance {self}" 90 | ) 91 | 92 | @abc.abstractmethod 93 | def execute(self, tmpdir: Path, command: Optional[List[str]] = None) -> int: 94 | """ 95 | Start the given environment and execute this python session. 96 | 97 | Optionally execute the given command in the environment. 98 | 99 | Args: 100 | tmpdir: filesystem path to an existing temporary directory 101 | command: optional list of command line arguments 102 | 103 | Returns: 104 | The exit code of the execution. 0 if successfull, else imply failure. 105 | """ 106 | pass # pragma: no cover 107 | 108 | def to_dict(self) -> Dict[str, Any]: 109 | """ 110 | Convert the instance to a python dict object. 111 | """ 112 | as_dict = dataclasses.asdict(self) 113 | # remove optional keys that don't have a value 114 | as_dict = { 115 | key: value 116 | for key, value in as_dict.items() 117 | if key or key in self.required_fields 118 | } 119 | return as_dict 120 | 121 | @classmethod 122 | def from_dict(cls, src_dict: Dict[str, Any]) -> "BaseLauncher": 123 | """ 124 | Generate an instance from a python dict object with a specific structure. 125 | """ 126 | return cls(**src_dict) 127 | -------------------------------------------------------------------------------- /tests/test_launchers_base.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import os 3 | from typing import Dict 4 | from typing import List 5 | 6 | import pytest 7 | 8 | from kloch.launchers import BaseLauncher 9 | from kloch.launchers import BaseLauncherSerialized 10 | from kloch.launchers.base._serialized import resolve_environ 11 | 12 | 13 | def test__resolve_environ(monkeypatch): 14 | monkeypatch.setenv("__TEST__", "SUCCESS") 15 | 16 | src_environ = { 17 | "PATH": ["$PATH", "D:\\some\\path"], 18 | "NOTRESOLVED": "foo;$$PATH;D:\\some\\path", 19 | "NUMBER": 1, 20 | "ANOTHERONE": "$__TEST__", 21 | "SUCCESSIVE": "$NUMBER", 22 | } 23 | result = resolve_environ(src_environ) 24 | assert len(result) == len(src_environ) 25 | assert result["PATH"] != f"$PATH{os.pathsep}D:\\some\\path" 26 | assert len(result["PATH"]) > len("D:\\some\\path") + 2 27 | assert result["PATH"].endswith(f"{os.pathsep}D:\\some\\path") 28 | assert result["NOTRESOLVED"] == f"foo;$PATH;D:\\some\\path" 29 | assert result["NUMBER"] == "1" 30 | assert result["ANOTHERONE"] == "SUCCESS" 31 | assert result["SUCCESSIVE"] == "1" 32 | 33 | 34 | def test__BaseLauncher__required_fields(): 35 | @dataclasses.dataclass 36 | class TestLauncher(BaseLauncher): 37 | params: List[str] = dataclasses.field(default_factory=list) 38 | # any required field must have a default value of None 39 | environ: Dict[str, str] = None 40 | 41 | required_fields = ["environ"] 42 | 43 | def execute(self, tmpdir, command=None): # pragma: no cover 44 | pass 45 | 46 | with pytest.raises(ValueError) as error: 47 | TestLauncher(params=["--verbose"]) 48 | assert "required field" in str(error.value) 49 | 50 | launcher = TestLauncher(params=["--verbose"], environ={"PATH": "ghghghgh"}) 51 | assert launcher.environ["PATH"] == "ghghghgh" 52 | 53 | del TestLauncher 54 | 55 | 56 | def test__BaseLauncherSerialized(data_dir, monkeypatch): 57 | src_dict = {} 58 | instance = BaseLauncherSerialized(src_dict) 59 | instance.validate() 60 | resolved = instance.resolved() 61 | assert len(resolved) == 2 62 | assert BaseLauncherSerialized.fields.environ in resolved 63 | assert resolved[BaseLauncherSerialized.fields.merge_system_environ] is False 64 | 65 | monkeypatch.chdir(data_dir) 66 | 67 | src_dict = { 68 | "environ": { 69 | "PATH": ["$PATH", "D:\\some\\testdir"], 70 | "NUMBER": 1, 71 | "SUCCESSIVE": "$NUMBER", 72 | }, 73 | "cwd": str(data_dir / "notexisting" / ".." / "$NUMBER"), 74 | } 75 | instance = BaseLauncherSerialized(src_dict) 76 | instance.validate() 77 | assert instance["environ"]["NUMBER"] == 1 78 | 79 | resolved = instance.resolved() 80 | assert resolved["environ"]["NUMBER"] == "1" 81 | assert resolved["environ"]["PATH"].endswith("testdir") 82 | assert resolved["cwd"] == str(data_dir / "1") 83 | 84 | launcher = instance.unserialize() 85 | assert isinstance(launcher, BaseLauncher) 86 | assert launcher.command == [] 87 | assert launcher.cwd == str(data_dir / "1") 88 | assert launcher.environ["NUMBER"] == "1" 89 | 90 | src_dict = {"command": ["arg1", "arg", 2]} 91 | instance = BaseLauncherSerialized(src_dict) 92 | with pytest.raises(AssertionError) as error: 93 | instance.validate() 94 | assert "must be str" in str(error.value) 95 | 96 | src_dict = {"command": ["arg1", "arg"]} 97 | instance = BaseLauncherSerialized(src_dict) 98 | instance.validate() 99 | 100 | 101 | def test__test__BaseLauncherSerialized__fields(): 102 | fields = BaseLauncherSerialized.fields.iterate() 103 | assert isinstance(fields[0], dataclasses.Field) 104 | 105 | 106 | def test__BaseLauncherSerialized__merge_system_environ(data_dir, monkeypatch): 107 | monkeypatch.setenv("__UNITTEST__", "SUCCESS") 108 | 109 | src_dict = { 110 | "environ": { 111 | "PATH": ["HELLOW", "$PATH", "D:\\some\\testdir"], 112 | "NUMBER": 1, 113 | "SUCCESSIVE": "$NUMBER", 114 | }, 115 | "cwd": str(data_dir / "notexisting" / ".." / "$NUMBER"), 116 | } 117 | instance = BaseLauncherSerialized(src_dict) 118 | assert len(instance["environ"]) == 3 119 | resolved = instance.resolved() 120 | assert len(resolved["environ"]) > 3 121 | assert resolved["environ"]["PATH"].startswith("HELLOW") 122 | assert "__UNITTEST__" in resolved["environ"] 123 | 124 | instance[BaseLauncherSerialized.fields.merge_system_environ] = False 125 | resolved = instance.resolved() 126 | assert len(resolved["environ"]) == 3 127 | -------------------------------------------------------------------------------- /tests/test_filesyntax_io.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | import kloch.filesyntax 6 | 7 | 8 | def test__is_file_environment_profile(data_dir): 9 | src_path = data_dir / "not-a-profile.txt" 10 | result = kloch.filesyntax.is_file_environment_profile(src_path) 11 | assert result is False 12 | 13 | src_path = data_dir / "fake-profile.yml" 14 | result = kloch.filesyntax.is_file_environment_profile(src_path) 15 | assert result is False 16 | 17 | src_path = data_dir / "profile.echoes.yml" 18 | result = kloch.filesyntax.is_file_environment_profile(src_path) 19 | assert result is True 20 | 21 | 22 | def test__read_profile_from_file__envvar(data_dir): 23 | profile_echoes_beta_paths = kloch.filesyntax.get_profile_file_path( 24 | "knots:echoes:beta", 25 | profile_locations=[data_dir], 26 | ) 27 | assert len(profile_echoes_beta_paths) == 1 28 | profile_echoes_beta = kloch.filesyntax.read_profile_from_file( 29 | profile_echoes_beta_paths[0], 30 | profile_locations=[data_dir], 31 | ) 32 | assert profile_echoes_beta.identifier == "knots:echoes:beta" 33 | assert profile_echoes_beta.inherit is None 34 | 35 | profile_echoes_path = kloch.filesyntax.get_profile_file_path( 36 | "knots:echoes", 37 | profile_locations=[data_dir], 38 | )[0] 39 | profile_echoes = kloch.filesyntax.read_profile_from_file( 40 | profile_echoes_path, 41 | profile_locations=[data_dir], 42 | ) 43 | assert profile_echoes.identifier == "knots:echoes" 44 | assert profile_echoes.inherit == profile_echoes_beta 45 | 46 | m_profile = profile_echoes.get_merged_profile() 47 | assert m_profile.identifier == "knots:echoes" 48 | assert m_profile.inherit is None 49 | 50 | 51 | def test__read_profile_from_id(data_dir): 52 | profile_echoes_beta = kloch.filesyntax.read_profile_from_id( 53 | "knots:echoes:beta", 54 | profile_locations=[data_dir], 55 | ) 56 | assert profile_echoes_beta.identifier == "knots:echoes:beta" 57 | assert profile_echoes_beta.inherit is None 58 | 59 | 60 | def test__read_profile_from_file__old(data_dir): 61 | profile_paths = kloch.filesyntax.get_profile_file_path( 62 | "version1", profile_locations=[data_dir] 63 | ) 64 | assert len(profile_paths) == 1 65 | with pytest.raises(kloch.filesyntax.ProfileAPIVersionError): 66 | kloch.filesyntax.read_profile_from_file(profile_paths[0]) 67 | 68 | 69 | def test__read_profile_from_file_nested(data_dir): 70 | profile_paths = kloch.filesyntax.get_profile_file_path( 71 | "knots:echoes:tmp", profile_locations=[data_dir] 72 | ) 73 | profile = kloch.filesyntax.read_profile_from_file( 74 | profile_paths[0], 75 | profile_locations=[data_dir], 76 | ) 77 | assert profile.identifier == "knots:echoes:tmp" 78 | assert profile.inherit is not None 79 | assert profile.inherit.identifier == "knots:echoes" 80 | assert profile.inherit.inherit is not None 81 | assert profile.inherit.inherit.identifier == "knots:echoes:beta" 82 | 83 | 84 | def test__serialize_profile(data_dir, tmp_path: Path): 85 | # test back and forth conversion 86 | profile_src = kloch.filesyntax.read_profile_from_id( 87 | "knots:echoes:beta", 88 | profile_locations=[data_dir], 89 | ) 90 | serialized = kloch.filesyntax.serialize_profile( 91 | profile_src, 92 | profile_locations=[data_dir], 93 | ) 94 | profile_new_path = tmp_path / "profile.yml" 95 | profile_new_path.write_text(serialized) 96 | 97 | profile_new = kloch.filesyntax.read_profile_from_file( 98 | profile_new_path, 99 | profile_locations=[data_dir], 100 | ) 101 | assert profile_src == profile_new 102 | 103 | 104 | def test__write_profile_to_file(data_dir, tmp_path: Path): 105 | # test back and forth conversion 106 | profile_src = kloch.filesyntax.read_profile_from_id( 107 | "knots:echoes:beta", 108 | profile_locations=[data_dir], 109 | ) 110 | profile_new_path = tmp_path / "profile.yml" 111 | kloch.filesyntax.write_profile_to_file( 112 | profile_src, 113 | file_path=profile_new_path, 114 | profile_locations=[data_dir], 115 | check_valid_id=False, 116 | ) 117 | assert profile_new_path.exists() 118 | profile_new = kloch.filesyntax.read_profile_from_file( 119 | profile_new_path, 120 | profile_locations=[data_dir], 121 | ) 122 | assert profile_src == profile_new 123 | 124 | with pytest.raises(kloch.filesyntax.ProfileIdentifierError): 125 | kloch.filesyntax.write_profile_to_file( 126 | profile_src, 127 | file_path=profile_new_path, 128 | profile_locations=[data_dir], 129 | check_valid_id=True, 130 | ) 131 | -------------------------------------------------------------------------------- /tests/test_e2e.py: -------------------------------------------------------------------------------- 1 | """ 2 | "end to end" test: as close as possible to real usage conditions 3 | 4 | TODO probably need to complexify the data even more 5 | """ 6 | 7 | import os 8 | import subprocess 9 | import sys 10 | 11 | import kloch 12 | 13 | 14 | def test_e2e_case1(data_dir, monkeypatch, tmp_path): 15 | """ 16 | Use environ to define kloch config 17 | """ 18 | test_data_dir = data_dir / "e2e-1" 19 | plugin_path = data_dir / "plugins-behr" 20 | cwd_dir = tmp_path / "cwd" 21 | cwd_dir.mkdir() 22 | session_dir = tmp_path / "session" 23 | 24 | environ = os.environ.copy() 25 | pythonpath = os.environ.get("PYTHONPATH") 26 | if pythonpath: 27 | environ["PYTHONPATH"] = f"{pythonpath}{os.pathsep}{plugin_path}" 28 | else: 29 | environ["PYTHONPATH"] = f"{plugin_path}" 30 | environ[kloch.Environ.CONFIG_PROFILE_ROOTS] = str(test_data_dir) 31 | environ[kloch.Environ.CONFIG_LAUNCHER_PLUGINS] = "kloch_behr" 32 | environ[kloch.Environ.CONFIG_CLI_SESSION_PATH] = str(session_dir) 33 | 34 | command = [ 35 | sys.executable, 36 | "-m", 37 | "kloch", 38 | "plugins", 39 | "--debug", 40 | ] 41 | result = subprocess.run(command, cwd=cwd_dir, env=environ) 42 | assert not result.returncode 43 | assert not list(cwd_dir.glob("*")) 44 | 45 | command = [ 46 | sys.executable, 47 | "-m", 48 | "kloch", 49 | "run", 50 | "prod", 51 | "--debug", 52 | "--", 53 | "testcommand", 54 | ] 55 | result = subprocess.run(command, cwd=cwd_dir, env=environ) 56 | assert not result.returncode 57 | assert not list(cwd_dir.glob("*")) 58 | 59 | profile_path = test_data_dir / "profile.prod.yml" 60 | 61 | command = [ 62 | sys.executable, 63 | "-m", 64 | "kloch", 65 | "run", 66 | str(profile_path), 67 | "--debug", 68 | "--", 69 | "testcommand", 70 | ] 71 | result = subprocess.run(command, cwd=cwd_dir, env=environ) 72 | assert not result.returncode 73 | 74 | 75 | def test_e2e_case2(data_dir, monkeypatch, tmp_path): 76 | """ 77 | Use a config file 78 | """ 79 | test_data_dir = data_dir / "e2e-1" 80 | plugin_path = data_dir / "plugins-behr" 81 | cwd_dir = tmp_path / "cwd" 82 | cwd_dir.mkdir() 83 | session_dir = tmp_path / "session" 84 | 85 | environ = os.environ.copy() 86 | pythonpath = os.environ.get("PYTHONPATH") 87 | if pythonpath: 88 | environ["PYTHONPATH"] = f"{pythonpath}{os.pathsep}{plugin_path}" 89 | else: 90 | environ["PYTHONPATH"] = f"{plugin_path}" 91 | environ[kloch.Environ.CONFIG_PATH] = str(test_data_dir / "config.yml") 92 | environ["KLOCHTEST_LOG_PATH"] = str(session_dir / ".log") 93 | environ["KLOCHTEST_SESSION_PATH"] = str(session_dir) 94 | 95 | command = [ 96 | sys.executable, 97 | "-m", 98 | "kloch", 99 | "plugins", 100 | "--debug", 101 | ] 102 | result = subprocess.run(command, cwd=cwd_dir, env=environ) 103 | assert not result.returncode 104 | assert not list(cwd_dir.glob("*")) 105 | 106 | command = [ 107 | sys.executable, 108 | "-m", 109 | "kloch", 110 | "run", 111 | "prod", 112 | "--debug", 113 | "--", 114 | "testcommand", 115 | ] 116 | result = subprocess.run(command, cwd=cwd_dir, env=environ) 117 | assert not result.returncode 118 | assert not list(cwd_dir.glob("*")) 119 | 120 | 121 | def test_e2e_case3(data_dir, monkeypatch, tmp_path): 122 | """ 123 | Use environ to define kloch config and test profile prodml 124 | """ 125 | test_data_dir = data_dir / "e2e-1" 126 | plugin_path = data_dir / "plugins-behr" 127 | cwd_dir = tmp_path / "cwd" 128 | cwd_dir.mkdir() 129 | session_dir = tmp_path / "session" 130 | 131 | environ = os.environ.copy() 132 | pythonpath = os.environ.get("PYTHONPATH") 133 | if pythonpath: 134 | environ["PYTHONPATH"] = f"{pythonpath}{os.pathsep}{plugin_path}" 135 | else: 136 | environ["PYTHONPATH"] = f"{plugin_path}" 137 | environ[kloch.Environ.CONFIG_PROFILE_ROOTS] = str(test_data_dir) 138 | environ[kloch.Environ.CONFIG_LAUNCHER_PLUGINS] = "kloch_behr" 139 | environ[kloch.Environ.CONFIG_CLI_SESSION_PATH] = str(session_dir) 140 | 141 | command = [ 142 | sys.executable, 143 | "-m", 144 | "kloch", 145 | "run", 146 | "prodml", 147 | "--debug", 148 | "--launcher", 149 | ".system", 150 | "--", 151 | "test_e2e_case3", 152 | ] 153 | result = subprocess.run( 154 | command, 155 | cwd=cwd_dir, 156 | env=environ, 157 | capture_output=True, 158 | text=True, 159 | ) 160 | print(result.stdout) 161 | print(result.stderr, file=sys.stderr) 162 | assert not result.returncode 163 | assert result.stdout.strip("\n").endswith("hello from test_e2e_case3") 164 | -------------------------------------------------------------------------------- /tests/test_launchers_serialized.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import kloch.launchers 4 | from kloch.launchers import LauncherSerializedDict 5 | from kloch.launchers import LauncherSerializedList 6 | from kloch.launchers import LauncherContext 7 | from kloch.launchers import LauncherPlatform 8 | 9 | LOGGER = logging.getLogger(__name__) 10 | 11 | 12 | def test__LauncherSerializedDict(): 13 | launcher_serial = LauncherSerializedDict( 14 | { 15 | "+=.python": { 16 | "python_file": "/foo", 17 | }, 18 | ".base": { 19 | "environ": { 20 | "PATH": ["$PATH", "/foo/bar"], 21 | "PROD": "unittest", 22 | }, 23 | }, 24 | }, 25 | ) 26 | launcher_classes = kloch.launchers.get_available_launchers_serialized_classes() 27 | launchers = launcher_serial.to_serialized_list(launcher_classes) 28 | assert len(launchers) == 2 29 | assert isinstance(launchers[0], kloch.launchers.PythonLauncherSerialized) 30 | assert launchers[0]["python_file"] == "/foo" 31 | assert isinstance(launchers[1], kloch.launchers.BaseLauncherSerialized) 32 | assert launchers[1]["environ"]["PATH"] == ["$PATH", "/foo/bar"] 33 | 34 | 35 | def test__LauncherSerializedDict__filter(): 36 | launcher_serial = LauncherSerializedDict( 37 | { 38 | ".system@os=windows": { 39 | "command": "powershell script.ps1", 40 | }, 41 | ".system@os=linux": { 42 | "command": "bash script.sh", 43 | }, 44 | ".base": { 45 | "environ": { 46 | "PATH": ["$PATH", "/foo/bar"], 47 | "PROD": "unittest", 48 | }, 49 | }, 50 | }, 51 | ) 52 | context = LauncherContext(platform=LauncherPlatform.windows) 53 | result = launcher_serial.get_filtered_context(context) 54 | assert result is not launcher_serial 55 | assert ".system@os=linux" in launcher_serial 56 | assert ".system@os=linux" not in result 57 | assert ".system@os=windows" in result 58 | assert ".base" in result 59 | 60 | launcher_serial = LauncherSerializedDict( 61 | { 62 | ".base@user=tester": { 63 | "envion": {"TESTING": "1"}, 64 | }, 65 | ".base": { 66 | "environ": { 67 | "PATH": ["$PATH", "/foo/bar"], 68 | "PROD": "unittest", 69 | }, 70 | }, 71 | }, 72 | ) 73 | context = LauncherContext(platform=LauncherPlatform.windows, user="tester") 74 | result = launcher_serial.get_filtered_context(context) 75 | assert ".base@user=tester" in result 76 | assert ".base" in result 77 | 78 | 79 | def test__LauncherSerializedDict__with_context_resolved(): 80 | launcher_serial = LauncherSerializedDict( 81 | { 82 | ".system@os=windows": { 83 | "command": "powershell script.ps1", 84 | }, 85 | ".system@os=linux": { 86 | "!=command": "bash script.sh", 87 | }, 88 | ".base@user=tester": { 89 | "environ": {"TESTING": "1"}, 90 | }, 91 | ".base": { 92 | "environ": { 93 | "PATH": ["$PATH", "/foo/bar"], 94 | "PROD": "unittest", 95 | }, 96 | }, 97 | }, 98 | ) 99 | expected = LauncherSerializedDict( 100 | { 101 | ".system": { 102 | "command": "powershell script.ps1", 103 | }, 104 | ".base": { 105 | "environ": { 106 | "TESTING": "1", 107 | "PATH": ["$PATH", "/foo/bar"], 108 | "PROD": "unittest", 109 | }, 110 | }, 111 | }, 112 | ) 113 | result = launcher_serial.with_context_resolved() 114 | assert result == expected 115 | 116 | 117 | def test__LauncherSerializedList(): 118 | src_list = [ 119 | kloch.launchers.BaseLauncherSerialized( 120 | { 121 | "environ": { 122 | "PATH": ["$PATH", "/foo/bar"], 123 | "PROD": "unittest", 124 | "SOME_LIST": [".base"], 125 | }, 126 | } 127 | ), 128 | kloch.launchers.PythonLauncherSerialized( 129 | { 130 | "python_file": "/foo", 131 | "+=environ": { 132 | "+=PATH": ["/rez"], 133 | "REZVERBOSE": 2, 134 | "SOME_LIST": ["rez"], 135 | }, 136 | } 137 | ), 138 | ] 139 | launcher_serial = LauncherSerializedList(src_list) 140 | # check copied on instancing 141 | src_list.append(5) 142 | assert 5 not in launcher_serial 143 | assert len(launcher_serial) == 2 144 | 145 | launcher_serial = launcher_serial.with_base_merged() 146 | assert len(launcher_serial) == 1 147 | environ = launcher_serial[0]["+=environ"] 148 | assert len(environ) == 4 149 | assert environ["+=PATH"] == ["$PATH", "/foo/bar", "/rez"] 150 | -------------------------------------------------------------------------------- /kloch/launchers/_serialized.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from typing import Dict 3 | from typing import List 4 | from typing import Type 5 | 6 | from kloch import MergeableDict 7 | from ._context import LauncherContext 8 | from ._context import unserialize_context_expression 9 | from ._context import resolve_context_expression 10 | from kloch.launchers import BaseLauncherSerialized 11 | 12 | 13 | class LauncherSerializedList(List[BaseLauncherSerialized]): 14 | def to_dict(self) -> Dict[str, Dict]: 15 | """ 16 | Convert the list to a builtin dict structure (noc ustom class used). 17 | 18 | Useful for serialization or to convert to a :any:`LauncherSerializedDict` instance. 19 | """ 20 | return {launcher.identifier: dict(launcher) for launcher in self} 21 | 22 | def with_base_merged(self) -> "LauncherSerializedList": 23 | """ 24 | Get a copy of this instance with the ``.base`` launcher merged with the other launchers. 25 | 26 | (This implies the returned instance does NOT have a ``.base`` key anymore) 27 | 28 | Returns: 29 | new instance with deepcopied structure. 30 | """ 31 | self_copy: LauncherSerializedList = copy.deepcopy(self) 32 | 33 | # extract the potential base that all launchers should inherit 34 | for launcher in self_copy: 35 | if launcher.__class__ is BaseLauncherSerialized: 36 | base_launcher = launcher 37 | self_copy.remove(base_launcher) 38 | break 39 | else: 40 | return self_copy 41 | 42 | return LauncherSerializedList( 43 | [copy.deepcopy(base_launcher) + launcher for launcher in self_copy] 44 | ) 45 | 46 | 47 | # revert to `MergeableDict[str, Dict]` when python-3.7 support dropped 48 | class LauncherSerializedDict(MergeableDict): 49 | """ 50 | A list of launchers instance serialized as a dict structure. 51 | 52 | The dict is expected to have the following root structure:: 53 | 54 | {"manager_name1": {...}, "manager_name2": {...}, ...} 55 | 56 | The dict structure include tokens that need to be resolved and indicate 57 | how to merge 2 :obj:`LauncherSerializedDict` instances together. 58 | See :any:`MergeableDict` for the full documentation on tokens. 59 | """ 60 | 61 | def get_filtered_context( 62 | self, context: LauncherContext 63 | ) -> "LauncherSerializedDict": 64 | """ 65 | Remove all launchers that doesn't match the given context. 66 | 67 | Returns: 68 | a new deepcopied instance with possibly lesser keys. 69 | """ 70 | newdict = copy.deepcopy(self) 71 | for launcher_identifier in self.keys(): 72 | launcher_context = unserialize_context_expression(launcher_identifier) 73 | if launcher_context != context: 74 | del newdict[launcher_identifier] 75 | 76 | return newdict 77 | 78 | def with_context_resolved(self) -> "LauncherSerializedDict": 79 | """ 80 | Merge all the same launchers with different contexts to a single launcher. 81 | """ 82 | toconcatenate: List[LauncherSerializedDict] = [] 83 | for launcher_identifier in self.keys(): 84 | resolved = resolve_context_expression(launcher_identifier) 85 | newdict = LauncherSerializedDict( 86 | {resolved: copy.deepcopy(self[launcher_identifier])} 87 | ) 88 | toconcatenate.append(newdict) 89 | 90 | if len(toconcatenate) == 0: 91 | return copy.deepcopy(self) 92 | elif len(toconcatenate) == 1: 93 | return toconcatenate[0] 94 | 95 | return sum(toconcatenate[1:], toconcatenate[0]) 96 | 97 | def to_serialized_list( 98 | self, 99 | launcher_classes: List[Type[BaseLauncherSerialized]], 100 | ) -> LauncherSerializedList: 101 | """ 102 | Convert the dict structure to a list of :any:`BaseLauncherSerialized` instances. 103 | 104 | Args: 105 | launcher_classes: 106 | list of launchers classes that can be possibly stored in this serialized dict. 107 | 108 | Raises: 109 | ValueError: If a launcher is serialized in this instance but is unknown. 110 | 111 | Returns: 112 | deepcopied dict structure as list of instances. 113 | """ 114 | _launcher_classes = { 115 | launcher.identifier: launcher for launcher in launcher_classes 116 | } 117 | launchers = [] 118 | for identifier, launcher_config in self.items(): 119 | _identifier = self.resolve_key_tokens(identifier) 120 | # it's not supposed to have context expression at this stage but we safe-proof it 121 | _identifier = resolve_context_expression(_identifier) 122 | launcher_class = _launcher_classes.get(_identifier) 123 | if not launcher_class: 124 | raise ValueError( 125 | f"No serialized-launcher with identifier '{_identifier}' found." 126 | f"Available launchers are '{', '.join(_launcher_classes.keys())}'" 127 | ) 128 | launcher = launcher_class(copy.deepcopy(launcher_config)) 129 | launchers.append(launcher) 130 | 131 | return LauncherSerializedList(launchers) 132 | -------------------------------------------------------------------------------- /doc/source/_extensions/execinject.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | import traceback 4 | 5 | from pathlib import Path 6 | from typing import Optional 7 | from typing import Union 8 | 9 | import docutils.nodes 10 | import docutils.statemachine 11 | from docutils.parsers.rst import Directive 12 | from docutils.parsers.rst import directives 13 | from sphinx.application import Sphinx 14 | from sphinx.util.typing import ExtensionMetadata 15 | 16 | 17 | DIRECTIVE_NAME = "exec-inject" 18 | 19 | 20 | class ExecutionError(RuntimeError): 21 | """ 22 | The python code could not be run sucessfully 23 | """ 24 | 25 | pass 26 | 27 | 28 | def prefixlines(source: str, prefix: str): 29 | return "\n".join([prefix + line for line in source.split("\n")]) 30 | 31 | 32 | def execute_python_code(code: Union[str, Path]) -> str: 33 | """ 34 | Execute the given python code in a python interpreter and return its stdout output. 35 | """ 36 | command = [sys.executable] 37 | if isinstance(code, Path): 38 | command.append(str(code)) 39 | else: 40 | command += ["-c", code.encode("utf-8")] 41 | 42 | result = subprocess.run( 43 | command, 44 | capture_output=True, 45 | text=True, 46 | check=False, 47 | encoding="utf-8", 48 | ) 49 | if result.returncode != 0: 50 | stdout = result.stdout.strip("\n") 51 | stderr = result.stderr.strip("\n") 52 | raise ExecutionError( 53 | f"# Error while executing code in subprocess:\n" 54 | f"[__stdout__]\n{prefixlines(stdout, '| ')}\n" 55 | f"[__stderr__]\n{prefixlines(stderr, '| ')}" 56 | ) 57 | 58 | result_text = result.stdout or "" + result.stderr or "" 59 | return result_text 60 | 61 | 62 | def generate_error_nodes( 63 | source_file_path: Path, 64 | line: int, 65 | traceback_txt: str, 66 | code_content: Optional[Union[str, Path]] = None, 67 | ): 68 | """ 69 | Replace the directive by a traceback block to debug the error that just happened. 70 | 71 | Args: 72 | source_file_path: path of the rst file with the original directive. 73 | line: number of the line in the rst file at which the directive is. 74 | traceback_txt: python formatted traceback to include in the block. 75 | code_content: 76 | optional code or file path that the directive try to execute but failed to. 77 | can be very verbose. 78 | """ 79 | nodes = [None] 80 | pnode = docutils.nodes.paragraph 81 | nodes += [ 82 | pnode( 83 | text=( 84 | f"Failed to execute directive {DIRECTIVE_NAME} in {Path(source_file_path).name}:{line}" 85 | ) 86 | ) 87 | ] 88 | if code_content: 89 | nodes += [docutils.nodes.literal_block(code_content, code_content)] 90 | nodes += [docutils.nodes.literal_block(traceback_txt, traceback_txt)] 91 | return [docutils.nodes.error(*nodes)] 92 | 93 | 94 | class ExecDirective(Directive): 95 | """ 96 | Execute the given python code and inject its stdout output directly in the rst document. 97 | 98 | Optionally load the python code from a given file. 99 | """ 100 | 101 | has_content = True 102 | option_spec = { 103 | "filename": directives.path, 104 | } 105 | 106 | @property 107 | def filename(self) -> Optional[Path]: 108 | return Path(self.options["filename"]) if "filename" in self.options else None 109 | 110 | def run(self): 111 | tab_width = self.options.get( 112 | "tab-width", self.state.document.settings.tab_width 113 | ) 114 | source_file_path = self.state_machine.input_lines.source( 115 | self.lineno - self.state_machine.input_offset - 1 116 | ) 117 | 118 | filepath = self.filename 119 | code_content = None 120 | 121 | try: 122 | if filepath: 123 | if not filepath.is_absolute(): 124 | filepath = Path(source_file_path).parent / filepath 125 | code_content = filepath 126 | else: 127 | code_content = "\n".join(self.content) 128 | 129 | text = execute_python_code(code_content) 130 | lines = docutils.statemachine.string2lines( 131 | text, 132 | tab_width, 133 | convert_whitespace=True, 134 | ) 135 | 136 | except Exception as error: 137 | self.reporter.error( 138 | f"{DIRECTIVE_NAME} directive failed to complete:\n" 139 | f"{prefixlines(str(error), '# ')}", 140 | line=self.lineno, 141 | ) 142 | traceback_txt = traceback.format_exc() 143 | return generate_error_nodes( 144 | source_file_path=source_file_path, 145 | line=self.lineno, 146 | traceback_txt=traceback_txt, 147 | code_content=code_content, 148 | ) 149 | 150 | self.state_machine.insert_input(lines, source_file_path) 151 | return [] 152 | 153 | 154 | def setup(app: Sphinx) -> ExtensionMetadata: 155 | """ 156 | Boilerplate required to be loaded by sphinx. 157 | """ 158 | app.add_directive(DIRECTIVE_NAME, ExecDirective) 159 | return { 160 | "version": "0.2", 161 | "parallel_read_safe": True, 162 | "parallel_write_safe": True, 163 | } 164 | -------------------------------------------------------------------------------- /kloch/launchers/_context.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import enum 3 | import getpass 4 | import logging 5 | import sys 6 | from typing import Dict 7 | from typing import Optional 8 | 9 | LOGGER = logging.getLogger(__name__) 10 | 11 | 12 | class LauncherPlatform(enum.Flag): 13 | """ 14 | Operating Systems that a launcher may support. 15 | """ 16 | 17 | linux = enum.auto() 18 | darwin = enum.auto() 19 | windows = enum.auto() 20 | 21 | @classmethod 22 | def current(cls): 23 | """ 24 | Get the member corresponding to the current runtime operating system. 25 | """ 26 | current = sys.platform 27 | if current.startswith("linux"): 28 | return cls.linux 29 | if current.startswith("darwin"): 30 | return cls.darwin 31 | if current in ("win32", "cygwin"): 32 | return cls.windows 33 | 34 | raise OSError(f"Unsupported operating system '{current}'") 35 | 36 | def is_linux(self) -> bool: 37 | return self == self.linux 38 | 39 | def is_mac(self) -> bool: 40 | return self == self.darwin 41 | 42 | def is_windows(self) -> bool: 43 | return self == self.windows 44 | 45 | 46 | _PLATFORM_MAPPING = { 47 | "linux": LauncherPlatform.linux, 48 | "windows": LauncherPlatform.windows, 49 | "mac": LauncherPlatform.darwin, 50 | } 51 | """ 52 | Mapping of serialized platform names to their corresponding enum instance. 53 | """ 54 | 55 | 56 | @dataclasses.dataclass 57 | class LauncherContext: 58 | """ 59 | Collection of system-contextual values in which a profile is being parsed. 60 | 61 | You can compare 2 context using the equal comparator however be aware that 62 | a None value (implies unset) will be considered equal to any other value. 63 | 64 | Example:: 65 | 66 | ctx1 = LauncherContext(platform=LauncherPlatform.linux, user="demo") 67 | ctx2 = LauncherContext(platform=LauncherPlatform.linux, user=None) 68 | assert ctx1 == ctx2 69 | 70 | """ 71 | 72 | platform: Optional[LauncherPlatform] = dataclasses.field( 73 | default=None, 74 | metadata={ 75 | "serialized_name": "os", 76 | "unserialize": lambda v: _PLATFORM_MAPPING[v], 77 | "doc": { 78 | "value": f"one of ``{'``, ``'.join(_PLATFORM_MAPPING.keys())}``", 79 | "description": "name of the operating system", 80 | }, 81 | }, 82 | ) 83 | """ 84 | name of the operating system 85 | """ 86 | 87 | user: Optional[str] = dataclasses.field( 88 | default=None, 89 | metadata={ 90 | "serialized_name": "user", 91 | "unserialize": str, 92 | "doc": { 93 | "value": f"arbitrary string", 94 | "description": "name of the user", 95 | }, 96 | }, 97 | ) 98 | """ 99 | name of the current user 100 | """ 101 | 102 | def __str__(self) -> str: 103 | return f"os={self.platform.name};user={self.user}" 104 | 105 | def __eq__(self, other): 106 | c = self.__class__ 107 | if not isinstance(other, c): 108 | raise TypeError(f"Cannot concatenate {c} and {type(other)}") 109 | 110 | for field in dataclasses.fields(c): 111 | selfvalue = getattr(self, field.name) 112 | othervalue = getattr(other, field.name) 113 | # we discard the comparison when a field is not set 114 | if selfvalue is None or othervalue is None: 115 | continue 116 | 117 | if selfvalue != othervalue: 118 | return False 119 | 120 | return True 121 | 122 | @classmethod 123 | def create_from_system(cls): 124 | """ 125 | Fill the instance with values retrieved from the current system context. 126 | """ 127 | return cls( 128 | platform=LauncherPlatform.current(), 129 | user=getpass.getuser(), 130 | ) 131 | 132 | 133 | _FIELDS_MAPPING: Dict[str, dataclasses.Field] = { 134 | field.metadata["serialized_name"]: field 135 | for field in dataclasses.fields(LauncherContext) 136 | } 137 | 138 | 139 | _ESCAPE_CHARACTER = "%!TMP}]" 140 | 141 | 142 | def _escape(source: str) -> str: 143 | return source.replace("@@", _ESCAPE_CHARACTER) 144 | 145 | 146 | def _unescape(source: str) -> str: 147 | return source.replace(_ESCAPE_CHARACTER, "@") 148 | 149 | 150 | def resolve_context_expression(source: str) -> str: 151 | """ 152 | Remove the profile context expression from the given source string. 153 | """ 154 | escaped = _escape(source) 155 | if "@" not in escaped: 156 | return _unescape(escaped) 157 | return _unescape(source.split("@")[0]) 158 | 159 | 160 | def unserialize_context_expression(source: str) -> LauncherContext: 161 | """ 162 | Generate a profile context based on its serialized expression form. 163 | 164 | The expression take the following pattern:: 165 | 166 | .(@key=value)* 167 | 168 | Where ``.`` means any character, ``()*`` means it can be repated multiple times. 169 | 170 | Args: 171 | source: 172 | abitrary string which contain an expression to unserialize, 173 | may be an empty string. 174 | 175 | Returns: 176 | a new profile context instance (that may be empty). 177 | """ 178 | escaped = _escape(source) 179 | 180 | if "@" not in escaped: 181 | return LauncherContext() 182 | 183 | members = escaped.split("@")[1:] 184 | asdict = {} 185 | 186 | for member in members: 187 | member = _unescape(member) 188 | key, value = member.split("=") 189 | field = _FIELDS_MAPPING.get(key) 190 | if not field: 191 | raise KeyError( 192 | f"Unsupported context key name '{key}'; must one of {_FIELDS_MAPPING}" 193 | ) 194 | field_name = field.name 195 | unserialize = field.metadata["unserialize"] 196 | # we explicitly allow a field to be defined multiple time, 197 | # its last definition will take precedence in value. 198 | asdict[field_name] = unserialize(value) 199 | 200 | return LauncherContext(**asdict) 201 | -------------------------------------------------------------------------------- /tests/test_dictmerge.py: -------------------------------------------------------------------------------- 1 | from kloch._dictmerge import MergeableDict 2 | 3 | 4 | def test__MergeableDict__type(): 5 | d1 = {"+=foo": [1, 2]} 6 | dm1 = MergeableDict({"+=foo": [1, 2]}) 7 | assert d1 == dm1 8 | 9 | # check copy at instancing 10 | dm2 = MergeableDict(d1) 11 | assert dm2 is not d1 12 | dm2["added"] = True 13 | assert "added" not in d1 14 | 15 | # check no deepcopy 16 | arg1 = [1, 2] 17 | d1 = MergeableDict({"foo": arg1}) 18 | assert d1["foo"] is arg1 19 | 20 | 21 | def test__MergeableDict__get(): 22 | arg1 = [1, 2] 23 | d1 = MergeableDict({"+=foo": arg1}) 24 | assert d1.get("foo") is None 25 | assert d1.get("foo", default=5) is 5 26 | assert d1.get("foo", default=5, ignore_tokens=True) is arg1 27 | 28 | arg1 = [1, 2] 29 | d1 = MergeableDict({"foo": arg1}) 30 | assert d1.get("foo") is arg1 31 | assert d1.get("foo", default=5) is arg1 32 | assert d1.get("+=foo", default=5, ignore_tokens=True) is arg1 33 | 34 | 35 | def test__MergeableDict__add(): 36 | dict_root = MergeableDict( 37 | { 38 | "rezenv": { 39 | "config": {"exclude": "whatever"}, 40 | "==requires": { 41 | "echoes": "2", 42 | "maya": "2023", 43 | }, 44 | "+=tests": { 45 | "+=foo": [1, 2, 3], 46 | "deeper!": {"as deep": [1, 2]}, 47 | }, 48 | }, 49 | "testenv": {"command": "echo $cwd"}, 50 | } 51 | ) 52 | dict_leaf = MergeableDict( 53 | { 54 | "+=rezenv": { 55 | "==config": {"include": "yes"}, 56 | "requires": { 57 | "-=maya": "_", 58 | "-=notAdded": "_", 59 | "added": "1.2", 60 | "!=echoes": "3", 61 | "!=knots": "1", 62 | }, 63 | "+=tests": { 64 | "foo": [4, 5, 6], 65 | "+=new-echoes-key": {"working": True}, 66 | "==deeper!": {"+=as deep": [0, 0]}, 67 | }, 68 | } 69 | } 70 | ) 71 | dict_expected = MergeableDict( 72 | { 73 | "+=rezenv": { 74 | "==config": {"include": "yes"}, 75 | "requires": { 76 | "echoes": "2", 77 | "added": "1.2", 78 | "!=knots": "1", 79 | }, 80 | "+=tests": { 81 | "foo": [1, 2, 3, 4, 5, 6], 82 | "+=new-echoes-key": {"working": True}, 83 | "==deeper!": {"+=as deep": [0, 0]}, 84 | }, 85 | }, 86 | "testenv": {"command": "echo $cwd"}, 87 | } 88 | ) 89 | result = dict_root + dict_leaf 90 | assert result == dict_expected 91 | 92 | expected = { 93 | "rezenv": { 94 | "config": {"include": "yes"}, 95 | "requires": { 96 | "echoes": "2", 97 | "added": "1.2", 98 | "knots": "1", 99 | }, 100 | "tests": { 101 | "foo": [1, 2, 3, 4, 5, 6], 102 | "new-echoes-key": {"working": True}, 103 | "deeper!": {"as deep": [0, 0]}, 104 | }, 105 | }, 106 | "testenv": {"command": "echo $cwd"}, 107 | } 108 | result = result.resolved() 109 | assert result == expected 110 | assert not isinstance(result, MergeableDict) 111 | 112 | 113 | def test__MergeableDict__add__subclass(): 114 | class MyDict(MergeableDict): 115 | pass 116 | 117 | class ConfigSystem(MergeableDict): 118 | pass 119 | 120 | md1 = MyDict({"+=requires": {"maya": "2020", "houdini": "20"}}) 121 | assert isinstance(md1, MyDict) 122 | 123 | md2 = ConfigSystem({"+=requires": {"maya": "2023", "nuke": "20"}}) 124 | 125 | mm = md1 + md2 126 | assert isinstance(mm, MyDict) 127 | 128 | 129 | def test__MergeableDict__add__different_type(): 130 | dm1 = MergeableDict( 131 | { 132 | "+=rezenv": { 133 | "+=config": {"exclude": "whatever"}, 134 | "requires": ["houdini-20"], 135 | "bar": 5, 136 | "+=params": ["--debug"], 137 | } 138 | } 139 | ) 140 | dm2 = MergeableDict( 141 | { 142 | "+=rezenv": { 143 | "+=config": None, 144 | "+=requires": "maya-2023", 145 | "-=bar": "test?", 146 | "params": "overriden !", 147 | } 148 | } 149 | ) 150 | dmmerged = dm1 + dm2 151 | assert dmmerged["+=rezenv"]["+=config"] is None 152 | assert dmmerged["+=rezenv"]["+=requires"] == "maya-2023" 153 | assert "-=bar" not in dmmerged["+=rezenv"] 154 | assert "bar" not in dmmerged["+=rezenv"] 155 | assert dmmerged["+=rezenv"]["params"] == "overriden !" 156 | 157 | 158 | def test__MergeableDict__add__nested(): 159 | dm1 = MergeableDict( 160 | { 161 | "+=rezenv": MergeableDict( 162 | { 163 | "+=config": {"exclude": "whatever"}, 164 | } 165 | ), 166 | } 167 | ) 168 | assert isinstance(dm1["+=rezenv"], MergeableDict) 169 | 170 | dm2 = MergeableDict( 171 | { 172 | "+=rezenv": MergeableDict( 173 | { 174 | "+=config": {"include": "IAMINCLUDED"}, 175 | "+=requires": ["maya-2023"], 176 | } 177 | ), 178 | } 179 | ) 180 | dmmerged = dm1 + dm2 181 | assert isinstance(dmmerged["+=rezenv"], MergeableDict) 182 | assert dmmerged["+=rezenv"]["+=config"] == { 183 | "exclude": "whatever", 184 | "include": "IAMINCLUDED", 185 | } 186 | 187 | 188 | def test__MergeableDict__add__order(): 189 | dm1 = MergeableDict( 190 | { 191 | "+=rezenv": { 192 | "+=config": {"exclude": "whatever"}, 193 | "+=requires": ["foo"], 194 | } 195 | } 196 | ) 197 | 198 | dm2 = MergeableDict( 199 | { 200 | "+=rezenv": { 201 | "+=config": {"exclude": "whatever"}, 202 | "+=requires": ["foo"], 203 | "newkey": True, 204 | } 205 | } 206 | ) 207 | dmmerged = dm1 + dm2 208 | assert list(dmmerged["+=rezenv"].keys()) == ["+=config", "+=requires", "newkey"] 209 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | 9 | ## [0.13.1] - 2025-02-10 10 | 11 | ### fixed 12 | 13 | - issue with deeply nested profiles (inheritance) 14 | 15 | ## [0.13.0] - 2025-01-30 16 | 17 | ### fixed 18 | 19 | - cli: issue with `list` command on inherited profiles 20 | 21 | ### added 22 | 23 | - launchers : `.system` launcher new field `command_as_str` 24 | - launchers : `.system` launcher new field `expand_first_arg` 25 | 26 | ### chores 27 | 28 | - Better launchers and launcher-plugin documentation. 29 | 30 | 31 | ## [0.12.0] - 2024-11-15 32 | 33 | ### added 34 | 35 | - context system: allow to specify suffix tokens on launchers name to filter 36 | their visiblity based on the system context on runtime. 37 | - cli: allow to specify profile to run/resolve as file paths 38 | - launchers: add `priority` key to let kloch guess which launcher to use 39 | when a profile has multiple of them. 40 | 41 | ### changed 42 | 43 | - ! `@python` launcher renamed to `.python` because of the new context system 44 | - ! `system` launcher renamed to `.system` for cohesion 45 | - updated profile version to 4 46 | - ! system launcher: add subprocess_kwargs field, shell=True is not default anymore. 47 | - improve error reporting when a launcher is invalid 48 | - minor logging improvements 49 | 50 | ### fixed 51 | 52 | - issues when picking a launcher through multiple in a profile 53 | 54 | ## [0.11.2] - 2024-10-26 55 | 56 | Add PyPI install instructions in documentation. 57 | 58 | ## [0.11.1] - 2024-10-26 59 | 60 | Fix PyPI publishing GitHub action. Doesn't affect code. 61 | 62 | ## [0.11.0] - 2024-10-26 63 | 64 | ### added 65 | 66 | - `cli.run_cli`, extracted from `__main__` 67 | - publish to PyPI and TestPyPI 68 | 69 | ### changed 70 | 71 | - ! full refacto of the plugin system; 72 | - `load_plugin_launchers` is now public 73 | - `load_plugin_launchers` return a new `LoadedPluginsLaunchers` object instance 74 | - remove `get_launcher_class` and `get_launcher_serialized_class`; not useful anymore 75 | - improve error handling (less verbose tracebacks) 76 | - moved all code in `__main__` to `cli` 77 | 78 | ### chores 79 | 80 | - doc: update the Usage page to reflect the change made in previous version 81 | - doc: fix missing public-api for ``kloch.filesyntax`` 82 | - ci: add ci to publish to TestPyPI and PyPI. 83 | - TestPyPI is only triggered on main branch pushes and ready-to-review PRs 84 | - PyPI is only triggered on GitHub release 85 | - added CONTRIBUTING.md 86 | 87 | ## [0.10.0] - 2024-08-04 88 | 89 | ### added 90 | 91 | - profile: new `!=` "if not exists" token 92 | - the log file parent directory is created if it doesn't exist 93 | - it's possible to add environment variable in the keys of the config file that are paths 94 | 95 | ### changed 96 | 97 | - ! cli: `--profile_paths` renamed to `--profile_roots` 98 | - ! profile: the default merge rule is now "append" instead of "override" 99 | - ! profile: renamed the root `base` key to `inherit` 100 | - logs go up to 262144 bytes before being rotated 101 | 102 | ## [0.9.0] - 2024-07-03 103 | 104 | ### added 105 | 106 | - relative path in the config file are made absolute to the config parent dir. 107 | 108 | ### fixed 109 | 110 | - keys in config file have their type properly casted to the expected type 111 | 112 | 113 | ## [0.8.2] - 2024-06-29 114 | 115 | ### fix 116 | 117 | - ci: `plugins` command not using self._config 118 | 119 | ## [0.8.1] - 2024-06-29 120 | 121 | ### chores 122 | 123 | - fixed documentation not building because of import errors 124 | 125 | ## [0.8.0] - 2024-06-29 126 | 127 | ### added 128 | 129 | - io: `read_profile_from_id()` 130 | - constants module with environment variables 131 | - logging to disk with `cli_logging_paths` config key 132 | - control for the "session directory" using config 133 | 134 | ### changed 135 | 136 | - !io: drop the global system to add profile locations, all profile location 137 | are added on each function call. 138 | - io: made `is_file_environment_profile` public 139 | - cli: add KlochConfig propagated through cli 140 | 141 | ### chores 142 | 143 | - fixing tests CI 144 | - pyproject use _extras_ instead of poetry groups 145 | - remove app packaging script: not used anymore 146 | 147 | 148 | ## [0.7.0] - 2024-05-27 149 | 150 | ### added 151 | 152 | - plugin system for launchers 153 | - individual config values can now be retrieved from environment variable 154 | 155 | ### changed 156 | 157 | - ! removed `rezenv` launcher to be extracted as plugin at https://github.com/knotsanimation/kloch-launcher-rezenv 158 | - logic change in the MergeableDict class when subclassing to return the 159 | proper subclass when adding 2 instances. 160 | 161 | ### chores 162 | 163 | - updated `Usage` documentation 164 | 165 | ## [0.6.0] - 2024-05-25 166 | 167 | ### added 168 | 169 | - `cli` add `python` subcommand to execute a file with the internal interpreter 170 | - `launchers` add a new `@python` launcher 171 | - `launchers` add `cwd` attribute 172 | - add configuration system 173 | 174 | ### changed 175 | 176 | - !! complex refacto of launcher serialization system. 177 | - `launchers` improve error message when missing a required field to a launcher 178 | - `launchers` improve debbug message of subprocess 179 | - `launchers` resolve `environ` earlier at init instead of in `execute()` 180 | - `launchers.rezenv` remove `requires` from required_fileds 181 | - ! `io` allow to specify a `profile_locations` arg 182 | 183 | ### fixed 184 | 185 | - `profile` merging of `.base` when token in launcher name 186 | - ensure python 3.7 compatibility (mostly invalid type hints) 187 | 188 | ### chores 189 | 190 | - improved unittesting 191 | - run unittest in CI 192 | 193 | 194 | ## [0.5.2] - 2024-05-06 195 | 196 | ### chores 197 | 198 | - added application packaging with `nuitka` in complement of `pyinstaller` 199 | - add branding (logo, typography, colors, ...) 200 | - add `copy-button` extension to sphinx doc 201 | - minor documentation hierarchy changes 202 | - improve index/README 203 | 204 | ## [0.5.1] - 2024-05-04 205 | 206 | ### added 207 | 208 | - `command` attribute to the `.base` launcher 209 | - available for all launcher, it is prioritary over the CLI command which is appened to it. 210 | 211 | 212 | ## [0.5.0] - 2024-05-04 213 | 214 | ### changed 215 | 216 | - ! renamed the core repository name to `kloch` (previously `kenvmanager`) 217 | - ! renamed profile magic name to `kloch_profile` (from `KenvEnvironmentProfile`) 218 | 219 | ## [0.4.0] - 2024-05-04 220 | 221 | ### changed 222 | 223 | - ! changed the concept of `managers` to `launchers`, everywhere. 224 | - ! bump the profile version to 2 225 | 226 | ## [0.3.0] - 2024-05-04 227 | 228 | ### added 229 | 230 | - allow to specify a `.base` manager that is merged with others on resolve. 231 | - add `kloch.get_available_managers_classes` 232 | - add a new `system` manager 233 | - allow environment variable expansion in `environ` 234 | - add path normalization in `environ` 235 | 236 | ### fixed 237 | 238 | - finalize the implementation of the `--` command in the CLI 239 | 240 | ### changed 241 | 242 | - ! `kloch.managers.rezenv` was made private 243 | 244 | ### chores 245 | 246 | - add custom `execinject` extension to build more dynamic documentation 247 | - use `execinject` to automatize the documentation of managers 248 | 249 | 250 | ## [0.2.0] - 2024-05-02 251 | 252 | Minor version bump for development purpose. No changes. 253 | 254 | ## [0.1.0] - 2024-05-02 255 | 256 | Initial release. 257 | -------------------------------------------------------------------------------- /kloch/launchers/_plugins.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import importlib 3 | import inspect 4 | import logging 5 | from typing import Dict 6 | from typing import Generic 7 | from typing import List 8 | from typing import Type 9 | from typing import TypeVar 10 | 11 | from kloch.launchers import BaseLauncher 12 | from kloch.launchers import BaseLauncherSerialized 13 | 14 | LOGGER = logging.getLogger(__name__) 15 | 16 | T = TypeVar("T") 17 | 18 | 19 | @dataclasses.dataclass 20 | class LoadedPluginsLaunchers(Generic[T]): 21 | launchers: List[T] 22 | """ 23 | List of external plugins that have been loaded 24 | """ 25 | 26 | missed: Dict[str, str] 27 | """ 28 | Given plugins that couldn't be loaded as mapping of {"module name": "error message"} 29 | """ 30 | 31 | given: List[str] 32 | """ 33 | Original list of modules name the plugins were extracted from. 34 | """ 35 | 36 | 37 | def load_plugin_launchers( 38 | module_names: List[str], 39 | subclass_type: Type[T], 40 | ) -> LoadedPluginsLaunchers[Type[T]]: 41 | """ 42 | Retrieve the launcher subclasses from the given module names. 43 | 44 | Import error are silenced and stored in the returned object. 45 | 46 | Args: 47 | module_names: list of importable python module names 48 | subclass_type: base class of the launchers to return 49 | 50 | Returns: 51 | An instance of plugins loaded 52 | """ 53 | plugins = [] 54 | missed = {} 55 | 56 | for module_name in module_names: 57 | try: 58 | module = importlib.import_module(module_name) 59 | except (ModuleNotFoundError, ImportError) as error: 60 | missed[module_name] = str(error) 61 | continue 62 | 63 | module_content = inspect.getmembers(module, inspect.isclass) 64 | module_launchers = [ 65 | obj[1] 66 | for obj in module_content 67 | if issubclass(obj[1], subclass_type) and not obj[1] is subclass_type 68 | ] 69 | if not module_launchers: 70 | missed[module_name] = ( 71 | f"Module doesn't have any subclass of '{subclass_type}'" 72 | ) 73 | continue 74 | 75 | for launcher in module_launchers: 76 | if launcher in plugins: 77 | LOGGER.warning( 78 | f"Got duplicated plugin '{launcher}' from module '{module_name}': skipping" 79 | ) 80 | continue 81 | plugins.append(launcher) 82 | 83 | return LoadedPluginsLaunchers( 84 | launchers=plugins, 85 | missed=missed, 86 | given=module_names.copy(), 87 | ) 88 | 89 | 90 | def _assert(condition: bool, message: str): 91 | if not condition: 92 | raise AssertionError(message) 93 | 94 | 95 | def _check_launcher_implementation(launcher: Type[BaseLauncher]): 96 | """ 97 | Raise an AssertionError if the given launcher subclass is not properly implemented. 98 | 99 | Args: 100 | launcher: a BaseLauncher class/subclass to check 101 | """ 102 | if launcher is not BaseLauncher: 103 | _assert( 104 | launcher.name != BaseLauncher.name, 105 | f"Implementation '{launcher.__name__}' forgot to override 'name' attribute.", 106 | ) 107 | 108 | _assert( 109 | issubclass(launcher, BaseLauncher), 110 | f"Implementation '{launcher.__name__}' must be a direct subclass of '{BaseLauncher.__name__}'.", 111 | ) 112 | 113 | 114 | def _check_launcher_serialized_implementation(launcher: Type[BaseLauncherSerialized]): 115 | """ 116 | Raise an AssertionError if the given serialized launcher subclass is not properly implemented. 117 | 118 | Args: 119 | launcher: a BaseLauncherSerialized class/subclass to check 120 | """ 121 | _assert( 122 | issubclass(launcher, BaseLauncherSerialized), 123 | f"Implementation '{launcher.__name__}' must be a direct subclass of '{BaseLauncherSerialized.__name__}'.", 124 | ) 125 | 126 | def missing(field: str) -> str: 127 | return ( 128 | f"Implementation '{launcher.__name__}' is missing the '{field}' attribute." 129 | ) 130 | 131 | _assert(hasattr(launcher, "fields"), missing("fields")) 132 | _assert(hasattr(launcher, "source"), missing("source")) 133 | _assert(hasattr(launcher, "identifier"), missing("identifier")) 134 | _assert(hasattr(launcher, "summary"), missing("summary")) 135 | _assert(hasattr(launcher, "description"), missing("description")) 136 | 137 | def wrong_override(field: str) -> str: 138 | return f"Implementation '{launcher.__name__}' doesn't override the base '{field}' attribute." 139 | 140 | if launcher is not BaseLauncherSerialized: 141 | _assert( 142 | launcher.source is not BaseLauncherSerialized, 143 | wrong_override("source"), 144 | ) 145 | _assert( 146 | launcher.fields is not BaseLauncherSerialized.fields, 147 | wrong_override("fields"), 148 | ) 149 | _assert( 150 | launcher.summary is not BaseLauncherSerialized.summary, 151 | wrong_override("summary"), 152 | ) 153 | _assert( 154 | launcher.description is not BaseLauncherSerialized.description, 155 | wrong_override("description"), 156 | ) 157 | _assert( 158 | launcher.identifier is not BaseLauncherSerialized.identifier, 159 | wrong_override("identifier"), 160 | ) 161 | 162 | fields_ser = dataclasses.fields(launcher.fields) 163 | fields_source = dataclasses.fields(launcher.source) 164 | 165 | _assert( 166 | len(fields_ser) >= len(fields_source), 167 | f"Implementation '{launcher.__name__}' must define as much or more fields than its source '{launcher.source}'", 168 | ) 169 | 170 | for field_ser in fields_ser: 171 | _assert( 172 | "required" in field_ser.metadata, 173 | f"Implementation '{launcher.__name__}' field '{field_ser.name}' is missing a 'required' attribute in its metadata.", 174 | ) 175 | _assert( 176 | "description" in field_ser.metadata, 177 | f"Implementation '{launcher.__name__}' field '{field_ser.name}' is missing a 'description' attribute in its metadata.", 178 | ) 179 | 180 | 181 | class PluginModuleError(Exception): 182 | pass 183 | 184 | 185 | class PluginImplementationError(Exception): 186 | pass 187 | 188 | 189 | def check_launcher_plugins( 190 | plugins: LoadedPluginsLaunchers[Type[BaseLauncherSerialized]], 191 | ) -> List[Exception]: 192 | """ 193 | Return any issue/error the given launcher may have. 194 | 195 | This function is not supposed to raise if a launcher is invalid. 196 | 197 | Args: 198 | plugins: collection of plugin launcher loaded from external modules. 199 | 200 | Returns: 201 | list of errors found in given loaded plugins, empty if no errors. 202 | """ 203 | errors: List[Exception] = [] 204 | if plugins.missed: 205 | for module, error in plugins.missed.items(): 206 | error = PluginModuleError(f"Module '{module}' was not loaded: {error}") 207 | errors.append(error) 208 | 209 | for launcher in plugins.launchers: 210 | try: 211 | _check_launcher_serialized_implementation(launcher) 212 | except AssertionError as error: 213 | error = PluginImplementationError(str(error)) 214 | errors.append(error) 215 | 216 | launchers = [launcher.source for launcher in plugins.launchers] 217 | 218 | try: 219 | for launcher in launchers: 220 | _check_launcher_implementation(launcher) 221 | except AssertionError as error: 222 | error = PluginImplementationError(str(error)) 223 | errors.append(error) 224 | 225 | return errors 226 | -------------------------------------------------------------------------------- /kloch/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple configuration system for the Kloch runtime. 3 | """ 4 | 5 | import dataclasses 6 | import logging 7 | import os 8 | from pathlib import Path 9 | from typing import Dict 10 | from typing import List 11 | from typing import Optional 12 | from typing import TypeVar 13 | from typing import Union 14 | 15 | import yaml 16 | 17 | from kloch.constants import Environ 18 | from kloch._utils import expand_envvars 19 | 20 | LOGGER = logging.getLogger(__name__) 21 | 22 | T = TypeVar("T") 23 | 24 | 25 | def _placeholder_caster(x: T, *args, **kwargs) -> T: 26 | return x 27 | 28 | 29 | def _cast_list_split(src_str: str) -> List[str]: 30 | return src_str.split(",") 31 | 32 | 33 | def _cast_path(src_str: str) -> Path: 34 | return Path(src_str) 35 | 36 | 37 | def _cast_path_list_split(src_str: str) -> List[Path]: 38 | return [Path(path) for path in src_str.split(os.pathsep)] 39 | 40 | 41 | # caster for yaml configs: 42 | 43 | 44 | def _ensure_path_absolute(path: Path, absolute_root: Path) -> Path: 45 | if path.is_absolute(): 46 | return path 47 | return Path(absolute_root, path).resolve() 48 | 49 | 50 | def _resolve_path_env_vars(path: Path) -> Path: 51 | resolved = expand_envvars(str(path)) 52 | return Path(resolved) 53 | 54 | 55 | def _make_config_caster(caster): 56 | def _config_caster(v, *args, **kwargs): 57 | return caster(v) 58 | 59 | return _config_caster 60 | 61 | 62 | def _cast_config_path(src_str: str, config_dir: Path) -> Path: 63 | casted = _resolve_path_env_vars(Path(src_str)) 64 | casted = _ensure_path_absolute(casted, config_dir) 65 | return casted 66 | 67 | 68 | def _cast_config_path_list(src_list: List[str], config_dir: Path) -> List[Path]: 69 | return [_cast_config_path(path, config_dir) for path in src_list] 70 | 71 | 72 | @dataclasses.dataclass 73 | class KlochConfig: 74 | """ 75 | Configure kloch using a simple key/value pair dataclass system. 76 | """ 77 | 78 | launcher_plugins: List[str] = dataclasses.field( 79 | default_factory=list, 80 | metadata={ 81 | "documentation": ( 82 | "A list of importable python module names containing new launchers to support.\n\n" 83 | "If specified in environment variable, this must be a comma-separated list of str like ``module1,module2,module3``" 84 | ), 85 | "config_cast": _make_config_caster(list), 86 | "environ": Environ.CONFIG_LAUNCHER_PLUGINS, 87 | "environ_cast": _cast_list_split, 88 | }, 89 | ) 90 | 91 | cli_logging_paths: List[Path] = dataclasses.field( 92 | default_factory=list, 93 | metadata={ 94 | "documentation": ( 95 | "Filesystem path to one or multiple log file that might exists.\n" 96 | "If specified all the logging will also be wrote to those files.\n" 97 | "The log path parent directory is created if it doesn't exists.\n\n" 98 | "Logs are rotated between 2 files of 262.14Kb max.\n\n" 99 | "If specified from the environment, it must a list of path separated " 100 | "by the default system path separator (windows = ``;``, linux = ``:``)" 101 | ), 102 | "config_cast": _cast_config_path_list, 103 | "environ": Environ.CONFIG_CLI_LOGGING_PATHS, 104 | "environ_cast": _cast_path_list_split, 105 | }, 106 | ) 107 | 108 | cli_logging_format: str = dataclasses.field( 109 | default="{levelname: <7} | {asctime} [{name}] {message}", 110 | metadata={ 111 | "documentation": ( 112 | "Formatting to use for all logged messages. See python logging module documentation.\n" 113 | "The tokens must use the ``{`` style." 114 | ), 115 | "config_cast": _make_config_caster(str), 116 | "environ": Environ.CONFIG_CLI_LOGGING_FORMAT, 117 | "environ_cast": str, 118 | }, 119 | ) 120 | 121 | cli_logging_default_level: Union[int, str] = dataclasses.field( 122 | default="INFO", 123 | metadata={ 124 | "documentation": ( 125 | "Logging level to use if None have been specified.\n" 126 | "Can be an int or a level name as string as long as it is understandable" 127 | " by ``logging.getLevelName``." 128 | ), 129 | "config_cast": _make_config_caster(str), 130 | "environ": Environ.CONFIG_CLI_LOGGING_DEFAULT_LEVEL, 131 | "environ_cast": str, 132 | }, 133 | ) 134 | 135 | cli_session_dir: Optional[Path] = dataclasses.field( 136 | default=None, 137 | metadata={ 138 | "documentation": ( 139 | "Filesystem path to a directory that might exists.\n" 140 | "The directory is used to store temporarly any file generated during the executing of a launcher.\n" 141 | "If not specified, a system's default temporary location is used." 142 | ), 143 | "config_cast": _cast_config_path, 144 | "environ": Environ.CONFIG_CLI_SESSION_PATH, 145 | "environ_cast": _cast_path, 146 | }, 147 | ) 148 | 149 | cli_session_dir_lifetime: float = dataclasses.field( 150 | default=240.0, 151 | metadata={ 152 | "documentation": ( 153 | "Amount in hours before a session directory must be deleted.\n" 154 | "Note the deleting is performed only the next time kloch is started so it " 155 | "is possible a session directory exist longer if kloch is not launched for a while." 156 | ), 157 | "config_cast": _make_config_caster(float), 158 | "environ": Environ.CONFIG_CLI_SESSION_LIFETIME, 159 | "environ_cast": float, 160 | }, 161 | ) 162 | 163 | profile_roots: List[Path] = dataclasses.field( 164 | default_factory=list, 165 | metadata={ 166 | "documentation": ( 167 | "Filesystem path to one or multiple directory that might exists.\n" 168 | "The directories contain profile valid to be discoverable.\n\n" 169 | "If specified from the environment, it must a list of path separated " 170 | "by the default system path separator (windows = ``;``, linux = ``:``)" 171 | ), 172 | "config_cast": _cast_config_path_list, 173 | "environ": Environ.CONFIG_PROFILE_ROOTS, 174 | "environ_cast": _cast_path_list_split, 175 | }, 176 | ) 177 | 178 | @classmethod 179 | def from_file(cls, file_path: Path) -> "KlochConfig": 180 | """ 181 | Generate an instance from a serialized file. 182 | """ 183 | with file_path.open("r", encoding="utf-8") as file: 184 | asdict: Dict = yaml.safe_load(file) 185 | 186 | casters = { 187 | field.name: field.metadata["config_cast"] 188 | for field in dataclasses.fields(cls) 189 | } 190 | # cast config value to expected type 191 | # all paths type are made absolute using the config parent dir 192 | asdict = { 193 | k: casters.get(k, _placeholder_caster)(v, file_path.parent) 194 | for k, v in asdict.items() 195 | } 196 | return cls(**asdict) 197 | 198 | @classmethod 199 | def from_environment(cls) -> "KlochConfig": 200 | """ 201 | Generate an instance from a serialized file specified in an environment variable. 202 | """ 203 | environ = os.getenv(Environ.CONFIG_PATH) 204 | 205 | asdict = {} 206 | if environ: 207 | base = cls.from_file(Path(environ)) 208 | asdict = dataclasses.asdict(base) 209 | 210 | for field in dataclasses.fields(cls): 211 | env_var_name = field.metadata["environ"] 212 | env_var_value = os.getenv(env_var_name) 213 | if env_var_value is not None: 214 | value = field.metadata["environ_cast"](env_var_value) 215 | asdict[field.name] = value 216 | 217 | return cls(**asdict) 218 | 219 | @classmethod 220 | def get_field(cls, field_name: str) -> Optional[dataclasses.Field]: 221 | """ 222 | Return the dataclass field that match the given name else None. 223 | """ 224 | fields = dataclasses.fields(cls) 225 | field = [field for field in fields if field.name == field_name] 226 | return field[0] if field else None 227 | 228 | 229 | def get_config() -> KlochConfig: 230 | """ 231 | Get the current kloch configuration extracted from the environment. 232 | 233 | A default configuration is generated if no configuration file is specified. 234 | 235 | Returns: 236 | a new config instance 237 | """ 238 | return KlochConfig.from_environment() 239 | -------------------------------------------------------------------------------- /kloch/filesyntax/_io.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | from typing import Dict 4 | from typing import List 5 | from typing import Optional 6 | 7 | import yaml 8 | 9 | from ._profile import LauncherSerializedDict 10 | from ._profile import EnvironmentProfile 11 | 12 | 13 | LOGGER = logging.getLogger(__name__) 14 | 15 | 16 | KENV_PROFILE_MAGIC = "kloch_profile" 17 | KENV_PROFILE_VERSION = 4 18 | 19 | 20 | class ProfileAPIVersionError(Exception): 21 | """ 22 | Issue with the '__magic__' attribute of a profile. 23 | """ 24 | 25 | pass 26 | 27 | 28 | class ProfileInheritanceError(Exception): 29 | """ 30 | Issue with the 'inherit' attribute of a profile. 31 | """ 32 | 33 | pass 34 | 35 | 36 | class ProfileIdentifierError(Exception): 37 | """ 38 | Issue with the 'identifier' attribute of a profile. 39 | """ 40 | 41 | pass 42 | 43 | 44 | def is_file_environment_profile(file_path: Path) -> bool: 45 | """ 46 | Return True if the given file is an Environment Profile. 47 | 48 | Args: 49 | file_path: filesystem path to an existing file 50 | """ 51 | if not file_path.suffix == ".yml": 52 | return False 53 | 54 | with file_path.open("r", encoding="utf-8") as file: 55 | content = yaml.safe_load(file) 56 | 57 | if not content: 58 | return False 59 | 60 | return content.get("__magic__", "").startswith(KENV_PROFILE_MAGIC) 61 | 62 | 63 | def get_all_profile_file_paths(locations: Optional[List[Path]] = None) -> List[Path]: 64 | """ 65 | Get all the environment-profile file paths as registred by the user. 66 | 67 | Args: 68 | locations: list of filesystem path to directory that might exist 69 | """ 70 | locations = locations or [] 71 | return [ 72 | path 73 | for location in locations 74 | for path in location.glob("*.yml") 75 | if is_file_environment_profile(path) 76 | ] 77 | 78 | 79 | def _get_profile_identifier(file_path: Path) -> str: 80 | with file_path.open("r", encoding="utf-8") as file: 81 | asdict: Dict = yaml.safe_load(file) 82 | return asdict["identifier"] 83 | 84 | 85 | def get_profile_file_path( 86 | profile_id: str, 87 | profile_locations: Optional[List[Path]] = None, 88 | ) -> List[Path]: 89 | """ 90 | Get the filesystem location to the profile(s) with the given name. 91 | 92 | Args: 93 | profile_id: identifier that must match returned profiles. 94 | profile_locations: 95 | list of filesystem path to potential existing directories containing profiles. 96 | 97 | Returns: 98 | list of filesystem path to existing files . Might be empty. 99 | """ 100 | profile_paths = get_all_profile_file_paths(locations=profile_locations) 101 | profiles: List[Path] = [ 102 | path for path in profile_paths if _get_profile_identifier(path) == profile_id 103 | ] 104 | return profiles 105 | 106 | 107 | def read_profile_from_file( 108 | file_path: Path, 109 | profile_locations: Optional[List[Path]] = None, 110 | ) -> EnvironmentProfile: 111 | """ 112 | Generate an instance from a serialized file on disk. 113 | 114 | Raises: 115 | ProfileAPIVersionError: 116 | ProfileInheritanceError: 117 | 118 | Args: 119 | file_path: 120 | filesystem path to an existing valid profile file. 121 | profile_locations: 122 | list of filesystem path to potential existing directories containing profiles. 123 | """ 124 | with file_path.open("r", encoding="utf-8") as file: 125 | asdict: Dict = yaml.safe_load(file) 126 | 127 | profile_version = int(asdict["__magic__"].split(":")[-1]) 128 | if not profile_version == KENV_PROFILE_VERSION: 129 | raise ProfileAPIVersionError( 130 | f"Cannot read profile with version <{profile_version}> while current " 131 | f"API version is <{KENV_PROFILE_VERSION}>." 132 | ) 133 | del asdict["__magic__"] 134 | 135 | super_name: Optional[str] = asdict.get("inherit", None) 136 | if super_name: 137 | super_paths = get_profile_file_path( 138 | super_name, 139 | profile_locations=profile_locations, 140 | ) 141 | if len(super_paths) >= 2: 142 | raise ProfileInheritanceError( 143 | f"Found multiple profile with identifier '{super_name}' " 144 | f"specified from profile '{file_path}': {super_paths}." 145 | ) 146 | if not super_paths: 147 | raise ProfileInheritanceError( 148 | f"No profile found with identifier '{super_name}' " 149 | f"specified from profile '{file_path}'." 150 | ) 151 | 152 | super_profile = read_profile_from_file( 153 | file_path=super_paths[0], 154 | profile_locations=profile_locations, 155 | ) 156 | asdict["inherit"] = super_profile 157 | 158 | launchers = LauncherSerializedDict(asdict["launchers"]) 159 | asdict["launchers"] = launchers 160 | 161 | profile = EnvironmentProfile.from_dict(asdict) 162 | return profile 163 | 164 | 165 | def read_profile_from_id( 166 | profile_id: str, 167 | profile_locations: Optional[List[Path]] = None, 168 | ) -> EnvironmentProfile: 169 | """ 170 | Generate a profile instance from a serialized file on disk retrieved using the given identifier. 171 | 172 | Raises error if the profile file is not built properly. 173 | 174 | This a convenient function wrapping :func:`read_profile_from_file` and 175 | :func:`get_profile_file_path` and assuming that no profile with the same 176 | identifier exist in the locations. 177 | 178 | Args: 179 | profile_id: identifier that must match the profile. 180 | profile_locations: 181 | list of filesystem path to potential existing directories containing profiles. 182 | 183 | Returns: 184 | a profile instance 185 | """ 186 | profile_paths = get_profile_file_path( 187 | profile_id=profile_id, 188 | profile_locations=profile_locations, 189 | ) 190 | profile = read_profile_from_file( 191 | file_path=profile_paths[0], 192 | profile_locations=profile_locations, 193 | ) 194 | return profile 195 | 196 | 197 | def serialize_profile( 198 | profile: EnvironmentProfile, 199 | profile_locations: Optional[List[Path]] = None, 200 | ) -> str: 201 | """ 202 | Convert the instance to a serialized dictionnary intended to be written on disk. 203 | 204 | Raises: 205 | ProfileInheritanceError: if the inherited profile specified is not found on disk 206 | """ 207 | asdict = {"__magic__": f"{KENV_PROFILE_MAGIC}:{KENV_PROFILE_VERSION}"} 208 | asdict.update(profile.to_dict()) 209 | 210 | super_profile: Optional[EnvironmentProfile] = asdict.get("inherit", None) 211 | if super_profile: 212 | super_path = get_profile_file_path( 213 | profile_id=super_profile.identifier, 214 | profile_locations=profile_locations, 215 | ) 216 | if not super_path: 217 | raise ProfileInheritanceError( 218 | f"Profile '{super_profile.identifier}' specified for inheritance on " 219 | f"profile '{profile.identifier}' cannot be found on disk." 220 | ) 221 | asdict["inherit"] = super_profile.identifier 222 | 223 | # remove custom class wrapper 224 | asdict["launchers"] = dict(asdict["launchers"]) 225 | 226 | return yaml.dump(asdict, sort_keys=False) 227 | 228 | 229 | def write_profile_to_file( 230 | profile: EnvironmentProfile, 231 | file_path: Path, 232 | profile_locations: Optional[List[Path]] = None, 233 | check_valid_id: bool = True, 234 | extra_comments: List[str] = None, 235 | ) -> Path: 236 | """ 237 | Convert the instance to a serialized file on disk. 238 | 239 | Raises: 240 | ProfileIdentifierError: if check_valid_id=True and the profile identifier is not unique 241 | 242 | Args: 243 | profile: profile instance to write to disk 244 | file_path: 245 | filesystem path to a file that might exist. 246 | parent location is expected to exist. 247 | check_valid_id: 248 | if True, ensure the identifier of the profile is unique among all ``profile_locations`` 249 | profile_locations: 250 | list of filesystem path to potential existing directories containing profiles. 251 | extra_comments: optional lines of comments to put in the yaml header 252 | """ 253 | if check_valid_id: 254 | profile_paths = get_profile_file_path( 255 | profile_id=profile.identifier, 256 | profile_locations=profile_locations, 257 | ) 258 | if profile_paths and file_path not in profile_paths: 259 | raise ProfileIdentifierError( 260 | f"Found multiple profile with identifier '{profile.identifier}'." 261 | ) 262 | 263 | serialized = serialize_profile(profile, profile_locations=profile_locations) 264 | 265 | extra_comments = extra_comments or [] 266 | extra_comments = "# " + "\n# ".join(extra_comments) 267 | serialized = extra_comments + "\n" + serialized 268 | 269 | file_path.write_text(serialized) 270 | return file_path 271 | -------------------------------------------------------------------------------- /kloch/_dictmerge.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import enum 3 | import logging 4 | from typing import Any 5 | from typing import Callable 6 | from typing import Dict 7 | from typing import Optional 8 | from typing import Tuple 9 | from typing import TypeVar 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | class MergeRule(enum.IntEnum): 15 | """ 16 | Define how 2 python objects should be merged together. 17 | """ 18 | 19 | override = enum.auto() 20 | append = enum.auto() 21 | remove = enum.auto() 22 | ifnotexists = enum.auto() 23 | 24 | 25 | def refacto_dict( 26 | src_dict: Dict, 27 | callback: Callable[[Any, Any], Tuple[Any, Any]], 28 | recursive=True, 29 | ) -> Dict: 30 | """ 31 | Iterate through all key/value pairs of the given dict and execute the given callback 32 | at each step which return a new key/value pair for the output dict. 33 | 34 | Args: 35 | src_dict: dict obejct ot iterate 36 | callback: Callable expecting 2args: key, value and should return a tuple of new_key, new_value 37 | recursive: True to recursively process child dict 38 | 39 | Returns: 40 | a new dict instance 41 | """ 42 | new_dict = {} 43 | for key, value in src_dict.items(): 44 | if isinstance(value, dict) and recursive: 45 | value = refacto_dict(value, callback=callback, recursive=True) 46 | new_key, new_value = callback(key, value) 47 | new_dict[new_key] = new_value 48 | return new_dict 49 | 50 | 51 | def deepmerge_dicts( 52 | over_content: Dict[str, Any], 53 | base_content: Dict, 54 | merge_rule_callback: Optional[Callable[[str], MergeRule]] = None, 55 | key_resolve_callback: Optional[Callable[[str], str]] = None, 56 | ) -> Dict[str, Any]: 57 | """ 58 | Recursively merge the 2 given dict "over one another". 59 | 60 | Intended to work best with dict having the same key/value structure. 61 | 62 | The merging rules are defined by the given callbacks. If no callback is provided 63 | the default rule is to override. 64 | 65 | The following object types supports the ``append`` rule: 66 | 67 | - `list`: with a ``.extend()`` behavior 68 | - `dict`: deepmerged recursively 69 | 70 | For any other type, the `over`'s value override the `base`'s value. 71 | """ 72 | new_content = copy.deepcopy(base_content) 73 | merge_rule_callback = merge_rule_callback or (lambda k: MergeRule.override) 74 | key_resolve_callback = key_resolve_callback or (lambda k: k) 75 | 76 | for over_key, over_value in over_content.items(): 77 | merge_rule = merge_rule_callback(over_key) 78 | 79 | # put base and over at same level by resolving both 80 | base_keys_resolved = { 81 | key_resolve_callback(bk): bk for bk, _ in base_content.items() 82 | } 83 | over_key_resolved = key_resolve_callback(over_key) 84 | 85 | base_key: Optional[str] = base_keys_resolved.get(over_key_resolved, None) 86 | base_value: Optional[Any] = base_content[base_key] if base_key else None 87 | 88 | if merge_rule == merge_rule.remove: 89 | if base_key: 90 | del new_content[base_key] 91 | continue 92 | 93 | if merge_rule == merge_rule.override: 94 | if base_key: 95 | del new_content[base_key] 96 | new_content[over_key] = over_value 97 | continue 98 | 99 | if merge_rule == merge_rule.ifnotexists: 100 | if base_key: 101 | new_content[base_key] = base_value 102 | else: 103 | new_content[over_key] = over_value 104 | continue 105 | 106 | # reaching here implies `merge_rule.append` 107 | 108 | if isinstance(over_value, dict) and isinstance(base_value, dict): 109 | new_value = deepmerge_dicts( 110 | over_value, 111 | base_content=base_value, 112 | merge_rule_callback=merge_rule_callback, 113 | key_resolve_callback=key_resolve_callback, 114 | ) 115 | 116 | elif isinstance(over_value, list) and isinstance(base_value, list): 117 | new_value = [] + base_value + over_value 118 | 119 | else: 120 | new_value = copy.deepcopy(over_value) 121 | 122 | if base_key: 123 | # we have merged `base` value with `over` so we can remove it safely 124 | del new_content[base_key] 125 | new_content[over_key] = new_value 126 | continue 127 | 128 | return new_content 129 | 130 | 131 | def _remove_prefix(text: str, prefix: str) -> str: 132 | if text.startswith(prefix): 133 | return text[len(prefix) :] 134 | return text 135 | 136 | 137 | T = TypeVar("T", bound="MergeableDict") 138 | 139 | 140 | class MergeableDict(dict): 141 | """ 142 | A dict that can be deep-merged with another dict. 143 | 144 | The merging algorithm is defined in :obj:`deepmerge_dicts`. 145 | 146 | You can define the merging granularity on a per-key basis by adding token prefix 147 | to your keys. 148 | 149 | Available tokens are found in :obj:`MergeableDict.tokens`. 150 | 151 | Example: 152 | 153 | .. exec_code:: 154 | :caption_output: results: 155 | :language_output: python 156 | 157 | from kloch import MergeableDict 158 | dict_1 = MergeableDict({"+=config": {"cache": True, "level": 3, "port": "A46"}}) 159 | dict_2 = MergeableDict({"+=config": {"cache": False, "-=level": 3}}) 160 | print(dict_1 + dict_2) 161 | 162 | """ 163 | 164 | class tokens: 165 | append = "+=" 166 | remove = "-=" 167 | override = "==" 168 | ifnotexists = "!=" 169 | 170 | def __add__(self: T, other: T) -> T: 171 | """ 172 | Merge the 2 dict structure together with ``other`` merged over ``self``. 173 | 174 | The returned class type is of the left member of the + operation. 175 | 176 | Returns: 177 | new instance with deepcopied structure. 178 | """ 179 | if not isinstance(other, MergeableDict): 180 | raise TypeError( 181 | f"Cannot concatenate object of type {type(other)} with {type(self)}" 182 | ) 183 | 184 | new_content = deepmerge_dicts( 185 | over_content=other, 186 | base_content=self, 187 | key_resolve_callback=self.resolve_key_tokens, 188 | merge_rule_callback=self.get_merge_rule, 189 | ) 190 | return self.__class__(new_content) 191 | 192 | @classmethod 193 | def resolve_key_tokens(cls, key: str) -> str: 194 | """ 195 | Ensure the given key has all potential tokens removed. 196 | """ 197 | resolved = key 198 | for token in [ 199 | cls.tokens.append, 200 | cls.tokens.remove, 201 | cls.tokens.override, 202 | cls.tokens.ifnotexists, 203 | ]: 204 | resolved = _remove_prefix(resolved, token) 205 | return resolved 206 | 207 | @classmethod 208 | def get_merge_rule(cls, key: str) -> MergeRule: 209 | """ 210 | Extract the :obj:`MergeRule` for the given key based on its potential token. 211 | """ 212 | if key.startswith(cls.tokens.append): 213 | return MergeRule.append 214 | if key.startswith(cls.tokens.remove): 215 | return MergeRule.remove 216 | if key.startswith(cls.tokens.override): 217 | return MergeRule.override 218 | if key.startswith(cls.tokens.ifnotexists): 219 | return MergeRule.ifnotexists 220 | return MergeRule.append 221 | 222 | def get(self, key, default=None, ignore_tokens: bool = False): 223 | """ 224 | Args: 225 | key: key's value to retrieve 226 | default: value to return if key is not in the dict 227 | ignore_tokens: 228 | if True, both key and self are used with a resolved variant (without tokens). 229 | For example ``.get("config",ignore_tokens=True)`` would still return 230 | ``{...}`` if ``self=={"+=config":{...}}``. 231 | """ 232 | 233 | new_key = key 234 | if ignore_tokens: 235 | new_key = self.resolve_key_tokens(key) 236 | resolved_mapping = { 237 | self.resolve_key_tokens(child_key): child_key for child_key in self 238 | } 239 | if new_key in resolved_mapping: 240 | new_key = resolved_mapping[new_key] 241 | else: 242 | new_key = key 243 | 244 | return super().get(new_key, default) 245 | 246 | def resolved(self) -> Dict: 247 | """ 248 | Get the dict structure with all tokens resolved. 249 | 250 | Without tokens, the returned object become a regular dict instance. 251 | 252 | Returns: 253 | deepcopied dict structure. 254 | """ 255 | 256 | def process_pair(key: str, value: str): 257 | new_key = self.resolve_key_tokens(key) 258 | return new_key, value 259 | 260 | new_content = refacto_dict( 261 | src_dict=copy.deepcopy(self), 262 | callback=process_pair, 263 | recursive=True, 264 | ) 265 | return dict(new_content) 266 | --------------------------------------------------------------------------------