├── .github └── workflows │ ├── ci.yml │ ├── doc.yml │ └── publish.yml ├── .gitignore ├── .pylintrc ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── check_version.py ├── docs ├── .nojekyll ├── README.md ├── _sidebar.md ├── development │ ├── README.md │ ├── custom-fromconfig │ │ ├── README.md │ │ ├── config.py │ │ └── custom.py │ ├── custom-launcher │ │ ├── README.md │ │ ├── config.yaml │ │ ├── launcher.yaml │ │ ├── model.py │ │ └── print_command.py │ ├── custom-parser │ │ ├── README.md │ │ └── lorem_ipsum.py │ ├── publish-extensions.md │ └── testing.md ├── examples │ ├── README.md │ ├── change-parser │ │ ├── README.md │ │ ├── config.yaml │ │ ├── launcher.yaml │ │ └── model.py │ ├── combine-configs │ │ ├── .gitignore │ │ ├── README.md │ │ ├── config.yaml │ │ ├── convert.py │ │ ├── ml.py │ │ ├── model.yaml │ │ ├── optimizer.yaml │ │ ├── params │ │ │ ├── big.yaml │ │ │ └── small.yaml │ │ └── trainer.yaml │ ├── configure-launcher │ │ ├── README.md │ │ ├── config.yaml │ │ ├── launcher_default.yaml │ │ ├── launcher_dry.yaml │ │ ├── launcher_logging.yaml │ │ └── model.py │ ├── hyper-params │ │ ├── README.md │ │ ├── config.yaml │ │ ├── hp.py │ │ ├── hparams.yaml │ │ └── model.py │ ├── machine-learning │ │ ├── README.md │ │ ├── ml.py │ │ ├── model.yaml │ │ ├── optimizer.yaml │ │ ├── params │ │ │ ├── big.yaml │ │ │ └── small.yaml │ │ └── trainer.yaml │ └── manual-parsing │ │ ├── README.md │ │ ├── config.yaml │ │ ├── manual.py │ │ ├── model.py │ │ └── params.yaml ├── extensions │ ├── README.md │ ├── hparams-yarn-mlflow │ │ ├── README.md │ │ ├── config.yaml │ │ ├── hparams.yaml │ │ ├── launcher.yaml │ │ ├── launcher_advanced.yaml │ │ ├── model.py │ │ └── monkeypatch_fromconfig.py │ ├── mlflow │ │ ├── README.md │ │ ├── config.yaml │ │ ├── launcher.yaml │ │ ├── launcher_artifacts.yaml │ │ ├── launcher_artifacts_params.yaml │ │ ├── launcher_params.yaml │ │ ├── launcher_start.yaml │ │ ├── model.py │ │ └── params.yaml │ ├── requirements.txt │ └── yarn │ │ ├── README.md │ │ ├── config.yaml │ │ ├── launcher.yaml │ │ ├── model.py │ │ ├── monkeypatch_fromconfig.py │ │ └── params.yaml ├── getting-started │ ├── README.md │ ├── cheat-sheet │ │ ├── README.md │ │ ├── config.yaml │ │ ├── launcher.yaml │ │ └── model.py │ ├── install.md │ ├── quickstart │ │ ├── README.md │ │ ├── config.yaml │ │ ├── manual.py │ │ ├── model.py │ │ └── params.yaml │ └── why-fromconfig.md ├── images │ └── fromconfig.svg ├── index.html └── usage-reference │ ├── README.md │ ├── command-line.md │ ├── fromconfig │ ├── README.md │ ├── example.py │ └── example_kwargs.py │ ├── launcher │ ├── README.md │ ├── config.yaml │ ├── hparams.yaml │ ├── launcher.yaml │ └── model.py │ ├── overview.md │ └── parser │ ├── README.md │ ├── default.py │ ├── parser_evaluate_call.py │ ├── parser_evaluate_import.py │ ├── parser_evaluate_lazy.py │ ├── parser_evaluate_lazy_with_memoization.py │ ├── parser_evaluate_partial.py │ ├── parser_omegaconf.py │ └── parser_singleton.py ├── fromconfig ├── __init__.py ├── cli │ ├── __init__.py │ └── main.py ├── core │ ├── __init__.py │ ├── base.py │ └── config.py ├── launcher │ ├── __init__.py │ ├── base.py │ ├── default.py │ ├── dry.py │ ├── hparams.py │ ├── local.py │ ├── logger.py │ └── parser.py ├── parser │ ├── __init__.py │ ├── base.py │ ├── default.py │ ├── evaluate.py │ ├── omega.py │ └── singleton.py ├── utils │ ├── __init__.py │ ├── libimport.py │ ├── nest.py │ ├── strenum.py │ ├── testing.py │ └── types.py └── version.py ├── mypy.ini ├── requirements-test.txt ├── requirements.txt ├── setup.py └── tests ├── __init__.py └── unit ├── __init__.py ├── cli ├── __init__.py └── test_cli_main.py ├── core ├── __init__.py ├── test_core_base.py └── test_core_config.py ├── launcher ├── __init__.py ├── test_launcher_base.py ├── test_launcher_default.py ├── test_launcher_dry.py ├── test_launcher_hparams.py ├── test_launcher_local.py ├── test_launcher_logger.py └── test_launcher_parser.py ├── parser ├── __init__.py ├── test_parser_base.py ├── test_parser_default.py ├── test_parser_evaluate.py ├── test_parser_omegaconf.py └── test_parser_singleton.py └── utils ├── __init__.py ├── test_utils_libimport.py ├── test_utils_nest.py ├── test_utils_strenum.py └── test_utils_types.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Continuous integration 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: [3.6, 3.7, 3.8, 3.9] 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install, lint and unit tests 24 | run: | 25 | make venv-lint-test 26 | -------------------------------------------------------------------------------- /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Documentation to fromconfig.github.io 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Copy docs to fromconfig.github.io 14 | uses: cpina/github-action-push-to-another-repository@main 15 | env: 16 | API_TOKEN_GITHUB: ${{ secrets.FROMCONFIG_TOKEN }} 17 | with: 18 | source-directory: 'docs' 19 | destination-github-username: 'fromconfig' 20 | destination-repository-name: 'fromconfig.github.io' 21 | user-email: action@github.com 22 | target-branch: 'master' 23 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python 🐍 distributions 📦 to PyPI 2 | 3 | on: push 4 | 5 | jobs: 6 | build-n-publish: 7 | name: Build and publish Python 🐍 distributions 📦 to PyPI 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@master 11 | - name: Set up Python 3.6 12 | uses: actions/setup-python@v1 13 | with: 14 | python-version: 3.6 15 | - name: Build a binary wheel and a source tarball 16 | run: | 17 | make build-dist 18 | - name: Check version for publishing on PyPI 19 | if: startsWith(github.event.ref, 'refs/tags') 20 | run: | 21 | python check_version.py 22 | echo "Publishing !!!!!" 23 | - name: Publish distribution 📦 to PyPI 24 | if: ${{ success() && startsWith(github.event.ref, 'refs/tags') }} 25 | uses: pypa/gh-action-pypi-publish@master 26 | with: 27 | password: ${{ secrets.pypi_password }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .venv 3 | .pytest_cache 4 | .coverage 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test reports 44 | htmlcov/ 45 | .tox/ 46 | .cache 47 | nosetests.xml 48 | *.cover 49 | .hypothesis/ 50 | pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # MlFlow stuff: 65 | mlruns 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | docs/_source/ 73 | docs/_autosummary 74 | docs/API/_autosummary 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | 113 | # generated by build 114 | junit-py3-test.xml 115 | pylama-parent.ini 116 | tox-parent.ini 117 | adapt_template.sh 118 | 119 | # Ignore backup files 120 | *~ 121 | 122 | # Eclipse / VS Code 123 | .classpath 124 | .project 125 | .settings 126 | .vscode 127 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org). 6 | 7 | ## [0.7.2] - 2022-10-12 8 | 9 | ### Added 10 | ### Changed 11 | ### Deprecated 12 | ### Removed 13 | ### Fixed 14 | - Fix bug while parsing dummy mappings (i.e. dict) introduced with lazy args 15 | 16 | ### Security 17 | 18 | 19 | ## [0.7.1] - 2022-09-28 20 | 21 | ### Added 22 | ### Changed 23 | ### Deprecated 24 | ### Removed 25 | ### Fixed 26 | - Incorrectly parsed command line parameter for overrides using whitespace (`--key value`) 27 | 28 | ### Security 29 | 30 | 31 | ## [0.7.0] - 2022-08-26 32 | 33 | ### Added 34 | - Add lazy evaluation 35 | 36 | ### Changed 37 | ### Deprecated 38 | - Upgrade black that was not compatible with the latest versions of Python 3.9 39 | 40 | ### Removed 41 | ### Fixed 42 | ### Security 43 | 44 | 45 | ## [0.6.0] - 2021-09-06 46 | 47 | ### Added 48 | ### Changed 49 | - Update omegaconf to allow interpolation in resolvers 50 | 51 | ### Deprecated 52 | ### Removed 53 | ### Fixed 54 | ### Security 55 | 56 | 57 | ## [0.5.1] - 2021-05-11 58 | 59 | ### Added 60 | ### Changed 61 | - The documentation is moved to https://fromconfig.github.io 62 | 63 | ### Deprecated 64 | ### Removed 65 | ### Fixed 66 | - Incorrect default for the `DefaultLauncher` (invert log and parse step) 67 | 68 | ### Security 69 | 70 | 71 | ## [0.5.0] - 2021-04-30 72 | 73 | ### Added 74 | - `NAME` support in extensions for multiple launchers. 75 | - Better header in `hparams` 76 | 77 | ### Changed 78 | - Order of steps in `DefaultLauncher` is now `sweep, log, parse, run`. 79 | 80 | ### Deprecated 81 | ### Removed 82 | - `log_config` in logging launcher. 83 | 84 | ### Fixed 85 | ### Security 86 | 87 | 88 | 89 | ## [0.4.1] - 2021-04-28 90 | 91 | ### Added 92 | - The `_attr_` key can be used with an extension name in `Launcher.fromconfig`. 93 | 94 | ### Changed 95 | - The `fromconfig` method of the `FromConfig` base class signature is now generic (`config` does not have to be a Mapping). 96 | - The `Parser` `__call__` signature is also more generic (accepts `Any` instead of `Mapping`) 97 | 98 | ### Deprecated 99 | ### Removed 100 | - The steps syntax in `Launcher.fromconfig` is now in `DefaultLauncher` 101 | 102 | ### Fixed 103 | - Better type handling for the `OmegaConfParser` (only attempts to find resolvers if `config` is a mapping) 104 | 105 | ### Security 106 | 107 | 108 | 109 | ## [0.4.0] - 2021-04-27 110 | 111 | ### Added 112 | ### Changed 113 | ### Deprecated 114 | ### Removed 115 | - `ReferenceParser`: less powerful than the `OmegaConfParser` but same functionality, only caused confusion. 116 | 117 | ### Fixed 118 | ### Security 119 | 120 | 121 | ## [0.3.3] - 2021-04-23 122 | 123 | ### Added 124 | - Improved error message when trying to reload a `.jsonnet` file but `jsonnet` is not installed. 125 | - include `!include` and merge `<<:` support for YAML files 126 | - Custom resolvers easy registration for `OmegaConfParser` 127 | - Default resolver `now` for `OmegaConf` 128 | 129 | ### Changed 130 | - `jsonnet` import does not log an error if jsonnet is not available. 131 | 132 | ### Deprecated 133 | ### Removed 134 | ### Fixed 135 | ### Security 136 | 137 | 138 | ## [0.3.2] - 2021-04-22 139 | 140 | ### Added 141 | ### Changed 142 | ### Deprecated 143 | ### Removed 144 | ### Fixed 145 | - docs/README.md must be in data files in setup.py 146 | 147 | ### Security 148 | 149 | 150 | ## [0.3.1] - 2021-04-21 151 | 152 | ### Added 153 | - `DryLauncher` to dry-run config. 154 | - Skip launcher instantiation if config is `None` 155 | 156 | ### Changed 157 | ### Deprecated 158 | ### Removed 159 | ### Fixed 160 | - Wrong order when merging configs in `cli.main` 161 | 162 | ### Security 163 | 164 | 165 | ## [0.3.0] - 2021-04-20 166 | 167 | ### Added 168 | - `Launcher` base class with `HParamsLauncher`, `ParserLauncher`, `LoggingLauncher` and `LocalLauncher` 169 | - CLI mechanism for argument parsing and `fire.Fire` integration (now in Launcher) 170 | - Overrides support via key-value parameters for the CLI thanks to `utils.expand` 171 | - `ChainParser` to easily chain parsers. 172 | 173 | ### Changed 174 | - The `FromConfig` class now has a default implementation of `fromconfig` 175 | - The `DefaultParser` now inherits `ChainParser` and its implementation is moved to `parser/default.py` 176 | 177 | ### Deprecated 178 | ### Removed 179 | - `jsonnet` requirement in the setup.py (it should be optional as it tends to cause issues in some users). 180 | 181 | ### Fixed 182 | ### Security 183 | 184 | 185 | 186 | ## [0.2.2] - 2021-03-31 187 | 188 | ### Added 189 | ### Changed 190 | - `fromconfig.Config` now inherits `dict` instead of `collections.UserDict` to support JSON serialization. 191 | 192 | ### Deprecated 193 | ### Removed 194 | ### Fixed 195 | ### Security 196 | 197 | 198 | ## [0.2.0] - 2021-03-18 199 | 200 | ### Added 201 | ### Changed 202 | - Improved readme 203 | 204 | ### Deprecated 205 | ### Removed 206 | ### Fixed 207 | ### Security 208 | 209 | ## [0.1.0] - 2021-03-18 210 | 211 | ### Added 212 | - Initial commit 213 | 214 | ### Changed 215 | ### Deprecated 216 | ### Removed 217 | ### Fixed 218 | ### Security 219 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | install: ## [Local development] Upgrade pip, install requirements, install package. 2 | python -m pip install -U pip setuptools wheel 3 | python -m pip install -r requirements.txt 4 | python -m pip install -e . 5 | 6 | serve-doc: ## [Documentation] Serve documentation on http://localhost:3000 7 | cd docs && python -m http.server 3000 8 | 9 | install-dev: ## [Local development] Install test requirements 10 | python -m pip install -r requirements-test.txt 11 | 12 | lint: ## [Local development] Run mypy, pylint and black 13 | python -m mypy fromconfig 14 | python -m pylint fromconfig 15 | python -m black --check -l 120 fromconfig 16 | 17 | black: ## [Local development] Auto-format python code using black 18 | python -m black -l 120 . 19 | 20 | test: ## [Local development] Run unit tests, doctest and notebooks 21 | python -m pytest -v --cov=fromconfig --cov-report term-missing --cov-fail-under 95 tests 22 | python -m pytest --doctest-modules -v fromconfig 23 | $(MAKE) examples 24 | 25 | examples: ## [Doc] Run all examples 26 | cd docs/getting-started/quickstart && fromconfig config.yaml params.yaml - model - train 27 | cd docs/getting-started/quickstart && python manual.py 28 | cd docs/getting-started/cheat-sheet && fromconfig config.yaml launcher.yaml - model - train 29 | cd docs/examples/manual-parsing && python manual.py 30 | cd docs/examples/hyper-params && fromconfig config.yaml hparams.yaml - model - train 31 | cd docs/examples/hyper-params && python hp.py 32 | cd docs/examples/change-parser && fromconfig config.yaml launcher.yaml - model - train 33 | cd docs/examples/configure-launcher && fromconfig config.yaml --launcher.run=dry - model - train 34 | cd docs/examples/configure-launcher && fromconfig config.yaml launcher_dry.yaml - model - train 35 | cd docs/examples/configure-launcher && fromconfig config.yaml --logging.level=20 - model - train 36 | cd docs/examples/configure-launcher && fromconfig config.yaml launcher_logging.yaml - model - train 37 | cd docs/examples/machine-learning && fromconfig trainer.yaml model.yaml optimizer.yaml params/small.yaml - trainer - run 38 | cd docs/examples/machine-learning && fromconfig trainer.yaml model.yaml optimizer.yaml params/big.yaml - trainer - run 39 | cd docs/examples/combine-configs && fromconfig config.yaml params/small.yaml - trainer - run 40 | cd docs/examples/combine-configs && fromconfig config.yaml params/big.yaml - trainer - run 41 | cd docs/examples/combine-configs && python convert.py trainer.yaml trainer.json 42 | cd docs/usage-reference/fromconfig && python example.py 43 | cd docs/usage-reference/fromconfig && python example_kwargs.py 44 | cd docs/usage-reference/parser && python default.py 45 | cd docs/usage-reference/parser && python parser_evaluate_call.py 46 | cd docs/usage-reference/parser && python parser_evaluate_import.py 47 | cd docs/usage-reference/parser && python parser_evaluate_partial.py 48 | cd docs/usage-reference/parser && python parser_omegaconf.py 49 | cd docs/usage-reference/parser && python parser_singleton.py 50 | cd docs/usage-reference/launcher && fromconfig config.yaml hparams.yaml launcher.yaml - model - train 51 | cd docs/usage-reference/launcher && fromconfig --hparams.a=1,2 --hparams.b=3,4 52 | cd docs/development/custom-fromconfig && python config.py 53 | cd docs/development/custom-fromconfig && python custom.py 54 | cd docs/development/custom-parser && python lorem_ipsum.py 55 | cd docs/development/custom-launcher && fromconfig config.yaml launcher.yaml - model - train 56 | cd docs/extensions && python -m pip install -r requirements.txt 57 | cd docs/extensions/mlflow && fromconfig config.yaml params.yaml --launcher.log=mlflow - model - train 58 | cd docs/extensions/mlflow && fromconfig config.yaml params.yaml launcher.yaml - model - train 59 | cd docs/extensions/mlflow && fromconfig config.yaml params.yaml launcher_start.yaml - model - train 60 | cd docs/extensions/mlflow && fromconfig config.yaml params.yaml launcher_artifacts.yaml - model - train 61 | cd docs/extensions/mlflow && fromconfig config.yaml params.yaml launcher_params.yaml - model - train 62 | cd docs/extensions/mlflow && fromconfig config.yaml params.yaml launcher_artifacts_params.yaml - model - train 63 | cd docs/extensions/yarn && python monkeypatch_fromconfig.py config.yaml params.yaml --launcher.run=yarn - model - train 64 | cd docs/extensions/yarn && python monkeypatch_fromconfig.py config.yaml params.yaml launcher.yaml - model - train 65 | cd docs/extensions/hparams-yarn-mlflow && python monkeypatch_fromconfig.py config.yaml hparams.yaml --launcher.log=mlflow --launcher.run=yarn,mlflow,local - model - train 66 | cd docs/extensions/hparams-yarn-mlflow && python monkeypatch_fromconfig.py config.yaml hparams.yaml launcher.yaml - model - train 67 | cd docs/extensions/hparams-yarn-mlflow && python monkeypatch_fromconfig.py config.yaml hparams.yaml launcher_advanced.yaml - model - train 68 | 69 | venv-lint-test: ## [Continuous integration] Install in venv and run lint and test 70 | python -m venv .env && . .env/bin/activate && make install install-dev lint test && rm -rf .env 71 | 72 | build-dist: ## [Continuous integration] Build package for pypi 73 | python -m venv .env 74 | . .env/bin/activate && pip install -U pip setuptools wheel 75 | . .env/bin/activate && python setup.py sdist 76 | rm -rf .env 77 | 78 | .PHONY: help 79 | 80 | help: # Run `make help` to get help on the make commands 81 | @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 82 | -------------------------------------------------------------------------------- /check_version.py: -------------------------------------------------------------------------------- 1 | """Check version and git tag script.""" 2 | 3 | from pathlib import Path 4 | import re 5 | import sys 6 | import subprocess 7 | 8 | 9 | if __name__ == "__main__": 10 | # Read package version 11 | with Path("fromconfig/version.py").open(encoding="utf-8") as file: 12 | metadata = dict(re.findall(r'__([a-z]+)__\s*=\s*"([^"]+)"', file.read())) 13 | version = metadata["version"] 14 | 15 | # Read git tag 16 | with subprocess.Popen(["git", "describe", "--tags"], stdout=subprocess.PIPE) as process: 17 | tagged_version = process.communicate()[0].strip().decode(encoding="utf-8") 18 | 19 | # Exit depending on version and tagged_version 20 | if version == tagged_version: 21 | print(f"Tag and version are the same ({version}) !") 22 | sys.exit(0) 23 | else: 24 | print(f"Tag {tagged_version} and version {version} are not the same !") 25 | sys.exit(1) 26 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/docs/.nojekyll -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

FromConfig

4 | 5 |

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |

16 | 17 |

A library to instantiate any Python object from configuration files.

18 | 19 |

Documentation and Github

20 | 21 |

22 | pip install fromconfig and Get Started 23 |

24 | 25 |

26 | 27 |

28 | -------------------------------------------------------------------------------- /docs/_sidebar.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | * Getting started 4 | 5 | * [Install](getting-started/install) 6 | * [Quickstart](getting-started/quickstart/) 7 | * [Cheat Sheet](getting-started/cheat-sheet/) 8 | * [Why From Config ?](getting-started/why-fromconfig) 9 | 10 | * Examples 11 | 12 | * [Manual Parsing](examples/manual-parsing/) 13 | * [Hyper Params](examples/hyper-params/) 14 | * [Change Parser](examples/change-parser/) 15 | * [Configure Launcher](examples/configure-launcher/) 16 | * [Machine Learning](examples/machine-learning/) 17 | * [Combine Configs](examples/combine-configs/) 18 | 19 | * Extensions 20 | 21 | * [MlFlow](extensions/mlflow/) 22 | * [Yarn](extensions/yarn/) 23 | * [HParams + Yarn + MlFlow](extensions/hparams-yarn-mlflow/) 24 | 25 | * Usage Reference 26 | 27 | * [Overview](usage-reference/overview) 28 | * [Command Line](usage-reference/command-line) 29 | * [fromconfig](usage-reference/fromconfig/) 30 | * [Parser](usage-reference/parser/) 31 | * [Launcher](usage-reference/launcher/) 32 | 33 | * Development 34 | 35 | * [Testing](development/testing) 36 | * [Custom FromConfig](development/custom-fromconfig/) 37 | * [Custom Parser](development/custom-parser/) 38 | * [Custom Launcher](development/custom-launcher/) 39 | * [Publish Extensions](development/publish-extensions) 40 | -------------------------------------------------------------------------------- /docs/development/README.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | * [Testing](development/testing) 4 | * [Custom FromConfig](development/custom-fromconfig/) 5 | * [Custom Parser](development/custom-parser/) 6 | * [Custom Launcher](development/custom-launcher/) 7 | * [Publish Extensions](development/publish-extensions) 8 | 9 | -------------------------------------------------------------------------------- /docs/development/custom-fromconfig/README.md: -------------------------------------------------------------------------------- 1 | # Custom FromConfig 2 | 3 | The logic to instantiate objects from config dictionaries is always the same. 4 | 5 | It resolves the class, function or method `attr` from the `_attr_` key, recursively call `fromconfig` on all the other key-values to get a `kwargs` dictionary of objects, and call `attr(*args, **kwargs)`. 6 | 7 | It is possible to customize the behavior of `fromconfig` by inheriting the `FromConfig` class. 8 | 9 | For example 10 | 11 | [custom.py](custom.py ':include :type=code python') 12 | 13 | One custom `FromConfig` class is provided in `fromconfig` which makes it possible to stop the instantiation and keep config dictionaries as config dictionaries. 14 | 15 | For example 16 | 17 | [config.py](config.py ':include :type=code python') 18 | -------------------------------------------------------------------------------- /docs/development/custom-fromconfig/config.py: -------------------------------------------------------------------------------- 1 | """Config example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | if __name__ == "__main__": 7 | config = {"_attr_": "fromconfig.Config", "_config_": {"_attr_": "list"}} 8 | print(fromconfig.fromconfig(config)) # {'_attr_': 'list'} 9 | -------------------------------------------------------------------------------- /docs/development/custom-fromconfig/custom.py: -------------------------------------------------------------------------------- 1 | """Custom FromConfig implementation.""" 2 | 3 | import fromconfig 4 | 5 | 6 | class MyClass(fromconfig.FromConfig): 7 | """Custom FromConfig implementation.""" 8 | 9 | def __init__(self, x): 10 | self.x = x 11 | 12 | @classmethod 13 | def fromconfig(cls, config): 14 | # NOTE: config is not recursively instantiated yet 15 | config = {key: fromconfig.fromconfig(value) for key, value in config.items()} 16 | if "x" not in config: 17 | return cls(0) 18 | else: 19 | return cls(**config) 20 | 21 | 22 | if __name__ == "__main__": 23 | cfg = {} 24 | got = MyClass.fromconfig(cfg) 25 | assert isinstance(got, MyClass) 26 | assert got.x == 0 27 | -------------------------------------------------------------------------------- /docs/development/custom-launcher/README.md: -------------------------------------------------------------------------------- 1 | # Custom Launcher 2 | 3 | Another flexibility provided by `fromconfig` is the ability to write custom `Launcher` classes. 4 | 5 | The `Launcher` base class is simple 6 | 7 | ```python 8 | class Launcher(FromConfig, ABC): 9 | """Base class for launchers.""" 10 | 11 | def __init__(self, launcher: "Launcher"): 12 | self.launcher = launcher 13 | 14 | def __call__(self, config: Any, command: str = ""): 15 | """Launch implementation. 16 | 17 | Parameters 18 | ---------- 19 | config : Any 20 | The config 21 | command : str, optional 22 | The fire command 23 | """ 24 | raise NotImplementedError() 25 | ``` 26 | 27 | For example, let's implement a `Launcher` that simply prints the command (and does nothing else). 28 | 29 | [print_command.py](print_command.py ':include :type=code python') 30 | 31 | Given the following config files 32 | 33 | `config.yaml` 34 | 35 | [config.yaml](config.yaml ':include :type=code yaml') 36 | 37 | `launcher.yaml` 38 | 39 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 40 | 41 | and module `model.py` 42 | 43 | 44 | [model.py](model.py ':include :type=code python') 45 | 46 | Run 47 | 48 | ``` 49 | fromconfig config.yaml launcher.yaml - model - train 50 | ``` 51 | 52 | You should see 53 | 54 | ``` 55 | model - train 56 | Training model with learning_rate 0.1 57 | ``` 58 | -------------------------------------------------------------------------------- /docs/development/custom-launcher/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: 0.1 4 | -------------------------------------------------------------------------------- /docs/development/custom-launcher/launcher.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # The other steps (sweep, log, parse) will use defaults. 7 | # The log step uses our custom launcher. 8 | launcher: 9 | log: 10 | _attr_: print_command.PrintCommandLauncher 11 | -------------------------------------------------------------------------------- /docs/development/custom-launcher/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/development/custom-launcher/print_command.py: -------------------------------------------------------------------------------- 1 | """Custom Launcher that prints the command.""" 2 | 3 | from typing import Any 4 | 5 | import fromconfig 6 | 7 | 8 | class PrintCommandLauncher(fromconfig.launcher.Launcher): 9 | def __call__(self, config: Any, command: str = ""): 10 | print(command) 11 | # Launcher are nested by default 12 | self.launcher(config=config, command=command) 13 | -------------------------------------------------------------------------------- /docs/development/custom-parser/README.md: -------------------------------------------------------------------------------- 1 | # Custom Parser 2 | 3 | One of `fromconfig`'s strength is its flexibility when it comes to the config syntax. 4 | 5 | To reduce the config boilerplate, it is possible to add a new `Parser` to support a new syntax. 6 | 7 | Let's cover a dummy example : let's say we want to replace all empty strings with "lorem ipsum". 8 | 9 | [lorem_ipsum.py](lorem_ipsum.py ':include :type=code python') 10 | -------------------------------------------------------------------------------- /docs/development/custom-parser/lorem_ipsum.py: -------------------------------------------------------------------------------- 1 | """Custom Parser that replaces empty string by a default string.""" 2 | 3 | from typing import Any 4 | 5 | import fromconfig 6 | 7 | 8 | class LoremIpsumParser(fromconfig.parser.Parser): 9 | """Custom Parser that replaces empty string by a default string.""" 10 | 11 | def __init__(self, default: str = "lorem ipsum"): 12 | self.default = default 13 | 14 | def __call__(self, config: Any): 15 | def _map_fn(value): 16 | if isinstance(value, str) and not value: 17 | return self.default 18 | return value 19 | 20 | # Utility to apply a function to all nodes of a nested dict 21 | # in a depth-first search 22 | return fromconfig.utils.depth_map(_map_fn, config) 23 | 24 | 25 | if __name__ == "__main__": 26 | cfg = {"x": "Hello World", "y": ""} 27 | parser = LoremIpsumParser() 28 | parsed = parser(cfg) 29 | assert parsed["y"] == parser.default 30 | -------------------------------------------------------------------------------- /docs/development/publish-extensions.md: -------------------------------------------------------------------------------- 1 | # Publish Extensions 2 | 3 | ## Discovery 4 | 5 | Once you've implemented a custom [`Launcher`](#usage-reference/launcher/), you can share it as a `fromconfig` extension. 6 | 7 | To do so, publish a new package on `PyPI` that has a specific entry point that maps to a module defined in your package in which one or more `Launcher` classes is defined. 8 | 9 | To add an entry point, update the `setup.py` by adding 10 | 11 | ```python 12 | setuptools.setup( 13 | ... 14 | entry_points={"fromconfig0": ["your_extension_name = your_extension_module"]}, 15 | ) 16 | ``` 17 | 18 | Each `Launcher` class defined in the entry-point can define a class attribute `NAME` that uniquely identifies the launcher. 19 | 20 | For example, if the entry point name (`your_extension_name`) is `debug`, the following launcher will be available under the name `debug.print_command`. 21 | 22 | ```python 23 | """Custom Launcher that prints the command.""" 24 | 25 | from typing import Any 26 | 27 | import fromconfig 28 | 29 | 30 | class PrintCommandLauncher(fromconfig.launcher.Launcher): 31 | 32 | NAME = "print_command" 33 | 34 | def __call__(self, config: Any, command: str = ""): 35 | print(command) 36 | # Launcher are nested by default 37 | self.launcher(config=config, command=command) 38 | ``` 39 | 40 | 41 | If you don't specify the `NAME` attribute, the entry point name will be used. 42 | 43 | If your extension implements more than one launcher, you need to specify the `NAME` of each `Launcher` class (except one) otherwise there will be a name conflict. 44 | 45 | 46 | ## Implementation 47 | 48 | Make sure to look at the available launchers defined directly in `fromconfig`. 49 | 50 | It is recommended to keep the number of `__init__` arguments as low as possible (if any) and instead retrieve parameters from the `config` itself at run time. A rule of thumb is to use `__init__` arguments only if the launcher is meant to be called multiple times with different options. 51 | 52 | If your `Launcher` class is not meant to wrap another `Launcher` class (that's the case of the `LocalLauncher` for example), make sure to override the `__init__` method like so 53 | 54 | ```python 55 | def __init__(self, launcher: Launcher = None): 56 | if launcher is not None: 57 | raise ValueError(f"Cannot wrap another launcher but got {launcher}") 58 | super().__init__(launcher=launcher) # type: ignore 59 | ``` 60 | 61 | Once your extension is available, update `fromconfig` documentation and add an example in [extensions](extensions/). 62 | -------------------------------------------------------------------------------- /docs/development/testing.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | To install development tools 4 | 5 | ```bash 6 | make install-dev 7 | ``` 8 | 9 | To lint the code (mypy, pylint and black) 10 | 11 | ```bash 12 | make lint 13 | ``` 14 | 15 | To format the code with black 16 | 17 | ```bash 18 | make black 19 | ``` 20 | 21 | To run tests 22 | 23 | ```bash 24 | make test 25 | ``` 26 | -------------------------------------------------------------------------------- /docs/examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | * [Manual Parsing](examples/manual-parsing/) 4 | * [Hyper Params](examples/hyper-params/) 5 | * [Change Parser](examples/change-parser/) 6 | * [Configure Launcher](examples/configure-launcher/) 7 | * [Machine Learning](examples/machine-learning/) 8 | 9 | -------------------------------------------------------------------------------- /docs/examples/change-parser/README.md: -------------------------------------------------------------------------------- 1 | # Change Parser 2 | 3 | ## Recommended Method 4 | 5 | You can change the parser used by the `fromconfig` CLI. 6 | 7 | For example, let's say we only want to apply the `SingletonParser` (instead of the `DefaultParser` that is applied by default). 8 | 9 | Given the module and config 10 | 11 | `model.py` 12 | 13 | [model.py](model.py ':include :type=code python') 14 | 15 | `config.yaml` 16 | 17 | [config.yaml](config.yaml ':include :type=code yaml') 18 | 19 | Create a new config file 20 | 21 | `launcher.yaml` 22 | 23 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 24 | 25 | 26 | and run 27 | 28 | ```bash 29 | fromconfig config.yaml launcher.yaml - model - train 30 | ``` 31 | 32 | You should see 33 | 34 | ``` 35 | Training model with learning_rate ${params.learning_rate} 36 | ``` 37 | 38 | As expected, the `OmegaConfParser` (part of the `DefaultParser` and responsible for resolving the `${params.learning_rate}` interpolation) is not applied, and instead, only the `SingletonParser` is applied. 39 | 40 | 41 | ## Custom Launcher 42 | 43 | If you use a custom launcher, note that you may be disabling parsing as it is merely one of the steps executed by the `DefaultLauncher`. The `ParserLauncher` part of the `DefaultLauncher` instantiates the `parser` from the config's `parser` key and applies parsing during launch. 44 | 45 | As long as the `ParserLauncher` is ran by your custom `Launcher`, the recommended method should work. 46 | 47 | You can also [write your own `Launcher`](development/custom-launcher/) that will be responsible for parsing and [configure `fromconfig` to use your launcher](examples/configure-launcher/). 48 | -------------------------------------------------------------------------------- /docs/examples/change-parser/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${params.learning_rate}" 4 | 5 | params: 6 | learning_rate: 0.1 7 | -------------------------------------------------------------------------------- /docs/examples/change-parser/launcher.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Parser used by the ParserLauncher (part of the DefaultLauncher) 7 | parser: 8 | _attr_: fromconfig.parser.SingletonParser 9 | -------------------------------------------------------------------------------- /docs/examples/change-parser/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore json files generated by convert 2 | *.json 3 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/README.md: -------------------------------------------------------------------------------- 1 | # Combine Configs 2 | 3 | ## Dynamically 4 | 5 | As shown in the [Machine Learning](examples/machine-learning/) example, it is practical in many scenarios to be able to combine different config files dynamically. 6 | 7 | Given a module `ml.py` defining model, optimizer and trainer classes 8 | 9 | [ml.py](ml.py ':include :type=code python') 10 | 11 | And the following config files 12 | 13 | `trainer.yaml` 14 | 15 | [trainer.yaml](trainer.yaml ':include :type=code yaml') 16 | 17 | `model.yaml` 18 | 19 | [model.yaml](model.yaml ':include :type=code yaml') 20 | 21 | `optimizer.yaml` 22 | 23 | [optimizer.yaml](optimizer.yaml ':include :type=code yaml') 24 | 25 | `params/big.yaml` 26 | 27 | [params/big.yaml](params/big.yaml ':include :type=code yaml') 28 | 29 | `params/small.yaml` 30 | 31 | [params/small.yaml](params/small.yaml ':include :type=code yaml') 32 | 33 | 34 | It is possible to launch two different trainings with different set of hyper-parameters with 35 | 36 | ```bash 37 | fromconfig trainer.yaml model.yaml optimizer.yaml params/small.yaml - trainer - run 38 | fromconfig trainer.yaml model.yaml optimizer.yaml params/big.yaml - trainer - run 39 | ``` 40 | 41 | which should print 42 | 43 | ``` 44 | Training Model(dim=10) with Optimizer(learning_rate=0.01) 45 | Training Model(dim=100) with Optimizer(learning_rate=0.001) 46 | ``` 47 | 48 | ## Organize and Include 49 | 50 | One caveat of being able to split your configuration across different files is that the command line starts to grow in size, deserving the purpose of simplicity. 51 | 52 | Of course, we could try to maintain at most one config file for the things that are not supposed to change, and another set of files that represent variations of the global configuration. However, it would be to the detriment of readability, since having self-contained and short config files configuring one component is a huge advantage. 53 | 54 | To get the best of both worlds, `fromconfig.load` supports a [special YAML `Loader` that enables includes](https://stackoverflow.com/questions/528281/how-can-i-include-a-yaml-file-inside-another). Similar to JSONNET (and its [import mechanism](https://jsonnet.org/learning/tutorial.html)), it is possible to statically include a YAML file into another. 55 | 56 | For example, `config.yaml` merges the content of all `trainer.yaml`, `model.yaml` and `optimizer.yaml` files 57 | 58 | [config.yaml](config.yaml ':include :type=code yaml') 59 | 60 | It is then equivalent to launch 61 | 62 | ```bash 63 | fromconfig config.yaml params/small.yaml - trainer - run 64 | ``` 65 | 66 | The `<<:` ([The merge key Language-Independent Type for YAML](https://yaml.org/type/merge.html)) indicates that the dictionary from the included file should be merged with the top-level (there is some magic in fromconfig that should make this work). 67 | 68 | Note that because there is no guarantee of ordering, __key conflicts and overrides are not supported__, meaning that the dictionaries you merge with the top level should have unique keys, shared neither by the current file nor any other included files meant to be merged in the same dictionary. 69 | 70 | If you want to do some advanced manipulation and merging, it is highly recommended to use `JSONNET` instead. 71 | 72 | Though it is not possible to import YAML files from JSONNET ([yet](https://github.com/google/jsonnet/issues/460)), you can easily convert YAML files to JSON and import the JSON files instead. 73 | 74 | ## Converting YAML to JSON 75 | 76 | If you need to convert config formats, you can do it in Python with 77 | 78 | [convert.py](convert.py ':include :type=code python') 79 | 80 | and run 81 | 82 | ```bash 83 | python convert.py trainer.yaml trainer.json 84 | ``` 85 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/config.yaml: -------------------------------------------------------------------------------- 1 | # Include files (!include path) and merge into top level dictionary (<<:) 2 | # WARNING: Key Conflicts are not authorized 3 | <<: !include trainer.yaml 4 | <<: !include model.yaml 5 | <<: !include optimizer.yaml 6 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/convert.py: -------------------------------------------------------------------------------- 1 | """Convert file format.""" 2 | 3 | import fire 4 | 5 | import fromconfig 6 | 7 | 8 | def convert(path_input, path_output): 9 | """Convert input into output with load and dump.""" 10 | fromconfig.dump(fromconfig.load(path_input), path_output) 11 | 12 | 13 | if __name__ == "__main__": 14 | fire.Fire(convert) 15 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/ml.py: -------------------------------------------------------------------------------- 1 | """Machine Learning Example code. 2 | 3 | Example 4 | ------- 5 | fromconfig trainer.yaml model.yaml optimizer.yaml params/small.yaml - trainer - run 6 | fromconfig trainer.yaml model.yaml optimizer.yaml params/big.yaml - trainer - run 7 | """ 8 | from dataclasses import dataclass 9 | 10 | 11 | @dataclass 12 | class Model: 13 | """Dummy Model class.""" 14 | 15 | dim: int 16 | 17 | 18 | @dataclass 19 | class Optimizer: 20 | """Dummy Optimizer class.""" 21 | 22 | learning_rate: float 23 | 24 | 25 | class Trainer: 26 | """Dummy Trainer class.""" 27 | 28 | def __init__(self, model, optimizer): 29 | self.model = model 30 | self.optimizer = optimizer 31 | 32 | def run(self): 33 | print(f"Training {self.model} with {self.optimizer}") 34 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/model.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: "ml.Model" 3 | dim: "${params.dim}" 4 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/optimizer.yaml: -------------------------------------------------------------------------------- 1 | optimizer: 2 | _attr_: "ml.Optimizer" 3 | learning_rate: "${params.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/params/big.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | dim: 100 3 | learning_rate: 0.001 4 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/params/small.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | dim: 10 3 | learning_rate: 0.01 4 | -------------------------------------------------------------------------------- /docs/examples/combine-configs/trainer.yaml: -------------------------------------------------------------------------------- 1 | trainer: 2 | _attr_: "ml.Trainer" 3 | model: "${model}" 4 | optimizer: "${optimizer}" 5 | -------------------------------------------------------------------------------- /docs/examples/configure-launcher/README.md: -------------------------------------------------------------------------------- 1 | # Configure Launcher 2 | 3 | The easiest way to configure the launcher is to change one of the 4 steps. 4 | 5 | The default is configured in the following way 6 | 7 | [launcher_default.yaml](launcher_default.yaml ':include :type=code yaml') 8 | 9 | ## Dry run 10 | 11 | Let's see how to swap the `LocalLauncher` (configured by `run: local`) with the `DryLauncher` (configured by `run: dry`). 12 | 13 | For example, given the following module and config file 14 | 15 | `model.py` 16 | 17 | [model.py](model.py ':include :type=code python') 18 | 19 | `config.yaml` 20 | 21 | [config.yaml](config.yaml ':include :type=code yaml') 22 | 23 | run 24 | 25 | ```bash 26 | fromconfig config.yaml --launcher.run=dry - model - train 27 | ``` 28 | 29 | which prints the config and command but does not instantiate or run any method. 30 | 31 | ``` 32 | {'model': {'_attr_': 'model.Model', 'learning_rate': 0.1}} 33 | model - train 34 | ``` 35 | 36 | Note that it is equivalent to adding a `launcher_dry.yaml` config file 37 | 38 | [launcher_dry.yaml](launcher_dry.yaml ':include :type=code yaml') 39 | 40 | and running 41 | 42 | ```bash 43 | fromconfig config.yaml launcher_dry.yaml - model - train 44 | ``` 45 | 46 | ## Configure Logging 47 | 48 | The logging launcher (responsible for basic logging, configured by `log: logging`) can be configured with the `logging.level` parameter. 49 | 50 | For example, 51 | 52 | ```bash 53 | fromconfig config.yaml --logging.level=20 - model - train 54 | ``` 55 | 56 | prints 57 | 58 | ``` 59 | INFO:model:Training model with learning_rate 0.1 60 | ``` 61 | 62 | Note that this is equivalent to adding a `launcher_logging.yaml` config file 63 | 64 | [launcher_logging.yaml](launcher_logging.yaml ':include :type=code yaml') 65 | 66 | and running 67 | 68 | ```bash 69 | fromconfig config.yaml launcher_logging.yaml - model - train 70 | ``` 71 | 72 | ## Advanced 73 | 74 | It is also possible to use a [custom launcher](development/custom-launcher/) and / or [customize how the different launchers are composed](usage-reference/launcher/). 75 | -------------------------------------------------------------------------------- /docs/examples/configure-launcher/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: 0.1 4 | -------------------------------------------------------------------------------- /docs/examples/configure-launcher/launcher_default.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | launcher: 7 | sweep: hparams # Use HParamsLauncher for the sweep step 8 | log: logging # Use LoggingLauncher for the log step 9 | parse: parser # Use ParserLauncher for the parse step 10 | run: local # Use LocalLauncher for the run step 11 | -------------------------------------------------------------------------------- /docs/examples/configure-launcher/launcher_dry.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # The other steps (sweep, log, parse) will use defaults. 7 | launcher: 8 | run: dry # Use DryLauncher for the run step 9 | -------------------------------------------------------------------------------- /docs/examples/configure-launcher/launcher_logging.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Used by the LoggingLauncher to set the logging verbosity level 7 | logging: 8 | level: 20 9 | -------------------------------------------------------------------------------- /docs/examples/configure-launcher/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | import logging 4 | 5 | 6 | LOGGER = logging.getLogger(__name__) 7 | 8 | 9 | class Model: 10 | def __init__(self, learning_rate: float): 11 | self.learning_rate = learning_rate 12 | 13 | def train(self): 14 | LOGGER.info(f"Training model with learning_rate {self.learning_rate}") 15 | -------------------------------------------------------------------------------- /docs/examples/hyper-params/README.md: -------------------------------------------------------------------------------- 1 | # Hyper Params 2 | 3 | 4 | ## Using the `HParamsLauncher` 5 | 6 | By default, the [`DefaultLauncher`](usage-reference/launcher/) used by the CLI includes the `HParamsLauncher`, that launches multiple runs using the grid specified by the `hparams` entry. 7 | 8 | Given the following module and config files (similar to the [quickstart](getting-started/quickstart/), we only changed `params` into `hparams`) 9 | 10 | `model.py` 11 | 12 | [model.py](model.py ':include :type=code python') 13 | 14 | `config.yaml` 15 | 16 | [config.yaml](config.yaml ':include :type=code yaml') 17 | 18 | `hparams.yaml` 19 | 20 | [hparams.yaml](hparams.yaml ':include :type=code yaml') 21 | 22 | run 23 | 24 | ```bash 25 | fromconfig config.yaml hparams.yaml - model - train 26 | ``` 27 | 28 | You should see plenty of logs and two trainings 29 | 30 | ``` 31 | ========================[learning_rate=0.01]============================ 32 | Training model with learning_rate 0.01 33 | ========================[learning_rate=0.001]=========================== 34 | Training model with learning_rate 0.001 35 | ``` 36 | 37 | You can also specify the grid of hyper-parameter via CLI overrides. 38 | 39 | For example, 40 | 41 | ```bash 42 | fromconfig config.yaml --hparams.learning_rate=0.1,0.01 - model - train 43 | ``` 44 | 45 | ## Manually 46 | 47 | If you want to do something more custom, you can also easily manipulate the config manually. 48 | 49 | For example, 50 | 51 | [hp.py](hp.py ':include :type=code python') 52 | 53 | 54 | ## Custom Launcher 55 | 56 | You can also [write your own `Launcher`](development/custom-launcher/) and [configure `fromconfig` to use your launcher](examples/configure-launcher/). 57 | -------------------------------------------------------------------------------- /docs/examples/hyper-params/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${hparams.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/examples/hyper-params/hp.py: -------------------------------------------------------------------------------- 1 | """Manual Hyper Parameter search example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | if __name__ == "__main__": 7 | config = fromconfig.load("config.yaml") 8 | parser = fromconfig.parser.DefaultParser() 9 | for learning_rate in [0.01, 0.1]: 10 | params = {"learning_rate": learning_rate} 11 | parsed = parser({**config, "hparams": params}) 12 | model = fromconfig.fromconfig(parsed["model"]) 13 | model.train() 14 | # Clear the singletons if any as we most likely don't want 15 | # to share between configs 16 | fromconfig.parser.singleton.clear() 17 | -------------------------------------------------------------------------------- /docs/examples/hyper-params/hparams.yaml: -------------------------------------------------------------------------------- 1 | hparams: 2 | learning_rate: [0.01, 0.001] 3 | -------------------------------------------------------------------------------- /docs/examples/hyper-params/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning 2 | 3 | `fromconfig` is particularly well suited for Machine Learning as it is common to have a lot of different parameters, sometimes far down the call stack, and different configurations of these hyper-parameters. 4 | 5 | Given a module `ml.py` defining model, optimizer and trainer classes 6 | 7 | [ml.py](ml.py ':include :type=code python') 8 | 9 | And the following config files 10 | 11 | `trainer.yaml` 12 | 13 | [trainer.yaml](trainer.yaml ':include :type=code yaml') 14 | 15 | `model.yaml` 16 | 17 | [model.yaml](model.yaml ':include :type=code yaml') 18 | 19 | `optimizer.yaml` 20 | 21 | [optimizer.yaml](optimizer.yaml ':include :type=code yaml') 22 | 23 | `params/big.yaml` 24 | 25 | [params/big.yaml](params/big.yaml ':include :type=code yaml') 26 | 27 | `params/small.yaml` 28 | 29 | [params/small.yaml](params/small.yaml ':include :type=code yaml') 30 | 31 | 32 | It is possible to launch two different trainings with different set of hyper-parameters with 33 | 34 | ```bash 35 | fromconfig trainer.yaml model.yaml optimizer.yaml params/small.yaml - trainer - run 36 | fromconfig trainer.yaml model.yaml optimizer.yaml params/big.yaml - trainer - run 37 | ``` 38 | 39 | which should print 40 | 41 | ``` 42 | Training Model(dim=10) with Optimizer(learning_rate=0.01) 43 | Saving Model(dim=10) to models/2021-04-23-12-05-47 44 | Training Model(dim=100) with Optimizer(learning_rate=0.001) 45 | Saving Model(dim=100) to models/2021-04-23-12-05-48 46 | ``` 47 | 48 | We used the custom resolver `now` with the `OmegaConfParser` to generate a path for the model (see `"models/${now:}"`). Read more about the interpolation mechanism (you can register your own resolvers) in the [Usage Reference](usage-reference/parser/). 49 | 50 | Note that it is encouraged to save these config files with the experiment's files to get full reproducibility. [MlFlow](https://mlflow.org) is an open-source platform that tracks your experiments by logging metrics and artifacts. 51 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/ml.py: -------------------------------------------------------------------------------- 1 | """Machine Learning Example code.""" 2 | 3 | from dataclasses import dataclass 4 | 5 | 6 | @dataclass 7 | class Model: 8 | """Dummy Model class.""" 9 | 10 | dim: int 11 | 12 | 13 | @dataclass 14 | class Optimizer: 15 | """Dummy Optimizer class.""" 16 | 17 | learning_rate: float 18 | 19 | 20 | class Trainer: 21 | """Dummy Trainer class.""" 22 | 23 | def __init__(self, model, optimizer, path): 24 | self.model = model 25 | self.optimizer = optimizer 26 | self.path = path 27 | 28 | def run(self): 29 | print(f"Training {self.model} with {self.optimizer}") 30 | print(f"Saving {self.model} to {self.path}") 31 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/model.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: "ml.Model" 3 | dim: "${params.dim}" 4 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/optimizer.yaml: -------------------------------------------------------------------------------- 1 | optimizer: 2 | _attr_: "ml.Optimizer" 3 | learning_rate: "${params.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/params/big.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | dim: 100 3 | learning_rate: 0.001 4 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/params/small.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | dim: 10 3 | learning_rate: 0.01 4 | -------------------------------------------------------------------------------- /docs/examples/machine-learning/trainer.yaml: -------------------------------------------------------------------------------- 1 | trainer: 2 | _attr_: "ml.Trainer" 3 | model: "${model}" 4 | optimizer: "${optimizer}" 5 | path: "models/${now:}" 6 | -------------------------------------------------------------------------------- /docs/examples/manual-parsing/README.md: -------------------------------------------------------------------------------- 1 | # Manual Parsing 2 | 3 | It is possible to manipulate configs directly in the code without using the `fromconfig` CLI. 4 | 5 | For example, 6 | 7 | [Manual Example](manual.py ':include :type=code python') 8 | -------------------------------------------------------------------------------- /docs/examples/manual-parsing/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${params.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/examples/manual-parsing/manual.py: -------------------------------------------------------------------------------- 1 | """Manual Parsing Example.""" 2 | 3 | import functools 4 | 5 | import fromconfig 6 | 7 | import model 8 | 9 | 10 | if __name__ == "__main__": 11 | # Load configs from yaml and merge into one dictionary 12 | paths = ["config.yaml", "params.yaml"] 13 | configs = [fromconfig.load(path) for path in paths] 14 | config = functools.reduce(fromconfig.utils.merge_dict, configs) 15 | 16 | # Parse the config (resolve interpolation) 17 | parser = fromconfig.parser.DefaultParser() 18 | parsed = parser(config) 19 | 20 | # Instantiate one of the keys 21 | instance = fromconfig.fromconfig(parsed["model"]) 22 | assert isinstance(instance, model.Model) 23 | instance.train() 24 | 25 | # You can also use the DefaultLauncher that replicates the CLI 26 | launcher = fromconfig.launcher.DefaultLauncher() 27 | launcher(config, "model - train") 28 | -------------------------------------------------------------------------------- /docs/examples/manual-parsing/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/examples/manual-parsing/params.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | learning_rate: 0.001 3 | -------------------------------------------------------------------------------- /docs/extensions/README.md: -------------------------------------------------------------------------------- 1 | # Extensions 2 | 3 | * [MlFlow](extensions/mlflow/) 4 | * [Yarn](extensions/yarn/) 5 | * [HParams Yarn MlFlow](extensions/hparams-yarn-mlflow/) 6 | 7 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/README.md: -------------------------------------------------------------------------------- 1 | # HParams + Yarn + MlFlow 2 | 3 | 4 | ## Install 5 | 6 | ```bash 7 | pip install fromconfig_yarn fromconfig_mlflow 8 | ``` 9 | 10 | ## Quickstart 11 | 12 | To activate both yarn and MlFlow, simply add `--launcher.log=mlflow --launcher.run=yarn,mlflow,local` to your command. 13 | 14 | ```bash 15 | fromconfig config.yaml hparams.yaml --launcher.log=mlflow --launcher.run=yarn,mlflow,local - model - train 16 | ``` 17 | 18 | Simply specifying `--launcher.run=yarn` would not be sufficient. Adding `mlflow,local` restarts the MlFlow run on the distant machine, thanks to the MlFlow environment variables that are automatically set and forwarded by default. 19 | 20 | With 21 | 22 | `model.py` 23 | 24 | [model.py](model.py ':include :type=code python') 25 | 26 | `config.yaml` 27 | 28 | [config.yaml](config.yaml ':include :type=code yaml') 29 | 30 | `hparams.yaml` 31 | 32 | [hparams.yaml](hparams.yaml ':include :type=code yaml') 33 | 34 | It should print 35 | 36 | ``` 37 | ============================================[learning_rate=0.1]============================================ 38 | Started run: http://127.0.0.1:5000/experiments/0/runs/1c8b62d08d134cdda4e0ac545e8a804c 39 | Uploading PEX and running on YARN 40 | Active run found: http://127.0.0.1:5000/experiments/0/runs/1c8b62d08d134cdda4e0ac545e8a804c 41 | Training model with learning_rate 0.1 42 | ===========================================[learning_rate=0.01]=========================================== 43 | Started run: http://127.0.0.1:5000/experiments/0/runs/3a78d8c011884e2fb8d4b2813cf398dc 44 | Uploading PEX and running on YARN 45 | Active run found: http://127.0.0.1:5000/experiments/0/runs/3a78d8c011884e2fb8d4b2813cf398dc 46 | Training model with learning_rate 0.01 47 | ``` 48 | 49 | You can also monkeypatch the relevant functions to "fake" the Hadoop environment with 50 | 51 | ```bash 52 | python monkeypatch_fromconfig.py config.yaml hparams.yaml --launcher.log=mlflow --launcher.run=yarn,mlflow,local - model - train 53 | ``` 54 | 55 | You can also use a `launcher.yaml` file 56 | 57 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 58 | 59 | And launch with 60 | 61 | ```bash 62 | fromconfig config.yaml hparams.yaml launcher.yaml - model - train 63 | ``` 64 | 65 | 66 | ## Advanced 67 | 68 | You can configure each launcher more precisely. See the [mlflow](/extensions/mlflow/) and [yarn](/extensions/yarn/) references. 69 | 70 | For example, 71 | 72 | [launcher_advanced.yaml](launcher_advanced.yaml ':include :type=code yaml') 73 | 74 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/config.yaml: -------------------------------------------------------------------------------- 1 | # Configure model 2 | model: 3 | _attr_: model.Model # Full import string to the class to instantiate 4 | learning_rate: "${hparams.learning_rate}" # Other key value parameter 5 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/hparams.yaml: -------------------------------------------------------------------------------- 1 | # Configure hyper parameters, use interpolation ${hparams.key} to use them 2 | hparams: 3 | learning_rate: [0.1, 0.01] 4 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/launcher.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure mlflow 11 | mlflow: 12 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 13 | # experiment_name: "test-experiment" # Which experiment to use 14 | # run_id: 12345 # To restore a previous run 15 | # run_name: test # To give a name to your new run 16 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 17 | # include_keys: # Only log params that match *model* 18 | # - model 19 | 20 | # Configure Yarn 21 | yarn: 22 | # name: test 23 | # zip_file: "path/to/env.pex" # Reuse existing pex 24 | # package_path: "hdfs://root/user/..." # Target path (where to upload the env.pex) 25 | # num_cores: 8 # Number of executor's cores 26 | # memory: "32 GiB" # Executor's memory 27 | # jvm_memory_in_gb: 8 # JVM memory 28 | # env_vars: # Environment variables to forward to yarn 29 | # - CUDA_VISIBLE_DEVICES 30 | # - MLFLOW_RUN_ID 31 | # - MLFLOW_TRACKING_URI 32 | 33 | # Configure launcher 34 | launcher: 35 | log: 36 | - logging 37 | - mlflow 38 | parse: 39 | - mlflow.log_artifacts 40 | - parser 41 | - mlflow.log_params 42 | run: 43 | - yarn 44 | - mlflow 45 | - local 46 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/launcher_advanced.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure mlflow 11 | mlflow: 12 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 13 | # experiment_name: "test-experiment" # Which experiment to use 14 | # run_id: 12345 # To restore a previous run 15 | # run_name: test # To give a name to your new run 16 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 17 | # include_keys: # Only log params that match *model* 18 | # - model 19 | 20 | # Configure Yarn 21 | yarn: 22 | # name: test 23 | # zip_file: "path/to/env.pex" # Reuse existing pex 24 | # package_path: "hdfs://root/user/..." # Target path (where to upload the env.pex) 25 | # num_cores: 8 # Number of executor's cores 26 | # memory: "32 GiB" # Executor's memory 27 | # jvm_memory_in_gb: 8 # JVM memory 28 | # env_vars: # Environment variables to forward to yarn 29 | # - CUDA_VISIBLE_DEVICES 30 | # - MLFLOW_RUN_ID 31 | # - MLFLOW_TRACKING_URI 32 | 33 | # Configure launcher 34 | launcher: 35 | log: 36 | - logging 37 | # Start MlFlow Run and set environment variables 38 | - _attr_: mlflow 39 | set_env_vars: true 40 | set_run_id: false 41 | parse: 42 | # Log both the command and the config before parsing 43 | - _attr_: mlflow.log_artifacts 44 | path_command: launch.sh 45 | path_config: config.yaml 46 | - parser 47 | # Only log the parsed config 48 | - _attr_: mlflow.log_artifacts 49 | path_command: null 50 | path_config: parsed.yaml 51 | # Log the flattened config as run parameters 52 | - mlflow.log_params 53 | run: 54 | - yarn 55 | - mlflow 56 | - local 57 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | import mlflow 4 | 5 | 6 | class Model: 7 | def __init__(self, learning_rate: float): 8 | self.learning_rate = learning_rate 9 | 10 | def train(self): 11 | print(f"Training model with learning_rate {self.learning_rate}") 12 | if mlflow.active_run(): 13 | mlflow.log_metric("learning_rate", self.learning_rate) 14 | -------------------------------------------------------------------------------- /docs/extensions/hparams-yarn-mlflow/monkeypatch_fromconfig.py: -------------------------------------------------------------------------------- 1 | """Monkey Patch to fake yarn environment. 2 | 3 | Usage 4 | ----- 5 | python monkeypatch_fromconfig.py config.yaml launcher.yaml - model - train 6 | """ 7 | 8 | import contextlib 9 | import cluster_pack 10 | import skein 11 | from collections import namedtuple 12 | from cluster_pack.skein import skein_launcher 13 | 14 | 15 | @contextlib.contextmanager 16 | def _MonkeyClient(): # pylint: disable=invalid-name 17 | """Monkey skein Client.""" 18 | 19 | Client = namedtuple("Client", "application_report") 20 | Report = namedtuple("Report", "tracking_url") 21 | 22 | def application_report(app_id): 23 | return Report(app_id) 24 | 25 | yield Client(application_report) 26 | 27 | 28 | def _monkey_submit_func(func, args, **kwargs): 29 | """Monkey patch submit function.""" 30 | # pylint: disable=unused-argument 31 | print("Uploading PEX and running on YARN") 32 | func(*args) 33 | 34 | 35 | setattr(cluster_pack, "upload_env", lambda *_, **__: None) 36 | setattr(cluster_pack, "upload_zip", lambda *_, **__: None) 37 | setattr(skein, "Client", _MonkeyClient) 38 | setattr(skein_launcher, "submit_func", _monkey_submit_func) 39 | 40 | 41 | if __name__ == "__main__": 42 | from fromconfig.cli.main import main 43 | 44 | main() 45 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/README.md: -------------------------------------------------------------------------------- 1 | # FromConfig MlFlow 2 | [![pypi](https://img.shields.io/pypi/v/fromconfig-mlflow.svg)](https://pypi.python.org/pypi/fromconfig-mlflow) 3 | [![ci](https://github.com/criteo/fromconfig-mlflow/workflows/Continuous%20integration/badge.svg)](https://github.com/criteo/fromconfig-mlflow/actions?query=workflow%3A%22Continuous+integration%22) 4 | 5 | A [fromconfig](https://github.com/criteo/fromconfig) `Launcher` for [MlFlow](https://www.mlflow.org) support. 6 | 7 | 8 | 9 | ## Install 10 | 11 | ```bash 12 | pip install fromconfig_mlflow 13 | ``` 14 | 15 | 16 | ## Quickstart 17 | 18 | To activate `MlFlow` login, simply add `--launcher.log=mlflow` to your command 19 | 20 | ```bash 21 | fromconfig config.yaml params.yaml --launcher.log=mlflow - model - train 22 | ``` 23 | 24 | With 25 | 26 | `model.py` 27 | 28 | [model.py](model.py ':include :type=code python') 29 | 30 | `config.yaml` 31 | 32 | [config.yaml](config.yaml ':include :type=code yaml') 33 | 34 | `params.yaml` 35 | 36 | [params.yaml](params.yaml ':include :type=code yaml') 37 | 38 | 39 | It should print 40 | 41 | ``` 42 | Started run: http://127.0.0.1:5000/experiments/0/runs/7fe650dd99574784aec1e4b18fceb73f 43 | Training model with learning_rate 0.001 44 | ``` 45 | 46 | If you navigate to `http://127.0.0.1:5000/experiments/0/runs/7fe650dd99574784aec1e4b18fceb73f` you should see the logged `learning_rate` metric. 47 | 48 | 49 | ## MlFlow server 50 | 51 | To setup a local MlFlow tracking server, run 52 | 53 | ```bash 54 | mlflow server 55 | ``` 56 | 57 | which should print 58 | 59 | ``` 60 | [INFO] Starting gunicorn 20.0.4 61 | [INFO] Listening at: http://127.0.0.1:5000 62 | ``` 63 | 64 | We will assume that the tracking URI is `http://127.0.0.1:5000` from now on. 65 | 66 | 67 | 68 | ## Configure MlFlow 69 | 70 | You can set the tracking URI either via an environment variable or via the config. 71 | 72 | To set the `MLFLOW_TRACKING_URI` environment variable 73 | 74 | ```bash 75 | export MLFLOW_TRACKING_URI=http://127.0.0.1:5000 76 | ``` 77 | 78 | Alternatively, you can set the `mlflow.tracking_uri` config key either via command line with 79 | 80 | ```bash 81 | fromconfig config.yaml params.yaml --launcher.log=mlflow --mlflow.tracking_uri="http://127.0.0.1:5000" - model - train 82 | ``` 83 | 84 | or in a config file with 85 | 86 | `launcher.yaml` 87 | 88 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 89 | 90 | and run 91 | 92 | ```bash 93 | fromconfig config.yaml params.yaml launcher.yaml - model - train 94 | ``` 95 | 96 | 97 | ## Artifacts and Parameters 98 | 99 | In this example, we add logging of the config and parameters. 100 | 101 | Re-using the [quickstart](#quickstart) code, modify the `launcher.yaml` file 102 | 103 | [launcher_artifacts_params.yaml](launcher_artifacts_params.yaml ':include :type=code yaml') 104 | 105 | and run 106 | 107 | ```bash 108 | fromconfig config.yaml params.yaml launcher.yaml - model - train 109 | ``` 110 | 111 | which prints 112 | 113 | ``` 114 | INFO:fromconfig_mlflow.launcher:Started run: http://127.0.0.1:5000/experiments/0/runs/ 115 | Training model with learning_rate 0.001 116 | ``` 117 | 118 | If you navigate to the MlFlow run URL, you should see 119 | - the original config, saved as `config.yaml` 120 | - the parameters, a flattened version of the *parsed* config (`model.learning_rate` is `0.001` and not `${params.learning_rate}`) 121 | 122 | 123 | 124 | ## Usage-Reference 125 | 126 | 127 | ### `StartRunLauncher` 128 | 129 | To configure MlFlow, add a `mlflow` entry to your config and set the following parameters 130 | 131 | - `run_id`: if you wish to restart an existing run 132 | - `run_name`: if you wish to give a name to your new run 133 | - `tracking_uri`: to configure the tracking remote 134 | - `experiment_name`: to use a different experiment than the custom 135 | experiment 136 | - `artifact_location`: the location of the artifacts (config files) 137 | 138 | Additionally, the launcher can be initialized with the following attributes 139 | 140 | - `set_env_vars`: if True (default is `True`), set `MLFLOW_RUN_ID` and `MLFLOW_TRACKING_URI` 141 | - `set_run_id`: if True (default is `False`), set `mlflow.run_id` in config. 142 | 143 | For example, 144 | 145 | [launcher_start.yaml](launcher_start.yaml ':include :type=code yaml') 146 | 147 | 148 | 149 | ### `LogArtifactsLauncher` 150 | 151 | The launcher can be initialized with the following attributes 152 | 153 | - `path_command`: Name for the command file. If `None`, don't log the command. 154 | - `path_config`: Name for the config file. If `None`, don't log the config. 155 | 156 | For example, 157 | 158 | [launcher_artifacts.yaml](launcher_artifacts.yaml ':include :type=code yaml') 159 | 160 | 161 | 162 | ### `LogParamsLauncher` 163 | 164 | The launcher will use `include_keys` and `ignore_keys` if present in the config in the `mlflow` key. 165 | 166 | - `ignore_keys` : If given, don't log some parameters that have some substrings. 167 | - `include_keys` : If given, only log some parameters that have some substrings. Also shorten the flattened parameter to start at the first match. For example, if the config is `{"foo": {"bar": 1}}` and `include_keys=("bar",)`, then the logged parameter will be `"bar"`. 168 | 169 | For example, 170 | 171 | [launcher_params.yaml](launcher_params.yaml ':include :type=code yaml') 172 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${params.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/launcher.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure mlflow 7 | mlflow: 8 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 9 | # experiment_name: "test-experiment" # Which experiment to use 10 | # run_id: 12345 # To restore a previous run 11 | # run_name: test # To give a name to your new run 12 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 13 | 14 | # Configure launcher 15 | launcher: 16 | log: mlflow 17 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/launcher_artifacts.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure mlflow 11 | mlflow: 12 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 13 | # experiment_name: "test-experiment" # Which experiment to use 14 | # run_id: 12345 # To restore a previous run 15 | # run_name: test # To give a name to your new run 16 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 17 | 18 | # Configure launcher 19 | launcher: 20 | log: 21 | - logging 22 | - mlflow 23 | parse: 24 | - _attr_: mlflow.log_artifacts 25 | path_command: launch.sh 26 | path_config: config.yaml 27 | - parser 28 | - _attr_: mlflow.log_artifacts 29 | path_command: null 30 | path_config: parsed.yaml 31 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/launcher_artifacts_params.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure mlflow 11 | mlflow: 12 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 13 | # experiment_name: "test-experiment" # Which experiment to use 14 | # run_id: 12345 # To restore a previous run 15 | # run_name: test # To give a name to your new run 16 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 17 | # include_keys: # Only log params that match *model* 18 | # - model 19 | 20 | # Configure launcher 21 | launcher: 22 | log: 23 | - logging 24 | - mlflow 25 | parse: 26 | - mlflow.log_artifacts 27 | - parser 28 | - mlflow.log_params 29 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/launcher_params.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure mlflow 11 | mlflow: 12 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 13 | # experiment_name: "test-experiment" # Which experiment to use 14 | # run_id: 12345 # To restore a previous run 15 | # run_name: test # To give a name to your new run 16 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 17 | include_keys: # Only log params that match *model* 18 | - model 19 | 20 | # Configure launcher 21 | launcher: 22 | log: 23 | - logging 24 | - mlflow 25 | parse: 26 | - parser 27 | - mlflow.log_params 28 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/launcher_start.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure mlflow 11 | mlflow: 12 | # tracking_uri: "http://127.0.0.1:5000" # Or set env variable MLFLOW_TRACKING_URI 13 | # experiment_name: "test-experiment" # Which experiment to use 14 | # run_id: 12345 # To restore a previous run 15 | # run_name: test # To give a name to your new run 16 | # artifact_location: "path/to/artifacts" # Used only when creating a new experiment 17 | 18 | # Configure Launcher 19 | launcher: 20 | log: 21 | - logging 22 | - _attr_: mlflow 23 | set_env_vars: true 24 | set_run_id: true 25 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | import mlflow 4 | 5 | 6 | class Model: 7 | def __init__(self, learning_rate: float): 8 | self.learning_rate = learning_rate 9 | 10 | def train(self): 11 | print(f"Training model with learning_rate {self.learning_rate}") 12 | if mlflow.active_run(): 13 | mlflow.log_metric("learning_rate", self.learning_rate) 14 | -------------------------------------------------------------------------------- /docs/extensions/mlflow/params.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | learning_rate: 0.001 3 | -------------------------------------------------------------------------------- /docs/extensions/requirements.txt: -------------------------------------------------------------------------------- 1 | fromconfig-mlflow>=0.3.0 2 | fromconfig-yarn 3 | -------------------------------------------------------------------------------- /docs/extensions/yarn/README.md: -------------------------------------------------------------------------------- 1 | # FromConfig Yarn 2 | [![pypi](https://img.shields.io/pypi/v/fromconfig-yarn.svg)](https://pypi.python.org/pypi/fromconfig-yarn) 3 | [![ci](https://github.com/criteo/fromconfig-yarn/workflows/Continuous%20integration/badge.svg)](https://github.com/criteo/fromconfig-yarn/actions?query=workflow%3A%22Continuous+integration%22) 4 | 5 | A [fromconfig](https://github.com/criteo/fromconfig) `Launcher` for yarn execution. 6 | 7 | 8 | ## Install 9 | 10 | ```bash 11 | pip install fromconfig_yarn 12 | ``` 13 | 14 | 15 | ## Quickstart 16 | 17 | To run on yarn, simply add `--launcher.run=yarn` to your command 18 | 19 | ```bash 20 | fromconfig config.yaml params.yaml --launcher.run=yarn - model - train 21 | ``` 22 | 23 | With 24 | 25 | `model.py` 26 | 27 | [model.py](model.py ':include :type=code python') 28 | 29 | `config.yaml` 30 | 31 | [config.yaml](config.yaml ':include :type=code yaml') 32 | 33 | `params.yaml` 34 | 35 | [params.yaml](params.yaml ':include :type=code yaml') 36 | 37 | It should print 38 | 39 | ``` 40 | INFO skein.Driver: Driver started, listening on 12345 41 | INFO:fromconfig_yarn.launcher:Uploading pex to viewfs://root/user/path/to/pex 42 | INFO:cluster_pack.filesystem:Resolved base filesystem: 43 | INFO:cluster_pack.uploader:Zipping and uploading your env to viewfs://root/user/path/to/pex 44 | INFO skein.Driver: Uploading application resources to viewfs://root/user/... 45 | INFO skein.Driver: Submitting application... 46 | INFO impl.YarnClientImpl: Submitted application application_12345 47 | INFO:fromconfig_yarn.launcher:TRACKING_URL: http://12.34.56/application_12345 48 | ``` 49 | 50 | You can also monkeypatch the relevant functions to "fake" the Hadoop environment with 51 | 52 | ```bash 53 | python monkeypatch_fromconfig.py config.yaml params.yaml --launcher.run=yarn - model - train 54 | ``` 55 | 56 | ## Usage Reference 57 | 58 | To configure Yarn, add a `yarn` entry to your config. 59 | 60 | You can set the following parameters. 61 | 62 | - `env_vars`: A list of environment variables to forward to the container(s) 63 | - `hadoop_file_systems`: The list of available filesystems 64 | - `ignored_packages`: The list of packages not to include in the environment 65 | - `jvm_memory_in_gb`: The JVM memory (default, `8`) 66 | - `memory`: The executor's memory (default, `32 GiB`) 67 | - `num_cores`: The executor's number of cores (default, `8`) 68 | - `package_path`: The HDFS location where to save the environment 69 | - `zip_file`: The path to an existing `pex` file, either local or on HDFS 70 | - `name`: The application name 71 | 72 | Using a `launcher.yaml` file to specify the launcher and more advanced parameters 73 | 74 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 75 | 76 | launch with 77 | 78 | ```bash 79 | fromconfig config.yaml params.yaml launcher.yaml - model - train 80 | ``` 81 | -------------------------------------------------------------------------------- /docs/extensions/yarn/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${params.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/extensions/yarn/launcher.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure logging 7 | logging: 8 | level: 20 9 | 10 | # Configure yarn 11 | yarn: 12 | # name: test-fromconfig # Application's name on Yarn 13 | # zip_file: "path/to/env.pex" # Reuse existing pex 14 | # package_path: "hdfs://root/user/..." # Target path (where to upload the env.pex) 15 | # num_cores: 8 # Number of executor's cores 16 | # memory: "32 GiB" # Executor's memory 17 | # jvm_memory_in_gb: 8 # JVM memory 18 | # env_vars: # Environment variables to forward to yarn 19 | # - CUDA_VISIBLE_DEVICES 20 | # - MLFLOW_RUN_ID 21 | # - MLFLOW_TRACKING_URI 22 | 23 | # Configure launcher 24 | launcher: 25 | run: yarn 26 | -------------------------------------------------------------------------------- /docs/extensions/yarn/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/extensions/yarn/monkeypatch_fromconfig.py: -------------------------------------------------------------------------------- 1 | """Monkey Patch to fake yarn environment. 2 | 3 | Usage 4 | ----- 5 | python monkeypatch_fromconfig.py config.yaml params.yaml launcher.yaml - model - train 6 | """ 7 | 8 | import contextlib 9 | import cluster_pack 10 | import skein 11 | from collections import namedtuple 12 | from cluster_pack.skein import skein_launcher 13 | 14 | 15 | @contextlib.contextmanager 16 | def _MonkeyClient(): # pylint: disable=invalid-name 17 | """Monkey skein Client.""" 18 | 19 | Client = namedtuple("Client", "application_report") 20 | Report = namedtuple("Report", "tracking_url") 21 | 22 | def application_report(app_id): 23 | return Report(app_id) 24 | 25 | yield Client(application_report) 26 | 27 | 28 | def _monkey_submit_func(func, args, **kwargs): 29 | """Monkey patch submit function.""" 30 | # pylint: disable=unused-argument 31 | print("Uploading PEX and running on YARN") 32 | func(*args) 33 | 34 | 35 | setattr(cluster_pack, "upload_env", lambda *_, **__: None) 36 | setattr(cluster_pack, "upload_zip", lambda *_, **__: None) 37 | setattr(skein, "Client", _MonkeyClient) 38 | setattr(skein_launcher, "submit_func", _monkey_submit_func) 39 | 40 | 41 | if __name__ == "__main__": 42 | from fromconfig.cli.main import main 43 | 44 | main() 45 | -------------------------------------------------------------------------------- /docs/extensions/yarn/params.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | learning_rate: 0.001 3 | -------------------------------------------------------------------------------- /docs/getting-started/README.md: -------------------------------------------------------------------------------- 1 | # Getting-Started 2 | 3 | * [Install](getting-started/install) 4 | * [Quickstart](getting-started/quickstart/) 5 | * [Cheat Sheet](getting-started/cheat-sheet/) 6 | * [Why From Config ?](getting-started/why-fromconfig) 7 | -------------------------------------------------------------------------------- /docs/getting-started/cheat-sheet/README.md: -------------------------------------------------------------------------------- 1 | # Cheat Sheet 2 | 3 | ## Syntax and default options 4 | 5 | [`fromconfig.fromconfig`](usage-reference/fromconfig/) special keys 6 | 7 | 8 | | Key | Value Example | Use | 9 | |------------|---------------------|---------------------------------------------------| 10 | | `"_attr_"` | `"foo.bar.MyClass"` | Full import string of a class, function or method | 11 | | `"_args_"` | `[1, 2]` | Positional arguments | 12 | 13 | [`fromconfig.parser.DefaultParser`](usage-reference/parser/) supported syntax 14 | 15 | | Key | Value | Use | 16 | |-----------------|-----------------------------------|----------------------------------------| 17 | | `"_singleton_"` | `"my_singleton_name"` | Creates a singleton identified by name | 18 | | `"_eval_"` | `"call"`, `"import"`, `"partial"` | Evaluation modes | 19 | | | `"${params.url}:${params.port}"` | Interpolation via OmegaConf | 20 | 21 | [`fromconfig.launcher.DefaultLauncher`](usage-reference/launcher/) options (keys at config's toplevel) 22 | 23 | 24 | | Key | Value Example | Use | 25 | |-------------|----------------------------------------------------|---------------------------------------------| 26 | | `"logging"` | `{"level": 20}` | Change logging level to 20 (`logging.INFO`) | 27 | | `"parser"` | `{"_attr_": "fromconfig.parser.DefaultParser"}` | Configure which parser is used | 28 | | `"hparams"` | `{"learning_rate": [0.1, 0.001]}` | Hyper-parameter search (use references like `${hparams.learning_rate}` in other parts of the config) | 29 | 30 | 31 | ## Config Example 32 | 33 | As an example, let's consider a `model.py` module 34 | 35 | [model.py](model.py ':include :type=code python') 36 | 37 | with the following config files 38 | 39 | `config.yaml` 40 | 41 | [config.yaml](config.yaml ':include :type=code yaml') 42 | 43 | `launcher.yaml` 44 | 45 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 46 | 47 | In a terminal, run 48 | 49 | ```bash 50 | fromconfig config.yaml launcher.yaml - model - train 51 | ``` 52 | -------------------------------------------------------------------------------- /docs/getting-started/cheat-sheet/config.yaml: -------------------------------------------------------------------------------- 1 | # Configure model 2 | model: 3 | _attr_: model.Model # Full import string to the class to instantiate 4 | _args_: ["${hparams.dim}"] # Positional arguments 5 | _singleton_: "model_${hparams.dim}_${hparams.learning_rate}" # All ${model} interpolation will instantiate the same object with that name 6 | _eval_: "call" # Optional ("call" is the default behavior) 7 | learning_rate: "${hparams.learning_rate}" # Other key value parameter 8 | -------------------------------------------------------------------------------- /docs/getting-started/cheat-sheet/launcher.yaml: -------------------------------------------------------------------------------- 1 | # ==================== FromConfig Launcher Config ====================== 2 | # These parameters are used by FromConfig to instantiate a launcher. 3 | # The launcher is responsible for parsing, logging, instantiating and 4 | # executing the config and Fire command. 5 | # ====================================================================== 6 | # Configure hyper parameters, use interpolation ${hparams.key} to use them 7 | hparams: 8 | learning_rate: [0.1, 0.001] 9 | dim: [10, 100] 10 | 11 | # Configure logging level (set to logging.INFO) 12 | logging: 13 | level: 20 14 | 15 | # Configure parser (optional, using this parser is the default behavior) 16 | parser: 17 | _attr_: "fromconfig.parser.DefaultParser" 18 | 19 | # Configure launcher (optional, the following config creates the same launcher as the default behavior) 20 | launcher: 21 | sweep: hparams 22 | log: logging 23 | parse: parser 24 | run: local 25 | -------------------------------------------------------------------------------- /docs/getting-started/cheat-sheet/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, dim: int, learning_rate: float): 6 | self.dim = dim 7 | self.learning_rate = learning_rate 8 | 9 | def train(self): 10 | print(f"Training model({self.dim}) with learning_rate {self.learning_rate}") 11 | -------------------------------------------------------------------------------- /docs/getting-started/install.md: -------------------------------------------------------------------------------- 1 | # Install 2 | 3 | ## Install with pip 4 | 5 | ```bash 6 | pip install fromconfig 7 | ``` 8 | 9 | ## Install from source 10 | 11 | To install the library from source in editable mode 12 | 13 | ```bash 14 | git clone https://github.com/criteo/fromconfig 15 | cd fromconfig 16 | make install 17 | ``` 18 | 19 | ## Optional dependencies 20 | 21 | If you want to support the [JSONNET](https://jsonnet.org) format, you need to install it explicitly (it is not included in `fromconfig`s requirements). 22 | 23 | First, make sure `JSONNET` is installed on your machine. 24 | 25 | With Homebrew 26 | 27 | ```bash 28 | brew install jsonnet 29 | ``` 30 | 31 | Install the python binding with 32 | 33 | ```bash 34 | pip install jsonnet 35 | ``` 36 | 37 | See [the official install instructions on GitHub](https://github.com/google/jsonnet). 38 | -------------------------------------------------------------------------------- /docs/getting-started/quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | `fromconfig` can configure any Python object, without any change to the code. 4 | 5 | As an example, let's consider a `model.py` module 6 | 7 | [model.py](model.py ':include :type=code python') 8 | 9 | with the following config files 10 | 11 | `config.yaml` 12 | 13 | [config.yaml](config.yaml ':include :type=code yaml') 14 | 15 | `params.yaml` 16 | 17 | [params.yaml](params.yaml ':include :type=code yaml') 18 | 19 | 20 | ## Command Line 21 | 22 | In a terminal, run 23 | 24 | ```bash 25 | fromconfig config.yaml params.yaml - model - train 26 | ``` 27 | 28 | which prints 29 | ``` 30 | Training model with learning_rate 0.1 31 | ``` 32 | 33 | The `fromconfig` command loads the config files, parses them, instantiates the result with `fromconfig.fromconfig` and then launches the `fire.Fire` command `- model - train` which roughly translates into "get the `model` key from the instantiated dictionary and execute the `train` method". 34 | 35 | __Warning__ It is not safe to use `fromconfig` with any config received from an untrusted source. Because it uses import strings to resolve attributes, it is as powerful as a plain python script and can execute arbitrary code. 36 | 37 | ## Manual 38 | 39 | You can also manipulate the configs manually. 40 | 41 | [manual.py](manual.py ':include :type=code python') 42 | -------------------------------------------------------------------------------- /docs/getting-started/quickstart/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${params.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/getting-started/quickstart/manual.py: -------------------------------------------------------------------------------- 1 | """Manual quickstart.""" 2 | 3 | import functools 4 | 5 | import fromconfig 6 | 7 | import model 8 | 9 | 10 | if __name__ == "__main__": 11 | # Load configs from yaml and merge into one dictionary 12 | paths = ["config.yaml", "params.yaml"] 13 | configs = [fromconfig.load(path) for path in paths] 14 | config = functools.reduce(fromconfig.utils.merge_dict, configs) 15 | 16 | # Parse the config (resolve interpolation) 17 | parser = fromconfig.parser.DefaultParser() 18 | parsed = parser(config) 19 | 20 | # Instantiate one of the keys 21 | instance = fromconfig.fromconfig(parsed["model"]) 22 | assert isinstance(instance, model.Model) 23 | instance.train() 24 | 25 | # You can also use the DefaultLauncher that replicates the CLI 26 | launcher = fromconfig.launcher.DefaultLauncher() 27 | launcher(config, "model - train") 28 | -------------------------------------------------------------------------------- /docs/getting-started/quickstart/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/getting-started/quickstart/params.yaml: -------------------------------------------------------------------------------- 1 | params: 2 | learning_rate: 0.1 3 | -------------------------------------------------------------------------------- /docs/getting-started/why-fromconfig.md: -------------------------------------------------------------------------------- 1 | # Why FromConfig ? 2 | 3 | `fromconfig` enables the instantiation of arbitrary trees of Python objects from config files. 4 | 5 | It echoes the `FromParams` base class of [AllenNLP](https://github.com/allenai/allennlp). 6 | 7 | It is particularly well suited for __Machine Learning__. Launching training jobs on remote clusters requires custom command lines, with arguments that need to be propagated through the call stack (e.g., setting parameters of a particular layer). The usual way is to write a custom command with a reduced set of arguments, combined by an assembler that creates the different objects. With `fromconfig`, the command line becomes generic, and all the specifics are kept in config files. As a result, this preserves the code from any backwards dependency issues and allows full reproducibility by saving config files as jobs' artifacts. It also makes it easier to merge different sets of arguments in a dynamic way through interpolation. 8 | 9 | `fromconfig` is based off the config system developed as part of the [deepr](https://github.com/criteo/deepr) library, a collections of utilities to define and train Tensorflow models in a Hadoop environment. 10 | 11 | Other relevant libraries are: 12 | 13 | * [fire](https://github.com/google/python-fire) automatically generate command line interface (CLIs) from absolutely any Python object. 14 | * [omegaconf](https://github.com/omry/omegaconf) YAML based hierarchical configuration system with support for merging configurations from multiple sources. 15 | * [hydra](https://hydra.cc/docs/intro/) A higher-level framework based off `omegaconf` to configure complex applications. 16 | * [gin](https://github.com/google/gin-config) A lightweight configuration framework based on dependency injection. 17 | * [thinc](https://thinc.ai/) A lightweight functional deep learning library that comes with an integrated config system 18 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | fromconfig 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /docs/usage-reference/README.md: -------------------------------------------------------------------------------- 1 | # Usage Reference 2 | 3 | 4 | * [Overview](usage-reference/overview) 5 | * [Command Line](usage-reference/command-line) 6 | * [fromconfig](usage-reference/fromconfig/) 7 | * [Parser](usage-reference/parser/) 8 | * [Launcher](usage-reference/launcher/) 9 | -------------------------------------------------------------------------------- /docs/usage-reference/command-line.md: -------------------------------------------------------------------------------- 1 | # Command Line 2 | 3 | ## Usage 4 | 5 | Usage : call `fromconfig` on any number of paths to config files, with optional key value overrides. Use the full expressiveness of [python Fire](https://github.com/google/python-fire) to manipulate the resulting instantiated object. 6 | 7 | ```bash 8 | fromconfig config.yaml params.yaml --key=value - name 9 | ``` 10 | 11 | Supported formats : YAML, JSON, and [JSONNET](https://jsonnet.org). 12 | 13 | The command line loads the different config files into Python dictionaries and merge them (if there is any key conflict, the config on the right overrides the ones from the left). 14 | 15 | It then instantiate the [`launcher`](usage-reference/launcher/) (using the `launcher` key if present in the config) and launches the config with the rest of the fire command. The `launcher` is responsible for parsing (resolving interpolation, etc.), and uses a [`Parser`](usage-reference/parser/). 16 | 17 | With [Python Fire](https://github.com/google/python-fire), you can manipulate the resulting instantiated dictionary via the command line by using the fire syntax. 18 | 19 | For example `fromconfig config.yaml - name` instantiates the dictionary defined in `config.yaml` and gets the value associated with the key `name`. 20 | 21 | 22 | ## Overrides 23 | 24 | You can provide additional key value parameters following the [Python Fire](https://github.com/google/python-fire) syntax as overrides directly via the command line. 25 | 26 | For example, using the [quickstart](getting-started/quickstart)'s material, running 27 | 28 | ```bash 29 | fromconfig config.yaml params.yaml --params.learning_rate=0.01 - model - train 30 | ``` 31 | 32 | will print 33 | 34 | ``` 35 | Training model with learning_rate 0.01 36 | ``` 37 | 38 | This is strictly equivalent to defining another config file (eg. `overrides.yaml`) 39 | 40 | ```yaml 41 | params: 42 | learning_rate: 0.01 43 | ``` 44 | 45 | and running 46 | 47 | ```bash 48 | fromconfig config.yaml params.yaml overrides.yaml - model - train 49 | ``` 50 | 51 | since the config files are merged from left to right, the files on the right overriding the existing keys from the left in case of conflict. 52 | -------------------------------------------------------------------------------- /docs/usage-reference/fromconfig/README.md: -------------------------------------------------------------------------------- 1 | # Config syntax 2 | 3 | The `fromconfig.fromconfig` function recursively instantiates objects from dictionaries. 4 | 5 | It uses two special keys 6 | 7 | - `_attr_`: (optional) full import string to any Python object. 8 | - `_args_`: (optional) positional arguments. 9 | 10 | For example, 11 | 12 | [example.py](example.py ':include :type=code python') 13 | 14 | `FromConfig` resolves the builtin type `str` from the `_attr_` key, and creates a new string with the positional arguments defined in `_args_`, in other words `str(1)` which return `'1'`. 15 | 16 | If the `_attr_` key is not given, then the dictionary is left as a dictionary (the values of the dictionary may be recursively instantiated). 17 | 18 | If other keys are available in the dictionary, they are treated as key-value arguments (`kwargs`). 19 | 20 | For example 21 | 22 | [example_kwargs.py](example_kwargs.py ':include :type=code python') 23 | 24 | Note that any mapping-like container is supported (there is no special "config" class in `fromconfig`). 25 | -------------------------------------------------------------------------------- /docs/usage-reference/fromconfig/example.py: -------------------------------------------------------------------------------- 1 | """Simple example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | if __name__ == "__main__": 7 | config = {"_attr_": "str", "_args_": [1]} 8 | assert fromconfig.fromconfig(config) == "1" 9 | -------------------------------------------------------------------------------- /docs/usage-reference/fromconfig/example_kwargs.py: -------------------------------------------------------------------------------- 1 | """Kwargs example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | class Point: 7 | def __init__(self, x, y): 8 | self.x = x 9 | self.y = y 10 | 11 | 12 | if __name__ == "__main__": 13 | config = {"_attr_": "Point", "x": 0, "y": 1} 14 | point = fromconfig.fromconfig(config) 15 | assert isinstance(point, Point) 16 | assert point.x == 0 17 | assert point.y == 1 18 | -------------------------------------------------------------------------------- /docs/usage-reference/launcher/README.md: -------------------------------------------------------------------------------- 1 | # Launcher 2 | 3 | 4 | ## Default 5 | 6 | When a `fromconfig` command is executed (example `fromconfig config.yaml params.yaml - model - train`), the config is loaded, a launcher is instantiated (possibly configured by the config itself if the `launcher` key is present in the config) and then the launcher "launches" the config with the remaining fire arguments. 7 | 8 | By default, 4 launchers are executed in the following order 9 | 10 | - [`fromconfig.launcher.HParamsLauncher`](#hparams): uses the `hparams` key of the config (if present) to launch multiple sub-configs from a grid of hyper-parameters. 11 | - [`fromconfig.launcher.LoggingLauncher`](#logging): setup the basic config of the `logging` module. 12 | - [`fromconfig.launcher.Parser`](#parser): applies a parser (by default, `DefaultParser`) to the config to resolve interpolation, singletons, etc. 13 | - [`fromconfig.launcher.LocalLauncher`](#local): runs `fire.Fire(fromconfig.fromconfig(config), command)` to instantiate and execute the config with the fire arguments (`command`, for example `model - train`). 14 | 15 | Let's see for example how to configure the logging level and perform an hyper-parameter search. 16 | 17 | Given the following module and config files (similar to the quickstart, we only changed `params` into `hparams`) 18 | 19 | `model.py` 20 | 21 | [model.py](model.py ':include :type=code python') 22 | 23 | `config.yaml` 24 | 25 | [config.yaml](config.yaml ':include :type=code yaml') 26 | 27 | `hparams.yaml` 28 | 29 | [hparams.yaml](hparams.yaml ':include :type=code yaml') 30 | 31 | 32 | `launcher.yaml` 33 | 34 | [launcher.yaml](launcher.yaml ':include :type=code yaml') 35 | 36 | run 37 | 38 | ```bash 39 | fromconfig config.yaml hparams.yaml launcher.yaml - model - train 40 | ``` 41 | 42 | which should print 43 | 44 | ``` 45 | ========================[learning_rate=0.1]==================================== 46 | Training model with learning_rate 0.1 47 | ========================[learning_rate=0.01]=================================== 48 | Training model with learning_rate 0.01 49 | ``` 50 | 51 | 52 | 53 | ## Launcher Configuration 54 | 55 | The launcher is instantiated from the `launcher` key if present in the config. 56 | 57 | For ease of use, multiple syntaxes are provided. 58 | 59 | 60 | ### Config Dict 61 | The `launcher` entry can be a config dictionary (with an `_attr_` key) that defines how to instantiate a `Launcher` instance (possibly custom). 62 | 63 | For example 64 | 65 | ```yaml 66 | launcher: 67 | _attr_: fromconfig.launcher.LocalLauncher 68 | ``` 69 | 70 | 71 | ### Name 72 | The `launcher` entry can be a `str`, corresponding to a name that maps to a `Launcher` class. The internal `Launcher` names are 73 | 74 | 75 | | Name | Class | 76 | |---------|---------------------------------------| 77 | | hparams | `fromconfig.launcher.HParamsLauncher` | 78 | | logging | `fromconfig.launcher.LoggingLauncher` | 79 | | parser | `fromconfig.launcher.ParserLauncher` | 80 | | local | `fromconfig.launcher.LocalLauncher` | 81 | | dry | `fromconfig.launcher.DryLauncher` | 82 | 83 | It is possible via extensions to add new `Launcher` classes to the list of available launchers (learn more in the examples section). 84 | 85 | 86 | ### List 87 | The `launcher` entry can be a list of [config dict](#config-dict) and/or [names](#name). In that case, the resulting launcher is a nested launcher instance of the different launchers. 88 | 89 | For example 90 | 91 | ```yaml 92 | launcher: 93 | - hparams 94 | - local 95 | ``` 96 | 97 | will result in `HParamsLauncher(LocalLauncher())`. 98 | 99 | 100 | ### Steps 101 | The `launcher` entry can also be a dictionary with 4 special keys for which the value can be any of config dict, name or list. 102 | 103 | - `sweep`: if not specified, will use [`hparams`](#hparams) 104 | - `log`: if not specified, will use [`logging`](#logging) 105 | - `parse`: if not specified, will use [`parser`](#parser) 106 | - `run`: if not specified, will use [`local`](#logging) 107 | 108 | Setting either all or a subset of these keys allows you to modify one of the 4 steps while still using the defaults for the rest of the steps. 109 | 110 | The result, again, is similar to the list mechanism, as a nested instance. 111 | 112 | For example 113 | 114 | ```yaml 115 | launcher: 116 | sweep: hparams 117 | log: logging 118 | parse: parser 119 | run: local 120 | ``` 121 | 122 | results in `HParamsLauncher(ParserLauncher(LoggingLauncher(LocalLauncher())))`. 123 | 124 | 125 | 126 | ## HParams 127 | 128 | The `HParamsLauncher` provides basic hyper parameter search support. It is active by default. 129 | 130 | In your config, simply add a `hparams` entry. Each key is the name of a hyper parameter. Each value should be an iterable of values to try. The `HParamsLauncher` retrieves these hyper-parameter values, iterates over the combinations (Cartesian product) and launches each config overriding the `hparams` entry with the actual values. 131 | 132 | For example 133 | 134 | ```yaml 135 | fromconfig --hparams.a=1,2 --hparams.b=3,4 136 | ``` 137 | 138 | Generates 139 | 140 | ``` 141 | hparams: {"a": 1, "b": 3} 142 | hparams: {"a": 1, "b": 4} 143 | hparams: {"a": 2, "b": 3} 144 | hparams: {"a": 2, "b": 4} 145 | ``` 146 | 147 | 148 | 149 | ## Parser 150 | 151 | The `ParserLauncher` applies parsing to the config. By default, it uses the `DefaultParser`. You can configure the parser with your custom parser by overriding the `parser` key of the config. 152 | 153 | For example 154 | 155 | ```yaml 156 | parser: 157 | _attr_: "fromconfig.parser.DefaultParser" 158 | ``` 159 | 160 | Will tell the `ParserLauncher` to instantiate the `DefaultParser`. 161 | 162 | 163 | ## Logging 164 | 165 | The `LoggingLauncher` can change the logging level (modifying the `logging.basicConfig` so this will apply to any other `logger` configured to impact the logging's root logger) and log a flattened view of the parameters. 166 | 167 | For example, to change the logging verbosity to `INFO` (20), simply do 168 | 169 | ```yaml 170 | logging: 171 | level: 20 172 | ``` 173 | 174 | 175 | 176 | ## Local 177 | 178 | The previous `Launcher`s were only either generating configs, parsing them, or logging them. To actually instantiate the object using `fromconfig` and manipulate the resulting object via the python Fire syntax, the default behavior is to use the `LocalLauncher`. 179 | 180 | If you wanted to execute the code remotely, you would have to swap the `LocalLauncher` by your custom `Launcher`. 181 | -------------------------------------------------------------------------------- /docs/usage-reference/launcher/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | _attr_: model.Model 3 | learning_rate: "${hparams.learning_rate}" 4 | -------------------------------------------------------------------------------- /docs/usage-reference/launcher/hparams.yaml: -------------------------------------------------------------------------------- 1 | hparams: 2 | learning_rate: [0.1, 0.01] 3 | -------------------------------------------------------------------------------- /docs/usage-reference/launcher/launcher.yaml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 20 3 | -------------------------------------------------------------------------------- /docs/usage-reference/launcher/model.py: -------------------------------------------------------------------------------- 1 | """Dummy Model.""" 2 | 3 | 4 | class Model: 5 | def __init__(self, learning_rate: float): 6 | self.learning_rate = learning_rate 7 | 8 | def train(self): 9 | print(f"Training model with learning_rate {self.learning_rate}") 10 | -------------------------------------------------------------------------------- /docs/usage-reference/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | The library is made of three main components. 4 | 5 | 1. A independent and lightweight __syntax__ to instantiate any Python object from dictionaries with `fromconfig.fromconfig(config)` (using special keys `_attr_` and `_args_`). See [fromconfig](usage-reference/fromconfig/). 6 | 2. A simple abstraction to parse configs before instantiation. This allows configs to remain short and readable with syntactic sugar to define singletons, perform interpolation, etc. See [Parser](usage-reference/parser/). 7 | 3. A composable, flexible, and customizable framework to manipulate configs and launch jobs on remote servers, log values to tracking platforms, etc. See [Launcher](usage-reference/launcher/). 8 | 9 | The [Command Line](usage-reference/command-line) combines these three components for ease of use. 10 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/README.md: -------------------------------------------------------------------------------- 1 | # Parser 2 | 3 | 4 | 5 | ## Default 6 | 7 | `FromConfig` comes with a default parser which sequentially applies 8 | 9 | - [`OmegaConfParser`](#omegaconf): can be practical for interpolation. 10 | - [`EvaluateParser`](#evaluate): syntactic sugar to configure `functool.partial` or simple imports. 11 | - [`SingletonParser`](#singleton): syntactic sugar to define singletons. 12 | 13 | For example, let's see how to create singletons and use interpolation. 14 | 15 | [default.py](default.py ':include :type=code python') 16 | 17 | 18 | 19 | ## OmegaConf 20 | 21 | [OmegaConf](https://omegaconf.readthedocs.io) is a YAML based hierarchical configuration system with support for merging configurations from multiple sources. The `OmegaConfParser` wraps some of its functionality (for example, variable interpolation). 22 | 23 | For example 24 | 25 | [parser_omegaconf.py](parser_omegaconf.py ':include :type=code python') 26 | 27 | This examples uses 28 | 29 | - interpolation, with `${host}` 30 | - custom interpolation with resolvers `now` and `random_hex`. You can register your own resolvers using the `resolvers` key of the config. The `resolvers` should be a mapping from resolver name to a function, import string or any config dictionary defining a function. Using a resolver is as simple as doing `${resolver_name:arg1,arg2,...}`. 31 | 32 | 33 | The following resolvers are available by default 34 | 35 | - `env`: retrieves the value of environment variables. For example `${env:USER}` evaluates to the `$USER` environment variable 36 | - `now`: generates the current date with `datetime.now().strftime(fmt)` where `fmt` can be provided as an argument, and defaults to `%Y-%m-%d-%H-%M-%S`. 37 | 38 | Learn more on the [OmegaConf documentation website](https://omegaconf.readthedocs.io). 39 | 40 | 41 | ## Evaluate 42 | 43 | The `EvaluateParser` makes it possible to simply import a class / function, or configure a constructor via a `functools.partial` call. 44 | 45 | The parser uses a special key `_eval_` with possible values 46 | 47 | __call__ 48 | 49 | Standard behavior, results in `attr(kwargs)`. 50 | 51 | For example, 52 | 53 | [parser_evaluate_call.py](parser_evaluate_call.py ':include :type=code python') 54 | 55 | __import__ 56 | 57 | Simply import the attribute, results in `attr`. 58 | 59 | For example, 60 | 61 | [parser_evaluate_import.py](parser_evaluate_import.py ':include :type=code python') 62 | 63 | __partial__ 64 | 65 | Delays the call, results in a `functools.partial(attr, *args, **kwargs)`. 66 | 67 | For example, 68 | 69 | [parser_evaluate_partial.py](parser_evaluate_partial.py ':include :type=code python') 70 | 71 | It is also possible to delay the call to the function argument if you want them to be evaluated at run time rather 72 | than when the configuration is parsed. 73 | 74 | __lazy__ 75 | 76 | Delays the evaluation when used as an argument of another function with __partial__ evaluation. Can be paired with memoization. 77 | 78 | For example, 79 | 80 | [parser_evaluate_lazy.py](parser_evaluate_lazy.py ':include :type=code python') 81 | 82 | and 83 | 84 | [parser_evaluate_lazy_with_memoization.py](parser_evaluate_lazy_with_memoization.py ':include :type=code python') 85 | 86 | 87 | ## Singleton 88 | 89 | To define singletons (typically an object used in multiple places), use the `SingletonParser`. 90 | 91 | For example, 92 | 93 | [parser_singleton.py](parser_singleton.py ':include :type=code python') 94 | 95 | Without the `_singleton_` entry, two different dictionaries would have been created. 96 | 97 | Note that using interpolation is not a solution to create singletons, as the interpolation mechanism only copies missing parts of the configs. 98 | 99 | The parser uses the special key `_singleton_` whose value is the name associated with the instance to resolve singletons at instantiation time. 100 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/default.py: -------------------------------------------------------------------------------- 1 | """DefaultParser example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | class Model: 7 | def __init__(self, model_dir): 8 | self.model_dir = model_dir 9 | 10 | 11 | class Trainer: 12 | def __init__(self, model): 13 | self.model = model 14 | 15 | 16 | if __name__ == "__main__": 17 | config = { 18 | "model": { 19 | "_attr_": "Model", 20 | "_singleton_": "my_model", # singleton 21 | "model_dir": "${data.root}/${data.model}", # interpolation 22 | }, 23 | "data": {"root": "/path/to/root", "model": "subdir/for/model"}, 24 | "trainer": {"_attr_": "Trainer", "model": "${model}"}, # interpolation 25 | } 26 | 27 | # Parse and instantiate 28 | parser = fromconfig.parser.DefaultParser() 29 | parsed = parser(config) 30 | instance = fromconfig.fromconfig(parsed) 31 | 32 | # Check result 33 | assert id(instance["model"]) == id(instance["trainer"].model) 34 | assert instance["model"].model_dir == "/path/to/root/subdir/for/model" 35 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_evaluate_call.py: -------------------------------------------------------------------------------- 1 | """EvaluateParser call example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | if __name__ == "__main__": 7 | config = {"_attr_": "str", "_eval_": "call", "_args_": ["hello world"]} 8 | parser = fromconfig.parser.EvaluateParser() 9 | parsed = parser(config) 10 | assert fromconfig.fromconfig(parsed) == "hello world" 11 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_evaluate_import.py: -------------------------------------------------------------------------------- 1 | """EvaluateParser import example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | if __name__ == "__main__": 7 | config = {"_attr_": "str", "_eval_": "import"} 8 | parser = fromconfig.parser.EvaluateParser() 9 | parsed = parser(config) 10 | assert fromconfig.fromconfig(parsed) is str 11 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_evaluate_lazy.py: -------------------------------------------------------------------------------- 1 | """EvaluateParser lazy example.""" 2 | 3 | import fromconfig 4 | 5 | ENVIRONMENT = {"ENV_VAR": "UNINITIALIZED"} 6 | 7 | if __name__ == "__main__": 8 | 9 | def initialize_environment(): 10 | ENVIRONMENT["ENV_VAR"] = "VALUE" 11 | 12 | def load_env_var(): 13 | return ENVIRONMENT["ENV_VAR"] 14 | 15 | def run_job(env_var): 16 | assert env_var == "VALUE" 17 | 18 | def pipeline(jobs): 19 | for job in jobs: 20 | job() 21 | 22 | # We want to configure a job that runs the following 23 | # >>> initialize_environment() 24 | # >>> run_job(load_env_var()) 25 | # without changing the function signatures 26 | 27 | config = { 28 | "_attr_": "pipeline", 29 | "_eval_": "partial", 30 | "jobs": [ 31 | {"_attr_": "initialize_environment", "_eval_": "partial"}, 32 | {"_attr_": "run_job", "_eval_": "partial", "env_var": {"_attr_": "load_env_var", "_eval_": "lazy"}}, 33 | ], 34 | } 35 | parser = fromconfig.parser.EvaluateParser() 36 | parsed = parser(config) 37 | pipeline = fromconfig.fromconfig(parsed) 38 | pipeline() 39 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_evaluate_lazy_with_memoization.py: -------------------------------------------------------------------------------- 1 | """EvaluateParser lazy with memoization example.""" 2 | 3 | 4 | import fromconfig 5 | 6 | ENVIRONMENT = {"ENV_VAR": "UNINITIALIZED", "N_HEAVY_COMPUTATIONS": 0} 7 | 8 | if __name__ == "__main__": 9 | 10 | def initialize_environment(): 11 | ENVIRONMENT["ENV_VAR"] = "VALUE" 12 | 13 | def load_env_var_and_perform_heavy_computations(): 14 | ENVIRONMENT["N_HEAVY_COMPUTATIONS"] += 1 15 | return ENVIRONMENT["ENV_VAR"] 16 | 17 | def run_job(env_var): 18 | assert env_var == "VALUE" 19 | 20 | def pipeline(jobs): 21 | for job in jobs: 22 | job() 23 | 24 | # We want to configure a job that runs the following 25 | # >>> initialize_environment() 26 | # >>> run_job(load_env_var_and_perform_heavy_computations()) 27 | # >>> run_job(load_env_var_and_perform_heavy_computations()) 28 | # where load_env_var_and_perform_heavy_computations is only evaluated once. 29 | 30 | config = { 31 | "_attr_": "pipeline", 32 | "_eval_": "partial", 33 | "jobs": [ 34 | {"_attr_": "initialize_environment", "_eval_": "partial"}, 35 | { 36 | "_attr_": "run_job", 37 | "_eval_": "partial", 38 | "env_var": { 39 | "_attr_": "load_env_var_and_perform_heavy_computations", 40 | "_eval_": "lazy", 41 | "_memoization_key_": "env_var", 42 | }, 43 | }, 44 | { 45 | "_attr_": "run_job", 46 | "_eval_": "partial", 47 | "env_var": { 48 | "_attr_": "load_env_var_and_perform_heavy_computations", 49 | "_eval_": "lazy", 50 | "_memoization_key_": "env_var", 51 | }, 52 | }, 53 | ], 54 | } 55 | parser = fromconfig.parser.EvaluateParser() 56 | parsed = parser(config) 57 | pipeline = fromconfig.fromconfig(parsed) 58 | pipeline() 59 | assert ENVIRONMENT["N_HEAVY_COMPUTATIONS"] == 1 60 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_evaluate_partial.py: -------------------------------------------------------------------------------- 1 | """EvaluateParser partial example.""" 2 | 3 | import functools 4 | 5 | import fromconfig 6 | 7 | 8 | if __name__ == "__main__": 9 | config = {"_attr_": "str", "_eval_": "partial", "_args_": ["hello world"]} 10 | parser = fromconfig.parser.EvaluateParser() 11 | parsed = parser(config) 12 | fn = fromconfig.fromconfig(parsed) 13 | assert isinstance(fn, functools.partial) 14 | assert fn() == "hello world" 15 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_omegaconf.py: -------------------------------------------------------------------------------- 1 | """OmegaConfParser example.""" 2 | 3 | import fromconfig 4 | import random 5 | 6 | 7 | def random_hex() -> str: 8 | return hex(hash(random.random())) 9 | 10 | 11 | if __name__ == "__main__": 12 | config = { 13 | "host": "localhost", 14 | "port": "8008", 15 | "url": "${host}:${port}", 16 | "path": "models/${now:}/${random_hex:}", # Use default resolver now + custom resolver 17 | "resolvers": {"random_hex": random_hex}, # Register custom resolver 18 | } 19 | parser = fromconfig.parser.OmegaConfParser() 20 | parsed = parser(config) 21 | print(parsed) 22 | assert parsed["url"] == "localhost:8008" 23 | -------------------------------------------------------------------------------- /docs/usage-reference/parser/parser_singleton.py: -------------------------------------------------------------------------------- 1 | """SingletonParser example.""" 2 | 3 | import fromconfig 4 | 5 | 6 | if __name__ == "__main__": 7 | config = { 8 | "x": {"_attr_": "dict", "_singleton_": "my_dict", "x": 1}, 9 | "y": {"_attr_": "dict", "_singleton_": "my_dict", "x": 1}, 10 | } 11 | parser = fromconfig.parser.SingletonParser() 12 | parsed = parser(config) 13 | instance = fromconfig.fromconfig(parsed) 14 | assert id(instance["x"]) == id(instance["y"]) 15 | -------------------------------------------------------------------------------- /fromconfig/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-import,missing-docstring 2 | 3 | from fromconfig.core import * 4 | from fromconfig import launcher 5 | from fromconfig import parser 6 | from fromconfig import utils 7 | from fromconfig.version import * 8 | -------------------------------------------------------------------------------- /fromconfig/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/fromconfig/cli/__init__.py -------------------------------------------------------------------------------- /fromconfig/cli/main.py: -------------------------------------------------------------------------------- 1 | """Main entry point.""" 2 | 3 | import functools 4 | import sys 5 | import logging 6 | from typing import Iterable, Mapping 7 | 8 | import fire 9 | 10 | import fromconfig 11 | 12 | 13 | LOGGER = logging.getLogger(__name__) 14 | 15 | 16 | def launch(paths: Iterable[str], overrides: Mapping, command: str): 17 | """Load configs, merge, get launcher from plugins and launch. 18 | 19 | Parameters 20 | ---------- 21 | paths : Iterable[str] 22 | Paths to config files. 23 | overrides : Mapping 24 | Optional key value parameters that overrides config files 25 | command : str 26 | Rest of the python Fire command 27 | """ 28 | configs = [fromconfig.load(path) for path in paths] + [fromconfig.utils.expand(overrides.items())] 29 | config = functools.reduce(fromconfig.utils.merge_dict, configs) 30 | launcher = fromconfig.launcher.DefaultLauncher.fromconfig(config.pop("launcher", {})) 31 | launcher(config=config, command=command) 32 | 33 | 34 | def parse_args(): 35 | """Parse arguments from command line using Fire.""" 36 | _paths, _overrides = [], {} # pylint: disable=invalid-name 37 | 38 | def _parse_args(*paths, **overrides): 39 | # Display Fire Help 40 | if not paths and not overrides: 41 | return _parse_args 42 | 43 | # Extract paths and overrides from arguments 44 | _paths.extend(paths) 45 | _overrides.update(overrides) 46 | 47 | # Do nothing with remaining arguments 48 | def _no_op(*_args, **_kwargs): 49 | return None if not (_args or _kwargs) else _no_op 50 | 51 | return _no_op 52 | 53 | argv = sys.argv[1:] 54 | fire.Fire(_parse_args, argv) 55 | idx_of_fire_separator = len(argv) if "-" not in argv else argv.index("-") 56 | command = " ".join(argv[idx_of_fire_separator + 1 :]) 57 | return _paths, _overrides, command 58 | 59 | 60 | def main(): 61 | """Main entry point.""" 62 | sys.path.append(".") # For local imports 63 | paths, overrides, command = parse_args() 64 | if paths or overrides: 65 | launch(paths, overrides, command) 66 | -------------------------------------------------------------------------------- /fromconfig/core/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-import,missing-docstring 2 | 3 | from fromconfig.core.base import Keys, FromConfig, fromconfig 4 | from fromconfig.core.config import Config, load, dump 5 | -------------------------------------------------------------------------------- /fromconfig/core/base.py: -------------------------------------------------------------------------------- 1 | """Base functionality.""" 2 | 3 | from abc import ABC 4 | from typing import Any 5 | import inspect 6 | import logging 7 | 8 | from fromconfig.utils import StrEnum, is_pure_iterable, is_mapping, from_import_string 9 | 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | class Keys(StrEnum): 15 | """Special Keys used by fromconfig. 16 | 17 | Attributes 18 | ---------- 19 | ARGS : str 20 | Name of the special key for the full import string. 21 | ATTR : str 22 | Name of the special key for positional arguments. 23 | """ 24 | 25 | ATTR = "_attr_" 26 | ARGS = "_args_" 27 | 28 | 29 | class FromConfig(ABC): 30 | """Abstract class for custom from_config implementations. 31 | 32 | Example 33 | ------- 34 | >>> import fromconfig 35 | >>> class MyClass(fromconfig.FromConfig): 36 | ... def __init__(self, x): 37 | ... self.x = x 38 | ... @classmethod 39 | ... def fromconfig(cls, config): 40 | ... if "x" not in config: 41 | ... return cls(0) 42 | ... else: 43 | ... return cls(**config) 44 | >>> config = {} 45 | >>> got = MyClass.fromconfig(config) 46 | >>> isinstance(got, MyClass) 47 | True 48 | >>> got.x 49 | 0 50 | """ 51 | 52 | @classmethod 53 | def fromconfig(cls, config: Any): 54 | """Subclasses must override. 55 | 56 | Parameters 57 | ---------- 58 | config : Any 59 | Config dictionary, non-instantiated. 60 | """ 61 | if not is_mapping(config): 62 | raise TypeError(f"Expected Mapping but got {type(config)}") 63 | args = fromconfig(config.get(Keys.ARGS, [])) 64 | kwargs = {key: fromconfig(value) for key, value in config.items() if key not in Keys} 65 | return cls(*args, **kwargs) # type: ignore 66 | 67 | 68 | def fromconfig(config: Any): 69 | """From config implementation. 70 | 71 | Example 72 | ------- 73 | Use the '_attr_' key to configure the class, function, variable or 74 | method to configure. It is generally the full import string or the 75 | name of the class for builtins. 76 | 77 | >>> import fromconfig 78 | >>> config = {"_attr_": "str", "_args_": [1]} 79 | >>> fromconfig.fromconfig(config) 80 | '1' 81 | 82 | A more complicated example is 83 | >>> import fromconfig 84 | >>> class Point: 85 | ... def __init__(self, x, y): 86 | ... self.x = x 87 | ... self.y = y 88 | >>> config = { 89 | ... "_attr_": "Point", 90 | ... "x": 0, 91 | ... "y": 0 92 | ... } 93 | >>> point = fromconfig.fromconfig(config) 94 | >>> isinstance(point, Point) and point.x == 0 and point.y == 0 95 | True 96 | 97 | Parameters 98 | ---------- 99 | config : Any 100 | Typically a dictionary 101 | """ 102 | if is_mapping(config): 103 | # Resolve attribute, check if subclass of FromConfig 104 | attr = from_import_string(config[Keys.ATTR]) if Keys.ATTR in config else None 105 | if inspect.isclass(attr) and issubclass(attr, FromConfig): 106 | return attr.fromconfig({key: value for key, value in config.items() if key != Keys.ATTR}) 107 | 108 | # Resolve and instantiate args and kwargs 109 | args = fromconfig(config.get(Keys.ARGS, [])) 110 | kwargs = {key: fromconfig(value) for key, value in config.items() if key not in Keys} 111 | 112 | # No attribute resolved, return args and kwargs 113 | if attr is None: 114 | return type(config)({Keys.ARGS: args, **kwargs}) if args else type(config)(kwargs) 115 | 116 | # If attribute resolved, call attribute with args and kwargs 117 | return attr(*args, **kwargs) 118 | 119 | if is_pure_iterable(config): 120 | return type(config)(fromconfig(item) for item in config) 121 | 122 | return config 123 | -------------------------------------------------------------------------------- /fromconfig/core/config.py: -------------------------------------------------------------------------------- 1 | """Config serialization utilities.""" 2 | 3 | from pathlib import Path 4 | from typing import Union, Any, IO, Dict 5 | import json 6 | import logging 7 | import os 8 | from operator import itemgetter 9 | import re 10 | import io 11 | 12 | import yaml 13 | 14 | from fromconfig.core import base 15 | from fromconfig.utils import try_import, merge_dict, is_pure_iterable, is_mapping 16 | 17 | 18 | _jsonnet = try_import("_jsonnet") 19 | 20 | 21 | LOGGER = logging.getLogger(__name__) 22 | 23 | 24 | class Config(base.FromConfig, dict): 25 | """Keep a dictionary as dict during a fromconfig call. 26 | 27 | Example 28 | ------- 29 | >>> import fromconfig 30 | >>> config = { 31 | ... "_attr_": "fromconfig.Config", 32 | ... "_config_": { 33 | ... "_attr_": "list" 34 | ... } 35 | ... } 36 | >>> parsed = fromconfig.fromconfig(config) 37 | >>> parsed 38 | {'_attr_': 'list'} 39 | """ 40 | 41 | @classmethod 42 | def fromconfig(cls, config: Any): 43 | if is_mapping(config): 44 | return cls(config.get("_config_", config)) 45 | return cls(config) 46 | 47 | 48 | _YAML_MERGE = "<<:" 49 | 50 | _YAML_INCLUDE = "!include" 51 | 52 | 53 | class IncludeLoader(yaml.SafeLoader): 54 | """YAML Loader with `!include` constructor to load files.""" 55 | 56 | def __init__(self, stream: IO) -> None: 57 | """Initialize Loader.""" 58 | try: 59 | self.root = os.path.split(stream.name)[0] 60 | except AttributeError: 61 | self.root = os.path.curdir 62 | super().__init__(stream) 63 | 64 | 65 | def include(loader, node: yaml.Node) -> Any: 66 | """Include file referenced at node.""" 67 | path = os.path.join(loader.root, loader.construct_scalar(node)) 68 | return load(path) 69 | 70 | 71 | IncludeLoader.add_constructor(_YAML_INCLUDE, include) 72 | 73 | 74 | def yaml_load(stream, Loader): # pylint: disable=invalid-name 75 | """Custom yaml load to handle !include and merges.""" 76 | 77 | def _expand_includes(s: IO) -> IO: 78 | """Expand includes before merging. 79 | 80 | The IncludeLoader does not work with the YAML merge key. 81 | 82 | The trick here is to force the include to be loaded into a new 83 | key, generated from the line number ('{idx}!include<<') 84 | 85 | Parameters 86 | ---------- 87 | s : io.BaseStream 88 | A data stream (typically an opened file) 89 | 90 | Returns 91 | ------- 92 | str 93 | """ 94 | content = "" 95 | for idx, line in enumerate(s.readlines()): 96 | if re.match(f"{_YAML_MERGE} *{_YAML_INCLUDE} *.*", line): 97 | line = re.sub(f"{_YAML_MERGE} ", line, f"{idx}{_YAML_INCLUDE}{_YAML_MERGE} ") 98 | content += line 99 | result = io.StringIO(content) 100 | setattr(result, "name", stream.name) # Forward name for Loader 101 | return result 102 | 103 | def _merge_includes(item: Any) -> Any: 104 | """Merge includes. 105 | 106 | After expand and YAML parsing, some of the keys are idx<<: which 107 | were originally intended to be merged to the top level. 108 | 109 | Parameters 110 | ---------- 111 | item : Any 112 | Any node of the parsed YAML stream. 113 | 114 | Returns 115 | ------- 116 | Any 117 | """ 118 | if is_mapping(item): 119 | result = {} # type: Dict[Any, Any] 120 | for key, value in sorted(item.items(), key=itemgetter(0)): 121 | if key.endswith(f"{_YAML_INCLUDE}<<"): 122 | if not is_mapping(value): 123 | raise TypeError(f"Expected Mapping-like object but got {value} ({type(value)}") 124 | for subkey, subvalue in value.items(): 125 | # Since YAML provides no guarantee of ordering 126 | # a merge with overrides would be ill-defined 127 | result = merge_dict(result, {subkey: _merge_includes(subvalue)}, allow_override=False) 128 | else: 129 | result[key] = _merge_includes(value) 130 | return result 131 | if is_pure_iterable(item): 132 | return [_merge_includes(it) for it in item] 133 | return item 134 | 135 | return _merge_includes(yaml.load(_expand_includes(stream), Loader)) 136 | 137 | 138 | def load(path: Union[str, Path]): 139 | """Load dictionary from path. 140 | 141 | Parameters 142 | ---------- 143 | path : Union[str, Path] 144 | Path to file (yaml, yml, json or jsonnet format) 145 | """ 146 | suffix = Path(path).suffix 147 | if suffix in (".yaml", ".yml"): 148 | with Path(path).open() as file: 149 | try: 150 | return yaml_load(file, IncludeLoader) 151 | except Exception as e: # pylint: disable=broad-except 152 | LOGGER.error(f"Unable to use custom yaml_load ({e}), using yaml.safe_load instead.") 153 | return yaml.safe_load(file) 154 | if suffix == ".json": 155 | with Path(path).open() as file: 156 | return json.load(file) 157 | if suffix == ".jsonnet": 158 | if _jsonnet is None: 159 | msg = f"jsonnet is not installed but the resolved path extension is {suffix}. " 160 | msg += "Visit https://jsonnet.org for installation instructions." 161 | raise ImportError(msg) 162 | return json.loads(_jsonnet.evaluate_file(str(path))) 163 | raise ValueError(f"Unable to resolve method for path {path}") 164 | 165 | 166 | def dump(config, path: Union[str, Path]): 167 | """Dump dictionary content to file in path. 168 | 169 | Parameters 170 | ---------- 171 | path : Union[str, Path] 172 | Path to file (yaml, yml, json or jsonnet format) 173 | """ 174 | suffix = Path(path).suffix 175 | if suffix in (".yaml", ".yml"): 176 | with Path(path).open("w") as file: 177 | yaml.dump(config, file) 178 | elif suffix in (".json", ".jsonnet"): 179 | with Path(path).open("w") as file: 180 | json.dump(config, file, indent=4) 181 | else: 182 | raise ValueError(f"Suffix {suffix} not recognized for path {path}") 183 | -------------------------------------------------------------------------------- /fromconfig/launcher/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-import,missing-docstring 2 | 3 | from fromconfig.launcher.base import Launcher 4 | from fromconfig.launcher.default import DefaultLauncher 5 | from fromconfig.launcher.dry import DryLauncher 6 | from fromconfig.launcher.local import LocalLauncher 7 | from fromconfig.launcher.logger import LoggingLauncher 8 | from fromconfig.launcher.hparams import HParamsLauncher 9 | from fromconfig.launcher.parser import ParserLauncher 10 | -------------------------------------------------------------------------------- /fromconfig/launcher/base.py: -------------------------------------------------------------------------------- 1 | """Base class for launchers.""" 2 | 3 | from abc import ABC 4 | from typing import Any, Dict, Type 5 | import inspect 6 | import logging 7 | import pkg_resources 8 | 9 | from fromconfig.core.base import fromconfig, FromConfig, Keys 10 | from fromconfig.utils.libimport import from_import_string 11 | from fromconfig.utils.nest import merge_dict 12 | from fromconfig.utils.types import is_pure_iterable, is_mapping 13 | from fromconfig.version import __major__ 14 | 15 | 16 | LOGGER = logging.getLogger(__name__) 17 | 18 | # Internal and external Launcher classes referenced by name 19 | _CLASSES: Dict[str, Type] = {} # Loaded during first _classes() call 20 | 21 | 22 | class Launcher(FromConfig, ABC): 23 | """Base class for launchers.""" 24 | 25 | def __init__(self, launcher: "Launcher"): 26 | self.launcher = launcher 27 | 28 | def __call__(self, config: Any, command: str = ""): 29 | """Launch implementation. 30 | 31 | Parameters 32 | ---------- 33 | config : Any 34 | The config 35 | command : str, optional 36 | The fire command 37 | """ 38 | raise NotImplementedError() 39 | 40 | def __repr__(self) -> str: 41 | return f"{self.__class__.__name__}({self.launcher})" 42 | 43 | @classmethod 44 | def fromconfig(cls, config: Any) -> "Launcher": 45 | """Custom fromconfig implementation. 46 | 47 | The config parameter can either be a plain config dictionary 48 | with special keys (like _attr_, etc.). In that case, it simply 49 | uses fromconfig to instantiate the corresponding class. 50 | 51 | It can also list launchers by name (set internally or plugin 52 | names for external launchers). In that case, it will compose 53 | the launchers (first item will be higher in the instance 54 | hierarchy). 55 | 56 | Parameters 57 | ---------- 58 | config : Mapping 59 | Typically a dictionary with keys run, log, parse and sweep. 60 | 61 | Returns 62 | ------- 63 | Launcher 64 | """ 65 | 66 | def _fromconfig(cfg, launcher: Launcher = None): 67 | # None case 68 | if cfg is None: 69 | return launcher 70 | 71 | # Already a Launcher 72 | if isinstance(cfg, Launcher): 73 | if launcher is not None: 74 | raise ValueError(f"Cannot wrap launchers, launcher is not None ({launcher}) but cfg={cfg}") 75 | return cfg 76 | 77 | # Launcher class name with no other parameters 78 | if isinstance(cfg, str): 79 | launcher_cls = _classes()[cfg] 80 | return launcher_cls(launcher=launcher) if launcher else launcher_cls() # type: ignore 81 | 82 | # List of launchers to compose (first item is highest) 83 | if is_pure_iterable(cfg): 84 | for item in reversed(cfg): 85 | launcher = _fromconfig(item, launcher) 86 | return launcher 87 | 88 | if is_mapping(cfg): 89 | # Resolve the class from ATTR key, default to parent 90 | if Keys.ATTR in cfg: 91 | if cfg[Keys.ATTR] in _classes(): 92 | launcher_cls = _classes()[cfg[Keys.ATTR]] 93 | else: 94 | launcher_cls = from_import_string(cfg[Keys.ATTR]) 95 | else: 96 | launcher_cls = cls 97 | 98 | # Special treatment for "launcher" key if present 99 | if "launcher" in cfg: 100 | if launcher is not None: 101 | raise ValueError(f"Cannot wrap launchers, launcher is not None ({launcher}) but cfg={cfg}") 102 | launcher = _fromconfig(cfg["launcher"]) 103 | cfg = cfg if not launcher else merge_dict(cfg, {"launcher": launcher}) 104 | 105 | # Instantiate positional and keyword arguments 106 | args = fromconfig(cfg.get(Keys.ARGS, [])) 107 | kwargs = {key: fromconfig(value) for key, value in cfg.items() if key not in Keys} 108 | return launcher_cls(*args, **kwargs) # type: ignore 109 | 110 | raise TypeError(f"Unable to instantiate launcher from {cfg} (unsupported type {type(cfg)})") 111 | 112 | return _fromconfig(config) 113 | 114 | 115 | def _classes() -> Dict[str, Type]: 116 | """Load and return internal and external classes.""" 117 | if not _CLASSES: 118 | _load() 119 | return _CLASSES 120 | 121 | 122 | def _load(): 123 | """Load internal and external classes into _CLASSES.""" 124 | # pylint: disable=import-outside-toplevel,cyclic-import 125 | # Import internal classes 126 | from fromconfig.launcher.hparams import HParamsLauncher 127 | from fromconfig.launcher.parser import ParserLauncher 128 | from fromconfig.launcher.logger import LoggingLauncher 129 | from fromconfig.launcher.local import LocalLauncher 130 | from fromconfig.launcher.dry import DryLauncher 131 | 132 | # Create references with default names 133 | _CLASSES["local"] = LocalLauncher 134 | _CLASSES["logging"] = LoggingLauncher 135 | _CLASSES["hparams"] = HParamsLauncher 136 | _CLASSES["parser"] = ParserLauncher 137 | _CLASSES["dry"] = DryLauncher 138 | 139 | # Load external classes, use entry point's name for reference 140 | for entry_point in pkg_resources.iter_entry_points(f"fromconfig{__major__}"): 141 | module = entry_point.load() 142 | for _, cls in inspect.getmembers(module, lambda m: inspect.isclass(m) and issubclass(m, Launcher)): 143 | name = f"{entry_point.name}.{cls.NAME}" if hasattr(cls, "NAME") else entry_point.name 144 | if name in _CLASSES: 145 | raise ValueError(f"Duplicate launcher name found {name} ({_CLASSES})") 146 | _CLASSES[name] = cls 147 | 148 | # Log loaded classes 149 | LOGGER.info(f"Loaded Launcher classes {_CLASSES}") 150 | -------------------------------------------------------------------------------- /fromconfig/launcher/default.py: -------------------------------------------------------------------------------- 1 | """Default Launcher.""" 2 | 3 | from collections import OrderedDict 4 | from typing import Any 5 | 6 | from fromconfig.launcher import base 7 | from fromconfig.launcher.hparams import HParamsLauncher 8 | from fromconfig.launcher.local import LocalLauncher 9 | from fromconfig.launcher.logger import LoggingLauncher 10 | from fromconfig.launcher.parser import ParserLauncher 11 | from fromconfig.utils.types import is_mapping 12 | 13 | 14 | # Special keys for a Launcher config split by steps with defaults 15 | _STEPS = OrderedDict( 16 | [ 17 | ("sweep", "hparams"), # Hyper Parameter Sweep 18 | ("log", "logging"), # Configure Logging 19 | ("parse", "parser"), # Parse config 20 | ("run", "local"), # Actually run the config 21 | ] 22 | ) 23 | 24 | 25 | class DefaultLauncher(base.Launcher): 26 | """Default Launcher. 27 | 28 | Attributes 29 | ---------- 30 | launcher : Launcher 31 | The wrapped launcher. 32 | """ 33 | 34 | def __init__(self, launcher: base.Launcher = None): 35 | super().__init__(launcher or HParamsLauncher(LoggingLauncher(ParserLauncher(LocalLauncher())))) 36 | 37 | def __call__(self, config: Any, command: str = ""): 38 | self.launcher(config=config, command=command) 39 | 40 | @classmethod 41 | def fromconfig(cls, config): 42 | """Custom fromconfig implementation. 43 | 44 | The config can be a dictionary with special keys run, log, parse 45 | and sweep. Each of these values can either be a plain dictionary 46 | or a Launcher class name. In that case, it is equivalent to a 47 | list of launchers defined in the sweep -> parse -> log -> run 48 | order. When overriding only a subset of these keys, the defaults 49 | will be used for the other steps. 50 | 51 | Example 52 | ------- 53 | In this example, we instantiate a Launcher that uses the default 54 | class for each of the launching steps. 55 | 56 | >>> import fromconfig 57 | >>> config = { 58 | ... "run": "local", 59 | ... "log": "logging", 60 | ... "parse": "parser", 61 | ... "sweep": "hparams" 62 | ... } 63 | >>> launcher = fromconfig.launcher.DefaultLauncher.fromconfig(config) 64 | 65 | By specifying twice "logging" for the "log" step the resulting 66 | launcher will wrap the LoggingLauncher twice (resulting in a 67 | double logging). 68 | 69 | >>> import fromconfig 70 | >>> config = { 71 | ... "log": ["logging", "logging"] 72 | ... } 73 | >>> launcher = fromconfig.launcher.DefaultLauncher.fromconfig(config) 74 | """ 75 | if is_mapping(config) and any(key in config for key in _STEPS): 76 | if not all(key in _STEPS for key in config): 77 | raise ValueError(f"Either all keys or none should be in {_STEPS} but got {config}") 78 | config = [config.get(key, default) for key, default in _STEPS.items()] 79 | 80 | return super().fromconfig(config) 81 | -------------------------------------------------------------------------------- /fromconfig/launcher/dry.py: -------------------------------------------------------------------------------- 1 | """Dry Launcher.""" 2 | 3 | from typing import Any 4 | import logging 5 | from pprint import pprint 6 | 7 | from fromconfig.launcher import base 8 | 9 | 10 | LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | class DryLauncher(base.Launcher): 14 | """Dry Launcher.""" 15 | 16 | def __init__(self, launcher: base.Launcher = None): 17 | super().__init__(launcher=launcher) # type: ignore 18 | 19 | def __call__(self, config: Any, command: str = ""): 20 | pprint(config) 21 | print(command) 22 | -------------------------------------------------------------------------------- /fromconfig/launcher/hparams.py: -------------------------------------------------------------------------------- 1 | """Hyper Params SweepLauncher.""" 2 | 3 | from typing import Any 4 | import itertools 5 | import logging 6 | import shutil 7 | 8 | from fromconfig.core.base import fromconfig 9 | from fromconfig.launcher import base 10 | from fromconfig.utils.nest import merge_dict 11 | from fromconfig.utils.types import is_mapping 12 | 13 | 14 | LOGGER = logging.getLogger(__name__) 15 | 16 | 17 | class HParamsLauncher(base.Launcher): 18 | """Hyper Params Launcher. 19 | 20 | Given a config, it extracts hyper parameters ranges by instantiating 21 | the `hparams` entry of the config. 22 | 23 | It then generates sets of parameters and merge their values into 24 | the `hparams` entry of the config. 25 | 26 | Attributes 27 | ---------- 28 | launcher : Launcher 29 | Launcher to launch each sub-job 30 | """ 31 | 32 | def __call__(self, config: Any, command: str = ""): 33 | """Generate configs via hyper-parameters.""" 34 | if not is_mapping(config): 35 | self.launcher(config=config, command=command) 36 | else: 37 | hparams = fromconfig(config.get("hparams") or {}) 38 | if not hparams: 39 | self.launcher(config=config, command=command) 40 | else: 41 | names = hparams.keys() 42 | for values in itertools.product(*[hparams[name] for name in names]): 43 | overrides = dict(zip(names, values)) 44 | print(header(overrides)) 45 | self.launcher(config=merge_dict(config, {"hparams": overrides}), command=command) 46 | 47 | 48 | def header(overrides) -> str: 49 | """Create header for experiment.""" 50 | # Get terminal number of columns 51 | try: 52 | columns, _ = shutil.get_terminal_size((80, 20)) 53 | except Exception: # pylint: disable=broad-except 54 | columns = 80 55 | 56 | # Join key-values and truncate if needed 57 | content = ", ".join(f"{key}={value}" for key, value in overrides.items()) 58 | if len(content) >= columns - 2: 59 | content = content[: columns - 2 - 3] + "." * 3 60 | content = f"[{content}]" 61 | 62 | # Add padding 63 | padding = "=" * max((columns - len(content)) // 2, 0) 64 | return f"{padding}{content}{padding}" 65 | -------------------------------------------------------------------------------- /fromconfig/launcher/local.py: -------------------------------------------------------------------------------- 1 | """Local Launcher.""" 2 | 3 | from typing import Any 4 | import fire 5 | import logging 6 | 7 | from fromconfig.core.base import fromconfig 8 | from fromconfig.launcher import base 9 | 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | class LocalLauncher(base.Launcher): 15 | """Local Launcher.""" 16 | 17 | def __init__(self, launcher: base.Launcher = None): 18 | if launcher is not None: 19 | raise ValueError(f"LocalLauncher cannot wrap another launcher but got {launcher}") 20 | super().__init__(launcher=launcher) # type: ignore 21 | 22 | def __call__(self, config: Any, command: str = ""): 23 | fire.Fire(fromconfig(config), command) 24 | -------------------------------------------------------------------------------- /fromconfig/launcher/logger.py: -------------------------------------------------------------------------------- 1 | """Logging Launcher.""" 2 | 3 | from typing import Any 4 | import logging 5 | 6 | from fromconfig.launcher import base 7 | from fromconfig.utils.types import is_mapping 8 | 9 | 10 | LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | class LoggingLauncher(base.Launcher): 14 | """Logging Launcher.""" 15 | 16 | def __init__(self, launcher: base.Launcher): 17 | super().__init__(launcher=launcher) 18 | 19 | def __call__(self, config: Any, command: str = ""): 20 | """Log parsed config params using logging module.""" 21 | # Extract params 22 | params = (config.get("logging") or {}) if is_mapping(config) else {} # type: ignore 23 | level = params.get("level", None) 24 | 25 | # Change verbosity level (applies to all loggers) 26 | if level is not None: 27 | logging.basicConfig(level=level) 28 | 29 | # Execute sub-launcher with no parser (already parsed) 30 | self.launcher(config=config, command=command) # type: ignore 31 | -------------------------------------------------------------------------------- /fromconfig/launcher/parser.py: -------------------------------------------------------------------------------- 1 | """Parse the config.""" 2 | 3 | from typing import Any 4 | import logging 5 | 6 | from fromconfig.core.base import fromconfig 7 | from fromconfig.launcher import base 8 | from fromconfig.parser.default import DefaultParser 9 | from fromconfig.parser.singleton import singleton 10 | from fromconfig.utils.types import is_mapping 11 | 12 | 13 | LOGGER = logging.getLogger(__name__) 14 | 15 | 16 | class ParserLauncher(base.Launcher): 17 | """Parse the config.""" 18 | 19 | def __call__(self, config: Any, command: str = ""): 20 | # Resolve parser 21 | if is_mapping(config): 22 | parser = fromconfig(config.pop("parser")) if "parser" in config else DefaultParser() 23 | else: 24 | parser = DefaultParser() 25 | LOGGER.debug(f"Resolved parser {parser}") 26 | 27 | # Launch 28 | self.launcher(config=parser(config) if callable(parser) else config, command=command) # type: ignore 29 | 30 | # Clear singleton to avoid leaks 31 | singleton.clear() 32 | -------------------------------------------------------------------------------- /fromconfig/parser/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-import,missing-docstring 2 | 3 | from fromconfig.parser.base import Parser, ChainParser 4 | from fromconfig.parser.evaluate import EvaluateParser 5 | from fromconfig.parser.default import DefaultParser 6 | from fromconfig.parser.omega import OmegaConfParser 7 | from fromconfig.parser.singleton import SingletonParser, singleton 8 | -------------------------------------------------------------------------------- /fromconfig/parser/base.py: -------------------------------------------------------------------------------- 1 | """Base functionality for parsers.""" 2 | 3 | from abc import ABC, abstractmethod 4 | from typing import Any 5 | 6 | 7 | class Parser(ABC): 8 | """Base parser class.""" 9 | 10 | @abstractmethod 11 | def __call__(self, config: Any): 12 | raise NotImplementedError() 13 | 14 | 15 | class ChainParser(Parser): 16 | """A parser that applies parsers sequentially.""" 17 | 18 | def __init__(self, *parsers: Parser): 19 | self.parsers = parsers 20 | 21 | def __call__(self, config: Any): 22 | parsed = config 23 | for parser in self: 24 | parsed = parser(parsed) 25 | return parsed 26 | 27 | def __iter__(self): 28 | return iter(self.parsers) 29 | 30 | def __repr__(self): 31 | return f"{self.__class__.__name__}({', '.join(map(repr, self))})" 32 | -------------------------------------------------------------------------------- /fromconfig/parser/default.py: -------------------------------------------------------------------------------- 1 | """Default Parser.""" 2 | 3 | from fromconfig.parser import base 4 | from fromconfig.parser.omega import OmegaConfParser 5 | from fromconfig.parser.evaluate import EvaluateParser 6 | from fromconfig.parser.singleton import SingletonParser 7 | 8 | 9 | class DefaultParser(base.ChainParser): 10 | """Create Default Parser. 11 | 12 | Example 13 | ------- 14 | >>> import fromconfig 15 | >>> class Model: 16 | ... def __init__(self, model_dir): 17 | ... self.model_dir = model_dir 18 | >>> class Trainer: 19 | ... def __init__(self, model): 20 | ... self.model = model 21 | >>> config = { 22 | ... "model": { 23 | ... "_attr_": "Model", 24 | ... "_singleton_": "my_model", 25 | ... "model_dir": "${data.root}/${data.model}" 26 | ... }, 27 | ... "data": { 28 | ... "root": "/path/to/root", 29 | ... "model": "subdir/for/model" 30 | ... }, 31 | ... "trainer": { 32 | ... "_attr_": "Trainer", 33 | ... "model": "${model}", 34 | ... } 35 | ... } 36 | >>> parser = fromconfig.parser.DefaultParser() 37 | >>> parsed = parser(config) 38 | >>> instance = fromconfig.fromconfig(parsed) 39 | >>> id(instance["model"]) == id(instance["trainer"].model) 40 | True 41 | >>> instance["model"].model_dir == "/path/to/root/subdir/for/model" 42 | True 43 | """ 44 | 45 | def __init__(self): 46 | super().__init__(OmegaConfParser(), EvaluateParser(), SingletonParser()) 47 | 48 | def __repr__(self): 49 | return self.__class__.__name__ 50 | -------------------------------------------------------------------------------- /fromconfig/parser/omega.py: -------------------------------------------------------------------------------- 1 | """Simple OmegaConf parser.""" 2 | 3 | from typing import Any 4 | from datetime import datetime 5 | 6 | from omegaconf import OmegaConf 7 | 8 | from fromconfig.core.base import fromconfig 9 | from fromconfig.parser import base 10 | from fromconfig.utils.libimport import from_import_string 11 | from fromconfig.utils.types import is_mapping, is_pure_iterable 12 | from fromconfig.utils.nest import merge_dict 13 | 14 | 15 | def now(fmt: str = "%Y-%m-%d-%H-%M-%S") -> str: 16 | return datetime.now().strftime(fmt) 17 | 18 | 19 | _RESOLVERS = {"now": now} # Default Resolvers 20 | 21 | 22 | class OmegaConfParser(base.Parser): 23 | """Simple OmegaConf parser. 24 | 25 | Example 26 | ------- 27 | >>> import fromconfig 28 | >>> config = { 29 | ... "host": "localhost", 30 | ... "port": "8008", 31 | ... "url": "${host}:${port}" 32 | ... } 33 | >>> parser = fromconfig.parser.OmegaConfParser() 34 | >>> parsed = parser(config) 35 | >>> parsed["url"] 36 | 'localhost:8008' 37 | 38 | You can configure custom resolvers for custom variable 39 | interpolation. For example 40 | 41 | >>> import fromconfig 42 | >>> def hello(s): 43 | ... return f"hello {s}" 44 | >>> config = { 45 | ... "hello_world": "${hello:world}", 46 | ... "date": "${now:}", 47 | ... "resolvers": {"hello": "hello"} 48 | ... } 49 | >>> parser = fromconfig.parser.OmegaConfParser() 50 | >>> parsed = parser(config) 51 | >>> assert parsed["hello_world"] == "hello world" # Custom resolver 52 | >>> assert "$" not in parsed["date"] # Make sure now was resolved 53 | """ 54 | 55 | def __call__(self, config: Any) -> Any: 56 | # Extract resolvers to register 57 | if is_mapping(config): 58 | resolvers = merge_dict(_RESOLVERS, config.get("resolvers") or {}) 59 | config = {key: value for key, value in config.items() if key != "resolvers"} 60 | else: 61 | resolvers = _RESOLVERS 62 | 63 | # Register resolvers 64 | for name, resolver in resolvers.items(): 65 | if isinstance(resolver, str): 66 | resolver = from_import_string(resolver) 67 | elif is_mapping(resolver): 68 | resolver = fromconfig(resolver) 69 | elif callable(resolver): 70 | pass 71 | else: 72 | raise TypeError(f"Unable to resolve {resolver}") 73 | OmegaConf.register_new_resolver(name, resolver) 74 | 75 | # Create config and parse 76 | if is_mapping(config) or is_pure_iterable(config): 77 | conf = OmegaConf.create(config) # type: ignore 78 | parsed = OmegaConf.to_container(conf, resolve=True) # type: ignore 79 | else: 80 | # Try to parse by wrapping in a list 81 | conf = OmegaConf.create([config]) # type: ignore 82 | parsed = OmegaConf.to_container(conf, resolve=True)[0] # type: ignore 83 | 84 | # Clear resolvers (avoid leaking module-level changes), return 85 | OmegaConf.clear_resolvers() 86 | return parsed 87 | -------------------------------------------------------------------------------- /fromconfig/parser/singleton.py: -------------------------------------------------------------------------------- 1 | """Singleton parser.""" 2 | 3 | from collections import UserDict 4 | from functools import partial 5 | from typing import Any, Callable 6 | 7 | from fromconfig.core import Keys 8 | from fromconfig.parser import base 9 | from fromconfig.utils import depth_map, is_mapping, from_import_string, to_import_string 10 | 11 | 12 | class _Singletons(UserDict): 13 | """Singleton register. 14 | 15 | Example 16 | ------- 17 | >>> from fromconfig.parser import singleton 18 | >>> def constructor(): 19 | ... return {"foo": "bar"} 20 | >>> d1 = singleton("d1", constructor) 21 | >>> d2 = singleton("d1", constructor) # constructor optional 22 | >>> id(d1) == id(d2) 23 | True 24 | """ 25 | 26 | def __call__(self, key, constructor: Callable[[], Any] = None): 27 | """Get or create singleton.""" 28 | if key not in self: 29 | if constructor is None: 30 | raise ValueError(f"Singleton {key} not found in {self}. Please specify constructor.") 31 | self[key] = constructor() 32 | return self[key] 33 | 34 | def __setitem__(self, key, value): 35 | if key in self: 36 | raise KeyError(f"Key {key} already in {self}. You must delete it first.") 37 | super().__setitem__(key, value) 38 | 39 | 40 | singleton = _Singletons() 41 | 42 | 43 | class SingletonParser(base.Parser): 44 | """Singleton parser. 45 | 46 | Examples 47 | -------- 48 | >>> import fromconfig 49 | >>> config = { 50 | ... "x": { 51 | ... "_attr_": "dict", 52 | ... "_singleton_": "my_dict", 53 | ... "x": 1 54 | ... }, 55 | ... "y": { 56 | ... "_attr_": "dict", 57 | ... "_singleton_": "my_dict", 58 | ... "x": 1 59 | ... } 60 | ... } 61 | >>> parser = fromconfig.parser.SingletonParser() 62 | >>> parsed = parser(config) 63 | >>> instance = fromconfig.fromconfig(parsed) 64 | >>> id(instance["x"]) == id(instance["y"]) 65 | True 66 | """ 67 | 68 | KEY = "_singleton_" 69 | 70 | def __call__(self, config: Any): 71 | """Parses config with _singleton_ key into valid config.""" 72 | 73 | def _map_fn(item): 74 | if is_mapping(item) and self.KEY in item: 75 | key = item[self.KEY] 76 | name = item[Keys.ATTR] 77 | args = item.get(Keys.ARGS, []) 78 | kwargs = {key: value for key, value in item.items() if key not in (self.KEY, Keys.ATTR, Keys.ARGS)} 79 | attr = {Keys.ATTR.value: to_import_string(from_import_string), "name": name} 80 | constructor = {Keys.ATTR.value: to_import_string(partial), Keys.ARGS.value: [attr, *args], **kwargs} 81 | return {Keys.ATTR.value: to_import_string(singleton), "key": key, "constructor": constructor} 82 | return item 83 | 84 | return depth_map(_map_fn, config) 85 | -------------------------------------------------------------------------------- /fromconfig/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-import,missing-docstring 2 | 3 | from fromconfig.utils.types import is_mapping, is_pure_iterable 4 | from fromconfig.utils.libimport import from_import_string, to_import_string, try_import 5 | from fromconfig.utils.strenum import StrEnum 6 | from fromconfig.utils.nest import flatten, expand, merge_dict, depth_map 7 | import fromconfig.utils.testing 8 | -------------------------------------------------------------------------------- /fromconfig/utils/libimport.py: -------------------------------------------------------------------------------- 1 | """Import utilities.""" 2 | 3 | from typing import Any 4 | import builtins 5 | import importlib 6 | import inspect 7 | import logging 8 | 9 | 10 | LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | def try_import(name, package=None): 14 | """Try import package name in package.""" 15 | try: 16 | return importlib.import_module(name, package=package) 17 | except ImportError: 18 | return None 19 | 20 | 21 | def to_import_string(attr: Any) -> str: 22 | """Retrieve import string from attribute. 23 | 24 | Parameters 25 | ---------- 26 | attr : Any 27 | Any python function, class or method 28 | """ 29 | if inspect.ismodule(attr): 30 | return attr.__name__ 31 | module = inspect.getmodule(attr) 32 | if module is None: 33 | raise ValueError(f"Unable to resolve module of {attr}") 34 | 35 | # If class, method or function, use __qualname__ 36 | if inspect.isclass(attr) or inspect.ismethod(attr) or inspect.isfunction(attr): 37 | if module is builtins: 38 | return attr.__qualname__ 39 | else: 40 | return f"{module.__name__}.{attr.__qualname__}" 41 | 42 | # Look for the instance's name in the user-defined module 43 | for name, member in inspect.getmembers(module): 44 | if id(member) == id(attr): 45 | return f"{module.__name__}.{name}" 46 | 47 | raise ValueError(f"Unable to resolve import string of {attr}") # pragma: no cover 48 | 49 | 50 | def from_import_string(name: str) -> Any: 51 | """Import module, class, method or attribute from string. 52 | 53 | Examples 54 | -------- 55 | >>> import fromconfig 56 | >>> fromconfig.utils.from_import_string("str") == str 57 | True 58 | 59 | Parameters 60 | ---------- 61 | name : str 62 | Import string or builtin name 63 | """ 64 | # Resolve import parts 65 | parts = [part for part in name.split(".") if part] 66 | if not parts: 67 | raise ImportError(f"No parts found (name='{name}', parts={parts})") 68 | 69 | # Import modules 70 | module, offset = None, 0 71 | for idx in range(1, len(parts)): 72 | try: 73 | module_name = ".".join(parts[:idx]) 74 | module = importlib.import_module(module_name) 75 | offset = idx 76 | except Exception as e: # pylint: disable=broad-except 77 | LOGGER.info(f"Exception while loading module from {module_name}: {e}") 78 | break 79 | 80 | # Get attribute from provided module, builtins or call stack modules 81 | for mod in [module, builtins] + [inspect.getmodule(fi.frame) for fi in inspect.stack()]: 82 | try: 83 | attr = mod 84 | for part in parts[offset:]: 85 | attr = getattr(attr, part) 86 | return attr 87 | except Exception as e: # pylint: disable=broad-except 88 | LOGGER.info(f"Exception while getting attribute from module {mod}: {e}") 89 | 90 | # Look for name in call stack globals 91 | for fi in inspect.stack(): 92 | for key, value in fi.frame.f_globals.items(): 93 | if key == name: 94 | return value 95 | 96 | raise ValueError(f"Unable to resolve attribute from import string '{name}'") 97 | -------------------------------------------------------------------------------- /fromconfig/utils/nest.py: -------------------------------------------------------------------------------- 1 | """Nest Utilities.""" 2 | 3 | import logging 4 | from typing import Callable, Dict, Any, Mapping, List, Tuple, Iterable, Union 5 | 6 | from fromconfig.utils.types import is_mapping, is_pure_iterable 7 | 8 | 9 | LOGGER = logging.getLogger(__name__) 10 | 11 | 12 | def depth_map(map_fn: Callable[[Any], Any], item: Any) -> Any: 13 | """Depth-first map implementation on dictionary, tuple and list. 14 | 15 | Examples 16 | -------- 17 | >>> import fromconfig 18 | >>> def add_one(x): 19 | ... return x + 1 if isinstance(x, int) else x 20 | >>> mapped = fromconfig.utils.depth_map(add_one, {"x": 1}) 21 | >>> mapped["x"] 22 | 2 23 | 24 | Parameters 25 | ---------- 26 | map_fn : Callable[[Any], Any] 27 | Map Function to apply to item and its children 28 | item : Any 29 | Any python object 30 | 31 | Returns 32 | ------- 33 | Any 34 | The result of applying map_fn to item and its children. 35 | """ 36 | # If mapping, try to create new mapping with mapped kwargs 37 | if is_mapping(item): 38 | return map_fn({key: depth_map(map_fn, value) for key, value in item.items()}) 39 | 40 | # If iterable, try to create new iterable with mapped args 41 | if is_pure_iterable(item): 42 | return map_fn([depth_map(map_fn, it) for it in item]) 43 | 44 | return map_fn(item) 45 | 46 | 47 | def merge_dict(item1: Mapping, item2: Mapping, allow_override: bool = True) -> Dict: 48 | """Merge item2 into item1. 49 | 50 | Examples 51 | -------- 52 | >>> import fromconfig 53 | >>> merged = fromconfig.utils.merge_dict({"x": 1}, {"y": 2}) 54 | >>> merged["x"] 55 | 1 56 | >>> merged["y"] 57 | 2 58 | 59 | Parameters 60 | ---------- 61 | item1 : Mapping 62 | Reference mapping 63 | item2 : Mapping 64 | Override mapping 65 | allow_override : bool, optional 66 | If True, allow keys to be present in both item1 and item2 67 | 68 | Returns 69 | ------- 70 | Mapping 71 | """ 72 | 73 | def _merge(it1: Any, it2: Any): 74 | """Recursive implementation.""" 75 | if is_mapping(it1): 76 | if not is_mapping(it2): 77 | raise TypeError(f"Incompatible types, {type(it2)} and {type(it2)}") 78 | 79 | # Build merged dictionary 80 | merged = {} 81 | for key in set(it1) | set(it2): 82 | if key in it1 and key in it2: 83 | if not allow_override: 84 | raise ValueError(f"Duplicate key found {key} and allow_override = False (not allowed)") 85 | merged[key] = _merge(it1[key], it2[key]) 86 | if key in it1 and key not in it2: 87 | merged[key] = it1[key] 88 | if key not in it1 and key in it2: 89 | merged[key] = it2[key] 90 | 91 | return merged 92 | 93 | return it2 94 | 95 | return _merge(item1, item2) 96 | 97 | 98 | def flatten(config: Any) -> List[Tuple[str, Any]]: 99 | """Flatten dictionary into a list of tuples key, value. 100 | 101 | 102 | Example 103 | ------- 104 | >>> import fromconfig 105 | >>> d = {"x": {"y": 0}} 106 | >>> fromconfig.utils.flatten(d) 107 | [('x.y', 0)] 108 | 109 | Parameters 110 | ---------- 111 | config : Any 112 | Typically a dictionary (possibly nested) 113 | 114 | Returns 115 | ------- 116 | List[Tuple[Optional[str], Any]] 117 | Each tuple is a flattened key with the corresponding value. 118 | """ 119 | 120 | def _flatten(item) -> List[Tuple[List[Union[str, int]], Any]]: 121 | if is_mapping(item): 122 | result = [] 123 | for key, values in item.items(): 124 | if "." in key: 125 | raise ValueError(f"Key {key} already has a `.`, unable to flatten.") 126 | for subkeys, value in values: 127 | result.append(([str(key)] + subkeys, value)) 128 | return result 129 | 130 | if is_pure_iterable(item): 131 | result = [] 132 | for idx, it in enumerate(item): 133 | for subkeys, value in it: 134 | result.append(([idx] + subkeys, value)) 135 | return result 136 | 137 | return [([], item)] 138 | 139 | flattened = depth_map(_flatten, config) 140 | return [(_to_dotlist(key), value) for key, value in flattened] 141 | 142 | 143 | def expand(flat: Iterable[Tuple[str, Any]]): 144 | """Expand flat dictionary into nested dictionary. 145 | 146 | Example 147 | ------- 148 | >>> import fromconfig 149 | >>> d = {"x.y": 0} 150 | >>> fromconfig.utils.expand(d.items()) 151 | {'x': {'y': 0}} 152 | 153 | Parameters 154 | ---------- 155 | flat : Iterable[Tuple[Optional[str], Any]] 156 | Iterable of flat keys, value 157 | """ 158 | 159 | def _expand(it: List[Tuple[List[Union[str, int]], Any]]) -> Dict: 160 | # Recursive step 161 | result = {} # type: ignore 162 | for keys, value in it: 163 | if not keys: 164 | if None in result: 165 | raise KeyError("More than one pure value found (not allowed)") 166 | result[None] = value 167 | else: 168 | key = keys[0] 169 | value = _expand([(keys[1:], value)]) 170 | result[key] = merge_dict(value, result.get(key, {})) 171 | return result 172 | 173 | def _normalize(it): 174 | if is_mapping(it) and any(isinstance(key, int) for key in it): 175 | return [it[idx] for idx in range(len(it))] 176 | if is_mapping(it) and any(key is None for key in it): 177 | if len(it) > 1: 178 | raise KeyError("More than one pure value found (not allowed)") 179 | return it[None] 180 | return it 181 | 182 | return depth_map(_normalize, _expand([(_from_dotlist(dotlist), value) for dotlist, value in flat])) 183 | 184 | 185 | def _to_dotlist(keys: List[Union[str, int]]) -> str: 186 | """Convert list of keys to dot-list. 187 | 188 | Parameters 189 | ---------- 190 | keys : List[Union[str, int]] 191 | List of keys. 192 | """ 193 | dotlist = "" 194 | for key in keys: 195 | if isinstance(key, str): 196 | dotlist = key if not dotlist else f"{dotlist}.{key}" 197 | elif isinstance(key, int): 198 | dotlist = f"{dotlist}[{key}]" 199 | else: 200 | raise TypeError(f"Unsupported key type {type(key)}") 201 | return dotlist 202 | 203 | 204 | def _from_dotlist(dotlist: str) -> List[Union[str, int]]: 205 | """Convert dot-list to list of keys. 206 | 207 | Parameters 208 | ---------- 209 | dotlist : str 210 | Dot-List 211 | """ 212 | keys: List[Union[str, int]] = [] 213 | for item in dotlist.split("."): 214 | for it in item.split("["): 215 | if it.endswith("]"): 216 | keys.append(int(it.rstrip("]"))) 217 | else: 218 | keys.append(it) 219 | return keys 220 | -------------------------------------------------------------------------------- /fromconfig/utils/strenum.py: -------------------------------------------------------------------------------- 1 | """String Enum.""" 2 | 3 | from enum import Enum, EnumMeta 4 | 5 | 6 | class StrEnumMeta(EnumMeta): 7 | """Meta class for String Enum.""" 8 | 9 | # pylint: disable=unsupported-membership-test,not-an-iterable 10 | 11 | def __contains__(cls, member): 12 | return any(key == member for key in cls) 13 | 14 | 15 | class StrEnum(str, Enum, metaclass=StrEnumMeta): 16 | """String Enum.""" 17 | -------------------------------------------------------------------------------- /fromconfig/utils/testing.py: -------------------------------------------------------------------------------- 1 | """Utilities for testing.""" 2 | 3 | 4 | def assert_launcher_is_discovered(name: str, expected): 5 | """Test that a launcher is found by name.""" 6 | # pylint: disable=import-outside-toplevel,cyclic-import 7 | from fromconfig.launcher.base import _classes 8 | 9 | assert _classes()[name] is expected 10 | -------------------------------------------------------------------------------- /fromconfig/utils/types.py: -------------------------------------------------------------------------------- 1 | """Types utilities.""" 2 | 3 | from collections.abc import Mapping, Iterable 4 | import logging 5 | 6 | 7 | LOGGER = logging.getLogger(__name__) 8 | 9 | 10 | def is_mapping(item) -> bool: 11 | return item is not None and isinstance(item, Mapping) 12 | 13 | 14 | def is_pure_iterable(item) -> bool: 15 | return item is not None and not isinstance(item, str) and isinstance(item, Iterable) and not is_mapping(item) 16 | -------------------------------------------------------------------------------- /fromconfig/version.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=all 2 | 3 | __version__ = "0.7.2" 4 | __author__ = "Criteo" 5 | 6 | __major__ = __version__.split(".")[0] 7 | __minor__ = __version__.split(".")[1] 8 | __patch__ = __version__.split(".")[2] 9 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options: 2 | 3 | [mypy] 4 | python_version = 3.6 5 | ignore_missing_imports = True 6 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | black==22.6.0 2 | mypy==0.790 3 | pylint==2.6.0 4 | pytest-cov==2.10.1 5 | pytest-xdist==2.1.0 6 | pytest==6.1.2 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fire==0.4.0 2 | jsonnet==0.17.0 3 | PyYAML==5.4.1 4 | omegaconf==2.1.1 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup script.""" 2 | 3 | from pathlib import Path 4 | import re 5 | import setuptools 6 | 7 | 8 | if __name__ == "__main__": 9 | # Read metadata from version.py 10 | with Path("fromconfig/version.py").open(encoding="utf-8") as file: 11 | metadata = dict(re.findall(r'__([a-z]+)__\s*=\s*"([^"]+)"', file.read())) 12 | 13 | # Read description from README 14 | with Path(Path(__file__).parent, "docs/README.md").open(encoding="utf-8") as file: 15 | long_description = file.read() 16 | 17 | # Run setup 18 | setuptools.setup( 19 | author=metadata["author"], 20 | version=metadata["version"], 21 | classifiers=[ 22 | "License :: OSI Approved :: Apache Software License", 23 | "Operating System :: OS Independent", 24 | "Programming Language :: Python :: 3.6", 25 | "Programming Language :: Python :: 3.7", 26 | "Programming Language :: Python :: 3.8", 27 | "Programming Language :: Python :: 3.9", 28 | "Intended Audience :: Developers", 29 | ], 30 | data_files=[(".", ["requirements.txt", "docs/README.md"])], 31 | dependency_links=[], 32 | description="A library to instantiate any Python object from configuration files", 33 | entry_points={"console_scripts": ["fromconfig = fromconfig.cli.main:main"]}, 34 | install_requires=["fire", "omegaconf", "pyyaml"], 35 | long_description=long_description, 36 | long_description_content_type="text/markdown", 37 | name="fromconfig", 38 | packages=setuptools.find_packages(), 39 | tests_require=["pytest"], 40 | url="https://github.com/criteo/fromconfig", 41 | ) 42 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/__init__.py -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/unit/cli/__init__.py -------------------------------------------------------------------------------- /tests/unit/cli/test_cli_main.py: -------------------------------------------------------------------------------- 1 | """Tests for cli.main.""" 2 | 3 | import fromconfig 4 | from fromconfig.cli.main import parse_args 5 | import sys 6 | from unittest.mock import patch 7 | 8 | import subprocess 9 | 10 | 11 | def capture(command): 12 | """Utility to execute and capture the result of a command.""" 13 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 14 | out, err = proc.communicate() 15 | return out, err, proc.returncode 16 | 17 | 18 | def test_cli_parse_args(): 19 | """Test cli.parse_args.""" 20 | argv = ["fromconfig", "config.yaml", "params.yaml", "-", "model", "-", "train"] 21 | with patch.object(sys, "argv", argv): 22 | paths, overrides, command = parse_args() 23 | expected_paths = ["config.yaml", "params.yaml"] 24 | expected_overrides = {} 25 | expected_command = "model - train" 26 | assert paths == expected_paths 27 | assert overrides == expected_overrides 28 | assert command == expected_command 29 | 30 | argv = ["fromconfig", "config.yaml", "--output", "/tmp", "-", "run"] 31 | with patch.object(sys, "argv", argv): 32 | paths, overrides, command = parse_args() 33 | expected_paths = ["config.yaml"] 34 | expected_overrides = {"output": "/tmp"} 35 | expected_command = "run" 36 | assert paths == expected_paths 37 | assert overrides == expected_overrides 38 | assert command == expected_command 39 | 40 | argv = ["fromconfig", "config.yaml", "--output=/tmp", "-", "run"] 41 | with patch.object(sys, "argv", argv): 42 | paths, overrides, command = parse_args() 43 | expected_paths = ["config.yaml"] 44 | expected_overrides = {"output": "/tmp"} 45 | expected_command = "run" 46 | assert paths == expected_paths 47 | assert overrides == expected_overrides 48 | assert command == expected_command 49 | 50 | 51 | def test_cli_main_nothing(): 52 | """Test that fromconfig with no argument does not error.""" 53 | out, err, exitcode = capture(["fromconfig"]) 54 | assert exitcode == 0, (out, err) 55 | assert all(word in out for word in [b"fromconfig", b"flags"]) 56 | assert err == b"" 57 | 58 | 59 | def test_cli_main(tmpdir): 60 | """Test cli.main.""" 61 | # Write parameters 62 | path_parameters = tmpdir.join("parameters.yaml") 63 | parameters = {"value": "hello world"} 64 | 65 | fromconfig.dump(parameters, path_parameters) 66 | 67 | # Write config 68 | path_config = tmpdir.join("config.yaml") 69 | config = {"run": {"_attr_": "print", "_args_": ["${value}"]}} 70 | fromconfig.dump(config, path_config) 71 | 72 | # Execute command and check result 73 | command = ["fromconfig", path_config, path_parameters, "-", "run"] 74 | out, err, exitcode = capture(command) 75 | assert exitcode == 0, (out, err) 76 | assert out == b"hello world\n" 77 | assert err == b"" 78 | -------------------------------------------------------------------------------- /tests/unit/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/unit/core/__init__.py -------------------------------------------------------------------------------- /tests/unit/core/test_core_base.py: -------------------------------------------------------------------------------- 1 | """Tests for core.base.""" 2 | 3 | import inspect 4 | from typing import Any 5 | 6 | import pytest 7 | 8 | import fromconfig 9 | 10 | 11 | class Custom(fromconfig.FromConfig): 12 | """Custom FromConfig class.""" 13 | 14 | def __init__(self, x): 15 | self.x = x 16 | 17 | def __eq__(self, other): 18 | return type(self) == type(other) and self.x == other.x # pylint: disable=unidiomatic-typecheck 19 | 20 | 21 | class CustomOverriden(Custom): 22 | """Custom FromConfig class with fromconfig implementation.""" 23 | 24 | @classmethod 25 | def fromconfig(cls, config: Any): 26 | return cls(config["_x"]) 27 | 28 | 29 | @pytest.mark.parametrize( 30 | "config, expected", 31 | [ 32 | pytest.param({"x": 1}, Custom(1), id="simple"), 33 | pytest.param(None, TypeError, id="simple"), 34 | pytest.param( 35 | {"x": {"_attr_": "tests.unit.core.test_core_base.Custom", "x": 2}}, Custom(Custom(2)), id="nested" 36 | ), 37 | ], 38 | ) 39 | def test_default_custom_fromconfig(config, expected): 40 | """Test default custom fromconfig.""" 41 | if inspect.isclass(expected) and issubclass(expected, Exception): 42 | with pytest.raises(expected): 43 | Custom.fromconfig(config) 44 | else: 45 | assert Custom.fromconfig(config) == expected 46 | 47 | 48 | @pytest.mark.parametrize( 49 | "config, expected", 50 | [ 51 | pytest.param( 52 | {"_attr_": "tests.unit.core.test_core_base.CustomOverriden", "_x": 1}, CustomOverriden(1), id="simple" 53 | ), 54 | pytest.param( 55 | {"_attr_": "fromconfig.Config", "_config_": {"_attr_": "str", "_args_": "hello"}}, 56 | fromconfig.Config(_attr_="str", _args_="hello"), 57 | id="config-with-attr", 58 | ), 59 | ], 60 | ) 61 | def test_core_fromconfig(config, expected): 62 | """Test core.fromconfig.""" 63 | assert fromconfig.fromconfig(config) == expected 64 | -------------------------------------------------------------------------------- /tests/unit/core/test_core_config.py: -------------------------------------------------------------------------------- 1 | """Tests for core.config.""" 2 | 3 | import json 4 | import yaml 5 | from pathlib import Path 6 | 7 | import pytest 8 | 9 | import fromconfig 10 | 11 | 12 | def test_core_config_no_jsonnet(tmpdir, monkeypatch): 13 | """Test jsonnet missing handling.""" 14 | monkeypatch.setattr(fromconfig.core.config, "_jsonnet", None) 15 | 16 | # No issue to dump even if missing 17 | config = {"x": 2} 18 | fromconfig.dump(config, str(tmpdir.join("config.jsonnet"))) 19 | fromconfig.dump(config, str(tmpdir.join("config.json"))) 20 | fromconfig.dump(config, str(tmpdir.join("config.yaml"))) 21 | fromconfig.dump(config, str(tmpdir.join("config.yml"))) 22 | 23 | # No issue to load non-jsonnet files 24 | assert fromconfig.load(str(tmpdir.join("config.json"))) == config 25 | assert fromconfig.load(str(tmpdir.join("config.yaml"))) == config 26 | assert fromconfig.load(str(tmpdir.join("config.yml"))) == config 27 | 28 | # Raise import error if reloading from jsonnet 29 | with pytest.raises(ImportError): 30 | fromconfig.load(str(tmpdir.join("config.jsonnet"))) 31 | 32 | 33 | def test_core_config(): 34 | """Test Config.""" 35 | config = fromconfig.Config(x=1) 36 | assert config["x"] == 1 37 | assert list(config) == ["x"] 38 | config["x"] = 2 39 | assert config["x"] == 2 40 | 41 | 42 | def test_core_config_is_json_serializable(): 43 | """Test that Config is json serializable.""" 44 | config = fromconfig.Config(x=1) 45 | assert json.dumps(config) == '{"x": 1}' 46 | 47 | 48 | @pytest.mark.parametrize( 49 | "path,serializer", 50 | [ 51 | pytest.param("config.json", json), 52 | pytest.param("config.jsonnet", json), 53 | pytest.param("config.yaml", yaml), 54 | pytest.param("config.yml", yaml), 55 | pytest.param("config.xml", None), 56 | ], 57 | ) 58 | def test_core_config_load_dump(path, serializer, tmpdir): 59 | """Test Config.load.""" 60 | config = {"x": 1} 61 | path = str(tmpdir.join(path)) 62 | 63 | if serializer is None: 64 | # Incorrect path (not supported) 65 | with pytest.raises(ValueError): 66 | fromconfig.dump(config, path) 67 | 68 | with pytest.raises(ValueError): 69 | fromconfig.load(path) 70 | 71 | else: 72 | # Dump config to file 73 | with Path(path).open("w") as file: 74 | if serializer is json: 75 | serializer.dump(config, file, indent=4) 76 | else: 77 | serializer.dump(config, file) 78 | 79 | # Read content of the dump 80 | with Path(path).open() as file: 81 | content = file.read() 82 | 83 | # Reload 84 | reloaded = fromconfig.load(path) 85 | assert reloaded == config 86 | 87 | # Dump with config method and check content is the same as before 88 | fromconfig.dump(reloaded, path) 89 | with Path(path).open() as file: 90 | assert file.read() == content 91 | 92 | 93 | @pytest.mark.parametrize("config, expected", [pytest.param("foo: bar", {"foo": "bar"})]) 94 | def test_core_config_include_loader_on_string(config, expected): 95 | """Test IncludeLoader.""" 96 | assert expected == yaml.load(config, fromconfig.core.config.IncludeLoader) 97 | 98 | 99 | @pytest.mark.parametrize( 100 | "files, expected", 101 | [ 102 | pytest.param( 103 | {"config.yaml": "foo: 1\nbar: !include bar.yaml", "bar.yaml": "2"}, {"foo": 1, "bar": 2}, id="simple" 104 | ), 105 | pytest.param( 106 | {"config.yaml": "foo: 1\n<<: !include bar.yaml", "bar.yaml": "bar: 2"}, 107 | {"foo": 1, "bar": 2}, 108 | id="simple-merge", 109 | ), 110 | pytest.param( 111 | {"config.yaml": "foo: 1\n<<: !include bar.yaml", "bar.yaml": "2"}, None, id="simple-merge-invalid" 112 | ), 113 | pytest.param( 114 | {"config.yaml": "foo: 1\nbar: !include bar/bar.yaml", "bar/bar.yaml": "2"}, 115 | {"foo": 1, "bar": 2}, 116 | id="nested", 117 | ), 118 | pytest.param( 119 | {"config.yaml": "foo: 1\n<<: !include bar/bar.yaml", "bar/bar.yaml": "bar: 2"}, 120 | {"foo": 1, "bar": 2}, 121 | id="nested-merge", 122 | ), 123 | pytest.param( 124 | { 125 | "config.yaml": "foo: 1\nbar: !include bar/bar.yaml", 126 | "bar/bar.yaml": "!include baz.yaml", 127 | "bar/baz.yaml": "2", 128 | }, 129 | {"foo": 1, "bar": 2}, 130 | id="nested-twice", 131 | ), 132 | pytest.param( 133 | { 134 | "config.yaml": "foo: 1\n<<: !include bar/bar.yaml", 135 | "bar/bar.yaml": "<<: !include baz.yaml", 136 | "bar/baz.yaml": "bar: 2", 137 | }, 138 | {"foo": 1, "bar": 2}, 139 | id="nested-twice-merge", 140 | ), 141 | ], 142 | ) 143 | def test_core_config_load_include_merge(files, expected, tmpdir): 144 | """Test include and merge functionality.""" 145 | for p, content in files.items(): 146 | Path(tmpdir, p).parent.mkdir(parents=True, exist_ok=True) 147 | with Path(tmpdir, p).open("w") as file: 148 | file.write(content) 149 | assert fromconfig.load(Path(tmpdir, "config.yaml")) == expected 150 | 151 | 152 | @pytest.mark.parametrize( 153 | "config, expected", 154 | [ 155 | pytest.param( 156 | {"_attr_": "str", "_args_": "hello"}, fromconfig.Config(_attr_="str", _args_="hello"), id="simple" 157 | ), 158 | pytest.param( 159 | {"_config_": {"_attr_": "str", "_args_": "hello"}}, 160 | fromconfig.Config(_attr_="str", _args_="hello"), 161 | id="config", 162 | ), 163 | pytest.param( 164 | [("_attr_", "str"), ("_args_", "hello")], 165 | fromconfig.Config(_attr_="str", _args_="hello"), 166 | id="list", 167 | ), 168 | ], 169 | ) 170 | def test_core_config_fromconfig(config, expected): 171 | """Test Config.fromconfig.""" 172 | assert fromconfig.Config.fromconfig(config) == expected 173 | -------------------------------------------------------------------------------- /tests/unit/launcher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/unit/launcher/__init__.py -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_base.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=protected-access 2 | """Test launcher.base.""" 3 | 4 | import pytest 5 | import pkg_resources 6 | from typing import Any 7 | 8 | import fromconfig 9 | 10 | from fromconfig.launcher import HParamsLauncher 11 | from fromconfig.launcher import ParserLauncher 12 | from fromconfig.launcher import LoggingLauncher 13 | from fromconfig.launcher import LocalLauncher 14 | from fromconfig.launcher import DryLauncher 15 | 16 | 17 | def assert_equal_launcher(a, b): 18 | assert type(a) is type(b) 19 | if a.launcher is not None or b.launcher is not None: 20 | assert_equal_launcher(a.launcher, b.launcher) 21 | 22 | 23 | def test_launcher_is_abstract(): 24 | with pytest.raises(NotImplementedError): 25 | fromconfig.launcher.Launcher(None)({}) 26 | 27 | 28 | @pytest.mark.parametrize( 29 | "name, expected", 30 | [ 31 | pytest.param("hparams", HParamsLauncher, id="hparams"), 32 | pytest.param("parser", ParserLauncher, id="parser"), 33 | pytest.param("logging", LoggingLauncher, id="logging"), 34 | pytest.param("local", LocalLauncher, id="local"), 35 | pytest.param("does not exists", KeyError, id="missing-key"), 36 | ], 37 | ) 38 | def test_launcher_classes(name, expected): 39 | """Test _get_cls.""" 40 | if issubclass(expected, fromconfig.launcher.Launcher): 41 | assert fromconfig.launcher.base._classes()[name] is expected 42 | else: 43 | with pytest.raises(expected): 44 | fromconfig.launcher.base._classes()[name] # pylint: disable=expression-not-assigned 45 | 46 | 47 | def test_launcher_classes_extension(monkeypatch): 48 | """Test launcher classes extension.""" 49 | 50 | fromconfig.launcher.base._CLASSES.clear() # Clear internal and external launchers 51 | 52 | class DummyModule: 53 | class DummyLauncher(fromconfig.launcher.Launcher): 54 | def __call__(self, config: Any, command: str = ""): 55 | ... 56 | 57 | class EntryPoint: 58 | def __init__(self, name, module): 59 | self.name = name 60 | self.module = module 61 | 62 | def load(self): 63 | return self.module 64 | 65 | # Test discovery 66 | monkeypatch.setattr(pkg_resources, "iter_entry_points", lambda *_: [EntryPoint("dummy", DummyModule)]) 67 | fromconfig.launcher.base._load() 68 | fromconfig.utils.testing.assert_launcher_is_discovered("dummy", DummyModule.DummyLauncher) 69 | assert fromconfig.launcher.base._classes()["dummy"] == DummyModule.DummyLauncher 70 | 71 | # Test that loading again causes errors because of duplicates 72 | with pytest.raises(ValueError): 73 | fromconfig.launcher.base._load() 74 | 75 | 76 | @pytest.mark.parametrize( 77 | "config, expected", 78 | [ 79 | pytest.param("local", LocalLauncher(), id="local"), 80 | pytest.param("dry", DryLauncher(), id="dry"), 81 | pytest.param(DryLauncher(), DryLauncher(), id="dry-instance"), 82 | pytest.param( 83 | ["hparams", "parser", "logging", "local"], 84 | HParamsLauncher(ParserLauncher(LoggingLauncher(LocalLauncher()))), 85 | id="hparams+parser+logging+local", 86 | ), 87 | pytest.param({"_attr_": "fromconfig.launcher.LocalLauncher"}, LocalLauncher(), id="fromconfig"), 88 | pytest.param({"_attr_": "local"}, LocalLauncher(), id="fromconfig-short"), 89 | pytest.param( 90 | {"_attr_": "fromconfig.launcher.HParamsLauncher", "launcher": "local"}, 91 | HParamsLauncher(LocalLauncher()), 92 | id="fromconfig+nested", 93 | ), 94 | pytest.param( 95 | [{"_attr_": "fromconfig.launcher.HParamsLauncher", "launcher": "local"}, "local"], 96 | ValueError, 97 | id="fromconfig+nested-duplicate", 98 | ), 99 | pytest.param( 100 | [HParamsLauncher(None), "local"], 101 | ValueError, 102 | id="fromconfig+nested+already-instantiated-cannot-wrap", 103 | ), 104 | pytest.param(1, TypeError, id="incorrect-type"), 105 | ], 106 | ) 107 | def test_launcher_fromconfig(config, expected): 108 | """Test custom fromconfig support.""" 109 | if isinstance(expected, fromconfig.launcher.Launcher): 110 | assert_equal_launcher(fromconfig.launcher.Launcher.fromconfig(config), expected) 111 | else: 112 | with pytest.raises(expected): 113 | fromconfig.launcher.Launcher.fromconfig(config) 114 | -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_default.py: -------------------------------------------------------------------------------- 1 | """Test for launcher.default.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | def assert_equal_launcher(a, b): 9 | assert type(a) is type(b) 10 | if a.launcher is not None or b.launcher is not None: 11 | assert_equal_launcher(a.launcher, b.launcher) 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "config, expected", 16 | [ 17 | pytest.param({"sweep": [], "log": [], "parse": []}, fromconfig.launcher.LocalLauncher(), id="only-run-list"), 18 | pytest.param( 19 | {"sweep": None, "log": None, "parse": None}, fromconfig.launcher.LocalLauncher(), id="only-run-none" 20 | ), 21 | pytest.param({"sweep": "hparams", "unknown-step": "local"}, ValueError, id="unknown-step"), 22 | ], 23 | ) 24 | def test_launcher_default_fromconfig(config, expected): 25 | """Test custom fromconfig support.""" 26 | if isinstance(expected, fromconfig.launcher.Launcher): 27 | assert_equal_launcher(fromconfig.launcher.DefaultLauncher.fromconfig(config), expected) 28 | else: 29 | with pytest.raises(expected): 30 | fromconfig.launcher.DefaultLauncher.fromconfig(config) 31 | -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_dry.py: -------------------------------------------------------------------------------- 1 | """Test launcher.dry.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "config, command", 10 | [ 11 | pytest.param({"foo": "bar"}, "foo", id="dict"), 12 | pytest.param(None, "", id="none"), 13 | pytest.param(["foo"], "", id="list"), 14 | ], 15 | ) 16 | def test_launcher_dry(config, command): 17 | """Test dry launcher.""" 18 | launcher = fromconfig.launcher.DryLauncher() 19 | launcher(config, command) 20 | -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_hparams.py: -------------------------------------------------------------------------------- 1 | """Test for launcher.hparams.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "config, command, expected", 10 | [ 11 | pytest.param( 12 | {"hparams": {"dim": [10, 100]}}, 13 | "command", 14 | [({"hparams": {"dim": 10}}, "command"), ({"hparams": {"dim": 100}}, "command")], 15 | id="dict", 16 | ), 17 | pytest.param([], "command", [([], "command")], id="list"), 18 | pytest.param(None, "command", [(None, "command")], id="none"), 19 | ], 20 | ) 21 | def test_launcher_hparams(config, command, expected): 22 | """Test HParamsLauncher.""" 23 | got = [] 24 | 25 | def got_launcher(config, command): 26 | got.append((config, command)) 27 | 28 | launcher = fromconfig.launcher.HParamsLauncher(got_launcher) 29 | launcher(config, command) 30 | assert got == expected 31 | -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_local.py: -------------------------------------------------------------------------------- 1 | """Test launcher.local.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | def test_launcher_local_init(): 9 | """Test LocalLauncher initialization.""" 10 | fromconfig.launcher.LocalLauncher() 11 | 12 | # Cannot nest two local launchers 13 | with pytest.raises(ValueError): 14 | fromconfig.launcher.LocalLauncher(fromconfig.launcher.LocalLauncher()) 15 | 16 | 17 | def test_launcher_local_basics(): 18 | """Test basic functionality of the local launcher.""" 19 | got = {} 20 | 21 | def run(): 22 | got.update({"run": True}) 23 | 24 | config = {"run": run} 25 | launcher = fromconfig.launcher.LocalLauncher() 26 | launcher(config, "run") 27 | assert got["run"] 28 | 29 | 30 | @pytest.mark.parametrize( 31 | "config, command", 32 | [pytest.param(None, "", id="none"), pytest.param([], "", id="list"), pytest.param({"run": None}, "run", id="dict")], 33 | ) 34 | def test_launcher_local_types(config, command): 35 | """Test that LocalLauncher accepts different types.""" 36 | launcher = fromconfig.launcher.LocalLauncher() 37 | launcher(config, command) 38 | -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_logger.py: -------------------------------------------------------------------------------- 1 | """Test launcher.logger.""" 2 | 3 | import logging 4 | 5 | import pytest 6 | 7 | import fromconfig 8 | 9 | 10 | @pytest.mark.parametrize( 11 | "config, expected", 12 | [ 13 | pytest.param({"foo": "bar"}, ({}, []), id="default"), 14 | pytest.param(None, ({}, []), id="none"), 15 | pytest.param({"foo": "bar", "logging": {"level": 20}}, ({"level": 20}, []), id="set-level"), 16 | ], 17 | ) 18 | def test_launcher_logger(config, expected, monkeypatch): 19 | """Basic Test for LoggingLauncher.""" 20 | basic_config = {} 21 | logs = [] 22 | 23 | class MonkeyLogger: 24 | def info(self, msg): 25 | logs.append(msg) 26 | 27 | def MonkeyBasicConfig(**kwargs): # pylint: disable=invalid-name 28 | basic_config.update(kwargs) 29 | 30 | monkeypatch.setattr(logging, "basicConfig", MonkeyBasicConfig) 31 | monkeypatch.setattr(fromconfig.launcher.logger, "LOGGER", MonkeyLogger()) 32 | 33 | launcher = fromconfig.launcher.LoggingLauncher(fromconfig.launcher.DryLauncher()) 34 | launcher(config) 35 | assert (basic_config, logs) == expected 36 | -------------------------------------------------------------------------------- /tests/unit/launcher/test_launcher_parser.py: -------------------------------------------------------------------------------- 1 | """Test launcher.parser.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "config, expected", 10 | [ 11 | pytest.param(None, None, id="none"), 12 | pytest.param({"foo": "bar"}, {"foo": "bar"}, id="dict"), 13 | pytest.param(["foo"], ["foo"], id="list"), 14 | pytest.param({"foo": "bar", "baz": "${foo}"}, {"foo": "bar", "baz": "bar"}, id="parsed"), 15 | pytest.param({"foo": "bar", "baz": "${foo}", "parser": None}, {"foo": "bar", "baz": "${foo}"}, id="no-parser"), 16 | ], 17 | ) 18 | def test_launcher_parser(config, expected): 19 | """Test ParserLauncher.""" 20 | got = {} 21 | 22 | def capture(config, command=""): 23 | # pylint: disable=unused-argument 24 | got.update({"result": config}) 25 | 26 | launcher = fromconfig.launcher.ParserLauncher(capture) 27 | launcher(config) 28 | assert got["result"] == expected 29 | -------------------------------------------------------------------------------- /tests/unit/parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/unit/parser/__init__.py -------------------------------------------------------------------------------- /tests/unit/parser/test_parser_base.py: -------------------------------------------------------------------------------- 1 | """Test for parser.base.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | def test_parser_parser_class_is_abstract(): 9 | """Test that from config is abstract.""" 10 | with pytest.raises(Exception): 11 | fromconfig.parser.Parser() # pylint: disable=abstract-class-instantiated 12 | 13 | with pytest.raises(NotImplementedError): 14 | fromconfig.parser.Parser.__call__(None, {"x": 1}) 15 | 16 | 17 | def test_parser_chain(): 18 | """Test Chain Parser.""" 19 | 20 | class MockParser(fromconfig.parser.Parser): 21 | def __init__(self, value): 22 | self.value = value 23 | 24 | def __repr__(self): 25 | return str(self.value) 26 | 27 | def __call__(self, config): 28 | config["value"] = self.value 29 | config["sum"] += self.value 30 | return config 31 | 32 | parser = fromconfig.parser.ChainParser(MockParser(1), MockParser(2)) 33 | got = parser({"value": None, "sum": 0}) 34 | assert got == {"value": 2, "sum": 3} 35 | assert repr(parser) == "ChainParser(1, 2)" 36 | -------------------------------------------------------------------------------- /tests/unit/parser/test_parser_default.py: -------------------------------------------------------------------------------- 1 | """Tests for parser.__init__.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "config, expected", [pytest.param({"_attr_": "list", "_eval_": "partial", "_args_": [[1]]}, lambda: [1])] 10 | ) 11 | def test_parser_default_parser(config, expected): 12 | """Test default parse.""" 13 | parser = fromconfig.parser.DefaultParser() 14 | parsed = parser(config) 15 | if callable(expected): 16 | assert fromconfig.fromconfig(parsed)() == expected() 17 | else: 18 | assert fromconfig.fromconfig(parsed) == expected 19 | -------------------------------------------------------------------------------- /tests/unit/parser/test_parser_evaluate.py: -------------------------------------------------------------------------------- 1 | """Tests for parser.evaluate.""" 2 | 3 | import pytest 4 | from unittest import mock 5 | 6 | import fromconfig 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "config, expected", 11 | [ 12 | pytest.param(None, None, id="none"), 13 | pytest.param({"a": 1, "b": 2}, {"a": 1, "b": 2}, id="dummy_mapping"), 14 | pytest.param({"_attr_": "str", "_eval_": "import"}, str, id="import"), 15 | pytest.param({"_attr_": "str", "_args_": ["hello"], "_eval_": "call"}, "hello", id="call"), 16 | pytest.param({"_attr_": "str", "_args_": ["hello"], "_eval_": "partial"}, lambda: "hello", id="partial"), 17 | pytest.param( 18 | {"_attr_": "str", "_args_": [{"hello": "world"}], "_eval_": "partial"}, 19 | lambda: "{'hello': 'world'}", 20 | id="partial_with_dummy_mapping", 21 | ), 22 | ], 23 | ) 24 | def test_parser_evaluate(config, expected): 25 | """Test parser.EvaluateParser.""" 26 | parser = fromconfig.parser.EvaluateParser() 27 | parsed = parser(config) 28 | if callable(expected): 29 | assert fromconfig.fromconfig(parsed)() == expected() 30 | else: 31 | assert fromconfig.fromconfig(parsed) == expected 32 | 33 | 34 | def fake_data_loader(path, number_of_line, is_server_up=False, use_custom_header=False): 35 | return ( 36 | f"Loaded {number_of_line} lines of data from {path}. Server was up: {is_server_up}, " 37 | f"custom header was used: {use_custom_header}" 38 | ) 39 | 40 | 41 | def find_latest_path(): 42 | """This function returns the path of the data to be loaded. 43 | This path could be unknown at the launch of the pipeline. 44 | """ 45 | return "latest/path/to/data" 46 | 47 | 48 | def ping_server(server_address): 49 | """This function pings the server to determine if it is up. 50 | It will be called when the data loader will be launched. 51 | """ 52 | return server_address == "localhost" 53 | 54 | 55 | @mock.patch(__name__ + ".find_latest_path", return_value=find_latest_path()) 56 | @mock.patch(__name__ + ".ping_server", return_value=ping_server("localhost")) 57 | def test_parser_evaluate_lazy(find_latest_path_mock, ping_server_mock): 58 | """Test that lazy arguments are evaluated after the function is called.""" 59 | 60 | config = { 61 | "_attr_": "fake_data_loader", 62 | "_args_": [{"_attr_": "find_latest_path", "_eval_": "lazy"}, 1000], 63 | "is_server_up": {"_attr_": "ping_server", "server_address": "localhost", "_eval_": "lazy"}, 64 | "use_custom_header": False, 65 | "_eval_": "partial", 66 | } 67 | parser = fromconfig.parser.EvaluateParser() 68 | parsed = parser(config) 69 | data_loader_job = fromconfig.fromconfig(parsed) 70 | # The lazy constructors should not have been called before the job has been launched 71 | find_latest_path_mock.assert_not_called() 72 | ping_server_mock.assert_not_called() 73 | 74 | expected_answer = ( 75 | "Loaded 1000 lines of data from latest/path/to/data. Server was up: True, custom header was used: False" 76 | ) 77 | assert data_loader_job() == expected_answer 78 | find_latest_path_mock.assert_called_once() 79 | ping_server_mock.assert_called_once() 80 | 81 | 82 | def fake_heavy_computations(): 83 | """This function performs heavy computations, therefore we want to evaluate it only once.""" 84 | return 2 85 | 86 | 87 | def multiply(a, b): 88 | return a * b 89 | 90 | 91 | @mock.patch(__name__ + ".fake_heavy_computations", return_value=fake_heavy_computations()) 92 | def test_parser_evaluate_lazy_with_memoization(heavy_computations_mock): 93 | """Test that lazy arguments with memoization are only evaluated once.""" 94 | config = { 95 | "_attr_": "multiply", 96 | "_eval_": "partial", 97 | "a": {"_attr_": "fake_heavy_computations", "_eval_": "lazy", "_memoization_key_": "fake_heavy_computations"}, 98 | "b": {"_attr_": "fake_heavy_computations", "_eval_": "lazy", "_memoization_key_": "fake_heavy_computations"}, 99 | } 100 | parser = fromconfig.parser.EvaluateParser() 101 | parsed = parser(config) 102 | multiply_partial = fromconfig.fromconfig(parsed) 103 | heavy_computations_mock.assert_not_called() 104 | assert multiply_partial() == 4 105 | heavy_computations_mock.assert_called_once() 106 | 107 | 108 | @pytest.mark.parametrize( 109 | "config, exception", 110 | [ 111 | pytest.param({"_attr_": "str", "_args_": ["hello"], "_eval_": "import"}, ValueError, id="import+args"), 112 | pytest.param({"_attr_": "dict", "x": 1, "_eval_": "import"}, ValueError, id="import+kwargs"), 113 | pytest.param({"x": 1, "_eval_": "import"}, KeyError, id="import+noattr"), 114 | pytest.param({"x": 1, "_eval_": "partial"}, KeyError, id="partial+noattr"), 115 | ], 116 | ) 117 | def test_parser_evaluate_exceptions(config, exception): 118 | """Test parser.EvaluateParser exceptions.""" 119 | parser = fromconfig.parser.EvaluateParser() 120 | with pytest.raises(exception): 121 | parser(config) 122 | -------------------------------------------------------------------------------- /tests/unit/parser/test_parser_omegaconf.py: -------------------------------------------------------------------------------- 1 | """Test for parser.omega""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "config, expected", 10 | [ 11 | pytest.param(None, None, id="none"), 12 | pytest.param({"foo": "bar"}, {"foo": "bar"}, id="none"), 13 | pytest.param({"foo": "bar", "baz": "${foo}"}, {"foo": "bar", "baz": "bar"}, id="vanilla"), 14 | ], 15 | ) 16 | def test_parser_omega(config, expected): 17 | """Test that OmegaConfParser accepts different types.""" 18 | parser = fromconfig.parser.OmegaConfParser() 19 | assert parser(config) == expected 20 | 21 | 22 | def hello(s): 23 | return f"hello {s}" 24 | 25 | 26 | @pytest.mark.parametrize( 27 | "resolvers, error", 28 | [ 29 | pytest.param({"hello": "hello"}, False), 30 | pytest.param({"hello": {"_attr_": "fromconfig.utils.from_import_string", "_args_": ["hello"]}}, False), 31 | pytest.param({"hello": ["hello"]}, True), 32 | ], 33 | ) 34 | def test_parser_omega_resolvers(resolvers, error): 35 | """Test OmegaConfParser.""" 36 | config = {"hello_world": "${hello:world}", "date": "${now:}", "resolvers": resolvers} 37 | parser = fromconfig.parser.OmegaConfParser() 38 | if error: 39 | with pytest.raises(Exception): 40 | parsed = parser(config) 41 | else: 42 | parsed = parser(config) 43 | assert parsed["hello_world"] == "hello world" # Custom resolver 44 | assert "$" not in parsed["date"] # Make sure now was resolved 45 | -------------------------------------------------------------------------------- /tests/unit/parser/test_parser_singleton.py: -------------------------------------------------------------------------------- 1 | """Test for parser.singleton.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | class Class: 9 | """Class.""" 10 | 11 | 12 | def test_parser_singleton_constructor(): 13 | """Test parser.singleton decorator.""" 14 | x = fromconfig.parser.singleton("name", Class) 15 | y = fromconfig.parser.singleton("name", Class) 16 | z = fromconfig.parser.singleton("name") 17 | assert id(x) == id(y) == id(z) 18 | 19 | 20 | def test_parser_singleton_constructor_missing(): 21 | """Test parser.singleton decorator with missing constructor.""" 22 | with pytest.raises(ValueError): 23 | fromconfig.parser.singleton("missing_constructor") 24 | 25 | 26 | def test_parser_singleton_constructor_no_override(): 27 | """Test parser.singleton decorator raises error with override.""" 28 | fromconfig.parser.singleton("name", Class) 29 | with pytest.raises(KeyError): 30 | fromconfig.parser.singleton["name"] = Class 31 | 32 | 33 | def test_parser_singleton(): 34 | """Test parser.SingletonParser.""" 35 | parser = fromconfig.parser.SingletonParser() 36 | config = { 37 | "x": {"_attr_": "dict", "_singleton_": "my_dict", "a": "a"}, 38 | "y": {"_attr_": "dict", "_singleton_": "my_dict", "a": "a"}, 39 | } 40 | parsed = parser(config) 41 | instance = fromconfig.fromconfig(parsed) 42 | x = instance["x"] 43 | y = instance["y"] 44 | expected = {"a": "a"} 45 | assert x == y == expected 46 | assert id(x) == id(y) 47 | -------------------------------------------------------------------------------- /tests/unit/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criteo/fromconfig/721ff5a72530b1718d2bcd884f673165f8ac937d/tests/unit/utils/__init__.py -------------------------------------------------------------------------------- /tests/unit/utils/test_utils_libimport.py: -------------------------------------------------------------------------------- 1 | """Tests for utils.libimport.""" 2 | 3 | import functools 4 | 5 | import pytest 6 | 7 | import fromconfig 8 | 9 | 10 | @pytest.mark.parametrize( 11 | "name, expected", [pytest.param("functools", functools), pytest.param("package_that_does_not_exist", None)] 12 | ) 13 | def test_try_import(name, expected): 14 | """Test utils.try_import.""" 15 | assert fromconfig.utils.try_import(name) == expected 16 | 17 | 18 | class Class: 19 | """Class.""" 20 | 21 | VARIABLE = "VARIABLE" 22 | 23 | def method(self): 24 | """Method.""" 25 | 26 | 27 | def function(): 28 | """Gunction.""" 29 | 30 | 31 | @pytest.mark.parametrize( 32 | "name,expected", 33 | [ 34 | pytest.param("", ImportError, id="empty"), 35 | pytest.param(".", ImportError, id="no-parts"), 36 | pytest.param("dict", dict, id="dict"), 37 | pytest.param("list", list, id="list"), 38 | pytest.param("function", function, id="local"), 39 | pytest.param("tests.unit.utils.test_utils_libimport.Class", Class, id="Class"), 40 | pytest.param("tests.unit.utils.test_utils_libimport.Class.method", Class.method, id="method"), 41 | pytest.param("tests.unit.utils.test_utils_libimport.Class.VARIABLE", Class.VARIABLE, id="VARIABLE"), 42 | pytest.param("tests.unit.utils.test_utils_libimport.function", function, id="function"), 43 | pytest.param("functools.partial", functools.partial, id="functools"), 44 | ], 45 | ) 46 | def test_utils_from_import_string(name, expected): 47 | """Test utils.from_import_string.""" 48 | if expected is ImportError: 49 | with pytest.raises(ImportError): 50 | fromconfig.utils.from_import_string(name) 51 | else: 52 | assert fromconfig.utils.from_import_string(name) == expected 53 | 54 | 55 | @pytest.mark.parametrize( 56 | "attr,name", 57 | [ 58 | pytest.param("A string", ValueError, id="string"), 59 | pytest.param(Class.VARIABLE, ValueError, id="VARIABLE"), 60 | pytest.param(fromconfig, "fromconfig", id="module"), 61 | pytest.param(str, "str", id="str"), 62 | pytest.param(dict, "dict", id="dict"), 63 | pytest.param(list, "list", id="list"), 64 | pytest.param(Class, "tests.unit.utils.test_utils_libimport.Class", id="Class"), 65 | pytest.param(Class.method, "tests.unit.utils.test_utils_libimport.Class.method", id="method"), 66 | pytest.param(function, "tests.unit.utils.test_utils_libimport.function", id="function"), 67 | pytest.param(functools.partial, "functools.partial", id="functools"), 68 | pytest.param(range(2), ValueError, id="generator"), 69 | ], 70 | ) 71 | def test_utils_to_import_string(attr, name): 72 | """Test utils.to_import_string.""" 73 | if name is ValueError: 74 | with pytest.raises(ValueError): 75 | fromconfig.utils.to_import_string(attr) 76 | else: 77 | assert fromconfig.utils.to_import_string(attr) == name 78 | -------------------------------------------------------------------------------- /tests/unit/utils/test_utils_nest.py: -------------------------------------------------------------------------------- 1 | """Test for utils.nest.""" 2 | 3 | import copy 4 | import numbers 5 | from functools import partial 6 | import pytest 7 | 8 | import fromconfig 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "config, expected", 13 | [ 14 | pytest.param({"x": 1}, [("x", 1)]), 15 | pytest.param({"x": {"y": 1}}, [("x.y", 1)]), 16 | pytest.param({"x": [1]}, [("x[0]", 1)]), 17 | ], 18 | ) 19 | def test_utils_flatten_expand(config, expected): 20 | """Test utils.flatten.""" 21 | assert fromconfig.utils.flatten(config) == expected 22 | assert fromconfig.utils.expand(expected) == config 23 | 24 | 25 | @pytest.mark.parametrize("config", [pytest.param({"a.b": {"c": "d"}}, id="key-has-dot")]) 26 | def test_utils_flatten_impossible(config): 27 | with pytest.raises(ValueError): 28 | fromconfig.utils.flatten(config) 29 | 30 | 31 | @pytest.mark.parametrize( 32 | "item1, item2, expected", 33 | [ 34 | pytest.param({"x": 1}, {"y": 2}, {"x": 1, "y": 2}, id="simple"), 35 | pytest.param({"x": 1}, {"x": 2}, {"x": 2}, id="override"), 36 | ], 37 | ) 38 | def test_utils_merge_dict(item1, item2, expected): 39 | """Test utils.merge_dict.""" 40 | assert fromconfig.utils.merge_dict(item1, item2, allow_override=True) == expected 41 | 42 | 43 | @pytest.mark.parametrize( 44 | "item1, item2, allow_override, error", 45 | [pytest.param({"x": 1}, [1], True, TypeError), pytest.param({"x": 1}, {"x": 2}, False, ValueError)], 46 | ) 47 | def test_utils_merge_dict_errors(item1, item2, allow_override, error): 48 | with pytest.raises(error): 49 | fromconfig.utils.merge_dict(item1, item2, allow_override=allow_override) 50 | 51 | 52 | def inc(item, value): 53 | """Increment item with value if item is Number.""" 54 | if isinstance(item, numbers.Number): 55 | return item + value 56 | return item 57 | 58 | 59 | @pytest.mark.parametrize( 60 | "map_fn, item, expected", 61 | [ 62 | pytest.param(partial(inc, value=1), 1, 2, id="int"), 63 | pytest.param(partial(inc, value=1), [1], [2], id="list"), 64 | pytest.param(partial(inc, value=1), {"x": 1}, {"x": 2}, id="dict"), 65 | pytest.param(partial(inc, value=1), {"x": [1]}, {"x": [2]}, id="dict+list"), 66 | pytest.param(partial(inc, value=1), {"x": {"y": 1}}, {"x": {"y": 2}}, id="dict+dict"), 67 | pytest.param(partial(inc, value=1), "a", "a", id="string"), 68 | pytest.param(partial(inc, value=1), {"x": 1, "a": "a"}, {"x": 2, "a": "a"}, id="dict+string"), 69 | ], 70 | ) 71 | def test_utils_depth_map(map_fn, item, expected): 72 | """Test utils.depth_map.""" 73 | original = copy.deepcopy(item) 74 | got = fromconfig.utils.depth_map(map_fn, item) 75 | assert got == expected, f"{got} != {expected}" 76 | assert original == item, "depth_map should not mutate" 77 | -------------------------------------------------------------------------------- /tests/unit/utils/test_utils_strenum.py: -------------------------------------------------------------------------------- 1 | """Tests for utils.strenum.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | class CustomEnum(fromconfig.utils.StrEnum): 9 | """Custom Enum.""" 10 | 11 | A = "A" 12 | B = "B" 13 | 14 | 15 | @pytest.mark.parametrize( 16 | "before, after", 17 | [ 18 | pytest.param("A", CustomEnum.A, id="stringA+enumA"), 19 | pytest.param("B", CustomEnum.B, id="stringB+enumB"), 20 | pytest.param("C", ValueError, id="stringC"), 21 | pytest.param(CustomEnum.A, CustomEnum.A, id="enumA+enumA"), 22 | pytest.param(CustomEnum.B, CustomEnum.B, id="enumB+enumB"), 23 | ], 24 | ) 25 | def test_utils_strenum_resolution(before, after): 26 | """Test utils.strenum resolution.""" 27 | if isinstance(after, CustomEnum): 28 | assert CustomEnum(before) is after # "is" stronger than == 29 | else: 30 | with pytest.raises(after): 31 | CustomEnum(before) 32 | 33 | 34 | @pytest.mark.parametrize( 35 | "left, right, expected", 36 | [ 37 | pytest.param("A", CustomEnum.A, True, id="stringA+enumA"), 38 | pytest.param("B", CustomEnum.B, True, id="stringB+enumB"), 39 | pytest.param("A", CustomEnum.B, False, id="stringA+enumB"), 40 | pytest.param("B", CustomEnum.A, False, id="stringB+enumA"), 41 | pytest.param("C", CustomEnum.A, False, id="stringC+enumA"), 42 | pytest.param("C", CustomEnum.B, False, id="stringC+enumB"), 43 | pytest.param(CustomEnum.A, CustomEnum.A, True, id="enumA+enumA"), 44 | pytest.param(CustomEnum.B, CustomEnum.B, True, id="enumB+enumB"), 45 | pytest.param(CustomEnum.A, CustomEnum.B, False, id="enumA+enumB"), 46 | pytest.param(CustomEnum.B, CustomEnum.A, False, id="enumB+enumA"), 47 | ], 48 | ) 49 | def test_utils_strenum_equality(left, right, expected): 50 | """Test utils.strenum equality.""" 51 | assert (left == right) == expected 52 | 53 | 54 | @pytest.mark.parametrize( 55 | "member, expected", 56 | [ 57 | pytest.param("A", True, id="stringA"), 58 | pytest.param("B", True, id="stringB"), 59 | pytest.param("C", False, id="stringC"), 60 | pytest.param(CustomEnum.A, True, id="enumA"), 61 | pytest.param(CustomEnum.B, True, id="enumB"), 62 | ], 63 | ) 64 | def test_utils_strenum_membership(member, expected): 65 | """Test utils.strenum membership.""" 66 | assert (member in CustomEnum) == expected 67 | -------------------------------------------------------------------------------- /tests/unit/utils/test_utils_types.py: -------------------------------------------------------------------------------- 1 | """Tests for utils.container.""" 2 | 3 | import pytest 4 | 5 | import fromconfig 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "obj, expected", 10 | [ 11 | pytest.param([], False, id="list"), 12 | pytest.param((), False, id="tuple"), 13 | pytest.param({}, True, id="dict"), 14 | pytest.param(fromconfig.Config(), True, id="Config"), 15 | ], 16 | ) 17 | def test_utils_is_mapping(obj, expected): 18 | """Test utils.is_mapping.""" 19 | assert fromconfig.utils.is_mapping(obj) == expected 20 | 21 | 22 | @pytest.mark.parametrize( 23 | "obj, expected", 24 | [ 25 | pytest.param([], True, id="list"), 26 | pytest.param({}, False, id="dict"), 27 | pytest.param("hello", False, id="string"), 28 | pytest.param((), True, id="tuple"), 29 | ], 30 | ) 31 | def test_utils_is_pure_iterable(obj, expected): 32 | """Test utils.is_pure_iterable.""" 33 | assert fromconfig.utils.is_pure_iterable(obj) == expected 34 | --------------------------------------------------------------------------------