├── .coveragerc ├── .editorconfig ├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ ├── codeql-analysis.yml │ ├── docs.yml │ └── publish-to-pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── AUTHORS.md ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── ThirdPartyNotices.txt ├── docs ├── .vitepress │ ├── components │ │ ├── ApiExplorer.vue │ │ ├── Pydoc.vue │ │ └── Tree.vue │ ├── config.js │ ├── public │ │ ├── CNAME │ │ ├── favicon.ico │ │ ├── icons │ │ │ ├── android-chrome-192x192.png │ │ │ ├── android-chrome-512x512.png │ │ │ ├── apple-touch-icon.png │ │ │ ├── favicon-16x16.png │ │ │ ├── favicon-32x32.png │ │ │ ├── mstile-150x150.png │ │ │ └── safari-pinned-tab.svg │ │ └── manifest.json │ ├── pydocgen.py │ └── theme │ │ ├── custom.css │ │ └── index.js ├── about │ └── approach.md ├── examples │ ├── aimstack-storage │ │ ├── aimstack.py │ │ ├── index.md │ │ └── test_aimstack.py │ ├── dependent-schedules │ │ ├── dependency_graph.py │ │ ├── index.md │ │ └── test_dependency_graph.py │ ├── dmosopt-component │ │ ├── dmosopt.py │ │ ├── index.md │ │ └── test_dmosopt.py │ ├── globus-storage │ │ ├── globus.py │ │ ├── index.md │ │ └── test_globus.py │ ├── index.md │ ├── mpi-execution │ │ ├── index.md │ │ ├── mpi.py │ │ └── test_mpi.py │ ├── require-execution │ │ ├── index.md │ │ ├── require.py │ │ └── test_require.py │ ├── scopes │ │ ├── group.py │ │ └── test_group_scope.py │ └── slurm-execution │ │ ├── index.md │ │ ├── slurm.py │ │ └── test_slurm.py ├── guide │ ├── cli.md │ ├── component.md │ ├── element.md │ ├── execution.md │ ├── installation.md │ ├── interface.md │ └── introduction.md ├── index.md ├── logo │ ├── logo.png │ └── logo.svg └── reference │ └── index.md ├── package-lock.json ├── package.json ├── poetry.lock ├── pyproject.toml ├── src └── machinable │ ├── __init__.py │ ├── cli.py │ ├── collection.py │ ├── component.py │ ├── config.py │ ├── element.py │ ├── errors.py │ ├── execution.py │ ├── index.py │ ├── interface.py │ ├── mixin.py │ ├── project.py │ ├── py.typed │ ├── query.py │ ├── schedule.py │ ├── schema.py │ ├── scope.py │ ├── storage.py │ ├── types.py │ └── utils.py └── tests ├── __init__.py ├── conftest.py ├── samples ├── importing │ ├── __init__.py │ ├── nested │ │ ├── base.py │ │ └── bottom.py │ └── top.py ├── in_session.py └── project │ ├── .gitignore │ ├── basic.py │ ├── count.py │ ├── dummy.py │ ├── empty.py │ ├── exec.py │ ├── fail.py │ ├── hello.py │ ├── interface │ ├── dummy.py │ ├── events_check.py │ ├── interrupted_lifecycle.py │ └── project.py │ ├── line.py │ ├── mixins │ ├── example.py │ └── extension.py │ ├── predicate.py │ └── scheduled.py ├── test_cli.py ├── test_collection.py ├── test_component.py ├── test_config.py ├── test_docs.py ├── test_element.py ├── test_execution.py ├── test_index.py ├── test_interface.py ├── test_mixin.py ├── test_project.py ├── test_query.py ├── test_schedule.py ├── test_scope.py ├── test_storage.py └── test_utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | source = machinable 5 | # omit = bad_file.py 6 | 7 | [paths] 8 | source = 9 | src/ 10 | */site-packages/ 11 | 12 | [report] 13 | show_missing = True 14 | # Regexes for lines to exclude from consideration 15 | exclude_lines = 16 | # Have to re-enable the standard pragma 17 | pragma: no cover 18 | 19 | # Don't complain about missing debug-only code: 20 | def __repr__ 21 | if self\.debug 22 | 23 | # Don't complain if tests don't hit defensive assertion code: 24 | raise AssertionError 25 | raise NotImplementedError 26 | 27 | # Don't complain if non-runnable code isn't run: 28 | if 0: 29 | if __name__ == .__main__.: 30 | if TYPE_CHECKING: 31 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Check http://editorconfig.org for more information 2 | # This is the main config file for this project: 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | indent_style = space 10 | indent_size = 2 11 | trim_trailing_whitespace = true 12 | 13 | [*.{py, pyi}] 14 | indent_style = space 15 | indent_size = 4 16 | 17 | [*.md] 18 | trim_trailing_whitespace = false 19 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | allow: 8 | - dependency-type: "direct" 9 | commit-message: 10 | prefix: ":arrow_up:" 11 | - package-ecosystem: "github-actions" 12 | directory: "/" 13 | schedule: 14 | interval: "weekly" 15 | allow: 16 | - dependency-type: "all" 17 | commit-message: 18 | prefix: ":arrow_up:" 19 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: ["3.8", "3.9", "3.10", "3.11"] 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v4.7.0 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install Poetry 23 | uses: snok/install-poetry@v1.3 24 | with: 25 | virtualenvs-create: true 26 | virtualenvs-in-project: true 27 | - name: Set up cache 28 | id: cached-poetry-dependencies 29 | uses: actions/cache@v3.3.1 30 | with: 31 | path: .venv 32 | key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} 33 | - name: Install dependencies 34 | run: poetry install --extras "all" 35 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' 36 | - name: black formatting check 37 | run: | 38 | source .venv/bin/activate 39 | black --config pyproject.toml --check ./ 40 | - name: pyuprade check 41 | run: | 42 | source .venv/bin/activate 43 | pyupgrade --py38-plus 44 | - name: isort check 45 | run: | 46 | source .venv/bin/activate 47 | isort . --settings-path pyproject.toml --check 48 | - name: editorconfig check 49 | run: | 50 | source .venv/bin/activate 51 | ec 52 | - name: Run tests 53 | run: | 54 | source .venv/bin/activate 55 | pytest 56 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | # The branches below must be a subset of the branches above 8 | branches: [ main ] 9 | schedule: 10 | - cron: '28 22 * * 0' 11 | 12 | jobs: 13 | analyze: 14 | name: Analyze 15 | runs-on: ubuntu-latest 16 | permissions: 17 | actions: read 18 | contents: read 19 | security-events: write 20 | 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | language: [ 'python' ] 25 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 26 | # Learn more: 27 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 28 | 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@v3 32 | 33 | # Initializes the CodeQL tools for scanning. 34 | - name: Initialize CodeQL 35 | uses: github/codeql-action/init@v2 36 | with: 37 | languages: ${{ matrix.language }} 38 | # If you wish to specify custom queries, you can do so here or in a config file. 39 | # By default, queries listed here will override any specified in a config file. 40 | # Prefix the list here with "+" to use these queries and those in the config file. 41 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 42 | 43 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 44 | # If this step fails, then you should remove it and run the build manually (see below) 45 | - name: Autobuild 46 | uses: github/codeql-action/autobuild@v2 47 | 48 | # ℹ️ Command-line programs to run using the OS shell. 49 | # 📚 https://git.io/JvXDl 50 | 51 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 52 | # and modify them (or add more) to build your code if your project 53 | # uses a compiled language 54 | 55 | #- run: | 56 | # make bootstrap 57 | # make release 58 | 59 | - name: Perform CodeQL Analysis 60 | uses: github/codeql-action/analyze@v2 61 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: [main] 5 | jobs: 6 | build-and-deploy: 7 | concurrency: ci-${{ github.ref }} 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v3 12 | - name: Set up Python 13 | uses: actions/setup-python@v4.7.0 14 | with: 15 | python-version: "3.x" 16 | - name: Install Poetry 17 | uses: snok/install-poetry@v1.3 18 | with: 19 | virtualenvs-create: true 20 | virtualenvs-in-project: true 21 | - name: Set up cache 22 | id: cached-poetry-dependencies 23 | uses: actions/cache@v3.3.1 24 | with: 25 | path: .venv 26 | key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} 27 | - name: Install dependencies 28 | run: poetry install --extras "all" 29 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' 30 | - name: Autogenerate reference documentation 31 | run: | 32 | source .venv/bin/activate 33 | python docs/.vitepress/pydocgen.py 34 | - name: Install vitepress 35 | uses: bahmutov/npm-install@v1.8.34 36 | - name: Build 37 | run: | 38 | /home/runner/work/machinable/machinable/node_modules/vitepress/bin/vitepress.js build docs 39 | cp -r docs/.vitepress/public/* docs/.vitepress/dist/ 40 | cp -r docs/logo docs/.vitepress/dist/ 41 | - name: Deploy 42 | uses: JamesIves/github-pages-deploy-action@v4.4.3 43 | with: 44 | branch: gh-pages 45 | folder: docs/.vitepress/dist 46 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Set up Python 13 | uses: actions/setup-python@v4.7.0 14 | with: 15 | python-version: "3.x" 16 | - name: Install Poetry 17 | uses: snok/install-poetry@v1.3 18 | - name: Build 19 | run: | 20 | poetry build 21 | - name: Publish distribution 📦 to PyPI 22 | if: startsWith(github.ref, 'refs/tags') 23 | uses: pypa/gh-action-pypi-publish@release/v1 24 | with: 25 | password: ${{ secrets.PYPI_API_TOKEN }} 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Test data 2 | /storage 3 | tests/.hypothesis 4 | **/vendor/.cache 5 | .hypothesis 6 | tests/samples/project/interface/remotes 7 | 8 | # Docs 9 | node_modules 10 | docs/.vitepress/dist 11 | docs/.vitepress/pydoc.js 12 | docs/.vitepress/.cache 13 | docs/.vitepress/cache 14 | 15 | # Temporary and binary files 16 | *~ 17 | .temp 18 | *.py[cod] 19 | *.so 20 | *.cfg 21 | !.isort.cfg 22 | !setup.cfg 23 | *.orig 24 | *.log 25 | *.pot 26 | __pycache__/* 27 | .cache/* 28 | .*.swp 29 | */.ipynb_checkpoints/* 30 | .DS_Store 31 | 32 | # Project files 33 | .vscode 34 | .ropeproject 35 | .project 36 | .pydevproject 37 | .settings 38 | .idea 39 | tags 40 | 41 | # Package files 42 | *.egg 43 | *.eggs/ 44 | .installed.cfg 45 | *.egg-info 46 | 47 | # Unittest and coverage 48 | htmlcov/* 49 | .coverage 50 | .coverage* 51 | .tox 52 | junit.xml 53 | coverage.xml 54 | .pytest_cache/ 55 | 56 | # Build and docs folder/files 57 | build/* 58 | dist/* 59 | sdist/* 60 | docs/api/* 61 | docs/_rst/* 62 | docs/_build/* 63 | cover/* 64 | MANIFEST 65 | 66 | # Per-project virtualenvs 67 | .venv*/ 68 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_stages: [commit, push] 2 | 3 | repos: 4 | - repo: local 5 | hooks: 6 | - id: black 7 | name: black 8 | entry: black --config pyproject.toml 9 | types: [python] 10 | language: system 11 | - repo: local 12 | hooks: 13 | - id: pyupgrade 14 | name: pyupgrade 15 | entry: pyupgrade --py38-plus 16 | types: [python] 17 | language: system 18 | - repo: local 19 | hooks: 20 | - id: isort 21 | name: isort 22 | entry: isort --settings-path pyproject.toml 23 | types: [python] 24 | language: system 25 | - repo: local 26 | hooks: 27 | - id: editorconfig 28 | name: editorconfig 29 | entry: ec 30 | types: [python] 31 | language: system 32 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Authors 2 | 3 | - Frithjof Gressmann (@frthjf) (Creator) 4 | 5 | ## Contributors 6 | 7 | - Seth Nabarro 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | 5 | # Unreleased 6 | 7 | No current changes 8 | 9 | # v4.10.6 10 | 11 | - Support sqlean.py 12 | - Include defaults in cachable key 13 | 14 | # v4.10.5 15 | 16 | - Upgrade omegaconf 17 | 18 | # v4.10.4 19 | 20 | - Respect .gitignore in code rsync 21 | - Prevents false positive config method matches 22 | - Adds CWD to PATH when using CLI 23 | - Load from directory in dispatch code 24 | 25 | # v4.10.3 26 | 27 | - Option to hide interfaces from get 28 | - Simplified `**kwargs` arguments in CLI 29 | - Adds save/load_attribute helper 30 | - Adds `@cachable` decorator utility 31 | 32 | # v4.10.2 33 | 34 | - Eager callable resolution in versions 35 | - Allow module dependencies in on_resolve_remotes 36 | - Fix CLI version parsing issue 37 | - Improved version call normalizer 38 | - Prevent recursions in self.future() calls 39 | 40 | # v4.10.1 41 | 42 | - Consistent `future()` behavior for Interface and Component 43 | - Support multi-line version arguments 44 | 45 | # v4.10.0 46 | 47 | - New configure and commit events 48 | - Support interface **kwargs in CLI 49 | - Adds `get.cached_or_fail` 50 | - Move `_machinable/project` to `interface/project` 51 | - Adds `Interface.future()` 52 | - Enable custom context lookups in index 53 | - Adds `utils.file_hash` 54 | - Adds `Execution().deferred()` to prevent automatic dispatch 55 | - Respect CLI context order 56 | 57 | # v4.9.2 58 | 59 | - Determine CLI target based on order to allow non-component targets 60 | - Ensure that config field is always reloaded from index to avoid incorrect recomputation 61 | 62 | # v4.9.1 63 | 64 | - Use text-based link relation by default 65 | 66 | # v4.9.0 67 | 68 | - Adds `Interface.related_iterator()` 69 | 70 | # v4.9.0 71 | 72 | - Use nanoseconds since epoch in timestamp 73 | - Adds `get.from_directory` and `get.by_id` to query 74 | - Introduces stable IDs based on the commit-context 75 | - Saves inverse relations and relationship meta-data in local directory 76 | - Adds index.import_directory method 77 | - Allows search by short ID instead of UUID 78 | - Adds storage upload/download methods 79 | 80 | # v4.8.4 81 | 82 | - Adds Interface.related() 83 | - Improved Globus storage example 84 | 85 | # v4.8.3 86 | 87 | - Ensure that committed in-session interface code is being updated 88 | 89 | # v4.8.2 90 | 91 | - Fix issue where extra data from schema is not reloaded from index 92 | 93 | # v4.8.1 94 | 95 | - Allow version extension via on_resolve_element 96 | - Resolve remotes during project element import 97 | - Allow arbitrary classes within on_resolve_element 98 | - Ignore errors during storage symlink creation 99 | 100 | # v4.8.0 101 | 102 | - Only unflatten version at the top-level by default 103 | - Adds `get.prefer_cached` modifier 104 | - Uses index and lazy directory fetch for faster interface retrieval 105 | - Adds storage relation symlink for easier directory navigation 106 | - Uses reversed uuid7 non-hex notation for easier tab-complete 107 | 108 | # v4.7.0 109 | 110 | - Support element list or instance in `get` for easy extension 111 | - Configurable project and python in component dispatch code 112 | - Adds shell helpers `utils.run_and_stream` and `utils.chmodx` 113 | - Supports get modifiers in CLI, closely matching the Python API 114 | 115 | # v4.6.3 116 | 117 | - Improves element string representation 118 | - Adds `execution.output_filepath` and `execution.component_directory` 119 | - Reject stale context matches in `index.find` 120 | 121 | # v4.6.2 122 | 123 | - Introduces cached `computed_resources` to supersede `compute_resources` 124 | - Fixes exception handling if raised within execution context 125 | 126 | # v4.6.1 127 | 128 | - Adds a priori modifiers `get.all`, `get.new` etc. 129 | - Remove experimental enderscore feature 130 | 131 | # v4.6.0 132 | 133 | - Leverages UUID7 for timestamp information 134 | - Drops support for EOL Python 3.7 135 | - Upgrades to pydantic v2 136 | - Drops default settings parser 137 | - Handles non-existing keys in Element.matches scope lookup 138 | - Propagate exceptions during mixin getattr lookup 139 | 140 | # v4.5.0 141 | 142 | - Adds scopes to support context annotations 143 | - Adds all() and new() interface query modifiers 144 | - Adds config `to_dict` helper 145 | - Gracefully end output streaming on keyboard interrupt 146 | 147 | # v4.4.0 148 | 149 | - Improved tracking of execution meta-data 150 | - Reliable `.execution` access to make None-checks obsolete 151 | - Adds `stream_output` to simplify live monitoring of execution logs 152 | - Allow predicate specification based on the element kind 153 | - Prevent index errors in multithreaded environments 154 | - Disables automatic gathering of project relationship 155 | 156 | # v4.3.1 157 | 158 | - Drops commandlib dependency 159 | - Fix Component.execution resolution for resource retrieval 160 | 161 | # v4.3.0 162 | 163 | - Generalized storage (#437) 164 | - Support for in-session element instances 165 | - Allow unstructured untyped dict-config 166 | - Simplified event structure 167 | 168 | # v4.2.0 169 | 170 | - Convert multiple storage into regular element (#426) 171 | 172 | # v4.1 173 | 174 | - Revamped CLI using element version syntax (#422) 175 | - Allow arbitrary plain file extensions when saving/loading files 176 | - Filesystem storage captures groups within JSON files rather than directory name 177 | - Represent independent schedule via `None` 178 | - Adds Execution.on_verify_schedule to verify execution schedule compatibility 179 | 180 | # v4.0 181 | 182 | - Complete rewrite using elementary approach. Check out the documentation to learn more. 183 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019-20 Frithjof Gressmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # machinable 2 | 3 |
4 | 5 |
6 | 7 |
8 | 9 | [![Build status](https://github.com/machinable-org/machinable/workflows/build/badge.svg)](https://github.com/machinable-org/machinable/actions?query=workflow%3Abuild) 10 | [![Dependencies Status](https://img.shields.io/badge/dependencies-up%20to%20date-brightgreen.svg)](https://github.com/machinable-org/machinable/pulls?utf8=%E2%9C%93&q=is%3Apr%20author%3Aapp%2Fdependabot) 11 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 12 | [![License](https://img.shields.io/github/license/machinable-org/machinable)](https://github.com/machinable-org/machinable/blob/main/LICENSE) 13 | 14 |
15 | 16 |
17 | 18 | **machinable** provides a system to manage research code effectively. Using a unified and modular representation, machinable can help structure your projects in a principled way so you can move quickly while enabling reuse and collaboration. 19 | 20 | Visit the [documentation](https://machinable.org/) to get started. 21 | -------------------------------------------------------------------------------- /ThirdPartyNotices.txt: -------------------------------------------------------------------------------- 1 | machinable THIRD-PARTY SOFTWARE NOTICES AND INFORMATION 2 | 3 | This project incorporates components from the projects listed below as indicated by comments in the code base. 4 | The original copyright notices and the licenses under which such components were received are set forth below. 5 | 6 | 1. sdispater/backpack version 0.1 (https://github.com/sdispater/backpack) 7 | 8 | NOTICES AND INFORMATION BEGIN HERE 9 | ================================== 10 | 11 | Copyright (c) 2015 Sébastien Eustace 12 | 13 | Permission is hereby granted, free of charge, to any person obtaining 14 | a copy of this software and associated documentation files (the 15 | "Software"), to deal in the Software without restriction, including 16 | without limitation the rights to use, copy, modify, merge, publish, 17 | distribute, sublicense, and/or sell copies of the Software, and to 18 | permit persons to whom the Software is furnished to do so, subject to 19 | the following conditions: 20 | 21 | The above copyright notice and this permission notice shall be 22 | included in all copies or substantial portions of the Software. 23 | 24 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 28 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 29 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 30 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 31 | 32 | ========================================= 33 | END OF sdispater/backpack NOTICES AND INFORMATION 34 | 35 | 2. drgrib/dotmap version 1.3.8 (https://github.com/drgrib/dotmap) 36 | 37 | NOTICES AND INFORMATION BEGIN HERE 38 | ================================== 39 | 40 | The MIT License (MIT) 41 | 42 | Copyright (c) 2015 Chris Redford 43 | 44 | Permission is hereby granted, free of charge, to any person obtaining a copy 45 | of this software and associated documentation files (the "Software"), to deal 46 | in the Software without restriction, including without limitation the rights 47 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 48 | copies of the Software, and to permit persons to whom the Software is 49 | furnished to do so, subject to the following conditions: 50 | 51 | The above copyright notice and this permission notice shall be included in all 52 | copies or substantial portions of the Software. 53 | 54 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 55 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 56 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 57 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 58 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 59 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 60 | SOFTWARE. 61 | 62 | ========================================= 63 | END OF drgrib/dotmap NOTICES AND INFORMATION 64 | 65 | 3. IDSIA/sacred version 0.7.5 (https://github.com/IDSIA/sacred) 66 | 67 | NOTICES AND INFORMATION BEGIN HERE 68 | ================================== 69 | 70 | The MIT License (MIT) 71 | 72 | Copyright (c) 2014 Klaus Greff 73 | 74 | Permission is hereby granted, free of charge, to any person obtaining a copy 75 | of this software and associated documentation files (the "Software"), to deal 76 | in the Software without restriction, including without limitation the rights 77 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 78 | copies of the Software, and to permit persons to whom the Software is 79 | furnished to do so, subject to the following conditions: 80 | 81 | The above copyright notice and this permission notice shall be included in 82 | all copies or substantial portions of the Software. 83 | 84 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 85 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 86 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 87 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 88 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 89 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 90 | THE SOFTWARE. 91 | 92 | ========================================= 93 | END OF IDSIA/sacred NOTICES AND INFORMATION 94 | 95 | 4. MasoniteFramework/orm version 1.0 (https://github.com/MasoniteFramework/orm) 96 | 97 | NOTICES AND INFORMATION BEGIN HERE 98 | ================================== 99 | 100 | MIT License 101 | 102 | Copyright (c) 2020 Joseph Mancuso 103 | 104 | Permission is hereby granted, free of charge, to any person obtaining a copy 105 | of this software and associated documentation files (the "Software"), to deal 106 | in the Software without restriction, including without limitation the rights 107 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 108 | copies of the Software, and to permit persons to whom the Software is 109 | furnished to do so, subject to the following conditions: 110 | 111 | The above copyright notice and this permission notice shall be included in all 112 | copies or substantial portions of the Software. 113 | 114 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 115 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 116 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 117 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 118 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 119 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 120 | SOFTWARE. 121 | 122 | ========================================= 123 | END OF MasoniteFramework/orm NOTICES AND INFORMATION 124 | -------------------------------------------------------------------------------- /docs/.vitepress/components/ApiExplorer.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 44 | -------------------------------------------------------------------------------- /docs/.vitepress/components/Pydoc.vue: -------------------------------------------------------------------------------- 1 | 4 | 39 | -------------------------------------------------------------------------------- /docs/.vitepress/components/Tree.vue: -------------------------------------------------------------------------------- 1 | 9 | 14 | -------------------------------------------------------------------------------- /docs/.vitepress/config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitepress' 2 | 3 | export default defineConfig({ 4 | // site config 5 | lang: 'en-US', 6 | title: 'machinable', 7 | description: 'A modular configuration system for research projects', 8 | head: [ 9 | ['link', { rel: 'icon', href: `/logo.png` }], 10 | ['link', { rel: 'manifest', href: '/manifest.json' }], 11 | ['meta', { name: 'theme-color', content: '#3eaf7c' }], 12 | ['meta', { name: 'apple-mobile-web-app-capable', content: 'yes' }], 13 | ['meta', { name: 'apple-mobile-web-app-status-bar-style', content: 'black' }], 14 | ['link', { rel: 'apple-touch-icon', href: `/icons/apple-touch-icon.png` }], 15 | ['link', { rel: 'mask-icon', href: '/icons/safari-pinned-tab.svg', color: '#3eaf7c' }], 16 | ['meta', { name: 'msapplication-TileImage', content: '/icons/msapplication-icon-144x144.png' }], 17 | ['meta', { name: 'msapplication-TileColor', content: '#000000' }] 18 | ], 19 | themeConfig: { 20 | logo: '/logo/logo.svg', 21 | nav: [ 22 | { text: 'Guide', link: '/guide/introduction' }, 23 | { text: 'Reference', link: '/reference/' }, 24 | { text: 'Examples', link: '/examples/' }, 25 | { 26 | text: 'About', 27 | items: [ 28 | { text: "Approach", link: '/about/approach' }, 29 | { text: 'Changelog', link: 'https://github.com/machinable-org/machinable/blob/main/CHANGELOG.md' } 30 | ] 31 | } 32 | ], 33 | sidebar: { 34 | '/guide/': [ 35 | { 36 | text: 'Getting Started', 37 | items: [ 38 | { 39 | text: 'Introduction', 40 | link: '/guide/introduction' 41 | }, 42 | { 43 | text: 'Installation', 44 | link: '/guide/installation' 45 | } 46 | ] 47 | }, 48 | { 49 | text: 'Concepts', 50 | items: [ 51 | { 52 | text: 'Element', 53 | link: '/guide/element' 54 | }, 55 | { 56 | text: 'Interface', 57 | link: '/guide/interface' 58 | }, 59 | { 60 | text: 'Component', 61 | link: '/guide/component' 62 | }, 63 | ] 64 | }, 65 | { 66 | text: 'Basics', 67 | items: [ 68 | { 69 | text: 'Execution', 70 | link: '/guide/execution' 71 | }, 72 | { 73 | text: 'CLI', 74 | link: '/guide/cli' 75 | }, 76 | ] 77 | }, 78 | ], 79 | '/examples/': [ 80 | { 81 | text: 'Storage', 82 | items: [ 83 | { 84 | text: 'Aimstack', 85 | link: '/examples/aimstack-storage/' 86 | }, 87 | { 88 | text: 'Globus', 89 | link: '/examples/globus-storage/' 90 | } 91 | ] 92 | }, 93 | { 94 | text: 'Execution', 95 | items: [ 96 | { 97 | text: 'MPI', 98 | link: '/examples/mpi-execution/' 99 | }, 100 | { 101 | text: 'Slurm', 102 | link: '/examples/slurm-execution/' 103 | }, 104 | { 105 | text: 'Require', 106 | link: '/examples/require-execution/' 107 | }, 108 | ] 109 | }, 110 | { 111 | text: 'Component', 112 | items: [ 113 | { 114 | text: 'dmosopt', 115 | link: '/examples/dmosopt-component/' 116 | }, 117 | ] 118 | }, 119 | ] 120 | }, 121 | footer: { 122 | message: 'MIT Licensed', 123 | copyright: 'Copyright © 2021-present' 124 | }, 125 | socialLinks: [ 126 | { icon: 'github', link: 'https://github.com/machinable-org/machinable' } 127 | ], 128 | 129 | editLink: { 130 | pattern: 'https://github.com/machinable-org/machinable/edit/main/docs/:path', 131 | text: 'Edit this page on GitHub' 132 | }, 133 | }, 134 | markdown: { 135 | lineNumbers: false 136 | } 137 | }) 138 | -------------------------------------------------------------------------------- /docs/.vitepress/public/CNAME: -------------------------------------------------------------------------------- 1 | machinable.org 2 | -------------------------------------------------------------------------------- /docs/.vitepress/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/favicon.ico -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/icons/android-chrome-192x192.png -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/icons/android-chrome-512x512.png -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/icons/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/icons/favicon-16x16.png -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/icons/favicon-32x32.png -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/mstile-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/.vitepress/public/icons/mstile-150x150.png -------------------------------------------------------------------------------- /docs/.vitepress/public/icons/safari-pinned-tab.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 7 | 8 | Created by potrace 1.11, written by Peter Selinger 2001-2013 9 | 10 | 12 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /docs/.vitepress/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "machinable", 3 | "short_name": "machinable", 4 | "icons": [ 5 | { 6 | "src": "/icons/android-chrome-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "/icons/android-chrome-512x512.png", 12 | "sizes": "512x512", 13 | "type": "image/png" 14 | } 15 | ], 16 | "start_url": "/index.html", 17 | "display": "standalone", 18 | "background_color": "#fff", 19 | "theme_color": "#2c3e50" 20 | } 21 | -------------------------------------------------------------------------------- /docs/.vitepress/pydocgen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Builds a JSON index of available APIs""" 4 | 5 | import types 6 | 7 | import inspect 8 | import json 9 | import os 10 | import pkgutil 11 | import pydoc 12 | 13 | ROOT = os.path.dirname( 14 | os.path.dirname(os.path.abspath(os.path.dirname(__file__))) 15 | ) 16 | 17 | 18 | def _isclass(object): 19 | return inspect.isclass( 20 | object 21 | ) # and not isinstance(object, types.GenericAlias) 22 | 23 | 24 | def docmodule(db, name, path, object): 25 | all = getattr(object, "__all__", None) 26 | 27 | # classes 28 | for key, value in inspect.getmembers(object, _isclass): 29 | if all is not None or (inspect.getmodule(value) or object) is object: 30 | if pydoc.visiblename(key, all, object): 31 | docclass(db, name, path + "." + key, value) 32 | 33 | # functions 34 | for key, value in inspect.getmembers(object, inspect.isroutine): 35 | if ( 36 | all is not None 37 | or inspect.isbuiltin(value) 38 | or inspect.getmodule(value) is object 39 | ): 40 | if pydoc.visiblename(key, all, object): 41 | docroutine(db, name, path + "." + key, value) 42 | 43 | db[path] = {"kind": "module", "name": name, "path": path} 44 | 45 | 46 | def docclass(db, name, path, object): 47 | try: 48 | realname = object.__name__ 49 | except AttributeError: 50 | realname = None 51 | name = name or realname 52 | bases = object.__bases__ 53 | 54 | def makename(c, m=object.__module__): 55 | return pydoc.classname(c, m) 56 | 57 | attrs = [ 58 | (name, kind, cls, value) 59 | for name, kind, cls, value in pydoc.classify_class_attrs(object) 60 | if pydoc.visiblename(name, obj=object) 61 | ] 62 | 63 | for attr in attrs: 64 | if attr[0].startswith("__"): 65 | continue 66 | # todo: fine grained docroutine 67 | if attr[1] in ["method", "class method", "static method"]: 68 | docroutine(db, name, path + "." + attr[0], attr[3]) 69 | 70 | db[path] = { 71 | "kind": "class", 72 | "realname": realname, 73 | "name": name, 74 | "path": path, 75 | "parents": list(map(makename, bases)), 76 | } 77 | 78 | 79 | def docroutine(db, name, path, object): 80 | try: 81 | realname = object.__name__ 82 | except AttributeError: 83 | realname = None 84 | name = name or realname 85 | db[path] = { 86 | "kind": "routine", 87 | "realname": realname, 88 | "name": name, 89 | "path": path, 90 | } 91 | 92 | 93 | def index(): 94 | db = {} 95 | for importer, modname, ispkg in pkgutil.walk_packages(["src"], ""): 96 | object, name = pydoc.resolve(modname) 97 | # This is an approximation of the reference implementation 98 | # for `pydoc.plaintext.document(object, name)` at 99 | # https://github.com/python/cpython/blob/e2591e4f5eb717922b2b33e201daefe4f99463dc/Lib/pydoc.py#L475 100 | if inspect.ismodule(object): 101 | docmodule(db, name, name, object) 102 | if _isclass(object): 103 | docclass(db, name, name, object) 104 | if inspect.isroutine(object): 105 | docroutine(db, name, name, object) 106 | 107 | return db 108 | 109 | 110 | def generate(): 111 | with open(os.path.join(ROOT, "docs/.vitepress/pydoc.js"), "w") as f: 112 | f.write(f"export default JSON.parse('{json.dumps(index())}');") 113 | 114 | 115 | if __name__ == "__main__": 116 | generate() 117 | -------------------------------------------------------------------------------- /docs/.vitepress/theme/custom.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --vp-c-brand: #0052b6; 3 | --vp-c-brand-light: #006992; 4 | --vp-button-brand-bg: #006992; 5 | --vp-button-brand-hover-bg: #0052b6; 6 | --vp-button-brand-border: #006992; 7 | --vp-button-brand-hover-border: #006992; 8 | } 9 | 10 | html.dark { 11 | --vp-c-brand: #60b0ff; 12 | --vp-c-brand-light: #80bfff; 13 | --vp-button-brand-bg: #006992; 14 | --vp-button-brand-hover-bg: #0052b6; 15 | --vp-button-brand-border: #006992; 16 | --vp-button-brand-hover-border: #006992; 17 | } 18 | -------------------------------------------------------------------------------- /docs/.vitepress/theme/index.js: -------------------------------------------------------------------------------- 1 | import DefaultTheme from 'vitepress/theme' 2 | import './custom.css' 3 | import Pydoc from '../components/Pydoc.vue' 4 | import Tree from '../components/Tree.vue' 5 | import pydocData from '../pydoc' 6 | export default { 7 | ...DefaultTheme, 8 | enhanceApp({ app }) { 9 | app.config.globalProperties.$pydocData = pydocData; 10 | app.component('Pydoc', Pydoc); 11 | app.component('Tree', Tree); 12 | } 13 | } 14 | 15 | -------------------------------------------------------------------------------- /docs/about/approach.md: -------------------------------------------------------------------------------- 1 | # About machinable's approach 2 | 3 | ::: tip Optional reading 4 | 5 | This background discusses the big-picture approach. For a hands-on tutorial, refer to the [guide](../guide/introduction.md). 6 | 7 | ::: 8 | 9 | 10 | ::: warning Coming soon 11 | 12 | This section is currently under construction 13 | 14 | ::: 15 | -------------------------------------------------------------------------------- /docs/examples/aimstack-storage/aimstack.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | import os 4 | 5 | import aim 6 | from machinable import Interface, Storage 7 | from pydantic import BaseModel, Field 8 | 9 | 10 | class Aimstack(Storage): 11 | class Config(BaseModel): 12 | repo: str = "./storage" 13 | system_tracking_interval: Optional[int] = None 14 | log_system_params: Optional[bool] = False 15 | include: List[str] = Field( 16 | default_factory=lambda: ["machinable.component"] 17 | ) 18 | 19 | def __init__(self, version=None): 20 | super().__init__(version=version) 21 | self._runs = {} 22 | self._repo = None 23 | 24 | @property 25 | def repo(self) -> aim.Repo: 26 | if self._repo is None: 27 | self._repo = aim.Repo(os.path.abspath(self.config.repo), init=True) 28 | 29 | return self._repo 30 | 31 | def contains(self, uuid: str) -> bool: 32 | try: 33 | query_res = self.repo.query_runs( 34 | f"run.uuid=='{uuid}'", report_mode=0 35 | ).iter_runs() 36 | runs = [item.run for item in query_res] 37 | except: 38 | runs = [] 39 | return len(runs) == 1 40 | 41 | def commit(self, interface: "Interface") -> None: 42 | # only track target interfaces 43 | if set(interface.lineage).isdisjoint(self.config.include): 44 | return 45 | 46 | self._runs[interface.uuid] = run = aim.Run( 47 | repo=os.path.abspath(self.config.directory), 48 | read_only=False, 49 | experiment=interface.module, 50 | force_resume=False, 51 | system_tracking_interval=self.config.system_tracking_interval, 52 | log_system_params=self.config.log_system_params, 53 | ) 54 | 55 | for k, v in interface.__model__.model_dump().items(): 56 | run[k] = v 57 | -------------------------------------------------------------------------------- /docs/examples/aimstack-storage/index.md: -------------------------------------------------------------------------------- 1 | # Aim storage 2 | 3 | Integration for the AI metadata tracker [AimStack](https://aimstack.io/). 4 | 5 | ## Usage example 6 | 7 | ```python 8 | from machinable import get 9 | 10 | get("aimstack", {"repo": "./path/to/aim-repo"}).__enter__() 11 | 12 | # your code 13 | ``` 14 | 15 | ## Source 16 | 17 | ::: code-group 18 | 19 | <<< @/examples/aimstack-storage/aimstack.py 20 | 21 | ::: 22 | -------------------------------------------------------------------------------- /docs/examples/aimstack-storage/test_aimstack.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from machinable import Index, Storage 3 | 4 | try: 5 | import aim 6 | except ImportError: 7 | aim = None 8 | 9 | 10 | @pytest.mark.skipif( 11 | aim is None, 12 | reason="Test requires aim environment", 13 | ) 14 | def test_aimstack_storage(tmp_path): 15 | from pathlib import Path 16 | 17 | tmp_path = Path("../aim-test-storage") 18 | 19 | index = Index( 20 | {"directory": str(tmp_path), "database": str(tmp_path / "test.sqlite")} 21 | ).__enter__() 22 | storage = Storage.instance( 23 | "aimstack", {"repo": index.config.directory} 24 | ).__enter__() 25 | 26 | storage.__exit__() 27 | -------------------------------------------------------------------------------- /docs/examples/dependent-schedules/dependency_graph.py: -------------------------------------------------------------------------------- 1 | from machinable import Schedule 2 | from machinable.collection import ComponentCollection 3 | 4 | 5 | class DependencyGraph(Schedule): 6 | def __call__(self, executables: ComponentCollection) -> ComponentCollection: 7 | schedule = ComponentCollection() 8 | done = set() 9 | 10 | def _resolve_dependencies(_executables): 11 | for e in reversed(_executables): 12 | if e.uses: 13 | _resolve_dependencies(e.uses) 14 | 15 | if e.cached(): 16 | done.add(e.uuid) 17 | continue 18 | 19 | if e.uuid not in done: 20 | schedule.append(e) 21 | done.add(e.uuid) 22 | 23 | # depth-first search 24 | _resolve_dependencies(executables) 25 | 26 | return schedule 27 | -------------------------------------------------------------------------------- /docs/examples/dependent-schedules/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/examples/dependent-schedules/index.md -------------------------------------------------------------------------------- /docs/examples/dependent-schedules/test_dependency_graph.py: -------------------------------------------------------------------------------- 1 | from machinable import Component, Element, Execution, Index, Project, get 2 | 3 | 4 | class Trace(Execution): 5 | def __init__(self, *args, **kwargs): 6 | super().__init__(*args, **kwargs) 7 | self.trace = [] 8 | 9 | def on_verify_schedule(self): 10 | return True 11 | 12 | def __call__(self): 13 | executables = self.pending_executables 14 | if self.schedule is not None: 15 | executables = self.schedule(self.executables) 16 | 17 | for executable in executables: 18 | executable.dispatch() 19 | self.trace.append(executable) 20 | 21 | @property 22 | def name_trace(self): 23 | return list(map(lambda x: x.__class__.__name__, self.trace)) 24 | 25 | 26 | def test_dependency_graph(tmp_path): 27 | class A(Component): 28 | pass 29 | 30 | class B(Component): 31 | pass 32 | 33 | class C(Component): 34 | pass 35 | 36 | class D(Component): 37 | pass 38 | 39 | class E(Component): 40 | pass 41 | 42 | with Index(str(tmp_path)): 43 | with Project("docs/examples/dependent-schedules"): 44 | with Trace(schedule=get("dependency_graph")) as execution: 45 | # 46 | # A 47 | # / \ 48 | # B C 49 | # \ / \ 50 | # D E 51 | 52 | a = get(A).launch() 53 | b = get(B, uses=[a]).launch() 54 | c = get(C, uses=[a]).launch() 55 | d = get(D, uses=[b, c]).launch() 56 | e = get(E, uses=[c]).launch() 57 | 58 | assert execution.name_trace == ["A", "C", "E", "B", "D"] 59 | 60 | with Trace(schedule=get("dependency_graph")) as execution: 61 | a2 = Element.make(A).launch() 62 | b = get(B, uses=[a]).launch() 63 | c = get(C, uses=[a]).launch() 64 | d2 = Element.make(D, uses=[b, c]).launch() 65 | e = get(E, uses=[c]).launch() 66 | assert execution.name_trace == ["D", "A"] 67 | -------------------------------------------------------------------------------- /docs/examples/dmosopt-component/index.md: -------------------------------------------------------------------------------- 1 | # dmosopt component 2 | 3 | Integration for [dmosopt](https://github.com/dmosopt/dmosopt). 4 | 5 | ## Usage example 6 | 7 | ```python 8 | from machinable import get 9 | 10 | with get("mpi", {"ranks": 8}): 11 | get("dmosopt", {'dopt_params': ...}).launch() 12 | ``` 13 | 14 | ## Source 15 | 16 | ::: code-group 17 | 18 | <<< @/examples/dmosopt-component/dmosopt.py 19 | 20 | ::: 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/examples/dmosopt-component/test_dmosopt.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from machinable import get 3 | 4 | 5 | def test_dmosopt_component(tmp_path): 6 | p = get("machinable.project", "docs/examples/dmosopt-component").__enter__() 7 | 8 | with get("machinable.index", str(tmp_path)): 9 | try: 10 | get("dmosopt").launch() 11 | except ModuleNotFoundError: 12 | pass 13 | 14 | p.__exit__() 15 | -------------------------------------------------------------------------------- /docs/examples/globus-storage/globus.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, List 2 | 3 | import os 4 | import time 5 | 6 | from globus_sdk import ( 7 | NativeAppAuthClient, 8 | RefreshTokenAuthorizer, 9 | TransferClient, 10 | TransferData, 11 | ) 12 | from globus_sdk.scopes import TransferScopes 13 | from globus_sdk.services.auth.errors import AuthAPIError 14 | from globus_sdk.services.transfer.errors import TransferAPIError 15 | from globus_sdk.tokenstorage import SimpleJSONFileAdapter 16 | from machinable import Storage 17 | from pydantic import BaseModel, Field 18 | 19 | if TYPE_CHECKING: 20 | from machinable import Interface 21 | 22 | 23 | class Globus(Storage): 24 | class Config(BaseModel): 25 | client_id: str = Field("???") 26 | remote_endpoint_id: str = Field("???") 27 | local_endpoint_id: str = Field("???") 28 | remote_endpoint_directory: str = Field("???") 29 | local_endpoint_directory: str = "~/" 30 | auth_filepath: str = "~/.globus-tokens.json" 31 | 32 | def __init__(self, version=None): 33 | super().__init__(version=version) 34 | self._auth_client = None 35 | self._auth_file = None 36 | self._authorizer = None 37 | self._transfer_client = None 38 | self.active_tasks = [] 39 | 40 | @property 41 | def auth_client(self): 42 | if self._auth_client is None: 43 | self._auth_client = NativeAppAuthClient(self.config.client_id) 44 | return self._auth_client 45 | 46 | @property 47 | def auth_file(self): 48 | if self._auth_file is None: 49 | self._auth_file = SimpleJSONFileAdapter( 50 | os.path.expanduser(self.config.auth_filepath) 51 | ) 52 | return self._auth_file 53 | 54 | @property 55 | def authorizer(self): 56 | if self._authorizer is None: 57 | if not self.auth_file.file_exists(): 58 | # do a login flow, getting back initial tokens 59 | self.auth_client.oauth2_start_flow( 60 | requested_scopes=[ 61 | f"{TransferScopes.all}[*https://auth.globus.org/scopes/{self.config.remote_endpoint_id}/data_access]", 62 | f"{TransferScopes.all}[*https://auth.globus.org/scopes/{self.config.local_endpoint_id}/data_access]", 63 | ], 64 | refresh_tokens=True, 65 | ) 66 | authorize_url = self.auth_client.oauth2_get_authorize_url() 67 | print( 68 | f"Please go to this URL and login:\n\n{authorize_url}\n", 69 | flush=True, 70 | ) 71 | time.sleep(0.05) 72 | auth_code = input("Please enter the code here: ").strip() 73 | tokens = self.auth_client.oauth2_exchange_code_for_tokens( 74 | auth_code 75 | ) 76 | self.auth_file.store(tokens) 77 | tokens = tokens.by_resource_server["transfer.api.globus.org"] 78 | else: 79 | # otherwise, we already did login; load the tokens 80 | tokens = self.auth_file.get_token_data( 81 | "transfer.api.globus.org" 82 | ) 83 | 84 | self._authorizer = RefreshTokenAuthorizer( 85 | tokens["refresh_token"], 86 | self.auth_client, 87 | access_token=tokens["access_token"], 88 | expires_at=tokens["expires_at_seconds"], 89 | on_refresh=self.auth_file.on_refresh, 90 | ) 91 | return self._authorizer 92 | 93 | @property 94 | def transfer_client(self): 95 | if self._transfer_client is None: 96 | self._transfer_client = TransferClient(authorizer=self.authorizer) 97 | return self._transfer_client 98 | 99 | def commit(self, interface: "Interface") -> str: 100 | try: 101 | src = os.path.abspath(interface.local_directory()) 102 | if not os.path.exists(src): 103 | raise RuntimeError("Interface must be committed before storage") 104 | 105 | # This is not a strict requirement since client might allow access 106 | # if os.path.normpath(src) != os.path.normpath(self.local_path(interface.uuid)): 107 | # raise RuntimeError("Interface directory must be in storage directory") 108 | 109 | task_data = TransferData( 110 | source_endpoint=self.config.local_endpoint_id, 111 | destination_endpoint=self.config.remote_endpoint_id, 112 | notify_on_succeeded=False, 113 | notify_on_failed=False, 114 | ) 115 | 116 | task_data.add_item( 117 | src, 118 | self.remote_path(interface.uuid), 119 | recursive=True, 120 | ) 121 | 122 | task_doc = self.transfer_client.submit_transfer(task_data) 123 | 124 | task_id = task_doc["task_id"] 125 | 126 | self.active_tasks.append(task_id) 127 | 128 | print(f"Submitted Globus commit, task_id={task_id}") 129 | 130 | # this operation is non-blocking by default 131 | 132 | return task_id 133 | except TransferAPIError as e: 134 | if ( 135 | e.code == "Conflict" 136 | and e.message 137 | == "A transfer with identical paths has not yet completed" 138 | ): 139 | return False 140 | elif e.code == "ConsentRequired": 141 | raise RuntimeError( 142 | f"You do not have the right permissions. Try removing {self.config.auth_filepath} and authenticating again with the appropriate identity provider." 143 | ) from e 144 | raise e 145 | except AuthAPIError as e: 146 | raise RuntimeError( 147 | f"Authentication failed. Try removing {self.config.auth_filepath} and authenticating again with the appropriate identity provider." 148 | ) from e 149 | 150 | def update(self, interface: "Interface") -> None: 151 | return self.commit(interface) 152 | 153 | def contains(self, uuid: str) -> bool: 154 | # check if folder exists on globus storage 155 | try: 156 | response = self.transfer_client.operation_ls( 157 | self.config.remote_endpoint_id, 158 | path=self.remote_path(uuid), 159 | show_hidden=True, 160 | ) 161 | except TransferAPIError as e: 162 | if e.code == "ClientError.NotFound": 163 | return False 164 | elif e.code == "ConsentRequired": 165 | raise RuntimeError( 166 | f"You do not have the right permissions. Try removing {self.config.auth_filepath} and authenticating again with the appropriate identity provider." 167 | ) from e 168 | raise e 169 | 170 | for item in response: 171 | if item["name"] == ".machinable": 172 | return True 173 | 174 | return False 175 | 176 | def retrieve( 177 | self, uuid: str, local_directory: str, timeout: int = 5 * 60 178 | ) -> bool: 179 | if not self.contains(uuid): 180 | return False 181 | 182 | task_data = TransferData( 183 | source_endpoint=self.config.remote_endpoint_id, 184 | destination_endpoint=self.config.local_endpoint_id, 185 | notify_on_succeeded=False, 186 | notify_on_failed=False, 187 | ) 188 | task_data.add_item( 189 | self.remote_path(uuid), 190 | os.path.abspath(local_directory), 191 | ) 192 | task_doc = self.transfer_client.submit_transfer(task_data) 193 | task_id = task_doc["task_id"] 194 | self.active_tasks.append(task_id) 195 | 196 | print(f"[Storage] Submitted Globus retrieve, task_id={task_id}") 197 | 198 | self.tasks_wait(timeout=timeout) 199 | 200 | return True 201 | 202 | def tasks_wait(self, timeout: int = 5 * 60) -> None: 203 | for task_id in self.active_tasks: 204 | print(f"[Storage] Waiting for Globus task {task_id} to complete") 205 | self.transfer_client.task_wait(task_id, timeout=timeout) 206 | print(f"[Storage] task_id={task_id} transfer finished") 207 | self.active_tasks = [] 208 | 209 | def local_path(self, *append): 210 | return os.path.join(self.config.local_endpoint_directory, *append) 211 | 212 | def remote_path(self, *append): 213 | return os.path.join(self.config.remote_endpoint_directory, *append) 214 | 215 | def search_for(self, interface) -> List[str]: 216 | response = self.transfer_client.operation_ls( 217 | self.config.remote_endpoint_id, 218 | path=self.remote_path(), 219 | show_hidden=True, 220 | ) 221 | found = [] 222 | for item in response: 223 | if interface.hash in item["name"]: 224 | found.append(item["name"]) 225 | 226 | return found 227 | -------------------------------------------------------------------------------- /docs/examples/globus-storage/index.md: -------------------------------------------------------------------------------- 1 | # Globus storage 2 | 3 | Integration for [Globus Compute](https://www.globus.org/). 4 | 5 | ## Usage example 6 | 7 | ```python 8 | from machinable import get 9 | 10 | storage = get("globus", {"client_id": ..., ...}).__enter__() 11 | 12 | # run some experiment 13 | component = ... 14 | 15 | # upload to globus 16 | storage.upload(component) 17 | 18 | # search for component in globus using hash 19 | matches = storage.search_for(experiment) 20 | print(matches) 21 | 22 | # download from globus 23 | storage.download(matches[0].uuid) 24 | ``` 25 | 26 | ## Source 27 | 28 | ::: code-group 29 | 30 | <<< @/examples/globus-storage/globus.py 31 | 32 | ::: 33 | -------------------------------------------------------------------------------- /docs/examples/globus-storage/test_globus.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import pytest 5 | from machinable import Component, Storage 6 | 7 | try: 8 | import globus_sdk 9 | except ImportError: 10 | globus_sdk = None 11 | 12 | 13 | @pytest.mark.skipif( 14 | globus_sdk is None or "MACHINABLE_GLOBUS_TEST_CONFIG" not in os.environ, 15 | reason="Test requires globus environment", 16 | ) 17 | def test_globus_storage(): 18 | storage = Storage.make( 19 | "globus", 20 | json.loads(os.environ.get("MACHINABLE_GLOBUS_TEST_CONFIG", "{}")), 21 | ).__enter__() 22 | try: 23 | c = Component() 24 | c.save_file(".test", "test") 25 | c.local_directory("test", create=True) 26 | c.save_file("test/tada", "test") 27 | c.save_file("test/.tada", "test") 28 | c.launch() 29 | 30 | storage.tasks_wait() 31 | 32 | assert storage.contains(c.uuid) 33 | 34 | fetched = os.path.abspath("./storage/fetched") 35 | os.makedirs(fetched, exist_ok=True) 36 | storage.retrieve(c.uuid, fetched) 37 | 38 | cf = Component.from_directory(fetched) 39 | assert cf.load_file(".test") == "test" 40 | assert cf.load_file("test/tada") == "test" 41 | assert cf.load_file("test/.tada") == "test" 42 | assert cf.execution.is_finished() 43 | finally: 44 | storage.__exit__() 45 | -------------------------------------------------------------------------------- /docs/examples/index.md: -------------------------------------------------------------------------------- 1 | # How-to examples 2 | 3 | This section presents code examples that demonstrate real-world usage and my be a good starting point when implementing your own projects. 4 | 5 | Please select an example from the menu. 6 | -------------------------------------------------------------------------------- /docs/examples/mpi-execution/index.md: -------------------------------------------------------------------------------- 1 | # MPI execution 2 | 3 | Integration to launch [MPI](https://en.wikipedia.org/wiki/Message_Passing_Interface) jobs. 4 | 5 | ## Usage example 6 | 7 | ```python 8 | from machinable import get 9 | 10 | with get("mpi", {"ranks": 8}): 11 | ... # your MPI ready component 12 | ``` 13 | 14 | ## Source 15 | 16 | ::: code-group 17 | 18 | <<< @/examples/mpi-execution/mpi.py 19 | 20 | ::: 21 | 22 | -------------------------------------------------------------------------------- /docs/examples/mpi-execution/mpi.py: -------------------------------------------------------------------------------- 1 | from typing import Literal, Optional, Union 2 | 3 | import os 4 | import sys 5 | 6 | from machinable import Execution 7 | from machinable.errors import ExecutionFailed 8 | from machinable.utils import chmodx, run_and_stream 9 | from pydantic import BaseModel, ConfigDict 10 | 11 | 12 | class MPI(Execution): 13 | class Config(BaseModel): 14 | model_config = ConfigDict(extra="forbid") 15 | 16 | preamble: Optional[str] = "" 17 | mpi: Optional[str] = "mpirun" 18 | python: Optional[str] = None 19 | resume_failed: Union[bool, Literal["new", "skip"]] = False 20 | dry: bool = False 21 | 22 | def on_compute_default_resources(self, executable): 23 | resources = {} 24 | 25 | ranks = executable.config.get("ranks", False) 26 | if ranks not in [None, False]: 27 | if ranks == -1: 28 | ranks = os.environ.get("MPI_RANKS", 0) 29 | if int(ranks) > 0: 30 | resources["-n"] = int(ranks) 31 | 32 | return resources 33 | 34 | def __call__(self): 35 | all_script = "#!/usr/bin/env bash\n" 36 | for executable in self.pending_executables: 37 | if self.config.resume_failed is not True: 38 | if ( 39 | executable.executions.filter( 40 | lambda x: x.is_incomplete(executable) 41 | ).count() 42 | > 0 43 | ): 44 | if self.config.resume_failed == "new": 45 | executable = executable.new().commit() 46 | elif self.config.resume_failed == "skip": 47 | continue 48 | else: 49 | msg = f"{executable.module} <{executable.id})> has previously been executed unsuccessfully. Set `resume_failed` to True, 'new' or 'skip' to handle resubmission." 50 | if self.config.dry: 51 | print("Dry run ... ", msg) 52 | continue 53 | 54 | raise ExecutionFailed(msg) 55 | 56 | resources = self.computed_resources(executable) 57 | mpi = executable.config.get("mpi", self.config.mpi) 58 | python = self.config.python or sys.executable 59 | 60 | script = "#!/usr/bin/env bash\n" 61 | 62 | if self.config.preamble: 63 | script += self.config.preamble 64 | 65 | # add debug information 66 | script += "\n" 67 | script += f"# {executable.module} <{executable.id}>\n" 68 | script += f"# {executable.local_directory()}>\n" 69 | script += "\n" 70 | 71 | script += executable.dispatch_code(python=python) 72 | 73 | script_file = chmodx( 74 | self.save_file( 75 | [executable.id, "mpi.sh"], 76 | script, 77 | ) 78 | ) 79 | 80 | if mpi is None: 81 | cmd = [] 82 | else: 83 | cmd = [mpi] 84 | for k, v in resources.items(): 85 | if v is None or v is True: 86 | cmd.append(k) 87 | else: 88 | if k.startswith("--"): 89 | cmd.append(f"{k}={v}") 90 | else: 91 | cmd.extend([k, str(v)]) 92 | 93 | cmd.append(script_file) 94 | 95 | self.save_file( 96 | [executable.id, "mpi.json"], 97 | data={ 98 | "cmd": cmd, 99 | "script": script, 100 | }, 101 | ) 102 | 103 | all_script += f"# {executable}\n" 104 | all_script += " ".join(cmd) + "\n\n" 105 | 106 | if self.config.dry: 107 | continue 108 | 109 | print(" ".join(cmd)) 110 | 111 | with open( 112 | self.local_directory(executable.id, "output.log"), 113 | "w", 114 | buffering=1, 115 | ) as f: 116 | try: 117 | run_and_stream( 118 | cmd, 119 | stdout_handler=lambda o: [ 120 | sys.stdout.write(o), 121 | f.write(o), 122 | ], 123 | stderr_handler=lambda o: [ 124 | sys.stderr.write(o), 125 | f.write(o), 126 | ], 127 | ) 128 | except KeyboardInterrupt as _ex: 129 | raise KeyboardInterrupt( 130 | "Interrupting `" + " ".join(cmd) + "`" 131 | ) from _ex 132 | 133 | sp = chmodx(self.save_file("mpi.sh", all_script)) 134 | 135 | if self.config.dry: 136 | print(f"# Dry run ... \n# ==============\n{sp}\n\n") 137 | print(all_script) 138 | -------------------------------------------------------------------------------- /docs/examples/mpi-execution/test_mpi.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | import pytest 4 | from machinable import Component, Execution, Index, Project 5 | 6 | try: 7 | import mpi4py 8 | except ImportError: 9 | mpi4py = None 10 | 11 | 12 | class MpiExample(Component): 13 | def __call__(self): 14 | print("Hello from MPI script") 15 | self.save_file("test.txt", "hello") 16 | 17 | 18 | @pytest.mark.skipif( 19 | not shutil.which("mpirun") or mpi4py is None, 20 | reason="Test requires MPI environment", 21 | ) 22 | def test_mpi_execution(tmp_path): 23 | with Index( 24 | {"directory": str(tmp_path), "database": str(tmp_path / "test.sqlite")} 25 | ): 26 | with Project("docs/examples/mpi-execution"): 27 | component = MpiExample() 28 | with Execution.get("mpi", resources={"-n": 2}): 29 | component.launch() 30 | assert component.execution.is_finished() 31 | assert component.load_file("test.txt") == "hello" 32 | -------------------------------------------------------------------------------- /docs/examples/require-execution/index.md: -------------------------------------------------------------------------------- 1 | # Require execution 2 | 3 | A way to assert that components have been cached. 4 | 5 | ## Usage example 6 | 7 | ```python 8 | from machinable import get 9 | 10 | with get("require"): 11 | ... # components to check 12 | ``` 13 | 14 | ## Source 15 | 16 | ::: code-group 17 | 18 | <<< @/examples/require-execution/require.py 19 | 20 | ::: 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/examples/require-execution/require.py: -------------------------------------------------------------------------------- 1 | from machinable import Execution 2 | 3 | 4 | class Require(Execution): 5 | class Config: 6 | prevent_commit: bool = True 7 | 8 | def add( 9 | self, 10 | executable, 11 | ): 12 | if not self.config.prevent_commit: 13 | return super().add(executable) 14 | 15 | if isinstance(executable, (list, tuple)): 16 | for _executable in executable: 17 | self.add(_executable) 18 | return self 19 | 20 | if not executable.cached(): 21 | raise RuntimeError( 22 | f"Execution required for {executable.module} <{executable.id}>" 23 | ) 24 | 25 | return super().add(executable) 26 | 27 | def on_before_dispatch(self): 28 | raise RuntimeError( 29 | "Execution is required:\n- " 30 | + "\n- ".join( 31 | self.pending_executables.map( 32 | lambda x: x.module + " <" + x.id + ">" 33 | ) 34 | ) 35 | ) 36 | -------------------------------------------------------------------------------- /docs/examples/require-execution/test_require.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from machinable import get 3 | 4 | 5 | def test_require_execution(tmp_path): 6 | p = get("machinable.project", "docs/examples/require-execution").__enter__() 7 | 8 | with get("machinable.index", str(tmp_path)): 9 | not_ready = get("machinable.component").commit() 10 | with pytest.raises(RuntimeError): 11 | with get("require"): 12 | not_ready.launch() 13 | 14 | ready = not_ready.launch() 15 | with get("require"): 16 | ready.launch() 17 | 18 | p.__exit__() 19 | -------------------------------------------------------------------------------- /docs/examples/scopes/group.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from datetime import datetime 4 | 5 | from machinable import schema 6 | from machinable.element import get_dump, get_lineage 7 | from machinable.scope import Scope 8 | 9 | 10 | def normgroup(group: Optional[str]) -> str: 11 | if group is None: 12 | return "" 13 | if not isinstance(group, str): 14 | raise ValueError(f"Invalid group. Expected str but got '{group}'") 15 | return group.lstrip("/") 16 | 17 | 18 | def resolve_group(group: str) -> Tuple[str, str]: 19 | group = normgroup(group) 20 | resolved = datetime.now().strftime(group) 21 | return group, resolved 22 | 23 | 24 | class Group(schema.Scope): 25 | kind: str = "Group" 26 | pattern: str 27 | path: Optional[str] = None 28 | 29 | 30 | class Group(Scope): 31 | """Group element""" 32 | 33 | kind = "Group" 34 | 35 | def __init__(self, group: Optional[str] = None): 36 | super().__init__(version=None) 37 | pattern, path = resolve_group(group) 38 | self.__model__ = schema.Group( 39 | kind=self.kind, 40 | module=self.__model__.module, 41 | config=self.__model__.config, 42 | version=self.__model__.version, 43 | pattern=pattern, 44 | path=path, 45 | lineage=get_lineage(self), 46 | ) 47 | self.__model__._dump = get_dump(self) 48 | 49 | @property 50 | def pattern(self) -> str: 51 | return self.__model__.pattern 52 | 53 | @property 54 | def path(self) -> str: 55 | return self.__model__.path 56 | 57 | def __call__(self): 58 | return {"group": self.path} 59 | -------------------------------------------------------------------------------- /docs/examples/scopes/test_group_scope.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from docs.examples.scopes.group import normgroup, resolve_group 3 | 4 | 5 | def test_normgroup(): 6 | assert normgroup(None) == "" 7 | assert normgroup("test") == "test" 8 | assert normgroup("/test") == "test" 9 | assert normgroup("/test/me") == "test/me" 10 | assert normgroup("test/") == "test/" 11 | 12 | with pytest.raises(ValueError): 13 | normgroup({"invalid"}) 14 | 15 | assert resolve_group("test") == ("test", "test") 16 | assert resolve_group("/test%Y") != ("/test%Y", "/test%Y") 17 | assert resolve_group("/test/me") == ("test/me", "test/me") 18 | -------------------------------------------------------------------------------- /docs/examples/slurm-execution/index.md: -------------------------------------------------------------------------------- 1 | # Slurm execution 2 | 3 | Integration to submit to the [Slurm](https://slurm.schedmd.com/documentation.html) scheduler. 4 | 5 | ## Usage example 6 | 7 | ```python 8 | from machinable import get 9 | 10 | with get("slurm", {"ranks": 8, 'preamble': 'mpirun'}): 11 | ... # your component 12 | ``` 13 | 14 | ## Source 15 | 16 | ::: code-group 17 | 18 | <<< @/examples/slurm-execution/slurm.py 19 | 20 | ::: 21 | -------------------------------------------------------------------------------- /docs/examples/slurm-execution/test_slurm.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import shutil 4 | import time 5 | from pathlib import Path 6 | 7 | import pytest 8 | from machinable import Component, Execution, Index, Project 9 | 10 | 11 | class SlurmComponent(Component): 12 | class Config: 13 | ranks: int = 1 14 | nodes: int = 1 15 | 16 | def __call__(self): 17 | print("Hello world from Slurm") 18 | self.save_file("test_run.json", {"success": True}) 19 | 20 | 21 | @pytest.mark.skipif( 22 | not shutil.which("sbatch") 23 | or "MACHINABLE_SLURM_TEST_RESOURCES" not in os.environ, 24 | reason="Test requires Slurm environment", 25 | ) 26 | def test_slurm_execution(tmp_path): 27 | component = SlurmComponent() 28 | directory = os.environ.get("MACHINABLE_SLURM_TEST_DIRECTORY", None) 29 | if directory is not None: 30 | tmp_path = Path(directory) / component.uuid 31 | with Index(str(tmp_path)): 32 | with Project("docs/examples/slurm-execution"): 33 | # standard submission 34 | with Execution.get( 35 | "slurm", 36 | {"confirm": False}, 37 | resources=json.loads( 38 | os.environ.get("MACHINABLE_SLURM_TEST_RESOURCES", "{}") 39 | ), 40 | ): 41 | component = SlurmComponent().launch() 42 | 43 | status = False 44 | for _ in range(60): 45 | if component.execution.is_finished(): 46 | assert ( 47 | "Hello world from Slurm" in component.execution.output() 48 | ) 49 | assert ( 50 | component.load_file("test_run.json")["success"] is True 51 | ) 52 | status = True 53 | break 54 | 55 | time.sleep(1) 56 | 57 | assert status, f"Timeout for {component.local_directory()}" 58 | 59 | # usage 60 | with Execution.get( 61 | "slurm", 62 | {"confirm": False}, 63 | resources=json.loads( 64 | os.environ.get("MACHINABLE_SLURM_TEST_RESOURCES", "{}") 65 | ), 66 | ) as e: 67 | A = SlurmComponent(uses=component).launch() 68 | A.save_file("name", "A") 69 | B = SlurmComponent().launch() 70 | B.save_file("name", "B") 71 | C = SlurmComponent(uses=[A, B]).launch() 72 | C.save_file("name", "C") 73 | 74 | status = False 75 | for _ in range(60): 76 | if C.execution.is_finished(): 77 | assert "Hello world from Slurm" in C.execution.output() 78 | assert C.load_file("test_run.json")["success"] is True 79 | status = True 80 | break 81 | 82 | time.sleep(1) 83 | 84 | assert status, f"Timeout for {C.local_directory()}" 85 | -------------------------------------------------------------------------------- /docs/guide/cli.md: -------------------------------------------------------------------------------- 1 | 2 | # Command-line interface 3 | 4 | Components can be launched directly from the command-line. The CLI works out of the box and closely mirrors the Python interface. To run a component, type its module name and method name, optionally followed by the configuration options, for example: 5 | ```bash 6 | machinable mnist_data batch_size=4 --launch 7 | ``` 8 | To use multiprocessing, you may type: 9 | ```bash 10 | machinable mnist_data batch_size=4 \ 11 | multiprocess processes=4 --launch 12 | ``` 13 | 14 | ### Creating aliases 15 | 16 | Generally, your command lines will likely look like the following: 17 | ```sh 18 | PYTHONPATH=.:$PYTHONPATH machinable get machinable.index directory=$STORAGE 19 | ``` 20 | This specifies to save and load results in the `$STORAGE` directory and it's useful to add an alias for this to your `.bashrc`: 21 | ```sh 22 | function ma { PYTHONPATH=.:$PYTHONPATH machinable get machinable.index directory=$STORAGE "$@"; } 23 | ``` 24 | so you can type 25 | ```sh 26 | ma 27 | ``` 28 | 29 | Note that `.` is a shorthand for `interface.`, e.g. typing `interface.example` is equivalent to `.example`. 30 | 31 | -------------------------------------------------------------------------------- /docs/guide/component.md: -------------------------------------------------------------------------------- 1 | # Component 2 | 3 | While interfaces are designed to associate data with code, machinable.Component are the special case that allows for execution. 4 | 5 | Consider the following example where we implement a preprocessing step to download a dataset: 6 | 7 | ```python 8 | from machinable import Component 9 | 10 | class MnistData(Component): 11 | """A dataset of handwritten characters""" 12 | Config = { 13 | "batch_size": 8, 14 | "name": "mnist" 15 | } 16 | 17 | def __call__(self): 18 | self.download_data(self.config.name, self.local_directory()) 19 | 20 | def download_data(dataset_name, target_directory): 21 | print(f"Downloading '{dataset_name}' ...") 22 | ... 23 | ``` 24 | 25 | 26 | ## Executing components 27 | 28 | Once implemented and configured, components can be executed by calling machinable.Component.launch: 29 | 30 | ```python 31 | >>> from machinable import get 32 | >>> mnist = get('mnist_data', {"batch_size": 4}) 33 | >>> mnist.launch() 34 | Downloading 'mnist' ... 35 | ``` 36 | 37 | If the execution is successful, the component is marked as finished. 38 | 39 | ```python 40 | >>> mnist.execution.is_finished() 41 | True 42 | ``` 43 | 44 | By design, component instances can only be executed once. They are automatically assigned a timestamp, random seed, as well as a nickname for easy identification. 45 | 46 | ```python 47 | >>> mnist.seed 48 | 1632827863 49 | ``` 50 | 51 | Invocations of `launch()` after successful execution, do not trigger another execution since the component is marked as cached. On the other hand, if the execution failed, calling `launch()` will resume the execution with the same configuration. 52 | -------------------------------------------------------------------------------- /docs/guide/element.md: -------------------------------------------------------------------------------- 1 | # Element 2 | 3 | At a basic level, machinable projects are regular Python projects consisting of *elements* - configurable classes that encapsulate parts of your code. 4 | For example, an element implementation of a dataset might look like this: 5 | 6 | ```python 7 | from machinable import Element # Element base class 8 | 9 | class MnistData(Element): 10 | """A dataset of handwritten characters""" 11 | Config = { 12 | "batch_size": 8, 13 | "name": "mnist" 14 | } 15 | ``` 16 | 17 | When inheriting from machinable.Element, you can specify default configuration values in the `Config` attribute placed at the top of the class definition. 18 | 19 | The parameters become available under `self.config` and can be accessed with object-notation (`self.config.my.value`) or dict-style access (`self.config['my']['value']`): 20 | 21 | ```python 22 | >>> MnistData().config.name 23 | 'mnist' 24 | ``` 25 | 26 | ## Versions 27 | 28 | You can override the default element configuration to instantiate different *versions* of the element: 29 | ```python 30 | >>> data = MnistData({"batch_size": 16}) 31 | >>> data.config.batch_size 32 | 16 33 | ``` 34 | 35 | Here, we passed the configuration update as a predefined dictionary, but it is equally possible to compute the versions dynamically, for example: 36 | 37 | ```python 38 | class MnistData(Element): 39 | """A dataset of handwritten characters""" 40 | Config = { 41 | "batch_size": 8, 42 | "name": "mnist" 43 | } 44 | 45 | def version_large(self): 46 | return { 47 | "batch_size": self.config.batch_size * 2 48 | } 49 | 50 | 51 | >>> MnistData("~large").config.batch_size 52 | 16 53 | ``` 54 | 55 | The `~{name}` indicates that the version is defined as a method and machinable will look up and call `version_{name}` to use the returned dictionary to update the default configuration. This works with parameters as well: 56 | 57 | ```python 58 | class MnistData(Element): 59 | """A dataset of handwritten characters""" 60 | Config = { 61 | "batch_size": 8, 62 | "name": "mnist" 63 | } 64 | 65 | def version_large(self, factor=2): 66 | return { 67 | "batch_size": int(self.config.batch_size * factor) 68 | } 69 | 70 | 71 | >>> MnistData("~large(3)").config.batch_size 72 | 24 73 | ``` 74 | 75 | Furthermore, it is possible to compose versions in a list, for example: 76 | 77 | ```python 78 | >>> MnistData(["~large(factor=0.5)", {"name": "halve"}]).config 79 | {'batch_size': 4, 'name': 'halve'} 80 | ``` 81 | 82 | The updates are merged from right to left, i.e. values appended to the list overwrite prior values: 83 | ```python 84 | >>> MnistData([{"name": "first"}, {"name": "second"}]).config.name 85 | 'second' 86 | ``` 87 | You can inspect and modify the current version using machinable.Element.version 88 | ```python 89 | >>> mnist = MnistData({"batch_size": 2}) 90 | >>> mnist.version() 91 | [{'batch_size': 2}] 92 | >>> mnist.version({"batch_size": 4}) 93 | [{'batch_size': 2}, {'batch_size': 4}] 94 | >>> mnist.config.batch_size 95 | 4 96 | >>> mnist.version([], overwrite=True) 97 | >>> mnist.config.batch_size 98 | 8 99 | ``` 100 | 101 | Notably, whatever version is being applied, machinable keeps track of the default configuration and applied configuration updates: 102 | 103 | ```python 104 | mnist = MnistData({"batch_size": 1}) 105 | >>> mnist.config._default_ 106 | {'batch_size': 8, 'name': 'mnist'} 107 | >>> mnist.config._update_ 108 | {'batch_size': 1} 109 | ``` 110 | 111 | Elements support many more advanced configuration features such as typing, validation, parameter documentation, computed values, etc., which will be covered in later sections of the Guide. For now, to summarize, elements are classes with default configurations that may be modified with a list of configuration updates. 112 | -------------------------------------------------------------------------------- /docs/guide/execution.md: -------------------------------------------------------------------------------- 1 | # Execution 2 | 3 | Components can be executed in different ways. You may, for example, like to run components using multiprocessing or execute in a cloud environment. However, instead of adding the execution logic directly to your component code, machinable makes it easy to separate concerns. You can encapsulate the execution implementation in its own execution class that can then be used to execute the component. 4 | 5 | To implement an execution, implement an interface that inherits from the machinable.Execution base class, for example: 6 | 7 | ::: code-group 8 | 9 | ```python [multiprocess.py] 10 | from multiprocessing import Pool 11 | 12 | from machinable import Execution 13 | 14 | 15 | class Multiprocess(Execution): 16 | Config = {"processes": 1} 17 | 18 | def __call__(self): 19 | pool = Pool(processes=self.config.processes, maxtasksperchild=1) 20 | try: 21 | pool.imap_unordered( 22 | lambda component: component.dispatch(), 23 | self.pending_executables, 24 | ) 25 | pool.close() 26 | pool.join() 27 | finally: 28 | pool.terminate() 29 | ``` 30 | 31 | ::: 32 | 33 | Much like a component, the execution class implements multiprocessing of the given `self.pending_executables` by dispatching them within a subprocess (`component.dispatch()`). 34 | 35 | As usual, we can instantiate this execution using the module convention: 36 | ```python 37 | from machinable import get 38 | 39 | multiprocess = get("multiprocess", {'processes': 2}) 40 | ``` 41 | 42 | Then, to use it, we can wrap the launch in the execution context: 43 | 44 | ```python 45 | with multiprocessing: 46 | mnist.launch() 47 | ``` 48 | 49 | Check out the [execution examples](../examples/) that include generally useful implementations you may like to use in your projects. 50 | -------------------------------------------------------------------------------- /docs/guide/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | machinable is available via [pip](https://pypi.org/project/machinable/). Install the current release 4 | 5 | ```bash 6 | $ pip install machinable 7 | ``` 8 | 9 | ::: info 10 | machinable currently supports Python 3.8 and higher 11 | ::: 12 | 13 | Note that machinable requires the sqlite json1 extension, otherwise, you will likely see the error message: 14 | `sqlite3.OperationalError: no such function: json_extract`. In this case, an easy way to obtain a suitable sqlite version is to install [sqlean.py](https://github.com/nalgeon/sqlean.py): 15 | 16 | 17 | ```bash 18 | $ pip install sqlean.py 19 | ``` 20 | 21 | 22 | -------------------------------------------------------------------------------- /docs/guide/interface.md: -------------------------------------------------------------------------------- 1 | # Interface 2 | 3 | [Elements](./element.md) by themselves are limited in that they are effectively stateless. You can construct and use them but any computed result or additional information will not be persisted. 4 | 5 | To enable storage and retrival we can use an machinable.Interface class. 6 | 7 | ```python 8 | from machinable import Interface 9 | 10 | class MnistData(Interface): 11 | """A dataset of handwritten characters""" 12 | Config = { 13 | "batch_size": 8, 14 | "name": "mnist" 15 | } 16 | ``` 17 | 18 | Interfaces inherit all the functionality of elements but can be committed and subsequently reloaded: 19 | 20 | ```python 21 | >>> mnist = MnistData() 22 | >>> mnist.commit() # persist this particular instance 23 | Interface [29f034] 24 | >>> mnist.local_directory() 25 | './storage/29f034ad2d1a46b8b71c9b30222b5b88' 26 | >>> Interface.from_directory('./storage/29f034ad2d1a46b8b71c9b30222b5b88') 27 | Interface [29f034] # reloaded interface 28 | ``` 29 | 30 | During commit, machinable collects information like a unique ID (e.g. `29f034`), the used configuration, and other meta-data and saves it in a unique storage (e.g. a local directory) from which it can be reloaded later. 31 | 32 | ## get 33 | 34 | In practice, however, it may be cumbersome to keep track of long IDs to reload existing interfaces. To avoid this issue, one of the fundamental ideas in the design of machinable is to make retrieval identical to initial instantiation. 35 | 36 | Specifically, to instantiate an interface (e.g. `MnistData()`) we can leverage the machinable.get function, which takes a class as the first argument and optional constructor arguments. 37 | 38 | ```python 39 | from machinable import get 40 | 41 | mnist = get(MnistData, {"batch_size": 8}) 42 | # -> this is equivalent to: MnistData({"batch_size": 8}) 43 | mnist.commit() 44 | ``` 45 | 46 | Now, if we later want to retrieve this instance, we can use the same code in place of a unique ID: 47 | 48 | ```python 49 | mnist_reloaded = get(MnistData, {"batch_size": 8}) 50 | 51 | assert mnist == mnist_reloaded 52 | ``` 53 | 54 | What is happening here is that machinable.get automatically searches the storage for an interface of type `MnistData` with a `batch_size` of `8`. If such an instance has not been committed yet (like when initially running the code), a new instance with this configuration will be returned. But if such an instance has previously been committed, it will simply be reloaded. 55 | 56 | ## The module convention 57 | 58 | As your project grows, the classes that you implement should be moved into their own Python module. You are free to structure your code as you see fit but there is one hard constraint that classes must be placed in their own modules. The project source code may, for instance, be organized like this: 59 | 60 | ``` 61 | example_project/ 62 | ├─ estimate_gravity.py # contains a data analysis component 63 | ├─ evolution/ 64 | | └─ simulate_offspring.py # contains a evolutionary simulation 65 | └─ main.py # main script to execute 66 | ``` 67 | 68 | The benefit of this requirement is that you can refer to the classes via their module import path. 69 | For example, using this *module convention*, you can simplify the instantiation of classes that are located in different modules: 70 | 71 | ::: code-group 72 | 73 | ```python [main.py (before)] 74 | from machinable import get 75 | 76 | from estimate_gravity import EstimateGravity 77 | from evolution.simulate_offspring import SimulateOffspring 78 | 79 | gravity = get(EstimateGravity) 80 | evolution = get(SimulateOffspring) 81 | ``` 82 | 83 | ```python [main.py (using the module convention)] 84 | from machinable import get 85 | 86 | gravity = get('estimate_gravity') 87 | evolution = get('evolution.simulate_offspring') 88 | ``` 89 | 90 | ::: 91 | 92 | Note that we do not refer to the classes by their name but just by the modules that contain them (since each module only contains one). As we will see later, importing and instantiating the classes this way has a lot of advantages, so it is the default way of instantiation in machinable projects. 93 | 94 | 95 | ## Saving and loading state 96 | 97 | While machinable automatically commits crucial information about the interface, you can use machinable.Interface.save_file and machinable.Interface.load_file to easily store and retrieve additional custom data in different file formats: 98 | 99 | ```python 100 | gravity.save_file('prediction.txt', 'a string') # text 101 | gravity.save_file('settings.json', {'neurons': [1, 2]}) # jsonable 102 | gravity.save_file('inputs.npy', np.array([1.0, 2.0])) # numpy 103 | gravity.save_file('results.p', results) # pickled 104 | 105 | >>> gravity.load_file('prediction.txt') 106 | 'a string' 107 | ``` 108 | 109 | This may be useful to save and restore some custom state of the interface. Furthermore, you are free to implement your own methods to persist data by writing and reading from the interface's machinable.Interface.local_directory: 110 | 111 | ```python 112 | import os 113 | mnist = get("mnist_data") 114 | with open(mnist.local_directory("download_script.sh"), "w") as f: 115 | f.write(...) 116 | os.chmod(f.name, 0o755) 117 | ``` 118 | 119 | Overall, interfaces make it easy to associate data with code as instantiation, storage and retrieval are managed automatically behind the scenes. 120 | -------------------------------------------------------------------------------- /docs/guide/introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | ## What is machinable? 4 | 5 | _machinable_ is a Python API for research code. It provides an object-oriented skeleton that helps you develop and experiment in a unified interface while handling tedious housekeeping behind the scenes. 6 | 7 | The key idea is to unify the running of code and the retrieval of produced results in one abstraction. A detailed discussion of this approach can be found in the [about section](../about/approach.md), but for now, here is a minimal example that illustrates the idea. 8 | 9 | 1. Write some code 10 | 11 | ::: code-group 12 | 13 | ```python [montecarlo.py] 14 | from random import random 15 | 16 | from pydantic import BaseModel, Field 17 | 18 | from machinable import Component 19 | 20 | 21 | class EstimatePi(Component): 22 | class Config(BaseModel): 23 | samples: int = 100 24 | 25 | def __call__(self): 26 | count = 0 27 | for _ in range(self.config.samples): 28 | x, y = random(), random() 29 | count += int((x**2 + y**2) <= 1) 30 | pi = 4 * count / self.config.samples 31 | 32 | self.save_file( 33 | "result.json", 34 | {"count": count, "pi": pi}, 35 | ) 36 | 37 | def summary(self): 38 | if self.execution.is_finished(): 39 | print( 40 | f"After {self.config.samples} samples, " 41 | f"PI is approximately {self.load_file('result.json')['pi']}." 42 | ) 43 | ``` 44 | 45 | 53 | 54 | ::: 55 | 56 | 2. Run and inspect it using a unified abstraction 57 | 58 | ::: code-group 59 | 60 | ```python [Python] 61 | from machinable import get 62 | 63 | # Imports component in `montecarlo.py` with samples=150; 64 | # if an component with this configuration exists, it 65 | # is automatically reloaded. 66 | experiment = get("montecarlo", {"samples": 150}) 67 | 68 | # Executes the component unless it's already been computed 69 | experiment.launch() 70 | 71 | experiment.summary() 72 | # >>> After 150 samples, PI is approximately 3.1466666666666665. 73 | ``` 74 | 75 | ```python [Jupyter] 76 | >>> from machinable import get 77 | >>> experiment = get("montecarlo", {"samples": 150}) 78 | >>> experiment.launch() 79 | Component <24aee0f> 80 | >>> experiment.summary() 81 | After 150 samples, PI is approximately 3.1466666666666665. 82 | >>> experiment.execution.nickname 83 | 'chocolate_mosquito' 84 | >>> experiment.finished_at().humanize() 85 | 'finished just now' 86 | >>> experiment.local_directory() 87 | './storage/24aee0fd05024400b116593d1436e9f5' 88 | ``` 89 | 90 | ```bash [CLI] 91 | $ machinable montecarlo samples=150 --launch --summary 92 | > After 150 samples, PI is approximately 3.1466666666666665. 93 | ``` 94 | 95 | ::: 96 | 97 | The above example demonstrates the two core principles of _machinable_ code: 98 | 99 | - **Enforced modularity** The Monte Carlo algorithm is encapsulated in its own module that can be instantiated with different configuration settings. 100 | - **Unified representation** Running code is handled through the same interface that is used to retrieve produced results; multiple invocations simply reload and display the results without re-running the experiment. 101 | 102 | You may already have questions - don't worry. We will cover the details in the rest of the documentation. For now, please read along so you can have a high-level understanding of what machinable offers. 103 | 104 | ## What it is not 105 | 106 | Research is extremely diverse so machinable primarily aims to be an **API-spec** that leaves concrete feature implementation to the user. Check out the [examples](../examples/) to learn what this looks like in practice. 107 | 108 | ## Where to go from here 109 | 110 | ::: info :gear:   Installation 111 | 112 | We recommend [installing machinable](./installation.md) to try things out while following along. 113 | 114 | ::: 115 | 116 | ::: info :student:   [Continue with the Guide](./element.md) 117 | 118 | Designed to learn concepts hands-on. Starts with the bare minimum of concepts necessary to start using machinable. Along the way, it will provide pointers to sections that discuss concepts in more detail or cover more advanced functionality. 119 | 120 | ::: 121 | 122 | ::: info :arrow_right:   [Check out the How-to guides](../examples/index.md) 123 | 124 | Explore real-world examples that demonstrate advanced concepts 125 | 126 | ::: 127 | 128 | ::: info :open_book:   [Consult the Reference](../reference/index.md) 129 | 130 | Describes available APIs in full detail. 131 | 132 | ::: 133 | 134 | 135 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: home 3 | 4 | hero: 5 | name: machinable 6 | text: research code 7 | tagline: A modular system to manage research code so you can move quickly while enabling reuse and collaboration. 8 | image: 9 | src: /logo/logo.png 10 | alt: machinable-logo 11 | actions: 12 | - theme: brand 13 | text: Get Started 14 | link: /guide/introduction 15 | - theme: alt 16 | text: View on GitHub 17 | link: https://github.com/machinable-org/machinable 18 | 19 | features: 20 | - icon: 🛠️ 21 | title: Unified representation 22 | details: Run code and inspect results using the same abstraction. Check out the example below ⏬ 23 | - icon: ⚡️ 24 | title: Designed for rapid iteration 25 | details: Spend more time experimenting while relying on machinable to keep things organized. 26 | - icon: 💡 27 | title: Hackable and interactive 28 | details: Tweak, extend, override while leveraging first-class support for Jupyter as well as the CLI. 29 | --- 30 | 31 | 32 |
33 | 34 |
35 | 36 |
37 | 38 |
39 | 40 | 41 |
42 | 43 |
44 |
45 | 46 | 47 | 48 |   💻 49 |
50 | 51 |
52 | 53 | ::: info Some research code 54 | 55 | Running code ... 56 | 57 | `python regression.py --rate=0.1 --logs=1 --name=run-01` 58 | 59 | ... and loading the corresponding results ... 60 | 61 | `python plot_regression_result.py --component=run-01` 62 | 63 | ... are distinct and often redundant. 64 | 65 | This means you have to manually keep track by remembering what the component with `rate=0.1` was called. 66 | 67 | ::: 68 | 69 |
70 | 71 | ::: tip machinable research code 72 | 73 | Running code ... 74 | 75 | `machinable regression rate=0.1 logs_=True --launch` 76 | 77 | ... and loading the corresponding results ... 78 | 79 | `machinable regression rate=0.1 logs_=True --launch --plot` 80 | 81 | ... are distinct but use the same abstraction. 82 | 83 | This means no need to worry about names as machinable automatically keeps track if you ran `rate=0.1` before. 84 | 85 | ::: 86 | 87 |
88 | 89 | :arrow_right: [Learn more about machinable's approach](./about/approach.md) 90 | 91 |
92 | 93 |
94 | 95 |
96 | 97 |
98 |
99 |
100 | 101 | logo 102 | 103 | 104 | 159 | -------------------------------------------------------------------------------- /docs/logo/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/docs/logo/logo.png -------------------------------------------------------------------------------- /docs/logo/logo.svg: -------------------------------------------------------------------------------- 1 | Asset 5 -------------------------------------------------------------------------------- /docs/reference/index.md: -------------------------------------------------------------------------------- 1 | # Reference documentation 2 | 3 | This page contains detailed API reference documentation. It is intended to be an in-depth resource for understanding the implementation details of machinable's interfaces. You may prefer reviewing the more explanatory [guide](../guide/introduction.md) before consulting this reference. 4 | 5 | ## API 6 | 7 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "docs:pydoc": "poetry run docs/.vitepress/pydocgen.py", 4 | "docs:dev": "vitepress dev docs", 5 | "docs:build": "vitepress build docs", 6 | "docs:serve": "vitepress serve docs" 7 | }, 8 | "dependencies": { 9 | "vitepress": "^1.0.0-beta.1", 10 | "vue": "^3.3.4" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "machinable" 3 | version = "4.10.6" 4 | description = "A modular configuration system for research projects" 5 | license = "MIT" 6 | authors = ["Frithjof Gressmann "] 7 | maintainers = ["Frithjof Gressmann "] 8 | readme = "README.md" 9 | homepage = "https://machinable.org" 10 | repository = "https://github.com/machinable-org/machinable" 11 | documentation = "https://machinable.org" 12 | keywords = ["machine-learning", "research"] 13 | classifiers = [ 14 | "Development Status :: 5 - Production/Stable", 15 | "Programming Language :: Python", 16 | "License :: OSI Approved :: MIT License", 17 | "Intended Audience :: Science/Research", 18 | ] 19 | include = ["CHANGELOG.md", "src/machinable/py.typed"] 20 | 21 | [tool.poetry.dependencies] 22 | python = "^3.8" 23 | flatten-dict = "^0.4" 24 | jsonlines = ">=3.1,<5.0" 25 | pydantic = ">=1,<3" 26 | arrow = "^1.3" 27 | importlib-metadata = { version = "^6.7", python = "<3.8" } 28 | omegaconf = "2.4.0.dev3" 29 | dill = "^0.3.9" 30 | typing-extensions = { version = "^4.7.0", python = "<3.11" } 31 | uuid7 = "^0.1.0" 32 | 33 | 34 | [tool.poetry.dev-dependencies] 35 | isort = "^5.13.1" 36 | pyupgrade = "^3.3" 37 | black = "^23.11.0" 38 | pytest = "^7.4" 39 | pre-commit = "^3.5.0" 40 | editorconfig-checker = "^2.7.3" 41 | pytest-cov = "^4.1.0" 42 | 43 | [tool.poetry.extras] 44 | all = ["numpy", "pandas"] 45 | 46 | [build-system] 47 | requires = ["poetry_core>=1.0.0"] 48 | build-backend = "poetry.core.masonry.api" 49 | 50 | [tool.poetry.scripts] 51 | machinable = "machinable.cli:main" 52 | 53 | [tool.black] 54 | # https://github.com/psf/black 55 | line-length = 80 56 | target_version = ["py36"] 57 | include = '\.pyi?$' 58 | exclude = ''' 59 | ( 60 | /( 61 | \.eggs 62 | | \.git 63 | | \.hg 64 | | \.mypy_cache 65 | | \.tox 66 | | \.venv 67 | | _build 68 | | buck-out 69 | | build 70 | | dist' 71 | )/ 72 | ) 73 | ''' 74 | 75 | [tool.isort] 76 | # https://github.com/timothycrosley/isort/ 77 | known_typing = "typing,types,typing_extensions,mypy,mypy_extensions" 78 | sections = "FUTURE,TYPING,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" 79 | include_trailing_comma = true 80 | default_section = "FIRSTPARTY" 81 | multi_line_output = 3 82 | force_grid_wrap = 0 83 | use_parentheses = true 84 | ensure_newline_before_comments = true 85 | line_length = 80 86 | 87 | [tool.pylint.'MESSAGES CONTROL'] 88 | disable = "protected-access, import-outside-toplevel, too-few-public-methods" 89 | -------------------------------------------------------------------------------- /src/machinable/__init__.py: -------------------------------------------------------------------------------- 1 | """machinable""" 2 | __all__ = [ 3 | "Element", 4 | "Interface", 5 | "Execution", 6 | "Component", 7 | "Project", 8 | "Storage", 9 | "Mixin", 10 | "Index", 11 | "Scope", 12 | "mixin", 13 | "Schedule", 14 | "get", 15 | ] 16 | __doc__ = """A modular system for machinable research code""" 17 | 18 | from importlib import metadata as importlib_metadata 19 | 20 | from machinable.cli import from_cli 21 | from machinable.component import Component 22 | from machinable.element import Element 23 | from machinable.execution import Execution 24 | from machinable.index import Index 25 | from machinable.interface import Interface 26 | from machinable.mixin import Mixin, mixin 27 | from machinable.project import Project 28 | from machinable.query import Query 29 | from machinable.schedule import Schedule 30 | from machinable.scope import Scope 31 | from machinable.storage import Storage 32 | 33 | 34 | def get_version() -> str: 35 | try: 36 | return importlib_metadata.version(__name__) 37 | except importlib_metadata.PackageNotFoundError: # pragma: no cover 38 | return "unknown" 39 | 40 | 41 | __version__: str = get_version() 42 | 43 | get = Query() 44 | -------------------------------------------------------------------------------- /src/machinable/cli.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, List, Optional 2 | 3 | import os 4 | import sys 5 | 6 | from omegaconf import OmegaConf 7 | 8 | if TYPE_CHECKING: 9 | from machinable.types import VersionType 10 | 11 | 12 | def parse(args: List) -> tuple: 13 | kwargs = [] 14 | methods = [] 15 | elements = [] 16 | dotlist = [] 17 | version = [] 18 | 19 | def _parse_dotlist(): 20 | if len(dotlist) == 0: 21 | return 22 | _ver = {} 23 | for k, v in OmegaConf.to_container( 24 | OmegaConf.from_dotlist(dotlist) 25 | ).items(): 26 | if k.startswith("**"): 27 | kwargs[-1][k[2:]] = v 28 | else: 29 | _ver[k] = v 30 | version.append(_ver) 31 | 32 | def _push(): 33 | if len(version) == 0: 34 | return 35 | 36 | if len(elements) > 0: 37 | elements[-1].extend(version) 38 | else: 39 | elements.append(version) 40 | 41 | for arg in args: 42 | if arg.startswith("~"): 43 | # version 44 | _parse_dotlist() 45 | dotlist = [] 46 | version.append(arg) 47 | elif arg.startswith("--"): 48 | # method 49 | methods.append((len(elements), arg[2:])) 50 | elif "=" in arg: 51 | # dotlist 52 | dotlist.append(arg) 53 | else: 54 | # module 55 | _parse_dotlist() 56 | _push() 57 | dotlist = [] 58 | version = [] 59 | # auto-complete `.project` -> `interface.project` 60 | if arg.startswith("."): 61 | arg = "interface" + arg 62 | elements.append([arg]) 63 | kwargs.append({}) 64 | 65 | _parse_dotlist() 66 | _push() 67 | 68 | return elements, kwargs, methods 69 | 70 | 71 | def from_cli(args: Optional[List] = None) -> "VersionType": 72 | if args is None: 73 | args = sys.argv[1:] 74 | 75 | elements, _, _ = parse(args) 76 | 77 | return sum(elements, []) 78 | 79 | 80 | def main(args: Optional[List] = None): 81 | import machinable 82 | 83 | if args is None: 84 | args = sys.argv[1:] 85 | 86 | if len(args) == 0: 87 | print("\nhelp") 88 | print("\nversion") 89 | print("\nget") 90 | return 0 91 | 92 | action, args = args[0], args[1:] 93 | 94 | if action == "help": 95 | h = "get" 96 | if len(args) > 0: 97 | h = args[0] 98 | 99 | if h == "get": 100 | print("\nmachinable get [element_module...] [version...] --method") 101 | print("\nExample:") 102 | print( 103 | "\tmachinable get my_component ~ver arg=1 nested.arg=2 --launch\n" 104 | ) 105 | return 0 106 | elif h == "version": 107 | print("\nmachinable version") 108 | return 0 109 | else: 110 | print("Unrecognized option") 111 | return 128 112 | 113 | if action == "version": 114 | version = machinable.get_version() 115 | print(version) 116 | return 0 117 | 118 | if action.startswith("get"): 119 | sys.path.append(os.getcwd()) 120 | 121 | get = machinable.get 122 | if action != "get": 123 | get = getattr(get, action.split(".")[-1]) 124 | 125 | elements, kwargs, methods = parse(args) 126 | contexts = [] 127 | component = None 128 | for i, (module, *version) in enumerate(elements): 129 | element = get(module, version, **kwargs[i]) 130 | if i == len(elements) - 1: 131 | component = element 132 | else: 133 | contexts.append(element.__enter__()) 134 | 135 | if component is None: 136 | raise ValueError("You have to provide at least one interface") 137 | 138 | for i, method in methods: 139 | # check if cli_{method} exists before falling back on {method} 140 | target = getattr( 141 | component, f"cli_{method}", getattr(component, method) 142 | ) 143 | if callable(target): 144 | target() 145 | else: 146 | print(target) 147 | 148 | for context in reversed(contexts): 149 | context.__exit__() 150 | 151 | return 0 152 | 153 | print("Invalid argument") 154 | return 128 155 | -------------------------------------------------------------------------------- /src/machinable/component.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, List, Optional, Union 2 | 3 | import inspect 4 | import os 5 | import random 6 | import sys 7 | import threading 8 | 9 | if sys.version_info >= (3, 11): 10 | from typing import Self 11 | else: 12 | from typing_extensions import Self 13 | 14 | from typing import Dict 15 | 16 | from machinable import errors, schema 17 | from machinable.collection import ComponentCollection, ExecutionCollection 18 | from machinable.element import _CONNECTIONS as connected_elements 19 | from machinable.element import get_dump, get_lineage 20 | from machinable.index import Index 21 | from machinable.interface import Interface, belongs_to, belongs_to_many 22 | from machinable.project import Project 23 | from machinable.storage import Storage 24 | from machinable.types import VersionType 25 | from machinable.utils import generate_seed 26 | 27 | if TYPE_CHECKING: 28 | from machinable.execution import Execution 29 | 30 | 31 | class Component(Interface): 32 | kind = "Component" 33 | default = None 34 | 35 | def __init__( 36 | self, 37 | version: VersionType = None, 38 | uses: Union[None, "Interface", List["Interface"]] = None, 39 | derived_from: Optional["Interface"] = None, 40 | seed: Union[int, None] = None, 41 | ): 42 | super().__init__(version=version, uses=uses, derived_from=derived_from) 43 | if seed is None: 44 | seed = generate_seed() 45 | self.__model__ = schema.Component( 46 | kind=self.kind, 47 | module=self.__model__.module, 48 | config=self.__model__.config, 49 | version=self.__model__.version, 50 | seed=seed, 51 | lineage=get_lineage(self), 52 | ) 53 | self.__model__._dump = get_dump(self) 54 | self._current_execution_context = None 55 | 56 | @property 57 | def current_execution_context(self) -> "Execution": 58 | if self._current_execution_context is None: 59 | from machinable.execution import Execution 60 | 61 | self._current_execution_context = Execution.get() 62 | return self._current_execution_context 63 | 64 | @belongs_to_many(key="execution_history") 65 | def executions() -> ExecutionCollection: 66 | from machinable.execution import Execution 67 | 68 | return Execution 69 | 70 | @property 71 | def components(self) -> "ComponentCollection": 72 | return ComponentCollection([self]) 73 | 74 | @property 75 | def execution(self) -> "Execution": 76 | from machinable.execution import Execution 77 | 78 | related = None 79 | if self.is_mounted(): 80 | # if mounted, search for related, most recent execution 81 | related = Index.get().find_related( 82 | relation="Execution.Component.execution_history", 83 | uuid=self.uuid, 84 | inverse=True, 85 | ) 86 | 87 | if related is not None and len(related) > 0: 88 | related = Interface.find_by_id( 89 | sorted(related, key=lambda x: x.timestamp, reverse=True)[ 90 | 0 91 | ].uuid 92 | ) 93 | else: 94 | related = None 95 | 96 | # use context if no related execution was found 97 | if related is None: 98 | if Execution.is_connected(): 99 | related = Execution.get() 100 | else: 101 | related = self.current_execution_context 102 | 103 | related.of(self) 104 | 105 | return related 106 | 107 | def launch(self) -> Self: 108 | from machinable.execution import Execution 109 | 110 | self.fetch() 111 | 112 | if Execution.is_connected(): 113 | # stage only, defer execution 114 | if not self.is_staged(): 115 | self.stage() 116 | Execution.get().add(self) 117 | else: 118 | self.current_execution_context.add(self) 119 | self.current_execution_context.dispatch() 120 | 121 | return self 122 | 123 | @property 124 | def seed(self) -> int: 125 | return self.__model__.seed 126 | 127 | @property 128 | def nickname(self) -> str: 129 | return self.__model__.nickname 130 | 131 | @classmethod 132 | def collect(cls, components) -> "ComponentCollection": 133 | return ComponentCollection(components) 134 | 135 | def dispatch(self) -> Self: 136 | """Dispatch the component lifecycle""" 137 | writes_meta_data = ( 138 | self.on_write_meta_data() is not False and self.is_mounted() 139 | ) 140 | try: 141 | self.on_before_dispatch() 142 | 143 | self.on_seeding() 144 | 145 | # meta-data 146 | if writes_meta_data: 147 | if not self.execution.is_started(): 148 | self.execution.update_status(status="started") 149 | else: 150 | self.execution.update_status(status="resumed") 151 | 152 | self.execution.save_file( 153 | [self.id, "host.json"], 154 | data=Project.get().provider().get_host_info(), 155 | ) 156 | 157 | def beat(): 158 | t = threading.Timer(15, beat) 159 | t.daemon = True 160 | t.start() 161 | self.on_heartbeat() 162 | if self.on_write_meta_data() is not False and self.is_mounted(): 163 | self.execution.update_status(status="heartbeat") 164 | return t 165 | 166 | heartbeat = beat() 167 | 168 | self.__call__() 169 | 170 | self.on_success() 171 | self.on_finish(success=True) 172 | 173 | if heartbeat is not None: 174 | heartbeat.cancel() 175 | 176 | if writes_meta_data: 177 | self.execution.update_status(status="finished") 178 | self.cached(True, reason="finished") 179 | 180 | self.on_after_dispatch(success=True) 181 | except BaseException as _ex: # pylint: disable=broad-except 182 | self.on_failure(exception=_ex) 183 | self.on_finish(success=False) 184 | self.on_after_dispatch(success=False) 185 | raise errors.ComponentException( 186 | f"{self.__class__.__name__} dispatch failed" 187 | ) from _ex 188 | finally: 189 | if writes_meta_data: 190 | # propagate changes 191 | for storage in Storage.connected(): 192 | storage.update(self) 193 | 194 | def cached( 195 | self, cached: Optional[bool] = None, reason: str = "user" 196 | ) -> bool: 197 | if cached is None: 198 | return self.load_file("cached", None) is not None 199 | elif cached is True: 200 | self.save_file("cached", str(reason)) 201 | return True 202 | elif cached is False: 203 | try: 204 | os.remove(self.local_directory("cached")) 205 | except OSError: 206 | pass 207 | 208 | return cached 209 | 210 | def dispatch_code( 211 | self, 212 | inline: bool = True, 213 | project_directory: Optional[str] = None, 214 | python: Optional[str] = None, 215 | ) -> Optional[str]: 216 | if project_directory is None: 217 | project_directory = Project.get().path() 218 | if python is None: 219 | python = sys.executable 220 | lines = ["from machinable import Project, Element, Component"] 221 | # context 222 | lines.append(f"Project('{project_directory}').__enter__()") 223 | for kind, elements in connected_elements.items(): 224 | if kind in ["Project", "Execution"]: 225 | continue 226 | for element in elements: 227 | jn = element.as_json().replace('"', '\\"').replace("'", "\\'") 228 | lines.append(f"Element.from_json('{jn}').__enter__()") 229 | # dispatch 230 | lines.append( 231 | f"component__ = Component.from_directory('{os.path.abspath(self.local_directory())}')" 232 | ) 233 | lines.append("component__.dispatch()") 234 | 235 | if inline: 236 | code = ";".join(lines) 237 | return f'{python} -c "{code}"' 238 | 239 | return "\n".join(lines) 240 | 241 | # life cycle 242 | 243 | def __call__(self) -> None: 244 | ... 245 | 246 | def on_before_dispatch(self) -> Optional[bool]: 247 | """Event triggered before the dispatch of the component""" 248 | 249 | def on_success(self): 250 | """Lifecycle event triggered iff execution finishes successfully""" 251 | 252 | def on_finish(self, success: bool): 253 | """Lifecycle event triggered right before the end of the component execution 254 | 255 | # Arguments 256 | success: Whether the execution finished sucessfully 257 | """ 258 | 259 | def on_failure(self, exception: Exception) -> None: 260 | """Lifecycle event triggered iff the execution finished with an exception 261 | 262 | # Arguments 263 | exception: Execution exception 264 | """ 265 | 266 | def on_after_dispatch(self, success: bool): 267 | """Lifecycle event triggered at the end of the dispatch. 268 | 269 | This is triggered independent of whether the execution has been successful or not. 270 | 271 | # Arguments 272 | success: Whether the execution finished sucessfully 273 | """ 274 | 275 | def on_seeding(self): 276 | """Lifecycle event to implement custom seeding using `self.seed`""" 277 | random.seed(self.seed) 278 | 279 | def on_write_meta_data(self) -> Optional[bool]: 280 | """Event triggered before meta-data such as creation time etc. is written to the storage 281 | 282 | Return False to prevent writing of meta-data 283 | """ 284 | 285 | def on_heartbeat(self) -> None: 286 | """Event triggered on heartbeat every 15 seconds""" 287 | -------------------------------------------------------------------------------- /src/machinable/config.py: -------------------------------------------------------------------------------- 1 | __all__ = ["to_dict"] 2 | from typing import TYPE_CHECKING, Any, Optional, Tuple, Union 3 | 4 | import collections 5 | import re 6 | import warnings 7 | from dataclasses import asdict, dataclass, is_dataclass 8 | from inspect import isclass 9 | 10 | import omegaconf 11 | from pydantic import BaseModel 12 | 13 | if TYPE_CHECKING: 14 | from machinable.element import Element 15 | 16 | 17 | class _ModelPrototype(BaseModel): 18 | pass 19 | 20 | 21 | def from_element(element: "Element") -> Tuple[dict, Optional[BaseModel]]: 22 | if not isclass(element): 23 | element = element.__class__ 24 | 25 | if not hasattr(element, "Config"): 26 | return {}, None 27 | 28 | config = getattr(element, "Config") 29 | 30 | # free-form 31 | if isinstance(config, collections.abc.Mapping): 32 | return config, None 33 | 34 | # pydantic model 35 | if isinstance(config, type(_ModelPrototype)): 36 | with warnings.catch_warnings(record=True): # ignore pydantic warnings 37 | return config().model_dump(), config 38 | 39 | # ordinary class 40 | if not is_dataclass(config): 41 | config = dataclass(config) 42 | 43 | # dataclass 44 | return asdict(config()), None 45 | 46 | 47 | def match_method(definition: str) -> Optional[Tuple[str, str]]: 48 | fn_match = re.match(r"^(?P\w+)\s?\((?P[^()]*)\)$", definition) 49 | if fn_match is None: 50 | return None 51 | 52 | function = fn_match.groupdict()["method"] 53 | args = fn_match.groupdict()["args"] 54 | return function, args 55 | 56 | 57 | def rewrite_config_methods( 58 | config: Union[collections.abc.Mapping, str, list, tuple] 59 | ) -> Any: 60 | if isinstance(config, list): 61 | return [rewrite_config_methods(v) for v in config] 62 | 63 | if isinstance(config, tuple): 64 | return (rewrite_config_methods(v) for v in config) 65 | 66 | if isinstance(config, collections.abc.Mapping): 67 | return {k: rewrite_config_methods(v) for k, v in config.items()} 68 | 69 | if isinstance(config, str) and match_method(config): 70 | # todo: take advatage of walrus operator in Python 3.8 71 | function, args = match_method(config) 72 | return '${config_method:"' + function + '","' + args + '"}' 73 | 74 | return config 75 | 76 | 77 | def to_dict(dict_like): 78 | if isinstance(dict_like, (omegaconf.DictConfig, omegaconf.ListConfig)): 79 | return omegaconf.OmegaConf.to_container(dict_like) 80 | 81 | if isinstance(dict_like, (list, tuple)): 82 | return dict_like.__class__([k for k in dict_like]) 83 | 84 | if not isinstance(dict_like, collections.abc.Mapping): 85 | return dict_like 86 | 87 | return {k: to_dict(v) for k, v in dict_like.items()} 88 | -------------------------------------------------------------------------------- /src/machinable/errors.py: -------------------------------------------------------------------------------- 1 | class MachinableError(Exception): 2 | """All of machinable exception inherit from this baseclass""" 3 | 4 | 5 | class ConfigurationError(MachinableError): 6 | """Invalid configuration 7 | 8 | Bases: MachinableError""" 9 | 10 | 11 | class DependencyMissing(MachinableError, ImportError): 12 | """Missing optional dependency 13 | 14 | Bases: ImportError""" 15 | 16 | 17 | class StorageError(MachinableError): 18 | """Storage error 19 | 20 | Bases: MachinableError""" 21 | 22 | 23 | class ExecutionFailed(MachinableError): 24 | """Execution failed 25 | 26 | Bases: MachinableError""" 27 | 28 | 29 | class ComponentException(MachinableError): 30 | """Component exception 31 | 32 | Bases: MachinableError""" 33 | -------------------------------------------------------------------------------- /src/machinable/mixin.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Any, Callable 2 | 3 | from functools import wraps 4 | from inspect import getattr_static 5 | 6 | if TYPE_CHECKING: 7 | from machinable.element import Element 8 | 9 | 10 | class Mixin: 11 | """Mixin base class""" 12 | 13 | 14 | class bind: 15 | """ 16 | Allows to dynamically extend object instances 17 | 18 | # Example 19 | ```python 20 | class Extension: 21 | def greet(self): 22 | # write an extension for Example class 23 | # note that self refers to the instance we are extending 24 | print(self.hello) 25 | 26 | class Example: 27 | def __init__(self): 28 | self.hello = 'hello world' 29 | # extend dynamically 30 | self.extension = Mixin(self, Extension, 'extension') 31 | 32 | Example().extension.greet() 33 | >>> 'hello world' 34 | ``` 35 | """ 36 | 37 | def __init__(self, target: Any, mixin_class: Any, name: str): 38 | self._binding_mixin_target = target 39 | self._binding_mixin_class = mixin_class 40 | self._binding_mixin_name = name 41 | 42 | def __getattr__(self, item): 43 | # forward dynamically into mix-in class 44 | attribute = getattr(self._binding_mixin_class, item, None) 45 | 46 | if attribute is None: 47 | raise AttributeError( 48 | f"'{self._binding_mixin_class.__name__}' has no attribute '{item}'" 49 | ) 50 | 51 | if isinstance(attribute, property): 52 | return attribute.fget(self._binding_mixin_target) 53 | 54 | if not callable(attribute): 55 | return attribute 56 | 57 | if isinstance( 58 | getattr_static(self._binding_mixin_class, item), staticmethod 59 | ): 60 | return attribute 61 | 62 | # if attribute is non-static method we decorate it to pass in the controller 63 | 64 | def bound_method(*args, **kwargs): 65 | return attribute(self._binding_mixin_target, *args, **kwargs) 66 | 67 | return bound_method 68 | 69 | 70 | def mixin(f: Callable) -> Any: 71 | @property 72 | @wraps(f) 73 | def _wrapper(self: "Element"): 74 | name = f.__name__ 75 | if name not in self.__mixins__: 76 | mixin_class = f(self) 77 | if isinstance(mixin_class, str): 78 | from machinable.project import Project, import_element 79 | 80 | mixin_class = import_element( 81 | Project.get().path(), mixin_class, Mixin 82 | ) 83 | 84 | self.__mixins__[name] = bind(self, mixin_class, name) 85 | 86 | # assign to __mixin__ for reference 87 | self.__mixin__ = self.__mixins__[name] 88 | 89 | return self.__mixins__[name] 90 | 91 | return _wrapper 92 | -------------------------------------------------------------------------------- /src/machinable/py.typed: -------------------------------------------------------------------------------- 1 | # PEP 561 2 | -------------------------------------------------------------------------------- /src/machinable/query.py: -------------------------------------------------------------------------------- 1 | from machinable.collection import InterfaceCollection 2 | from machinable.element import extend, normversion 3 | from machinable.interface import Interface 4 | from machinable.types import Optional, Union, VersionType 5 | 6 | 7 | class Query: 8 | def by_id(self, uuid: str) -> Optional[Interface]: 9 | return Interface.find_by_id(uuid) 10 | 11 | def from_directory(self, directory: str) -> Interface: 12 | return Interface.from_directory(directory) 13 | 14 | def __call__( 15 | self, 16 | module: Union[str, Interface, None] = None, 17 | version: VersionType = None, 18 | **kwargs, 19 | ) -> Interface: 20 | module, version = extend(module, version) 21 | return Interface.get(module, version, **kwargs) 22 | 23 | # modifiers 24 | 25 | def all( 26 | self, 27 | module: Union[None, str, Interface] = None, 28 | version: VersionType = None, 29 | **kwargs, 30 | ) -> "InterfaceCollection": 31 | module, version = extend(module, version) 32 | return Interface.find(module, version, **kwargs) 33 | 34 | def new( 35 | self, 36 | module: Union[None, str, Interface] = None, 37 | version: VersionType = None, 38 | **kwargs, 39 | ) -> Interface: 40 | module, version = extend(module, version) 41 | return Interface.make(module, version, **kwargs) 42 | -------------------------------------------------------------------------------- /src/machinable/schedule.py: -------------------------------------------------------------------------------- 1 | from machinable import schema 2 | from machinable.element import get_dump, get_lineage 3 | from machinable.interface import Interface 4 | from machinable.types import VersionType 5 | 6 | 7 | class Schedule(Interface): 8 | """Schedule base class""" 9 | 10 | kind = "Schedule" 11 | 12 | def __init__(self, version: VersionType = None): 13 | super().__init__(version) 14 | self.__model__ = schema.Schedule( 15 | kind=self.kind, 16 | module=self.__model__.module, 17 | config=self.__model__.config, 18 | version=self.__model__.version, 19 | lineage=get_lineage(self), 20 | ) 21 | self.__model__._dump = get_dump(self) 22 | -------------------------------------------------------------------------------- /src/machinable/schema.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union 2 | 3 | import time 4 | 5 | from machinable.utils import ( 6 | empty_uuid, 7 | generate_nickname, 8 | generate_seed, 9 | id_from_uuid, 10 | ) 11 | from pydantic import BaseModel, Field, PrivateAttr 12 | from uuid_extensions.uuid7 import timestamp_ns 13 | 14 | 15 | class Element(BaseModel): 16 | uuid: str = Field(default_factory=empty_uuid) 17 | kind: str = "Element" 18 | module: Optional[str] = None 19 | version: List[Union[str, Dict]] = [] 20 | config: Optional[Dict] = None 21 | predicate: Optional[Dict] = None 22 | context: Optional[Dict] = None 23 | lineage: Tuple[str, ...] = () 24 | 25 | @property 26 | def timestamp(self) -> int: 27 | try: 28 | return timestamp_ns(self.uuid, suppress_error=False) 29 | except ValueError: 30 | return 0 31 | 32 | @property 33 | def hash(self) -> str: 34 | return self.uuid[-12:] 35 | 36 | @property 37 | def id(self) -> str: 38 | return id_from_uuid(self.uuid) 39 | 40 | def extra(self) -> Dict: 41 | return {} 42 | 43 | 44 | class Storage(Element): 45 | kind: str = "Storage" 46 | 47 | 48 | class Index(Element): 49 | kind: str = "Index" 50 | 51 | 52 | class Scope(Element): 53 | kind: str = "Scope" 54 | 55 | 56 | class Interface(Element): 57 | kind: str = "Interface" 58 | _dump: Optional[bytes] = PrivateAttr(default=None) 59 | 60 | 61 | class Component(Interface): 62 | kind: str = "Component" 63 | seed: int = Field(default_factory=generate_seed) 64 | nickname: str = Field(default_factory=generate_nickname) 65 | 66 | def extra(self) -> Dict: 67 | return {"seed": self.seed, "nickname": self.nickname} 68 | 69 | 70 | class Project(Interface): 71 | kind: str = "Project" 72 | 73 | 74 | class Execution(Interface): 75 | kind: str = "Execution" 76 | seed: int = Field(default_factory=generate_seed) 77 | resources: Optional[Dict] = None 78 | 79 | def extra(self) -> Dict: 80 | return {"seed": self.seed, "resources": self.resources} 81 | 82 | 83 | class Schedule(Interface): 84 | kind: str = "Schedule" 85 | -------------------------------------------------------------------------------- /src/machinable/scope.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from machinable.config import to_dict 4 | from machinable.interface import Interface 5 | 6 | 7 | class Scope(Interface): 8 | kind = "Scope" 9 | default = None 10 | 11 | def __call__(self) -> Dict: 12 | return to_dict(self.config._update_) 13 | -------------------------------------------------------------------------------- /src/machinable/storage.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, List, Union 2 | 3 | import os 4 | 5 | from machinable import schema 6 | from machinable.element import get_lineage 7 | from machinable.index import Index 8 | from machinable.interface import Interface 9 | from machinable.types import VersionType 10 | from machinable.utils import load_file 11 | 12 | 13 | class Storage(Interface): 14 | kind = "Storage" 15 | default = None 16 | 17 | def __init__( 18 | self, 19 | version: VersionType = None, 20 | ): 21 | super().__init__(version=version) 22 | self.__model__ = schema.Storage( 23 | kind=self.kind, 24 | module=self.__model__.module, 25 | config=self.__model__.config, 26 | version=self.__model__.version, 27 | lineage=get_lineage(self), 28 | ) 29 | 30 | def upload( 31 | self, interface: "Interface", related: Union[bool, int] = True 32 | ) -> None: 33 | """ 34 | Upload the interface to the storage 35 | 36 | interface: Interface 37 | related: bool | int 38 | - 0/False: Do not save related interfaces 39 | - 1/True: Save immediately related interfaces 40 | - 2: Save related interfaces and their related interfaces 41 | """ 42 | if self.contains(interface.uuid): 43 | self.update(interface) 44 | else: 45 | self.commit(interface) 46 | 47 | if related: 48 | for r in interface.related(deep=int(related) == 2).all(): 49 | self.upload(r, related=False) 50 | 51 | def download( 52 | self, 53 | uuid: str, 54 | destination: Union[str, None] = None, 55 | related: Union[bool, int] = True, 56 | ) -> List[str]: 57 | """ 58 | Download to destination 59 | 60 | uuid: str 61 | Primary interface UUID 62 | destination: str | None 63 | If None, download will be imported into active index. If directory filepath, download will be placed 64 | in directory without import to index 65 | related: bool | int 66 | - 0/False: Do not upload related interfaces 67 | - 1/True: Upload immediately related interfaces 68 | - 2: Upload related interfaces and their related interfaces 69 | 70 | Returns: 71 | List of downloaded directories 72 | """ 73 | index = None 74 | download_directory = destination 75 | if destination is None: 76 | index = Index.get() 77 | download_directory = index.config.directory 78 | 79 | retrieved = [] 80 | 81 | # retrieve primary 82 | local_directory = os.path.join(download_directory, uuid) 83 | if self.retrieve(uuid, local_directory): 84 | retrieved.append(local_directory) 85 | 86 | # retrieve related 87 | if related: 88 | related_uuids = set() 89 | for r in load_file( 90 | [local_directory, "related", "metadata.jsonl"], [] 91 | ): 92 | related_uuids.add(r["uuid"]) 93 | related_uuids.add(r["related_uuid"]) 94 | 95 | for r in related_uuids: 96 | if r == uuid: 97 | continue 98 | retrieved.extend( 99 | self.download( 100 | r, 101 | destination, 102 | related=2 if int(related) == 2 else False, 103 | ) 104 | ) 105 | 106 | # import to index 107 | if index is not None: 108 | for directory in retrieved: 109 | index.import_directory(directory, relations=bool(related)) 110 | 111 | return retrieved 112 | 113 | def commit(self, interface: "Interface") -> None: 114 | ... 115 | 116 | def update(self, interface: "Interface") -> None: 117 | return self.commit(interface) 118 | 119 | def contains(self, uuid: str) -> bool: 120 | ... 121 | return False 122 | 123 | def retrieve(self, uuid: str, local_directory: str) -> bool: 124 | ... 125 | return False 126 | 127 | def search_for(self, interface: "Interface") -> List[str]: 128 | raise NotImplementedError() 129 | 130 | 131 | def fetch(uuid: str, directory: str) -> bool: 132 | available = False 133 | for storage in Storage.connected(): 134 | if storage.retrieve(uuid, directory): 135 | available = True 136 | break 137 | 138 | return available 139 | -------------------------------------------------------------------------------- /src/machinable/types.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Optional, Union 2 | 3 | from arrow.arrow import Arrow 4 | 5 | VersionType = Union[str, dict, None, List[Union[str, dict, None]]] 6 | ElementType = List[Union[str, dict]] 7 | DatetimeType = Arrow 8 | TimestampType = Union[float, int, DatetimeType] 9 | JsonableType = Dict[str, Union[str, float, int, None, DatetimeType]] 10 | Optional[List[Union[str, dict]]] 11 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/machinable-org/machinable/443884463fc2bec3b072d21492d7524f36bccc8d/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from machinable.index import Index 3 | 4 | 5 | @pytest.fixture() 6 | def tmp_storage(tmp_path): 7 | with Index( 8 | { 9 | "directory": str(tmp_path), 10 | "database": str(tmp_path / "index.sqlite"), 11 | } 12 | ) as index: 13 | yield index 14 | -------------------------------------------------------------------------------- /tests/samples/importing/__init__.py: -------------------------------------------------------------------------------- 1 | """Importing""" 2 | -------------------------------------------------------------------------------- /tests/samples/importing/nested/base.py: -------------------------------------------------------------------------------- 1 | from machinable import Element 2 | 3 | 4 | class BaseComponent(Element): 5 | """Base component""" 6 | -------------------------------------------------------------------------------- /tests/samples/importing/nested/bottom.py: -------------------------------------------------------------------------------- 1 | from .base import BaseComponent 2 | 3 | 4 | class BottomComponent(BaseComponent): 5 | """Bottom component""" 6 | -------------------------------------------------------------------------------- /tests/samples/importing/top.py: -------------------------------------------------------------------------------- 1 | from machinable import Element 2 | 3 | 4 | class TopComponent(Element): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/samples/in_session.py: -------------------------------------------------------------------------------- 1 | from machinable import Component 2 | from pydantic import BaseModel, Field 3 | 4 | 5 | class InSession(Component): 6 | class Config(BaseModel): 7 | a: int = Field(1, title="test") 8 | b: float = 0.1 9 | 10 | def __call__(self): 11 | print(self.config) 12 | -------------------------------------------------------------------------------- /tests/samples/project/.gitignore: -------------------------------------------------------------------------------- 1 | vendor/.cache/*.p 2 | vendor/fooba/vendor/.cache/*.p 3 | # Temporary and binary files 4 | *~ 5 | *.py[cod] 6 | *.so 7 | *.cfg 8 | !.isort.cfg 9 | !setup.cfg 10 | *.orig 11 | *.log 12 | *.pot 13 | __pycache__/* 14 | .cache/* 15 | .*.swp 16 | */.ipynb_checkpoints/* 17 | 18 | # Project files 19 | .ropeproject 20 | .project 21 | .pydevproject 22 | .settings 23 | .idea 24 | tags 25 | 26 | # Package files 27 | *.egg 28 | *.eggs/ 29 | .installed.cfg 30 | *.egg-info 31 | 32 | # Unittest and coverage 33 | htmlcov/* 34 | .coverage 35 | .tox 36 | junit.xml 37 | coverage.xml 38 | .pytest_cache/ 39 | .hypothesis 40 | 41 | # Build and docs folder/files 42 | build/* 43 | dist/* 44 | sdist/* 45 | docs/api/* 46 | docs/_rst/* 47 | docs/_build/* 48 | cover/* 49 | MANIFEST 50 | 51 | # Per-project virtualenvs 52 | .venv*/ 53 | 54 | 55 | # ---> macOS 56 | *.DS_Store 57 | .AppleDouble 58 | .LSOverride 59 | 60 | # Icon must end with two \r 61 | Icon 62 | 63 | 64 | # Thumbnails 65 | ._* 66 | 67 | # Files that might appear in the root of a volume 68 | .DocumentRevisions-V100 69 | .fseventsd 70 | .Spotlight-V100 71 | .TemporaryItems 72 | .Trashes 73 | .VolumeIcon.icns 74 | .com.apple.timemachine.donotpresent 75 | 76 | # Directories potentially created on remote AFP share 77 | .AppleDB 78 | .AppleDesktop 79 | Network Trash Folder 80 | Temporary Items 81 | .apdisk 82 | -------------------------------------------------------------------------------- /tests/samples/project/basic.py: -------------------------------------------------------------------------------- 1 | from machinable import Component 2 | 3 | 4 | class Basic(Component): 5 | def __init__(self, *args, **kwargs): 6 | super().__init__(*args, **kwargs) 7 | self._state = None 8 | 9 | def hello(self): 10 | return "there" 11 | 12 | def set_state(self, state): 13 | self._state = state 14 | 15 | def get_state(self): 16 | return self._state 17 | -------------------------------------------------------------------------------- /tests/samples/project/count.py: -------------------------------------------------------------------------------- 1 | from machinable import Component 2 | 3 | 4 | class Counter(Component): 5 | def __call__(self): 6 | self.save_file("count", self.count + 1) 7 | 8 | @property 9 | def count(self): 10 | return int(self.load_file("count", 0)) 11 | -------------------------------------------------------------------------------- /tests/samples/project/dummy.py: -------------------------------------------------------------------------------- 1 | from machinable import Component 2 | 3 | 4 | class Dummy(Component): 5 | class Config: 6 | a: int = 1 7 | ignore_me_: int = -1 8 | 9 | def name(self): 10 | return "dummy" 11 | -------------------------------------------------------------------------------- /tests/samples/project/empty.py: -------------------------------------------------------------------------------- 1 | class NotAnElement: 2 | pass 3 | -------------------------------------------------------------------------------- /tests/samples/project/exec.py: -------------------------------------------------------------------------------- 1 | from machinable import Execution 2 | 3 | 4 | class Test(Execution): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/samples/project/fail.py: -------------------------------------------------------------------------------- 1 | from machinable import Component 2 | 3 | 4 | class Fail(Component): 5 | def __call__(self): 6 | if not self.load_file("repaired", False): 7 | raise Exception("Fail") 8 | -------------------------------------------------------------------------------- /tests/samples/project/hello.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from machinable import Component 4 | 5 | 6 | class Hello(Component): 7 | @dataclass 8 | class Config: 9 | name: str = "World" 10 | 11 | def __call__(self): 12 | print(f"Hello {self.config.name}!") 13 | 14 | def resources(self): 15 | print(self.execution.computed_resources(self)) 16 | -------------------------------------------------------------------------------- /tests/samples/project/interface/dummy.py: -------------------------------------------------------------------------------- 1 | from machinable import Interface 2 | 3 | 4 | class Dummy(Interface): 5 | def __call__(self): 6 | print("Hello world!") 7 | 8 | def hello(self) -> str: 9 | return "world" 10 | -------------------------------------------------------------------------------- /tests/samples/project/interface/events_check.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional 2 | 3 | from machinable import Component, errors 4 | 5 | 6 | class EventsCheck(Component): 7 | def __init__(self, *args, **kwargs): 8 | super().__init__(*args, **kwargs) 9 | self.events = ["on_init"] 10 | 11 | def on_before_dispatch(self): 12 | self.events.append("on_dispatch") 13 | 14 | def on_seeding(self): 15 | self.events.append("on_seeding") 16 | return False 17 | 18 | def on_success(self): 19 | assert self.execution.is_started() 20 | self.events.append("on_success") 21 | 22 | def __call__(self) -> None: 23 | assert self.execution.is_active() 24 | self.events.append("on_call") 25 | 26 | def on_after_dispatch(self, success): 27 | self.events.append("on_after_dispatch") 28 | self.save_file("events.json", self.events) 29 | assert self.execution.is_finished() 30 | 31 | def on_failure(self, exception: errors.MachinableError): 32 | assert False 33 | -------------------------------------------------------------------------------- /tests/samples/project/interface/interrupted_lifecycle.py: -------------------------------------------------------------------------------- 1 | from machinable import Component 2 | 3 | 4 | class InterruptedLifecycle(Component): 5 | def __call__(self): 6 | self.local_directory("data", create=True) 7 | self.state = self.load_file("data/state.json", {"steps": 0}) 8 | 9 | for step in range(self.state["steps"], 10): 10 | self.state["steps"] = step + 1 11 | 12 | if step == 2: 13 | raise RuntimeError("Interrupt 1") 14 | 15 | if step == 6: 16 | raise RuntimeError("Interrupt 2") 17 | 18 | def on_finish(self, success): 19 | self.save_file("data/state.json", self.state) 20 | -------------------------------------------------------------------------------- /tests/samples/project/interface/project.py: -------------------------------------------------------------------------------- 1 | from machinable import Project 2 | 3 | 4 | class TestProject(Project): 5 | def config_global_conf(self, works=False): 6 | return works 7 | 8 | def version_global_ver(self, works=False): 9 | return works 10 | 11 | def on_resolve_element(self, module): 12 | if module == "@test": 13 | return "basic", None 14 | 15 | if module == "dummy_version_extend": 16 | return ["dummy", {"a": 100}], None 17 | 18 | return super().on_resolve_element(module) 19 | 20 | def get_host_info(self): 21 | info = super().get_host_info() 22 | info["dummy"] = "data" 23 | return info 24 | 25 | def on_resolve_remotes(self): 26 | return { 27 | "!hello": "file+" + self.path("hello.py"), 28 | "!hello-link": "link+" + self.path("hello.py"), 29 | "!invalid": "test", 30 | "!multi": ["file+" + self.path("hello.py"), "!hello-link"], 31 | "!multichain": ["file+" + self.path("hello.py"), "!multi"], 32 | "!multi-invalid": ["file+" + self.path("hello.py"), "!invalid"], 33 | } 34 | -------------------------------------------------------------------------------- /tests/samples/project/line.py: -------------------------------------------------------------------------------- 1 | from dummy import Dummy 2 | 3 | 4 | class Line(Dummy): 5 | def on_instantiate(self): 6 | self.msg_set_during_instantiation = "hello world" 7 | -------------------------------------------------------------------------------- /tests/samples/project/mixins/example.py: -------------------------------------------------------------------------------- 1 | from machinable import Element, mixin 2 | 3 | 4 | class Example(Element): 5 | class Config: 6 | a: str = "world" 7 | 8 | @mixin 9 | def test(self): 10 | return "mixins.extension" 11 | 12 | @mixin 13 | def dummy(self): 14 | return "dummy" 15 | 16 | def say(self) -> str: 17 | return self.test.hello() 18 | 19 | def say_bound(self): 20 | return self.bound_hello() 21 | 22 | def calling_into_the_void(self) -> str: 23 | return "success" 24 | -------------------------------------------------------------------------------- /tests/samples/project/mixins/extension.py: -------------------------------------------------------------------------------- 1 | from machinable import Mixin 2 | 3 | 4 | class Extension(Mixin): 5 | def hello(self) -> str: 6 | return self.there(self.config.a) 7 | 8 | def there(self, a) -> str: 9 | return "hello, " + a 10 | 11 | def bound_hello(self) -> str: 12 | return self.calling_into_the_void() 13 | 14 | def calling_into_the_void(self) -> str: 15 | return "no response" 16 | -------------------------------------------------------------------------------- /tests/samples/project/predicate.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Optional, Union 2 | 3 | from machinable import Component 4 | from machinable.interface import Interface 5 | from machinable.types import VersionType 6 | 7 | 8 | class PredicateComponent(Component): 9 | class Config: 10 | a: int = 1 11 | ignore_: int = 2 12 | 13 | def __init__(self, *args, test_context=None, **kwargs): 14 | super().__init__(*args, **kwargs) 15 | self._context = test_context 16 | 17 | def compute_context(self): 18 | context = super().compute_context() 19 | 20 | if self._context: 21 | context.pop("config") 22 | context.update(self._context) 23 | 24 | return context 25 | 26 | def on_compute_predicate(self): 27 | return {"test": "a"} 28 | -------------------------------------------------------------------------------- /tests/samples/project/scheduled.py: -------------------------------------------------------------------------------- 1 | from machinable import Schedule 2 | 3 | 4 | class Dummy(Schedule): 5 | def test(self): 6 | return True 7 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from machinable import Component, Project, from_cli, get_version 5 | from machinable.cli import main 6 | 7 | 8 | def test_cli_main(capfd, tmp_storage): 9 | # version 10 | assert main(["version"]) == 0 11 | out, err = capfd.readouterr() 12 | assert out == get_version() + "\n" 13 | 14 | # get 15 | with Project("tests/samples/project"): 16 | main(["get", "hello", "--launch"]) 17 | out, err = capfd.readouterr() 18 | assert out == "Hello World!\n" 19 | main(["get", "hello", "name=Test", "--launch"]) 20 | out, err = capfd.readouterr() 21 | assert out == "Hello Test!\n" 22 | main(["get", "hello", "name=Twice", "--launch", "--__call__"]) 23 | out, err = capfd.readouterr() 24 | assert out == "Hello Twice!\nHello Twice!\n" 25 | assert main(["get.new", "hello", "--launch"]) == 0 26 | 27 | with pytest.raises(ValueError): 28 | main(["get"]) 29 | 30 | out, err = capfd.readouterr() 31 | main(["get", "interface.dummy", "--__call__"]) 32 | out, err = capfd.readouterr() 33 | assert out == "Hello world!\n" 34 | main(["get", "interface.dummy", "hello", "name=there", "--__call__"]) 35 | out, err = capfd.readouterr() 36 | assert out == "Hello there!\n" 37 | 38 | main( 39 | [ 40 | "get", 41 | "machinable.execution", 42 | "**resources={'a': 1}", 43 | "hello", 44 | "name=there", 45 | "--resources", 46 | ] 47 | ) 48 | out, err = capfd.readouterr() 49 | assert out == "{'a': 1}\n" 50 | 51 | main( 52 | [ 53 | "get", 54 | "machinable.execution", 55 | "**resources={'a': 1}", 56 | "**resources={'a': 2}", 57 | "--__model__", 58 | ] 59 | ) 60 | out, err = capfd.readouterr() 61 | assert "resources={'a': 2}" in out 62 | 63 | out, err = capfd.readouterr() 64 | main( 65 | [ 66 | "get", 67 | "machinable.execution", 68 | "**resources={'a': 1}", 69 | "--__model__", 70 | ] 71 | ) 72 | out, err = capfd.readouterr() 73 | assert "resources={'a': 1}" in out 74 | 75 | # help 76 | assert main([]) == 0 77 | assert main(["help"]) == 0 78 | 79 | assert isinstance(from_cli(), list) 80 | assert from_cli([]) == [] 81 | assert from_cli(["~test", "a=1", "a.b=2"]) == ["~test", {"a": {"b": 2}}] 82 | assert from_cli(["test", "me"]) == ["test", "me"] 83 | 84 | 85 | def test_cli_to_cli(): 86 | assert Component().to_cli() == "machinable.component" 87 | assert ( 88 | Component(["~test", {"a": {"b": 1}}, "~foo"]).to_cli() 89 | == "machinable.component ~test a.b=1 ~foo" 90 | ) 91 | assert ( 92 | Component([{"a": {"b": 1}}, {"c": 1}]).to_cli() 93 | == "machinable.component a.b=1 c=1" 94 | ) 95 | assert ( 96 | Component({"a": "t m ."}).to_cli() == "machinable.component a='t m .'" 97 | ) 98 | 99 | 100 | def test_cli_installation(): 101 | assert os.system("machinable help") == 0 102 | assert os.system("machinable version") == 0 103 | assert os.WEXITSTATUS(os.system("machinable --invalid")) == 128 104 | -------------------------------------------------------------------------------- /tests/test_collection.py: -------------------------------------------------------------------------------- 1 | # This file contains modified 3rd party source code from 2 | # https://github.com/sdispater/backpack/blob/master/tests/collections/test_collection.py 3 | # The copyright and license agreement can be found in the ThirdPartyNotices.txt file at the root of this repository. 4 | 5 | from unittest import TestCase 6 | 7 | from machinable.collection import Collection, ComponentCollection, collect 8 | from machinable.component import Component 9 | from machinable.execution import Execution 10 | from machinable.project import Project 11 | 12 | 13 | def test_collect(): 14 | assert isinstance(collect([1, 2]), Collection) 15 | 16 | 17 | class Dummy(Component): 18 | class Config: 19 | m: int = -1 20 | 21 | 22 | def test_element_collection(tmp_storage): 23 | with Project("./tests/samples/project"): 24 | collection = Component.collect([Dummy({"m": i % 2}) for i in range(5)]) 25 | for i, e in enumerate(collection): 26 | e.save_file("i", i) 27 | assert isinstance(collection, ComponentCollection) 28 | 29 | collection.launch() 30 | assert all(collection.map(lambda x: x.execution.is_finished())) 31 | 32 | m = "tests.test_collection" 33 | assert len(collection.filter_by_context(m)) == 0 34 | assert len(collection.filter_by_context(m, {"m": 0})) == 3 35 | assert len(collection.filter_by_context(m, {"m": 1})) == 2 36 | 37 | assert collection.singleton(m, {"m": 1}).load_file("i") == "1" 38 | 39 | collection = Execution.collect([e.execution for e in collection]) 40 | 41 | assert len(collection.status("finished")) == 5 42 | assert len(collection.status("active")) == 0 43 | assert len(collection.status("started")) == 5 44 | assert len(collection.status("incomplete")) == 0 45 | assert len(collection.status("started").status("active")) == 0 46 | 47 | 48 | class CollectionTestCase(TestCase): 49 | def test_first_returns_first_item_in_collection(self): 50 | c = Collection(["foo", "bar"]) 51 | 52 | self.assertEqual("foo", c.first()) 53 | 54 | def test_last_returns_last_item_in_collection(self): 55 | c = Collection(["foo", "bar"]) 56 | 57 | self.assertEqual("bar", c.last()) 58 | 59 | def test_pop_removes_and_returns_last_item_or_specified_index(self): 60 | c = Collection(["foo", "bar"]) 61 | 62 | self.assertEqual("bar", c.pop()) 63 | self.assertEqual("foo", c.last()) 64 | 65 | c = Collection(["foo", "bar"]) 66 | 67 | self.assertEqual("foo", c.pop(0)) 68 | self.assertEqual("bar", c.first()) 69 | 70 | def test_empty_collection_is_empty(self): 71 | c = Collection() 72 | c2 = Collection([]) 73 | 74 | self.assertTrue(c.empty()) 75 | self.assertTrue(c2.empty()) 76 | 77 | def test_collection_is_constructed(self): 78 | c = Collection("foo") 79 | self.assertEqual(["foo"], c.all()) 80 | 81 | c = Collection(2) 82 | self.assertEqual([2], c.all()) 83 | 84 | c = Collection(False) 85 | self.assertEqual([False], c.all()) 86 | 87 | c = Collection(None) 88 | self.assertEqual([], c.all()) 89 | 90 | c = Collection() 91 | self.assertEqual([], c.all()) 92 | 93 | def test_offset_access(self): 94 | c = Collection(["foo", "bar"]) 95 | self.assertEqual("bar", c[1]) 96 | 97 | c[1] = "baz" 98 | self.assertEqual("baz", c[1]) 99 | 100 | del c[0] 101 | self.assertEqual("baz", c[0]) 102 | 103 | def test_forget(self): 104 | c = Collection(["foo", "bar", "boom"]) 105 | c.forget(0) 106 | self.assertEqual("bar", c[0]) 107 | c.forget(0, 1) 108 | self.assertTrue(c.empty()) 109 | 110 | def test_get_avg_items_from_collection(self): 111 | c = Collection([{"foo": 10}, {"foo": 20}]) 112 | self.assertEqual(15, c.avg("foo")) 113 | 114 | c = Collection([1, 2, 3, 4, 5]) 115 | self.assertEqual(3, c.avg()) 116 | 117 | c = Collection() 118 | self.assertIsNone(c.avg()) 119 | 120 | def test_collapse(self): 121 | obj1 = object() 122 | obj2 = object() 123 | 124 | c = Collection([[obj1], [obj2]]) 125 | self.assertEqual([obj1, obj2], c.collapse().all()) 126 | 127 | def test_collapse_with_nested_collection(self): 128 | c = Collection([Collection([1, 2, 3]), Collection([4, 5, 6])]) 129 | self.assertEqual([1, 2, 3, 4, 5, 6], c.collapse().all()) 130 | 131 | def test_contains(self): 132 | c = Collection([1, 3, 5]) 133 | 134 | self.assertTrue(c.contains(1)) 135 | self.assertFalse(c.contains(2)) 136 | self.assertTrue(c.contains(lambda x: x < 5)) 137 | self.assertFalse(c.contains(lambda x: x > 5)) 138 | self.assertIn(3, c) 139 | 140 | c = Collection([{"v": 1}, {"v": 3}, {"v": 5}]) 141 | self.assertTrue(c.contains("v", 1)) 142 | self.assertFalse(c.contains("v", 2)) 143 | 144 | obj1 = type("lamdbaobject", (object,), {})() 145 | obj1.v = 1 146 | obj2 = type("lamdbaobject", (object,), {})() 147 | obj2.v = 3 148 | obj3 = type("lamdbaobject", (object,), {})() 149 | obj3.v = 5 150 | c = Collection([{"v": 1}, {"v": 3}, {"v": 5}]) 151 | self.assertTrue(c.contains("v", 1)) 152 | self.assertFalse(c.contains("v", 2)) 153 | 154 | def test_countable(self): 155 | c = Collection(["foo", "bar"]) 156 | self.assertEqual(2, c.count()) 157 | self.assertEqual(2, len(c)) 158 | 159 | def test_diff(self): 160 | c = Collection(["foo", "bar"]) 161 | self.assertEqual(["foo"], c.diff(Collection(["bar", "baz"])).all()) 162 | 163 | def test_each(self): 164 | original = ["foo", "bar", "baz"] 165 | c = Collection(original) 166 | 167 | result = [] 168 | c.each(lambda x: result.append(x)) 169 | self.assertEqual(result, original) 170 | self.assertEqual(original, c.all()) 171 | 172 | def test_every(self): 173 | c = Collection([1, 2, 3, 4, 5, 6]) 174 | self.assertEqual([1, 3, 5], c.every(2).all()) 175 | self.assertEqual([2, 4, 6], c.every(2, 1).all()) 176 | 177 | def test_filter(self): 178 | c = Collection([{"id": 1, "name": "hello"}, {"id": 2, "name": "world"}]) 179 | self.assertEqual( 180 | [{"id": 2, "name": "world"}], 181 | c.filter(lambda item: item["id"] == 2).all(), 182 | ) 183 | 184 | c = Collection(["", "hello", "", "world"]) 185 | self.assertEqual(["hello", "world"], c.filter().all()) 186 | 187 | def test_where(self): 188 | c = Collection([{"v": 1}, {"v": 3}, {"v": 2}, {"v": 3}, {"v": 4}]) 189 | self.assertEqual([{"v": 3}, {"v": 3}], c.where("v", 3).all()) 190 | 191 | def test_implode(self): 192 | obj1 = type("lamdbaobject", (object,), {})() 193 | obj1.name = "john" 194 | obj1.email = "foo" 195 | c = Collection( 196 | [{"name": "john", "email": "foo"}, {"name": "jane", "email": "bar"}] 197 | ) 198 | self.assertEqual("foobar", c.implode("email")) 199 | self.assertEqual("foo,bar", c.implode("email", ",")) 200 | 201 | c = Collection(["foo", "bar"]) 202 | self.assertEqual("foobar", c.implode("")) 203 | self.assertEqual("foo,bar", c.implode(",")) 204 | 205 | def test_map(self): 206 | c = Collection([1, 2, 3, 4, 5]) 207 | self.assertEqual([3, 4, 5, 6, 7], c.map(lambda x: x + 2).all()) 208 | 209 | def test_merge(self): 210 | c = Collection([1, 2, 3]) 211 | c.merge([4, 5, 6]) 212 | self.assertEqual([1, 2, 3, 4, 5, 6], c.all()) 213 | 214 | c = Collection(Collection([1, 2, 3])) 215 | c.merge([4, 5, 6]) 216 | self.assertEqual([1, 2, 3, 4, 5, 6], c.all()) 217 | 218 | def test_prepend(self): 219 | c = Collection([4, 5, 6]) 220 | c.prepend(3) 221 | self.assertEqual([3, 4, 5, 6], c.all()) 222 | 223 | def test_append(self): 224 | c = Collection([3, 4, 5]) 225 | c.append(6) 226 | self.assertEqual([3, 4, 5, 6], c.all()) 227 | 228 | def test_pull(self): 229 | c = Collection([1, 2, 3, 4]) 230 | c.pull(2) 231 | self.assertEqual([1, 2, 4], c.all()) 232 | 233 | def test_put(self): 234 | c = Collection([1, 2, 4]) 235 | c.put(2, 3) 236 | self.assertEqual([1, 2, 3], c.all()) 237 | 238 | def test_reject(self): 239 | c = Collection([1, 2, 3, 4, 5, 6]) 240 | self.assertEqual([1, 2, 3], c.reject(lambda x: x > 3).all()) 241 | 242 | def test_reverse(self): 243 | c = Collection([1, 2, 3, 4]) 244 | self.assertEqual([4, 3, 2, 1], c.reverse().all()) 245 | 246 | def test_sort(self): 247 | c = Collection([5, 3, 1, 2, 4]) 248 | 249 | sorted = c.sort(lambda x: x) 250 | self.assertEqual([1, 2, 3, 4, 5], sorted.all()) 251 | 252 | def test_take(self): 253 | c = Collection([1, 2, 3, 4, 5, 6]) 254 | self.assertEqual([1, 2, 3], c.take(3).all()) 255 | self.assertEqual([4, 5, 6], c.take(-3).all()) 256 | 257 | def test_transform(self): 258 | c = Collection([1, 2, 3, 4]) 259 | c.transform(lambda x: x + 2) 260 | self.assertEqual([3, 4, 5, 6], c.all()) 261 | 262 | def test_zip(self): 263 | c = Collection([1, 2, 3]) 264 | self.assertEqual([(1, 4), (2, 5), (3, 6)], c.zip([4, 5, 6]).all()) 265 | 266 | def test_only(self): 267 | c = Collection([1, 2, 3, 4, 5]) 268 | self.assertEqual([2, 4], c.only(1, 3).all()) 269 | 270 | def test_without(self): 271 | c = Collection([1, 2, 3, 4, 5]) 272 | self.assertEqual([1, 3, 5], c.without(1, 3).all()) 273 | self.assertEqual([1, 2, 3, 4, 5], c.all()) 274 | 275 | def test_flatten(self): 276 | c = Collection({"foo": [5, 6], "bar": 7, "baz": {"boom": [1, 2, 3, 4]}}) 277 | 278 | self.assertEqual([1, 2, 3, 4, 5, 6, 7], c.flatten().sort().all()) 279 | 280 | c = Collection([1, [2, 3], 4]) 281 | self.assertEqual([1, 2, 3, 4], c.flatten().all()) 282 | -------------------------------------------------------------------------------- /tests/test_component.py: -------------------------------------------------------------------------------- 1 | import os 2 | import stat 3 | import subprocess 4 | import sys 5 | 6 | import pytest 7 | from machinable import ( 8 | Component, 9 | Execution, 10 | Project, 11 | Storage, 12 | errors, 13 | get, 14 | schema, 15 | ) 16 | from machinable.element import Element 17 | 18 | 19 | def test_component(tmp_storage): 20 | p = Project("./tests/samples/project").__enter__() 21 | component = Component.make("dummy") 22 | assert component.module == "dummy" 23 | assert isinstance(str(component), str) 24 | assert isinstance(repr(component), str) 25 | assert component.config.a == 1 26 | 27 | # version 28 | assert component.version() == [] 29 | assert component.version("test") == ["test"] 30 | assert component.version() == ["test"] 31 | assert component.version("replace", overwrite=True) == ["replace"] 32 | component.version({"a": -1}, overwrite=True) 33 | assert component.config.a == -1 34 | component.version({"a": 1}) 35 | assert component.config.a == 1 36 | 37 | component = Component.from_model(Component.model(component)) 38 | serialized = component.serialize() 39 | assert serialized["config"]["a"] == 1 40 | 41 | # write protection 42 | component = Component.make("dummy").commit() 43 | assert component.version() == [] 44 | with pytest.raises(errors.MachinableError): 45 | component.version(["modify"]) 46 | 47 | p.__exit__() 48 | 49 | 50 | def test_component_launch(tmp_storage): 51 | component = Component() 52 | assert not component.is_mounted() 53 | component.launch() 54 | assert component.is_mounted() 55 | assert component.execution.is_finished() 56 | 57 | # multiples 58 | component = Component() 59 | with Execution() as execution: 60 | component.launch() 61 | component.launch() 62 | component.launch() 63 | assert len(execution.executables) == 1 64 | 65 | with Execution(): 66 | e1 = Component().launch() 67 | e2 = Component().launch() 68 | assert e1.execution.is_finished() 69 | assert e2.execution.is_finished() 70 | assert e1.nickname != e2.nickname 71 | 72 | class Example(Component): 73 | def __call__(self): 74 | print("hello world") 75 | 76 | get(Example).launch() 77 | 78 | 79 | def test_component_relations(tmp_storage): 80 | with Project("./tests/samples/project") as project: 81 | component = Component.instance("basic") 82 | execution = Execution().add(component) 83 | component.push_related("project", project) 84 | execution.dispatch() 85 | 86 | assert component.project.name() == "project" 87 | assert component.execution.timestamp == execution.timestamp 88 | assert component.executions[0].timestamp == execution.timestamp 89 | assert len(component.uses) == 0 90 | 91 | with pytest.raises(errors.MachinableError): 92 | component.version("attempt_overwrite") 93 | 94 | derived = Component(derived_from=component) 95 | assert derived.ancestor is component 96 | derived_execution = Execution().add(derived).dispatch() 97 | 98 | # invalidate cache and reconstruct 99 | component.__related__ = {} 100 | component._relation_cache = {} 101 | execution.__related__ = {} 102 | execution._relation_cache = {} 103 | derived.__related__ = {} 104 | derived._relation_cache = {} 105 | derived_execution.__related__ = {} 106 | derived_execution._relation_cache = {} 107 | 108 | assert derived.ancestor.id == component.id 109 | assert derived.ancestor.hello() == "there" 110 | assert component.derived[0].id == derived.id 111 | 112 | derived = Component(derived_from=component) 113 | Execution().add(derived).dispatch() 114 | assert len(component.derived) == 2 115 | 116 | assert component.derive().id != component.id 117 | derived = component.derive(version=component.config) 118 | Execution().add(derived).dispatch() 119 | 120 | 121 | class DataElement(Element): 122 | class Config: 123 | dataset: str = "mnist" 124 | 125 | def hello(self): 126 | return "element" 127 | 128 | 129 | def test_component_lifecycle(tmp_storage): 130 | with Project("tests/samples/project"): 131 | # test dispatch lifecycle 132 | component = Component.make("interface.events_check") 133 | component.launch() 134 | assert len(component.load_file("events.json")) == 6 135 | 136 | 137 | class ExportComponent(Component): 138 | def __call__(self): 139 | print("Hello world") 140 | self.save_file("test_run.json", {"success": True}) 141 | 142 | 143 | def test_component_export(tmp_storage): 144 | component = ExportComponent() 145 | 146 | script = component.dispatch_code(inline=False) 147 | 148 | with pytest.raises(FileNotFoundError): 149 | exec(script) 150 | 151 | e = Execution().add(component).commit() 152 | 153 | script = component.dispatch_code(inline=False) 154 | 155 | assert not component.execution.is_started() 156 | 157 | exec(script) 158 | 159 | assert component.execution.is_finished() 160 | assert component.load_file("test_run.json")["success"] 161 | 162 | # inline 163 | component = ExportComponent() 164 | Execution().add(component).commit() 165 | script = component.dispatch_code(inline=True) 166 | script_filepath = component.save_file("run.sh", script) 167 | st = os.stat(script_filepath) 168 | os.chmod(script_filepath, st.st_mode | stat.S_IEXEC) 169 | 170 | output = subprocess.run( 171 | ["bash", script_filepath], capture_output=True, text=True, check=True 172 | ).stdout 173 | print(output) 174 | assert component.execution.is_finished() 175 | assert component.load_file("test_run.json")["success"] 176 | 177 | class OuterContext(Execution): 178 | def __call__(self): 179 | assert False, "Should not be called" 180 | 181 | c = ExportComponent().commit() 182 | with OuterContext(): 183 | script = c.dispatch_code(inline=False) 184 | exec(script) 185 | 186 | class EscapeTest(Component): 187 | Config = {"test": "method('valid_escape')"} 188 | 189 | def config_method(self, value): 190 | return value == "valid_escape" 191 | 192 | c = EscapeTest().commit() 193 | assert c.config.test 194 | exec(c.dispatch_code(inline=False)) 195 | 196 | assert c.dispatch_code(inline=True).find("\n") == -1 197 | 198 | out = c.dispatch_code(inline=True).replace(f'{sys.executable} -c "', "")[ 199 | :-1 200 | ] 201 | exec(out) 202 | 203 | 204 | def test_component_predicates(tmp_storage): 205 | p = Project("./tests/samples/project").__enter__() 206 | 207 | e1 = get("predicate", {"a": 2}) 208 | e1.launch() 209 | e2 = get("predicate", {"ignore_": 3}) 210 | e2.launch() 211 | assert e1 != e2 212 | e3 = get("predicate", {"a": 4}) 213 | e3.launch() 214 | assert e2 != e3 215 | 216 | p.__exit__() 217 | 218 | 219 | def test_component_interactive_session(tmp_storage): 220 | class T(Component): 221 | def is_valid(self): 222 | return True 223 | 224 | t = get(T) 225 | assert t.module == "__session__T" 226 | assert t.__model__._dump is not None 227 | 228 | # default launch 229 | t.launch() 230 | # serialization 231 | exec(t.dispatch_code(inline=False) + "\nassert component__.is_valid()") 232 | # retrieval 233 | assert t == get(T) 234 | 235 | # redefine 236 | 237 | class T(Component): 238 | def extended(self): 239 | return True 240 | 241 | def is_valid(self): 242 | return True 243 | 244 | rt = get(T) 245 | assert rt.extended() 246 | 247 | class TT(T): 248 | pass 249 | 250 | rtt = get(TT) 251 | assert rtt != rt 252 | 253 | from tests.samples.in_session import InSession 254 | 255 | t = get(InSession, {"a": 2}) 256 | t.launch() 257 | t2 = get(InSession, {"a": 2}) 258 | assert t == t2 259 | 260 | 261 | def test_component_from_index(tmp_storage): 262 | with Project("tests/samples/project"): 263 | c = Component.make("dummy", {"a": 9}).commit() 264 | cp = Component.find_by_id(c.uuid, fetch=False) 265 | assert c.seed == c.seed 266 | assert c.nickname == cp.nickname 267 | 268 | 269 | def test_component_future(tmp_storage): 270 | c = Component() 271 | assert c.future() is None 272 | assert len(c._futures_stack) == 0 273 | c.launch() 274 | assert c.future() is c 275 | with Execution().deferred() as execution: 276 | assert c.future() is None 277 | 278 | assert execution.executables[0] is c 279 | 280 | # future tracking 281 | class T(Component): 282 | def test(self): 283 | c = Component() 284 | assert c.future() is None 285 | assert len(c._futures_stack) == 0 286 | assert list(self._futures_stack) == [c.id] 287 | c.launch() 288 | assert c.future() is c 289 | assert len(self._futures_stack) == 0 290 | with Execution().deferred() as execution: 291 | assert c.future() is None 292 | 293 | assert execution.executables[0] is c 294 | 295 | def test_await(self): 296 | c = Component() 297 | c.launch() 298 | c = Component() 299 | assert c.future() is None 300 | return c.id 301 | 302 | t = T() 303 | assert t.future() is None 304 | t.test() 305 | assert len(t._futures_stack) == 0 306 | 307 | t.commit().cached(True) 308 | assert t.future() is t 309 | 310 | u = t.test_await() 311 | assert list(t._futures_stack) == [u] 312 | assert t.future() is None 313 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import omegaconf 2 | from machinable.config import ( 3 | from_element, 4 | match_method, 5 | rewrite_config_methods, 6 | to_dict, 7 | ) 8 | 9 | 10 | def test_config_from_element(): 11 | class Dummy: 12 | pass 13 | 14 | assert from_element(Dummy) == ({}, None) 15 | 16 | class HasConf: 17 | class Config: 18 | q: int = 1 19 | 20 | assert from_element(HasConf)[0]["q"] == 1 21 | assert from_element(HasConf())[0]["q"] == 1 22 | 23 | class DictConf: 24 | Config = {"a": 2} 25 | 26 | assert from_element(DictConf)[0]["a"] == 2 27 | 28 | 29 | def test_match_method(): 30 | assert match_method("find_me(a=1)") == ("find_me", "a=1") 31 | assert match_method("test(1)") == ("test", "1") 32 | assert match_method("foo") is None 33 | assert match_method("ma$formed()") is None 34 | assert match_method("foo(1) < 1") is None 35 | assert match_method("foo(1) + bar(2)") is None 36 | assert match_method(" foo(1)") is None 37 | 38 | 39 | def test_rewrite_config_methods(): 40 | rewrite_config_methods({"test": "test_me(1)"}) == { 41 | "test": "${config_method:test_me,1}" 42 | } 43 | 44 | 45 | def test_to_dict(): 46 | assert to_dict({"a": 1}) == {"a": 1} 47 | assert to_dict(omegaconf.DictConfig({"a": 1})) == {"a": 1} 48 | assert to_dict(omegaconf.ListConfig([1, 2, 3])) == [1, 2, 3] 49 | assert to_dict({"a": omegaconf.DictConfig({"b": 1})}) == {"a": {"b": 1}} 50 | assert to_dict([1, 2, 3]) == [1, 2, 3] 51 | assert to_dict((1, 2, 3)) == (1, 2, 3) 52 | assert to_dict(1) == 1 53 | assert to_dict("foo") == "foo" 54 | -------------------------------------------------------------------------------- /tests/test_docs.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | import os 4 | 5 | from machinable import Project 6 | from machinable.utils import import_from_directory 7 | 8 | 9 | def _parse_code_string(code_string: str) -> Tuple[Optional[str], Optional[str]]: 10 | code_string = code_string[3:] 11 | q = code_string.split("[") 12 | if len(q) == 1: 13 | return code_string.strip(), None 14 | else: 15 | language = q[0].strip() 16 | filename = q[1].strip()[:-1] 17 | return language, filename 18 | 19 | 20 | def test_docs(tmp_storage, tmp_path): 21 | wd = str(tmp_path / "snippets") 22 | os.makedirs(wd) 23 | 24 | # find all markdown files in docs 25 | # and extract code blocks 26 | for root, dirs, files in os.walk("docs"): 27 | for file in files: 28 | if not file.endswith(".md"): 29 | continue 30 | code_blocks = [] 31 | doc = os.path.join(root, file) 32 | print(f"Parsing {doc}") 33 | with open(doc) as f: 34 | lines = f.readlines() 35 | codeblock = None 36 | in_code_block = False 37 | in_code = False 38 | is_test = False 39 | for i, line in enumerate(lines): 40 | if line.startswith("::: code-group"): 41 | codeblock = { 42 | "fn": doc, 43 | "start": i + 1, 44 | "end": None, 45 | "code": [], 46 | } 47 | in_code_block = True 48 | elif line.startswith(":::") and in_code_block: 49 | codeblock["end"] = i + 1 50 | code_blocks.append(codeblock) 51 | in_code_block = False 52 | elif line.startswith("```") and in_code_block: 53 | if in_code: 54 | codeblock["code"][-1]["end"] = i + 1 55 | in_code = False 56 | else: 57 | lang, fn = _parse_code_string(line) 58 | if lang == "python": 59 | codeblock["code"].append( 60 | { 61 | "start": i + 1, 62 | "end": None, 63 | "filename": fn, 64 | "content": "", 65 | "is_test": is_test, 66 | } 67 | ) 68 | in_code = True 69 | elif in_code: 70 | codeblock["code"][-1]["content"] += line 71 | elif in_code_block and not in_code: 72 | if line.startswith("" in line: 75 | is_test = False 76 | for b, codeblock in enumerate(code_blocks): 77 | if not any([q["is_test"] for q in codeblock["code"]]): 78 | continue 79 | os.makedirs(os.path.join(wd, str(b))) 80 | tests = [] 81 | for c, code in enumerate(codeblock["code"]): 82 | if code["filename"] and code["filename"].endswith(".py"): 83 | code["module"] = os.path.splitext(code["filename"])[0] 84 | else: 85 | if not code["is_test"]: 86 | raise RuntimeError( 87 | "Non-test code block without filename" 88 | ) 89 | code["filename"] = f"test_{c+1}.py" 90 | code["module"] = f"test_{c+1}" 91 | with open( 92 | os.path.join(wd, str(b), code["filename"]), "w" 93 | ) as f: 94 | f.write(code["content"]) 95 | tests.append(code) 96 | 97 | with Project(os.path.join(wd, str(b))): 98 | for test in tests: 99 | # prettyprint test dict 100 | print(f"Running {test['filename']}") 101 | print(f"{test['start']}-{test['end']}") 102 | import_from_directory( 103 | test["module"], 104 | os.path.join(wd, str(b)), 105 | or_fail=True, 106 | ) 107 | -------------------------------------------------------------------------------- /tests/test_execution.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import random 4 | 5 | import pytest 6 | from machinable import Component, Execution, Project, Scope, errors, get 7 | from pydantic import BaseModel 8 | 9 | 10 | def test_execution(tmp_storage): 11 | # no-predicate by default 12 | e1 = get("machinable.execution", {"a": 1}).commit() 13 | e2 = get("machinable.execution", {"a": 1}).commit() 14 | assert e1 != e2 15 | 16 | execution = Execution() 17 | assert ( 18 | Execution.from_model(execution.__model__).timestamp 19 | == execution.timestamp 20 | ) 21 | 22 | e = Execution() 23 | assert str(e) == f"machinable.execution [{e.id}]" 24 | assert repr(e) == f"machinable.execution [{e.id}]" 25 | 26 | execution = Execution().add(Component()) 27 | assert len(execution.executables) == 1 28 | assert isinstance(execution.timestamp, int) 29 | 30 | # add 31 | component = Component() 32 | execution = Execution().add(component) 33 | assert len(execution.executables) == 1 34 | execution.dispatch() 35 | 36 | restored = Execution.find_by_id(execution.uuid) 37 | with pytest.raises(errors.MachinableError): 38 | restored.add(Component()) 39 | 40 | # host info 41 | assert execution.host_info["python_version"].startswith("3") 42 | 43 | # output 44 | c = Component().commit() 45 | e = Execution().add(c).commit() 46 | assert e.output(c) is None 47 | e.save_file([c.id, "output.log"], "test") 48 | assert e.output(c) == "test" 49 | 50 | assert e.output(c, incremental=True) == "test" 51 | e.save_file([c.id, "output.log"], "testt") 52 | assert e.output(c, incremental=True) == "t" 53 | assert e.output(c, incremental=True) == "" 54 | e.save_file([c.id, "output.log"], "testt more") 55 | assert e.output(c, incremental=True) == " more" 56 | 57 | # status 58 | e.update_status(c, "started") 59 | assert e.is_started(c) 60 | e.update_status(c, "heartbeat") 61 | assert e.is_active(c) 62 | e.update_status(c, "finished") 63 | assert e.is_finished(c) 64 | assert not e.is_incomplete(c) 65 | 66 | 67 | def test_execution_dispatch(tmp_storage): 68 | # prevent execution from component 69 | class T(Component): 70 | class Config(BaseModel): 71 | a: int = 1 72 | mode: str = "before" 73 | 74 | def on_before_dispatch(self): 75 | if self.config.mode == "before": 76 | raise ValueError("Prevent execution") 77 | 78 | def __call__(self): 79 | if self.config.mode == "runtime": 80 | raise RuntimeError("Should not execute") 81 | 82 | with pytest.raises(errors.ExecutionFailed): 83 | T().launch() 84 | 85 | with pytest.raises(errors.ExecutionFailed): 86 | T({"mode": "runtime"}).launch() 87 | 88 | # prevent commit for configuration errors 89 | with Project("./tests/samples/project"): 90 | valid = T() 91 | invalid = T({"a": []}) 92 | execution = Execution().add([valid, invalid]) 93 | with pytest.raises(errors.ConfigurationError): 94 | execution.dispatch() 95 | assert not valid.is_mounted() 96 | assert not invalid.is_mounted() 97 | 98 | 99 | def test_execution_deferral(tmp_storage): 100 | with Execution().deferred() as execution: 101 | component = Component().launch() 102 | 103 | assert not component.cached() 104 | assert not execution.is_started() 105 | execution.dispatch() 106 | assert execution.is_finished() 107 | assert component.cached() 108 | 109 | component = Component() 110 | assert not component.cached() 111 | with Execution().deferred() as execution: 112 | execution.deferred(False) 113 | component.launch() 114 | assert component.cached() 115 | 116 | 117 | def test_execution_context(tmp_storage): 118 | with Execution(schedule=None) as execution: 119 | e1 = Component() 120 | e1.launch() 121 | assert e1.execution == execution 122 | assert not e1.execution.is_started() 123 | e2 = Component() 124 | e2.launch() 125 | assert len(execution.executables) == 2 126 | assert e2.execution == execution 127 | assert not e2.execution.is_started() 128 | assert e1.execution.is_finished() 129 | assert e2.execution.is_finished() 130 | 131 | with Execution() as execution: 132 | e1 = Component() 133 | e1.launch() 134 | e2 = Component() 135 | e2.launch() 136 | assert e1.execution == execution 137 | assert e2.execution == execution 138 | assert not e1.execution.is_started() 139 | assert not e2.execution.is_started() 140 | assert e1.execution.is_finished() 141 | assert e2.execution.is_finished() 142 | 143 | 144 | def test_execution_resources(tmp_storage): 145 | component = Component() 146 | execution = Execution() 147 | # default resources are empty 148 | assert execution.computed_resources(component) == {} 149 | 150 | # default resources can be declared via a method 151 | class T(Execution): 152 | def on_compute_default_resources(self, _): 153 | return {"1": 2} 154 | 155 | execution = T() 156 | assert execution.computed_resources(component) == {"1": 2} 157 | # default resources are reused 158 | execution = T(resources={"test": "me"}) 159 | assert execution.__model__.resources["test"] == "me" 160 | assert execution.computed_resources(component) == {"1": 2, "test": "me"} 161 | # inheritance of default resources 162 | execution = T(resources={"3": 4}) 163 | assert execution.computed_resources(component) == {"1": 2, "3": 4} 164 | execution = T(resources={"3": 4, "_inherit_defaults": False}) 165 | assert execution.computed_resources(component) == {"3": 4} 166 | # inherit but ignore commented resources 167 | execution = T(resources={"3": 4, "#1": None}) 168 | assert execution.computed_resources(component) == {"3": 4} 169 | 170 | # interface 171 | with Execution(resources={"test": 1, "a": True}) as execution: 172 | component = Component() 173 | component.launch() 174 | assert component.execution.computed_resources()["test"] == 1 175 | 176 | with Execution(resources={"a": 3}) as execution: 177 | component.launch() 178 | # resources are still referring to prior execution 179 | assert component.execution.computed_resources()["a"] is True 180 | 181 | e2 = Component() 182 | e2.launch() 183 | assert e2.execution.computed_resources()["a"] == 3 184 | 185 | 186 | def test_interrupted_execution(tmp_storage): 187 | with Project("./tests/samples/project"): 188 | component = Component.make("interface.interrupted_lifecycle") 189 | try: 190 | component.launch() 191 | except errors.ExecutionFailed: 192 | pass 193 | 194 | assert component.execution.is_started() 195 | assert not component.execution.is_finished() 196 | 197 | # resume 198 | try: 199 | component.launch() 200 | except errors.ExecutionFailed: 201 | pass 202 | 203 | component.launch() 204 | assert component.execution.is_finished() 205 | 206 | 207 | def test_rerepeated_execution(tmp_storage): 208 | project = Project("./tests/samples/project").__enter__() 209 | 210 | class NoScope(Scope): 211 | def __call__(self) -> Dict: 212 | return {"random": random.randint(0, 99999)} 213 | 214 | # first execution 215 | with Execution() as execution1, NoScope(): 216 | c1 = get("count").launch() 217 | assert c1.count == 0 218 | assert c1.execution == execution1 219 | assert c1.execution.is_finished() 220 | assert c1.count == 1 221 | 222 | # second execution, nothing happens here 223 | with execution1: 224 | c1.launch() 225 | assert c1.execution == execution1 226 | assert c1.count == 1 227 | 228 | # add a new component to existing execution is not allowed 229 | with execution1: 230 | with NoScope(): 231 | c2 = get("count") 232 | with pytest.raises(errors.MachinableError): 233 | c2.launch() 234 | assert c2.count == 0 235 | assert not c2.is_committed() 236 | 237 | # resume execution 238 | with pytest.raises(errors.ExecutionFailed): 239 | with Execution() as execution2: 240 | with NoScope(): 241 | done = get("count").launch() 242 | with NoScope(): 243 | failed = get("fail").launch() 244 | assert done.execution.is_finished() 245 | assert not done.execution.is_resumed() 246 | assert not failed.execution.is_finished() 247 | 248 | failed.save_file("repaired", "yes") 249 | with execution2: 250 | done.launch() 251 | failed.launch() 252 | assert failed.execution.is_finished() 253 | assert failed.execution.is_resumed() 254 | assert len(execution2.executables) == 2 255 | 256 | # resume with another execution 257 | with pytest.raises(errors.ExecutionFailed): 258 | with Execution() as execution2: 259 | with NoScope(): 260 | done = get("count").launch() 261 | with NoScope(): 262 | failed = get("fail").launch() 263 | failed.save_file("repaired", "yes") 264 | with Execution() as execution3: 265 | failed.launch() 266 | assert ( 267 | failed.execution == execution3 268 | ), f"{failed.execution.uuid} != {execution3.uuid}" 269 | assert failed.execution.is_finished() 270 | assert not failed.execution.is_resumed() 271 | assert len(execution2.executables) == 2 272 | assert len(execution3.executables) == 1 273 | 274 | # attempted re-execution - silently ignored 275 | with Execution() as execution4: 276 | done.launch() 277 | assert done.count == 1 278 | assert not execution4.is_committed() 279 | with Execution() as execution5: 280 | done.launch() 281 | with NoScope(): 282 | done2 = get("count").launch() 283 | assert done.count == 1 284 | assert done2.count == 1 285 | assert len(execution5.executables) == 2 286 | 287 | project.__exit__() 288 | 289 | 290 | def test_execution_from_index(tmp_storage): 291 | with Project("tests/samples/project"): 292 | c = get("exec", resources={"a": 1}).commit() 293 | cp = Execution.find_by_id(c.uuid, fetch=False) 294 | assert c.seed == c.seed 295 | assert c.__model__.resources == cp.__model__.resources 296 | assert cp.__model__.resources["a"] == 1 297 | -------------------------------------------------------------------------------- /tests/test_index.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sqlite3 4 | 5 | import pytest 6 | from machinable import Component, index, schema 7 | 8 | 9 | def _is_migrated(db): 10 | return db.cursor().execute("PRAGMA user_version;").fetchone()[0] == 2 11 | 12 | 13 | def _matches(q, v): 14 | return {v.uuid for v in q} == {v.uuid for v in v} 15 | 16 | 17 | def test_index_migrate(): 18 | db = sqlite3.connect(":memory:") 19 | index.migrate(db) 20 | assert _is_migrated(db) 21 | db.close() 22 | 23 | 24 | def test_index_load(tmp_path): 25 | db = index.load(str(tmp_path / "index.sqlite"), create=True) 26 | assert os.path.exists(str(tmp_path / "index.sqlite")) 27 | assert _is_migrated(db) 28 | db.close() 29 | db = index.load( 30 | str(tmp_path / "non-existing" / "subdir" / "index.sqlite"), create=True 31 | ) 32 | assert os.path.exists( 33 | str(tmp_path / "non-existing" / "subdir" / "index.sqlite") 34 | ) 35 | assert _is_migrated(db) 36 | db.close() 37 | 38 | 39 | def test_index_commit(tmp_path): 40 | i = index.Index({"database": str(tmp_path / "index.sqlite")}) 41 | v = schema.Interface() 42 | e = ( 43 | v.uuid, 44 | "Interface", 45 | None, 46 | "null", 47 | "{}", 48 | "{}", 49 | "[]", 50 | "null", 51 | "[]", 52 | v.timestamp, 53 | "{}", 54 | "null", 55 | ) 56 | assert i.commit(v) is True 57 | with index.db(i.config.database) as db: 58 | assert db.cursor().execute("SELECT * FROM 'index';").fetchall() == [e] 59 | assert i.commit(v) is False 60 | assert db.cursor().execute("SELECT * FROM 'index';").fetchall() == [e] 61 | assert i.commit(schema.Interface()) is True 62 | assert ( 63 | len(db.cursor().execute("SELECT * FROM 'index';").fetchall()) == 2 64 | ) 65 | 66 | 67 | def test_index_create_relation(tmp_path, setup=False): 68 | i = index.Index({"database": str(tmp_path / "index.sqlite")}) 69 | v1, v2, v3, v4 = ( 70 | schema.Interface(), 71 | schema.Interface(), 72 | schema.Interface(), 73 | schema.Interface(), 74 | ) 75 | assert all([i.commit(v) for v in [v1, v2, v3, v4]]) 76 | i.create_relation("test_one", v1.uuid, v2.uuid) 77 | i.create_relation("test_one", v1.uuid, v2.uuid) # duplicate 78 | i.create_relation("test_many", v1.uuid, [v2.uuid, v3.uuid, v4.uuid]) 79 | i.create_relation("test_many_to_many", v1.uuid, [v2.uuid, v3.uuid]) 80 | i.create_relation("test_many_to_many", v2.uuid, [v3.uuid, v4.uuid]) 81 | 82 | if setup: 83 | return i, v1, v2, v3, v4 84 | 85 | with index.db(i.config.database) as db: 86 | assert ( 87 | len(db.cursor().execute("SELECT * FROM 'relations';").fetchall()) 88 | == 8 89 | ) 90 | 91 | 92 | def test_index_find(tmp_path): 93 | i = index.Index({"database": str(tmp_path / "index.sqlite")}) 94 | v = schema.Interface() 95 | assert i.commit(v) is True 96 | assert i.find_by_id(v.uuid) == v 97 | assert i.find_by_id("non-existing") is None 98 | 99 | 100 | def test_index_find_by_context(tmp_path): 101 | i = index.Index({"database": str(tmp_path / "index.sqlite")}) 102 | v = schema.Interface( 103 | context=dict(module="machinable", predicate={"a": 0, "b": 0}) 104 | ) 105 | i.commit(v) 106 | assert len(i.find_by_context(dict(module="machinable"))) == 1 107 | assert ( 108 | len(i.find_by_context(dict(module="machinable", predicate={"a": 1}))) 109 | == 0 110 | ) 111 | assert ( 112 | len(i.find_by_context(dict(module="machinable", predicate={"a": 0}))) 113 | == 1 114 | ) 115 | assert ( 116 | len( 117 | i.find_by_context( 118 | dict(module="machinable", predicate={"a": 0, "b": 1}) 119 | ) 120 | ) 121 | == 0 122 | ) 123 | assert ( 124 | len( 125 | i.find_by_context( 126 | dict(module="machinable", predicate={"a": 0, "b": 0}) 127 | ) 128 | ) 129 | == 1 130 | ) 131 | 132 | 133 | def test_index_find_by_hash(tmp_path): 134 | i = index.Index({"database": str(tmp_path / "index.sqlite")}) 135 | v = schema.Interface(module="machinable", predicate={"a": 0, "b": 0}) 136 | i.commit(v) 137 | assert i.find_by_hash(v.hash) == [v] 138 | 139 | v2 = schema.Interface(module="machinable") 140 | v2.uuid = v.uuid[:24] + v2.uuid[:12] 141 | i.commit(v2) 142 | assert i.find_by_hash(v2.hash) == [v2] 143 | 144 | v3 = schema.Interface(module="machinable", predicate={"a": 1}) 145 | i.commit(v3) 146 | 147 | assert i.find_by_hash("0" * 12) == [v, v3] 148 | 149 | 150 | def test_index_find_related(tmp_path): 151 | i, v1, v2, v3, v4 = test_index_create_relation(tmp_path, setup=True) 152 | 153 | q = i.find_related("test_one", v1.uuid) 154 | assert len(q) == 1 155 | assert q[0] == v2 156 | q = i.find_related("test_one", v2.uuid, inverse=True) 157 | assert len(q) == 1 158 | assert q[0] == v1 159 | 160 | q = i.find_related("test_many", v1.uuid) 161 | assert len(q) == 3 162 | assert _matches(q, [v2, v3, v4]) 163 | 164 | q = i.find_related("test_many", v2.uuid, inverse=True) 165 | assert len(q) == 1 166 | assert q[0] == v1 167 | 168 | q = i.find_related("test_many_to_many", v1.uuid) 169 | assert len(q) == 2 170 | assert _matches(q, [v2, v3]) 171 | 172 | q = i.find_related("test_many_to_many", v3.uuid, inverse=True) 173 | assert len(q) == 2 174 | assert _matches(q, [v1, v2]) 175 | 176 | 177 | def test_index_find(tmp_path): 178 | i = index.Index({"database": str(tmp_path / "index.sqlite")}) 179 | v = schema.Interface(module="machinable", predicate={"a": 0, "b": 0}) 180 | i.commit(v) 181 | assert i.find_by_hash(v.hash) == i.find(v, by="hash") 182 | assert i.find_by_id(v.uuid) == i.find(v, by="uuid")[0] 183 | assert i.find_by_id(v.id) == i.find(v, by="id")[0] 184 | 185 | with pytest.raises(ValueError): 186 | i.find(v.hash, by="invalid") 187 | 188 | 189 | def test_index_import_directory(tmp_path): 190 | local_index = index.Index(str(tmp_path / "local")) 191 | remote_index = index.Index(str(tmp_path / "remote")) 192 | 193 | with remote_index: 194 | a = Component().launch() 195 | b = Component({"a": 1}, uses=a).launch() 196 | a_source_dir = a.local_directory() 197 | b_source_dir = b.local_directory() 198 | assert b.uses[0] == a 199 | 200 | assert local_index.find_by_id(a.uuid) is None 201 | local_index.import_directory(a_source_dir, relations=False) 202 | local_index.import_directory(b_source_dir, file_importer=shutil.move) 203 | assert local_index.find_by_id(a.uuid) is not None 204 | assert local_index.find_by_id(b.uuid) is not None 205 | assert os.path.exists(local_index.local_directory(a.uuid)) 206 | assert os.path.exists(local_index.local_directory(b.uuid)) 207 | 208 | assert os.path.exists(a_source_dir) 209 | assert not os.path.exists(b_source_dir) 210 | 211 | assert local_index.find_related("Interface.Interface.using", b.uuid) 212 | 213 | with local_index: 214 | c = Component.find_by_id(b.uuid) 215 | r = c.uses 216 | assert r[0] == a 217 | -------------------------------------------------------------------------------- /tests/test_mixin.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from machinable import Element, Project 3 | 4 | 5 | def test_mixins(): 6 | with Project("./tests/samples/project"): 7 | element = Element.make("mixins.example") 8 | assert element.say() == "hello, world" 9 | assert element.say_bound() == "success" 10 | assert element.__mixin__.calling_into_the_void() == "no response" 11 | assert element.test.bound_hello() == "success" 12 | assert element.dummy.name() == "dummy" 13 | 14 | with pytest.raises(AttributeError): 15 | element.bla() 16 | with pytest.raises(AttributeError): 17 | element.there("") 18 | -------------------------------------------------------------------------------- /tests/test_project.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | import pytest 5 | from machinable import Component, Project 6 | 7 | 8 | def test_project(): 9 | project = Project() 10 | assert project.config.directory == os.getcwd() 11 | project = Project("tests/samples/project") 12 | assert project.name() == "project" 13 | assert project.path().endswith("samples/project") 14 | project.__enter__() 15 | assert Project.get().name() == "project" 16 | assert Project.get().module == "interface.project" 17 | project.__exit__() 18 | # note that this may fail if other tests have errors 19 | # and failed to clean up the project 20 | assert Project.get().module == "machinable.project" 21 | 22 | 23 | def test_project_events(tmp_storage): 24 | project = Project("tests/samples/project").__enter__() 25 | # global config 26 | assert Component.instance("dummy", {"a": "global_conf(2)"}).config.a == 2 27 | assert Component.instance("dummy", "~global_ver({'a': 3})").config.a == 3 28 | 29 | # module redirection 30 | assert Component.instance("@test").module == "basic" 31 | assert ( 32 | Component.instance("@test").hello() 33 | == Component.singleton("@test").hello() 34 | ) 35 | 36 | component = Component.instance("dummy") 37 | component.launch() 38 | 39 | # remotes 40 | shutil.rmtree("tests/samples/project/interface/remotes", ignore_errors=True) 41 | Component.instance("!hello")() 42 | Component.instance("!hello-link")() 43 | assert os.path.exists("tests/samples/project/interface/remotes") 44 | with pytest.raises(ValueError): 45 | Component.instance("!invalid") 46 | shutil.rmtree("tests/samples/project/interface/remotes", ignore_errors=True) 47 | Component.instance("!multi")() 48 | assert os.path.isfile("tests/samples/project/interface/remotes/!multi.py") 49 | assert os.path.isfile( 50 | "tests/samples/project/interface/remotes/!hello-link.py" 51 | ) 52 | shutil.rmtree("tests/samples/project/interface/remotes", ignore_errors=True) 53 | Component.instance("!multichain")() 54 | assert os.path.isfile("tests/samples/project/interface/remotes/!multi.py") 55 | assert os.path.isfile( 56 | "tests/samples/project/interface/remotes/!hello-link.py" 57 | ) 58 | with pytest.raises(ValueError): 59 | Component.instance("!multi-invalid") 60 | 61 | # extension 62 | assert Component.instance("dummy_version_extend").config.a == 100 63 | 64 | project.__exit__() 65 | -------------------------------------------------------------------------------- /tests/test_query.py: -------------------------------------------------------------------------------- 1 | # Note that most query testing can be found in test_interface_modifiers 2 | import pytest 3 | from machinable import Component, get 4 | 5 | 6 | def test_query_from_directory(tmp_storage): 7 | t = Component().launch() 8 | t2 = get.from_directory(t.local_directory()) 9 | assert t == t2 10 | 11 | 12 | def test_query_by_id(tmp_storage): 13 | t = Component().launch() 14 | t2 = get.by_id(t.uuid) 15 | assert t == t2 16 | t3 = get.by_id("nonexistent") 17 | assert t3 is None 18 | -------------------------------------------------------------------------------- /tests/test_schedule.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from machinable import Component, Execution, Project, Schedule, errors 3 | 4 | 5 | class Supported(Execution): 6 | def on_verify_schedule(self): 7 | return self.schedule.module == "scheduled" 8 | 9 | 10 | def test_schedule(tmp_storage): 11 | with Project("./tests/samples/project"): 12 | schedule = Schedule.instance("scheduled") 13 | assert schedule.test() 14 | 15 | # execution does not support schedule 16 | with pytest.raises(errors.ExecutionFailed): 17 | with Execution(schedule=schedule) as execution: 18 | Component().launch() 19 | 20 | # execution supports schedule 21 | with Supported(schedule=["scheduled"]) as execution: 22 | component = Component().launch() 23 | assert component.execution == execution 24 | assert component.execution.is_finished() 25 | assert execution.schedule.test() 26 | -------------------------------------------------------------------------------- /tests/test_scope.py: -------------------------------------------------------------------------------- 1 | from machinable import Interface, get 2 | from machinable.scope import Scope 3 | 4 | 5 | def test_scope_element(): 6 | scope = Scope() 7 | assert scope() == {} 8 | assert Scope({"test": 1})() == {"test": 1} 9 | assert Scope([{"a": 1}, {"a": 2}])() == {"a": 2} 10 | 11 | 12 | def test_scoping(tmp_storage): 13 | class T(Interface): 14 | Config = {"a": 1} 15 | 16 | e1 = get(T, {"a": 2}).commit() 17 | with Scope({"name": "test"}): 18 | e2 = get(T, {"a": 2}).commit() 19 | assert e2 != e1 20 | assert len(get(T, {"a": 2}).all()) == 2 21 | with Scope({"name": "test"}): 22 | assert get(T, {"a": 2}).commit() != e1 23 | 24 | e3 = get(T).commit() 25 | assert e1 != e2 != e3 26 | assert get(T, {"a": 2}) == e2 27 | 28 | assert (len(Scope().all())) == 0 29 | 30 | with Scope({"name": "test"}) as scope: 31 | assert (len(get.all())) == 1 32 | assert (len(Scope.get().all())) == 0 33 | assert len(scope.all()) == 0 34 | 35 | with Scope({"test": "isolation"}) as scope: 36 | assert len(get.all()) == 0 37 | -------------------------------------------------------------------------------- /tests/test_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | from machinable import Storage 5 | 6 | 7 | class CopyStorage(Storage): 8 | class Config: 9 | directory: str = "" 10 | 11 | def commit(self, interface) -> bool: 12 | directory = os.path.join(self.config.directory, interface.uuid) 13 | if not os.path.exists(directory): 14 | os.makedirs(directory) 15 | interface.to_directory(directory) 16 | 17 | def contains(self, uuid): 18 | return os.path.exists(os.path.join(self.config.directory, uuid)) 19 | 20 | def retrieve(self, uuid, local_directory) -> bool: 21 | if not self.contains(uuid): 22 | return False 23 | 24 | shutil.copytree( 25 | os.path.join(self.config.directory, uuid), 26 | local_directory, 27 | dirs_exist_ok=True, 28 | ) 29 | 30 | return True 31 | 32 | 33 | def test_storage(tmp_path): 34 | from machinable import Index, get 35 | 36 | primary = str(tmp_path / "primary") 37 | secondary = str(tmp_path / "secondary") 38 | 39 | i = Index( 40 | {"directory": primary, "database": str(tmp_path / "index.sqlite")} 41 | ).__enter__() 42 | 43 | st2 = CopyStorage({"directory": secondary}).__enter__() 44 | st1 = Storage().__enter__() 45 | 46 | project = get("machinable.project", "tests/samples/project").__enter__() 47 | 48 | interface1 = get("dummy").commit() 49 | interface2 = get("dummy", {"a": 5}).commit() 50 | 51 | assert os.path.exists(os.path.join(primary, interface1.uuid)) 52 | assert os.path.exists(os.path.join(secondary, interface1.uuid)) 53 | 54 | # delete primary source and reload from remote 55 | shutil.rmtree(primary) 56 | assert not os.path.exists(interface1.local_directory()) 57 | assert not os.path.exists(interface2.local_directory()) 58 | interface1_reload = get("dummy") 59 | interface1_reload.fetch() 60 | assert os.path.exists(interface1_reload.local_directory()) 61 | assert not os.path.exists(interface2.local_directory()) 62 | interface2_reload = get("dummy", {"a": 5}) 63 | interface2_reload.fetch() 64 | assert os.path.exists(interface2.local_directory()) 65 | 66 | project.__exit__() 67 | st1.__exit__() 68 | st2.__exit__() 69 | i.__exit__() 70 | 71 | 72 | def test_storage_upload_and_download(tmp_path): 73 | from machinable import Index, get 74 | 75 | primary = str(tmp_path / "primary") 76 | secondary = str(tmp_path / "secondary") 77 | 78 | i = Index(primary).__enter__() 79 | local = Index(str(tmp_path / "download")) 80 | 81 | storage = CopyStorage({"directory": secondary}) 82 | 83 | project = get("machinable.project", "tests/samples/project").__enter__() 84 | 85 | interface1 = get("dummy").launch() 86 | interface2 = get("dummy", {"a": 5}, uses=interface1).launch() 87 | 88 | assert not os.path.exists(tmp_path / "secondary" / interface2.uuid) 89 | storage.upload(interface2) 90 | assert os.path.exists(tmp_path / "secondary" / interface1.uuid) 91 | assert os.path.exists(tmp_path / "secondary" / interface2.uuid) 92 | 93 | assert not local.find(interface2) 94 | with local: 95 | downloads = storage.download(interface2.uuid, related=False) 96 | assert len(downloads) == 1 97 | assert os.path.exists(local.local_directory(interface2.uuid)) 98 | assert not os.path.exists(local.local_directory(interface1.uuid)) 99 | assert local.find(interface2) 100 | assert not local.find(interface1) 101 | 102 | downloads = storage.download(interface2.uuid, related=True) 103 | assert len(downloads) == 2 104 | assert os.path.exists(local.local_directory(interface1.uuid)) 105 | assert local.find(interface1) 106 | 107 | project.__exit__() 108 | i.__exit__() 109 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | import pytest 5 | from machinable import get_version, utils 6 | from machinable.element import Element 7 | 8 | 9 | def test_generate_nickname(): 10 | with pytest.raises(ValueError): 11 | utils.generate_nickname({}) 12 | with pytest.raises(KeyError): 13 | utils.generate_nickname("non-existent-category") 14 | assert len(utils.generate_nickname().split("_")) == 2 15 | assert utils.generate_nickname(["tree"]).find("_") == -1 16 | 17 | 18 | def test_filesystem_utils(tmpdir): 19 | # json 20 | filepath = str(tmpdir / "random/path/test.json") 21 | utils.save_file(filepath, {"jsonable": 1, "test": True}) 22 | r = utils.load_file(filepath) 23 | assert r["jsonable"] == 1 24 | assert r["test"] is True 25 | 26 | # jsonlines 27 | filepath = str(tmpdir / "random/data/jsonlines.jsonl") 28 | utils.save_file(filepath, {"jsonable": 1, "test": True}) 29 | utils.save_file(filepath, {"jsonable": 2, "test": True}, mode="a") 30 | r = utils.load_file(filepath) 31 | assert len(r) == 2 32 | assert r[1]["jsonable"] == 2 33 | assert r[0]["test"] == r[1]["test"] 34 | 35 | # text 36 | filepath = str(tmpdir / "random/test.diff") 37 | utils.save_file(filepath, "test") 38 | assert utils.load_file(filepath) == "test" 39 | filepath = str(tmpdir / "random/test_ext") 40 | utils.save_file(filepath, 1) 41 | assert utils.load_file(filepath) == "1" 42 | 43 | # pickle 44 | filepath = str(tmpdir / "test.p") 45 | utils.save_file(filepath, ["test"]) 46 | assert utils.load_file(filepath) == ["test"] 47 | 48 | # numpy 49 | # filepath = str(tmpdir / "number.npy") 50 | # utils.save_file(filepath, np.array([1, 2, 3])) 51 | # assert utils.load_file(filepath).sum() == 6 52 | 53 | assert utils.load_file("not_existing.txt", default="default") == "default" 54 | utils.save_file(str(tmpdir / "unsupported.extension"), 0.0) 55 | assert utils.load_file(str(tmpdir / "unsupported.extension")) == "0.0" 56 | 57 | 58 | def test_import_from_directory(): 59 | # relative imports 60 | assert type( 61 | utils.import_from_directory( 62 | "top", "./tests/samples/importing" 63 | ).TopComponent 64 | ) is type( 65 | utils.import_from_directory( 66 | "importing.top", "./tests/samples" 67 | ).TopComponent 68 | ) 69 | 70 | # import modules with and without __init__.py 71 | assert ( 72 | utils.import_from_directory("nested", "./tests/samples/importing") 73 | is not None 74 | ) 75 | assert ( 76 | utils.import_from_directory("importing", "./tests/samples").__doc__ 77 | == "Importing" 78 | ) 79 | 80 | assert utils.import_from_directory("non_existing", "./tests") is None 81 | with pytest.raises(ModuleNotFoundError): 82 | utils.import_from_directory("non_existing", "./tests", or_fail=True) 83 | 84 | 85 | def test_find_subclass_in_module(): 86 | assert utils.find_subclass_in_module(None, None) is None 87 | 88 | module = utils.import_from_directory("top", "./tests/samples/importing") 89 | assert type(utils.find_subclass_in_module(module, Element)) is type( 90 | module.TopComponent 91 | ) 92 | 93 | # ignores imported classes? 94 | module = utils.import_from_directory( 95 | "nested.bottom", "./tests/samples/importing" 96 | ) 97 | assert type(utils.find_subclass_in_module(module, Element)) is type( 98 | module.BottomComponent 99 | ) 100 | 101 | 102 | def test_unflatten_dict(): 103 | d = {"a.b": "c"} 104 | original = {"a.b": "c"} 105 | assert utils.unflatten_dict(d)["a"]["b"] == "c" 106 | assert d == original 107 | 108 | # recursive 109 | d = {"a.b": {"c.d": "e"}} 110 | original = {"a.b": {"c.d": "e"}} 111 | assert utils.unflatten_dict(d)["a"]["b"]["c"]["d"] == "e" 112 | assert d == original 113 | 114 | 115 | def test_machinable_version(): 116 | assert isinstance(get_version(), str) 117 | 118 | 119 | def test_git_utils(tmp_path): 120 | # create a repository 121 | repo_dir = str(tmp_path / "test_repo") 122 | os.makedirs(repo_dir, exist_ok=True) 123 | subprocess.run(["git", "init"], cwd=repo_dir, check=True) 124 | 125 | # get_diff 126 | assert utils.get_diff(str(tmp_path)) is None 127 | assert utils.get_diff(repo_dir) == "" 128 | 129 | with open(os.path.join(repo_dir, "test"), "w") as f: 130 | f.write("some test data") 131 | 132 | subprocess.run(["git", "add", "."], cwd=repo_dir, check=True) 133 | assert "some test data" in utils.get_diff(repo_dir) 134 | 135 | 136 | def test_mixins(): 137 | class MixinImplementation: 138 | attribute = "works" 139 | 140 | def is_bound(self, param): 141 | return "bound_to_" + self.flags.BOUND + "_" + str(param) 142 | 143 | def this_reference(self, param): 144 | return self.__mixin__.is_bound("and_referenced_" + str(param)) 145 | 146 | def this_attribute(self): 147 | return self.__mixin__.attribute 148 | 149 | def this_static(self, param): 150 | return self.__mixin__.static_method(param) 151 | 152 | @staticmethod 153 | def static_method(foo): 154 | return foo 155 | 156 | @property 157 | def key_propery(self): 158 | return 1 159 | 160 | 161 | def test_directory_version(): 162 | for case in [ 163 | None, 164 | {"directory": "yes"}, 165 | "~version", 166 | ]: 167 | assert utils.is_directory_version(case) is False 168 | for case in [ 169 | "~", 170 | ".", 171 | "./", 172 | "./version", 173 | "test", 174 | "test.me", 175 | "/path/to/version", 176 | "../test", 177 | ]: 178 | assert utils.is_directory_version(case) is True 179 | 180 | 181 | def test_joinpath(): 182 | assert utils.joinpath(["a", "b"]) == "a/b" 183 | assert utils.joinpath(["a", "b", "c"]) == "a/b/c" 184 | e = Element() 185 | assert utils.joinpath([e.id, "b"]) == f"{e.id}/b" 186 | assert utils.joinpath(["a", ""]) == "a/" 187 | assert utils.joinpath([None, "a", None, "b"]) == "a/b" 188 | 189 | 190 | def test_chmodx(tmp_path): 191 | script = utils.save_file(str(tmp_path / "test.sh"), 'echo "HELLO"') 192 | assert not os.access(script, os.X_OK) 193 | utils.chmodx(script) 194 | assert os.access(script, os.X_OK) 195 | 196 | 197 | def test_run_and_stream(tmp_path): 198 | script = utils.chmodx( 199 | utils.save_file(str(tmp_path / "test.sh"), 'echo "HELLO"') 200 | ) 201 | 202 | o = [] 203 | utils.run_and_stream( 204 | script, shell=True, stdout_handler=lambda x: o.append(x) 205 | ) 206 | assert o[0] == "HELLO\n" 207 | 208 | 209 | def test_file_hash(tmp_path): 210 | file = utils.save_file(str(tmp_path / "test.txt"), "test") 211 | assert utils.file_hash(file) == "a71079d42853" 212 | assert utils.file_hash(tmp_path / "test.txt") == "a71079d42853" 213 | with pytest.raises(FileNotFoundError): 214 | utils.file_hash("not-existing") 215 | 216 | 217 | @pytest.mark.parametrize( 218 | "input_code, expected", 219 | [ 220 | ("foo(1, bar=2)", "foo(1,bar=2)"), 221 | ("foo( 1, bar = 2 )", "foo(1,bar=2)"), 222 | ("\nfoo(\n1,\nbar = 2\n)\n", "foo(1,bar=2)"), 223 | ("foo(' hello ', bar=2)", "foo(' hello ',bar=2)"), 224 | ("foo(bar= 'world', baz =3)", "foo(bar='world',baz=3)"), 225 | ("~foo(bar= ' world',)", "~foo(bar=' world',)"), 226 | (" ~foo(bar= 'world')", "~foo(bar='world')"), 227 | ], 228 | ) 229 | def test_norm_version_call(input_code, expected): 230 | assert utils.norm_version_call(input_code) == expected 231 | --------------------------------------------------------------------------------