├── .gitattributes ├── .github └── workflows │ ├── conventional-prs.yml │ ├── release-please.yml │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── pyproject.toml ├── setup.cfg ├── snakemake_interface_executor_plugins ├── __init__.py ├── _common.py ├── cli.py ├── dag.py ├── executors │ ├── __init__.py │ ├── base.py │ ├── jobscript.sh │ ├── real.py │ └── remote.py ├── jobs.py ├── logging.py ├── persistence.py ├── registry │ ├── __init__.py │ └── plugin.py ├── scheduler.py ├── settings.py ├── utils.py └── workflow.py └── tests ├── test_py37.py └── tests.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # SCM syntax highlighting & preventing 3-way merges 2 | pixi.lock merge=binary linguist-language=YAML linguist-generated=true 3 | -------------------------------------------------------------------------------- /.github/workflows/conventional-prs.yml: -------------------------------------------------------------------------------- 1 | name: PR 2 | on: 3 | pull_request_target: 4 | types: 5 | - opened 6 | - reopened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | title-format: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: amannn/action-semantic-pull-request@v3.4.0 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | 6 | name: release-please 7 | 8 | jobs: 9 | release-please: 10 | runs-on: ubuntu-latest 11 | outputs: 12 | release_created: ${{ steps.release.outputs.release_created }} 13 | steps: 14 | - uses: GoogleCloudPlatform/release-please-action@v3 15 | id: release 16 | with: 17 | release-type: python 18 | package-name: snakemake-interface-executor-plugins 19 | 20 | publish-pypi: 21 | runs-on: ubuntu-latest 22 | needs: release-please 23 | permissions: 24 | id-token: write 25 | if: ${{ needs.release-please.outputs.release_created }} 26 | 27 | steps: 28 | - uses: actions/checkout@v4 29 | 30 | - name: Install Pixi 31 | uses: prefix-dev/setup-pixi@v0.8.3 32 | with: 33 | environments: publish 34 | pixi-version: v0.42.1 35 | 36 | - name: Build source and wheel distribution + check build 37 | # this will build the source and wheel into the dist/ directory 38 | run: | 39 | pixi run --environment publish check-build 40 | 41 | - name: Publish distribution to PyPI 42 | uses: pypa/gh-action-pypi-publish@release/v1 43 | env: 44 | PYPI_USERNAME: __token__ 45 | PYPI_PASSWORD: ${{ secrets.PYPI_TOKEN }} 46 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches-ignore: [] 9 | 10 | permissions: 11 | contents: read 12 | checks: write 13 | issues: write 14 | pull-requests: write 15 | 16 | jobs: 17 | quality-control: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Check out the code 21 | uses: actions/checkout@v4 22 | 23 | - name: Install Pixi 24 | uses: prefix-dev/setup-pixi@v0.8.3 25 | with: 26 | environments: dev 27 | pixi-version: v0.42.1 28 | 29 | - name: Ruff Format 30 | if: always() 31 | run: | 32 | pixi run --environment dev format --check 33 | 34 | - name: Ruff lint 35 | if: always() 36 | run: | 37 | pixi run --environment dev lint --diff 38 | # - name: Mypy 39 | # if: always() 40 | # run: | 41 | # pixi run --environment dev type-check 42 | 43 | - name: Collect QC 44 | run: echo "All quality control checks passed" 45 | 46 | testing: 47 | runs-on: ubuntu-latest 48 | steps: 49 | - uses: actions/checkout@v4 50 | 51 | - name: Install Pixi 52 | uses: prefix-dev/setup-pixi@v0.8.3 53 | with: 54 | environments: dev 55 | pixi-version: v0.42.1 56 | 57 | - name: Run tests 58 | run: pixi run --environment dev test --show-capture=all -s -vv 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | poetry.lock 163 | # pixi environments 164 | .pixi 165 | *.egg-info 166 | pixi.lock 167 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [9.3.5](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.3.4...v9.3.5) (2025-03-27) 4 | 5 | 6 | ### Bug Fixes 7 | 8 | * adapt CI job permissions to trusted publishing workflow ([47464a2](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/47464a24f859afca2393c81b057d285abd41418f)) 9 | 10 | ## [9.3.4](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.3.3...v9.3.4) (2025-03-26) 11 | 12 | 13 | ### Bug Fixes 14 | 15 | * if requested, properly encode Path types as base64 ([#84](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/84)) ([8518425](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/8518425006e31376a4ac775574848a4ea510e76d)). **Important** in case you use the setting `remote-job-local-storage-prefix`, you now no longer should escape environment variables that will be evaluated in the remote job with a leading backslash, see https://snakemake.github.io/snakemake-plugin-catalog/plugins/storage/fs.html#further-details. 16 | 17 | ## [9.3.3](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.3.2...v9.3.3) (2024-12-21) 18 | 19 | 20 | ### Bug Fixes 21 | 22 | * problem with spaces in path (by quoting CLI args) ([#79](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/79)) ([404cd30](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/404cd302c3427f0c92b252f9e5800160051731b4)) 23 | 24 | ## [9.3.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.3.1...v9.3.2) (2024-10-06) 25 | 26 | 27 | ### Bug Fixes 28 | 29 | * iteration over env vars ([#77](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/77)) ([07135e5](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/07135e5d712344c53b6f99b3824fb97ea03c801b)) 30 | 31 | ## [9.3.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.3.0...v9.3.1) (2024-10-04) 32 | 33 | 34 | ### Bug Fixes 35 | 36 | * use correct names when collecting builtin plugins ([#75](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/75)) ([359e8ed](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/359e8ed32990cbccbfeda7fe746ece19d12e75bf)) 37 | 38 | ## [9.3.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.2.0...v9.3.0) (2024-10-04) 39 | 40 | 41 | ### Features 42 | 43 | * load builtin snakemake executor plugins ([#73](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/73)) ([03ee96b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/03ee96be5047d68f7d9de951ab75458e7c79d3e3)) 44 | 45 | ## [9.2.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.1.1...v9.2.0) (2024-07-04) 46 | 47 | 48 | ### Features 49 | 50 | * add a default for the `can_transfer_local_files` interface ([#67](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/67)) ([793df28](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/793df28ba733eb462fba7824f46729af65a58dc4)) 51 | * support for commas in wildcards ([#56](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/56)) ([0e8ed82](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0e8ed82c2dc8338b402e646dde7ca48e02075922)) 52 | 53 | ## [9.1.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.1.0...v9.1.1) (2024-04-12) 54 | 55 | 56 | ### Bug Fixes 57 | 58 | * pass cores to remote jobs if they are set ([395af5e](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/395af5e05c8b09d107415159819d0e3cef58717f)) 59 | 60 | ## [9.1.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.0.2...v9.1.0) (2024-03-26) 61 | 62 | 63 | ### Features 64 | 65 | * add utils for encoding CLI args as base64 ([#64](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/64)) ([38a53ec](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/38a53ecec3af3fc45d2f962972460fa50258b2b1)) 66 | 67 | ## [9.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.0.1...v9.0.2) (2024-03-22) 68 | 69 | 70 | ### Bug Fixes 71 | 72 | * quote list of args ([#62](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/62)) ([656ba0a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/656ba0afb867301ccb48b24837fda1793e3281dc)) 73 | 74 | ## [9.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v9.0.0...v9.0.1) (2024-03-21) 75 | 76 | 77 | ### Bug Fixes 78 | 79 | * fix quoting of string arguments that are passed to spawned jobs ([#60](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/60)) ([d3d55a3](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/d3d55a32dbd78be679727c3f95cd42308a9597ab)) 80 | 81 | ## [9.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.2.0...v9.0.0) (2024-03-11) 82 | 83 | 84 | ### ⚠ BREAKING CHANGES 85 | 86 | * pass common settings to SpawedJobArgsFactory; shell command arg quoting fixes ([#58](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/58)) 87 | 88 | ### Features 89 | 90 | * pass common settings to SpawedJobArgsFactory; shell command arg quoting fixes ([#58](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/58)) ([867a027](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/867a027e8abbfa8937900b648aeade91b01c2c38)) 91 | 92 | ## [8.2.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.3...v8.2.0) (2024-01-16) 93 | 94 | 95 | ### Features 96 | 97 | * add ability to pass group args to remote jobs ([bcfd819](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/bcfd81953b3feeaac6669a3487cc1eab3d5a2727)) 98 | 99 | ## [8.1.3](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.2...v8.1.3) (2023-12-19) 100 | 101 | 102 | ### Bug Fixes 103 | 104 | * break circular import ([aed33aa](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/aed33aa2aba20e229398deb5ad486d3b0ec7e213)) 105 | 106 | ## [8.1.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.1...v8.1.2) (2023-12-12) 107 | 108 | 109 | ### Documentation 110 | 111 | * CommonSettings ([#50](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/50)) ([85b995d](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/85b995d726cd941ea0f6e43b6217e95140a82327)) 112 | 113 | ## [8.1.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.1.0...v8.1.1) (2023-12-08) 114 | 115 | 116 | ### Bug Fixes 117 | 118 | * allow value of none for shared fs usage setting ([d334869](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/d33486933f41f2bccd099e2cab90b2d1a854def2)) 119 | 120 | ## [8.1.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.0.2...v8.1.0) (2023-11-30) 121 | 122 | 123 | ### Features 124 | 125 | * add method for checking whether there is a common workdir assumed in storage settings ([29dc8dd](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/29dc8dd43ba4bd8d33eeda14a3dff9272d3751f0)) 126 | 127 | 128 | ### Bug Fixes 129 | 130 | * adapt to API changes ([21cae32](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/21cae32a8a69b58b732b773a849abfb02b533575)) 131 | 132 | ## [8.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.0.1...v8.0.2) (2023-11-20) 133 | 134 | 135 | ### Bug Fixes 136 | 137 | * fix arg passing ([caee462](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/caee46241f4fc639ed585761421d335f7783399c)) 138 | 139 | ## [8.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v8.0.0...v8.0.1) (2023-11-20) 140 | 141 | 142 | ### Bug Fixes 143 | 144 | * cleanup ci ([061ff4c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/061ff4ce7a6ef699dfff58c149195425bef13e86)) 145 | * fix method name ([16138ad](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/16138ad9e085a289590b8308cc954662e8df0ffe)) 146 | 147 | ## [8.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.3...v8.0.0) (2023-11-20) 148 | 149 | 150 | ### ⚠ BREAKING CHANGES 151 | 152 | * added common setting for defining whether workflow sources shall be deployed. 153 | 154 | ### Features 155 | 156 | * added common setting for defining whether workflow sources shall be deployed. ([04319bb](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/04319bbe410275eea28cbe47d2abfe9b0b50c3e5)) 157 | 158 | ## [7.0.3](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.2...v7.0.3) (2023-10-26) 159 | 160 | 161 | ### Bug Fixes 162 | 163 | * fix envvar declarations code ([fc31775](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/fc31775075f8b6ac6317b3762bcd385f31a8b746)) 164 | * improved precommand handling ([af1f010](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/af1f01006fd5e7a493659e0bcb80e570628e5176)) 165 | 166 | ## [7.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.1...v7.0.2) (2023-10-20) 167 | 168 | 169 | ### Bug Fixes 170 | 171 | * ignore errors when trying to delete tmpdir upon shutdown ([#39](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/39)) ([406422c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/406422c967ebd33227c34e257b0a1a5cdd0a3e4d)) 172 | 173 | ## [7.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v7.0.0...v7.0.1) (2023-10-17) 174 | 175 | 176 | ### Miscellaneous Chores 177 | 178 | * release 7.0.1 ([c51e47b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c51e47b5b3eed7a5d52e27dead1d51659563aa9d)) 179 | 180 | ## [7.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v6.0.0...v7.0.0) (2023-10-17) 181 | 182 | 183 | ### ⚠ BREAKING CHANGES 184 | 185 | * move behavior args into common settings and use __post_init__ method for additional initialization 186 | 187 | ### Features 188 | 189 | * move behavior args into common settings and use __post_init__ method for additional initialization ([c6cb3c9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c6cb3c9c02de7e7aeba241558ba549a65abcfc2b)) 190 | * support precommand ([32da209](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/32da20943b1afe8854566356ed448015e4f67e6c)) 191 | 192 | ## [6.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v5.0.2...v6.0.0) (2023-10-12) 193 | 194 | 195 | ### ⚠ BREAKING CHANGES 196 | 197 | * adapt to API changes 198 | 199 | ### Features 200 | 201 | * adapt to API changes ([f74151b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f74151b32a9b98a323bbbf88a818b7da5fe97427)) 202 | 203 | 204 | ### Bug Fixes 205 | 206 | * adapt to changes in snakemake ([e572f02](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/e572f02c7b5270fc02fc871ac6197575ce42ad5c)) 207 | * cleanup interfaces ([88c6554](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/88c65546db32fc5e48827173bb016d69691c41cb)) 208 | * udpate deps ([99b5b1e](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/99b5b1e61302d75fb6ca7a959fda18cceaf12703)) 209 | 210 | ## [5.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v5.0.1...v5.0.2) (2023-09-22) 211 | 212 | 213 | ### Documentation 214 | 215 | * mention poetry plugin ([733f2f9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/733f2f93b0e1fedb9aeda21ea6987b7b7059be11)) 216 | 217 | ## [5.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v5.0.0...v5.0.1) (2023-09-22) 218 | 219 | 220 | ### Bug Fixes 221 | 222 | * adapt to changes in snakemake-interface-common ([faa05a4](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/faa05a40068e656e533671324c4a3928158e652e)) 223 | * adapt to fixes in snakemake-interface-common ([2a92560](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/2a92560fa602cab4b3085643324bdaaa36d1ea42)) 224 | 225 | ## [5.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v4.0.1...v5.0.0) (2023-09-21) 226 | 227 | 228 | ### ⚠ BREAKING CHANGES 229 | 230 | * maintain Python 3.7 compatibility by moving settings base classes to the settings module 231 | 232 | ### Bug Fixes 233 | 234 | * maintain Python 3.7 compatibility by moving settings base classes to the settings module ([71c976e](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/71c976ea2a51afa418683effd3db9d80dca15150)) 235 | * use bugfix release of snakemake-interface-common ([2441fc3](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/2441fc36fc0cfc404aafeb0d8b86e7f107c7ebb6)) 236 | 237 | ## [4.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v4.0.0...v4.0.1) (2023-09-20) 238 | 239 | 240 | ### Bug Fixes 241 | 242 | * return correct value for next_seconds_between_status_checks ([0606922](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/06069228debfc55629f2eb6f2e88ac1b81ad90c8)) 243 | 244 | ## [4.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v3.0.2...v4.0.0) (2023-09-19) 245 | 246 | 247 | ### ⚠ BREAKING CHANGES 248 | 249 | * rename ExecutorJobInterface into JobExecutorInterface 250 | 251 | ### Code Refactoring 252 | 253 | * rename ExecutorJobInterface into JobExecutorInterface ([9f61b6a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/9f61b6a5f16ab39582429b813640fa08f3e0231c)) 254 | 255 | ## [3.0.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v3.0.1...v3.0.2) (2023-09-12) 256 | 257 | 258 | ### Bug Fixes 259 | 260 | * add error details in case of improper join_cli_args usage ([cb0245f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/cb0245fe47adfc73e07600821b5813687025ad9c)) 261 | 262 | ## [3.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v3.0.0...v3.0.1) (2023-09-11) 263 | 264 | 265 | ### Bug Fixes 266 | 267 | * avoid dependeing on argparse_dataclass fork ([0a1f02d](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0a1f02d5facf81a48ab687d8cb2809aebd6518d8)) 268 | * fix NoneType definition ([1654a41](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/1654a4140b7ee91c5e7f7370795fd67e5e70014b)) 269 | 270 | ## [3.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v2.0.0...v3.0.0) (2023-09-11) 271 | 272 | 273 | ### ⚠ BREAKING CHANGES 274 | 275 | * unify self.report_job_error and self.print_job_error. 276 | 277 | ### Features 278 | 279 | * add further metadata to ExecutorSettings ([30f0977](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/30f0977a646f13bc86a649e3e76ddfbf417f3ace)) 280 | * add get_items_by_category method to ExecutorSettings ([7f62bb9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/7f62bb9d15aa80f87964974d7a0bca504990e540)) 281 | * add support for env_var specification in ExecutorSettings ([a1e3123](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a1e3123a80db7b96bdb3cca11fa3faa21ab90ab3)) 282 | * unify self.report_job_error and self.print_job_error. ([2f24fb9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/2f24fb938cef05abf912e4a66a066fdce414f06b)) 283 | 284 | 285 | ### Documentation 286 | 287 | * update readme ([100bdc0](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/100bdc015ef2e8af4aa35fb2a027f46aeb73d244)) 288 | * update readme ([836b893](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/836b893287c8abed89dc738f9a3f48c335d9827a)) 289 | 290 | ## [2.0.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.2.0...v2.0.0) (2023-09-08) 291 | 292 | 293 | ### ⚠ BREAKING CHANGES 294 | 295 | * rename ExecutorPluginRegistry.get to get_plugin. 296 | * naming 297 | * improved API 298 | 299 | ### Features 300 | 301 | * add touch_exec ([0ac8b16](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0ac8b16e86419267e6cea49dee3451ed22fbde80)) 302 | * allow to set the next sleep time ([f8fde6c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f8fde6c0cbdbaf7cf164db65aae3ead5f5db919a)) 303 | * allow to specify the initial amount of seconds to sleep before checking job status ([0e88e6f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0e88e6ff4d3547c2c0991bc9d172413c9ed0d70b)) 304 | * improved API ([0226c9d](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/0226c9d2e7ab330f8552827f9714a43ff7f805c5)) 305 | * naming ([978f74c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/978f74cab5ab1e829f1636993dd46b8b51589ce8)) 306 | * rename ExecutorPluginRegistry.get to get_plugin. ([c1b50d9](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c1b50d9fb433d685749c10319827dea973caa8b2)) 307 | * simplify API ([3e4be2a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/3e4be2af66fd6fdbd3b0b60562023a4bdc64f92e)) 308 | 309 | 310 | ### Bug Fixes 311 | 312 | * API typing ([c9180fa](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/c9180fa55897e1b974edb068531f4cd6edce8d15)) 313 | 314 | ## [1.2.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.1.2...v1.2.0) (2023-09-05) 315 | 316 | 317 | ### Features 318 | 319 | * simplify executor API ([7479c1c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/7479c1c98f0fdf04ee77cea11feae4da2421ff90)) 320 | * various API improvements ([a2808ad](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a2808ad1ec480949e88efc442532952f42d8f450)) 321 | 322 | ## [1.1.2](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.1.1...v1.1.2) (2023-09-01) 323 | 324 | 325 | ### Bug Fixes 326 | 327 | * convert enum to cli choice ([a2f287c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a2f287c9cd66d4b4ccb847434ee9294c4749b233)) 328 | * various adaptations to changes in Snakemake 8.0 ([58ff504](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/58ff50488b49b1a34e41c4fa9812297430cc9672)) 329 | 330 | ## [1.1.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.1.0...v1.1.1) (2023-08-30) 331 | 332 | 333 | ### Bug Fixes 334 | 335 | * practical improvements ([f91133a](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f91133af160aef941e3663cfe3a50653589244f3)) 336 | 337 | ## [1.1.0](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.0.1...v1.1.0) (2023-08-28) 338 | 339 | 340 | ### Features 341 | 342 | * refactor and clean up interfaces ([#14](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/14)) ([fc28032](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/fc28032030204504e26c148e73ef8d85af9a5cf7)) 343 | 344 | ## [1.0.1](https://github.com/snakemake/snakemake-interface-executor-plugins/compare/v1.0.0...v1.0.1) (2023-08-02) 345 | 346 | 347 | ### Bug Fixes 348 | 349 | * release process fix ([b539c1b](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/b539c1b6795cfff9440bbd7e283e51c2df6518ba)) 350 | 351 | ## 1.0.0 (2023-08-02) 352 | 353 | 354 | ### Features 355 | 356 | * migrate interfaces from snakemake to this package ([#7](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/7)) ([cc3327c](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/cc3327c1e3020ff25f72f93f4c2711d7cadb11e6)) 357 | * migrate snakemake.common.Mode into this package ([b0aa928](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/b0aa928d30f4cf49d459b8fa6ed6904d269f9d27)) 358 | * object oriented plugin interface implementation ([a6923d2](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/a6923d2e5319124b5db2b72210280c43c5a47624)) 359 | * start of work to integrate functions ([#5](https://github.com/snakemake/snakemake-interface-executor-plugins/issues/5)) ([56f16d8](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/56f16d8f8ce9b0bf47d7d88be98548c6ed860970)) 360 | 361 | 362 | ### Bug Fixes 363 | 364 | * fix jobname checking logic ([1358f5f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/1358f5fd3070cf1c4f0b08e65b9cd805dd8a1e90)) 365 | * jobname checking ([f7a67d4](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/f7a67d4d6dfee279fa1ff088e1d1f9241dfdcbe0)) 366 | * remove superfluous attribute ([7d283f8](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/7d283f8551f0d2c80dadf87717d804422b3b5c09)) 367 | 368 | 369 | ### Performance Improvements 370 | 371 | * improve plugin lookup ([514514f](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/514514f22fcd3387915a65969ffd1d6c14c56964)) 372 | 373 | 374 | ### Miscellaneous Chores 375 | 376 | * release 1.0 ([59415f4](https://github.com/snakemake/snakemake-interface-executor-plugins/commit/59415f461616ab69668ea06c4a34932de70ea4bc)) 377 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Snakemake Community 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Stable interfaces and functionality for Snakemake executor plugins 2 | 3 | This package provides a stable interface for interactions between Snakemake and its executor plugins. 4 | 5 | Plugins should implement the following skeleton to comply with this interface. 6 | It is recommended to use Snakemake's poetry plugin to set up this skeleton (and automated testing) within a python package, see https://github.com/snakemake/poetry-snakemake-plugin. 7 | 8 | ```python 9 | from dataclasses import dataclass, field 10 | from typing import List, Generator, Optional 11 | from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo 12 | from snakemake_interface_executor_plugins.executors.remote import RemoteExecutor 13 | from snakemake_interface_executor_plugins.settings import ( 14 | ExecutorSettingsBase, CommonSettings 15 | ) 16 | from snakemake_interface_executor_plugins.workflow import WorkflowExecutorInterface 17 | from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface 18 | from snakemake_interface_executor_plugins.jobs import ( 19 | JobExecutorInterface, 20 | ) 21 | 22 | # Optional: 23 | # Define additional settings for your executor. 24 | # They will occur in the Snakemake CLI as --- 25 | # Omit this class if you don't need any. 26 | # Make sure that all defined fields are Optional and specify a default value 27 | # of None or anything else that makes sense in your case. 28 | @dataclass 29 | class ExecutorSettings(ExecutorSettingsBase): 30 | myparam: Optional[int] = field( 31 | default=None, 32 | metadata={ 33 | "help": "Some help text", 34 | # Optionally request that setting is also available for specification 35 | # via an environment variable. The variable will be named automatically as 36 | # SNAKEMAKE__, all upper case. 37 | # This mechanism should only be used for passwords and usernames. 38 | # For other items, we rather recommend to let people use a profile 39 | # for setting defaults 40 | # (https://snakemake.readthedocs.io/en/stable/executing/cli.html#profiles). 41 | "env_var": False, 42 | # Optionally specify a function that parses the value given by the user. 43 | # This is useful to create complex types from the user input. 44 | "parse_func": ..., 45 | # If a parse_func is specified, you also have to specify an unparse_func 46 | # that converts the parsed value back to a string. 47 | "unparse_func": ..., 48 | # Optionally specify that setting is required when the executor is in use. 49 | "required": True, 50 | # Optionally specify multiple args with "nargs": True 51 | }, 52 | ) 53 | 54 | 55 | # Required: 56 | # Specify common settings shared by various executors. 57 | common_settings = CommonSettings( 58 | # define whether your executor plugin executes locally 59 | # or remotely. In virtually all cases, it will be remote execution 60 | # (cluster, cloud, etc.). Only Snakemake's standard execution 61 | # plugins (snakemake-executor-plugin-dryrun, snakemake-executor-plugin-local) 62 | # are expected to specify False here. 63 | non_local_exec=True, 64 | # Whether the executor implies to not have a shared file system 65 | implies_no_shared_fs=True, 66 | # whether to deploy workflow sources to default storage provider before execution 67 | job_deploy_sources=True, 68 | # whether arguments for setting the storage provider shall be passed to jobs 69 | pass_default_storage_provider_args=True, 70 | # whether arguments for setting default resources shall be passed to jobs 71 | pass_default_resources_args=True, 72 | # whether environment variables shall be passed to jobs (if False, use 73 | # self.envvars() to obtain a dict of environment variables and their values 74 | # and pass them e.g. as secrets to the execution backend) 75 | pass_envvar_declarations_to_cmd=True, 76 | # whether the default storage provider shall be deployed before the job is run on 77 | # the remote node. Usually set to True if the executor does not assume a shared fs 78 | auto_deploy_default_storage_provider=True, 79 | # specify initial amount of seconds to sleep before checking for job status 80 | init_seconds_before_status_checks=0, 81 | ) 82 | 83 | 84 | # Required: 85 | # Implementation of your executor 86 | class Executor(RemoteExecutor): 87 | def __post_init__(self): 88 | # access workflow 89 | self.workflow 90 | # access executor specific settings 91 | self.workflow.executor_settings 92 | 93 | # IMPORTANT: in your plugin, only access methods and properties of 94 | # Snakemake objects (like Workflow, Persistence, etc.) that are 95 | # defined in the interfaces found in the 96 | # snakemake-interface-executor-plugins and the 97 | # snakemake-interface-common package. 98 | # Other parts of those objects are NOT guaranteed to remain 99 | # stable across new releases. 100 | 101 | # To ensure that the used interfaces are not changing, you should 102 | # depend on these packages as >=a.b.c, Generator[SubmittedJobInfo, None, None]: 125 | # Check the status of active jobs. 126 | 127 | # You have to iterate over the given list active_jobs. 128 | # If you provided it above, each will have its external_jobid set according 129 | # to the information you provided at submission time. 130 | # For jobs that have finished successfully, you have to call 131 | # self.report_job_success(active_job). 132 | # For jobs that have errored, you have to call 133 | # self.report_job_error(active_job). 134 | # This will also take care of providing a proper error message. 135 | # Usually there is no need to perform additional logging here. 136 | # Jobs that are still running have to be yielded. 137 | # 138 | # For queries to the remote middleware, please use 139 | # self.status_rate_limiter like this: 140 | # 141 | # async with self.status_rate_limiter: 142 | # # query remote middleware here 143 | # 144 | # To modify the time until the next call of this method, 145 | # you can set self.next_sleep_seconds here. 146 | ... 147 | 148 | def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): 149 | # Cancel all active jobs. 150 | # This method is called when Snakemake is interrupted. 151 | ... 152 | ``` 153 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "snakemake-interface-executor-plugins" 3 | version = "9.3.5" 4 | description = "This package provides a stable interface for interactions between Snakemake and its executor plugins." 5 | authors = [{ name = "Johannes Köster", email = "johannes.koester@uni-due.de" }] 6 | license = { text = "MIT" } 7 | readme = "README.md" 8 | requires-python = ">=3.11" 9 | dependencies = [ 10 | "snakemake-interface-common>=1.17.4", 11 | "throttler>=1.2.2", 12 | "argparse-dataclass>=2.0.0", 13 | ] 14 | 15 | [tool.coverage.run] 16 | omit = [".*", "*/site-packages/*"] 17 | 18 | [tool.coverage.report] 19 | fail_under = 63 20 | 21 | [build-system] 22 | build-backend = "poetry.core.masonry.api" 23 | requires = ["poetry-core"] 24 | 25 | [tool.pixi.project] 26 | channels = ["conda-forge"] 27 | platforms = ["osx-arm64", "linux-64"] 28 | 29 | [tool.pixi.pypi-dependencies] 30 | 31 | [tool.pixi.tasks] 32 | 33 | [tool.pixi.environments] 34 | dev = { features = ["dev"] } 35 | publish = { features = ["publish"] } 36 | 37 | [tool.pixi.feature.dev.pypi-dependencies] 38 | snakemake-interface-executor-plugins = { path = ".", editable = true } 39 | snakemake = { git = "https://github.com/snakemake/snakemake.git" } 40 | snakemake-executor-plugin-cluster-generic = { git = "https://github.com/snakemake/snakemake-executor-plugin-cluster-generic.git" } 41 | 42 | [tool.pixi.feature.dev.tasks.test] 43 | cmd = [ 44 | "pytest", 45 | "--cov=snakemake_interface_executor_plugins", 46 | "--cov-report=xml:coverage-report/coverage.xml", 47 | "--cov-report=term-missing", 48 | "tests/tests.py" 49 | ] 50 | description = "Run tests and generate coverage report" 51 | 52 | 53 | [tool.pixi.feature.dev.dependencies] 54 | pytest = ">=8.3.5,<9" 55 | ruff = ">=0.10.0,<0.11" 56 | mypy = ">=1.15.0,<2" 57 | pytest-cov = ">=6.0.0,<7" 58 | 59 | [tool.pixi.feature.dev.tasks] 60 | format = "ruff format snakemake_interface_executor_plugins" 61 | lint = "ruff check" 62 | type-check = "mypy snakemake_interface_executor_plugins/" 63 | qc = { depends-on = ["format", "lint", "type-check"] } 64 | 65 | [tool.mypy] 66 | ignore_missing_imports = true 67 | 68 | # Publish 69 | [tool.pixi.feature.publish.dependencies] 70 | twine = ">=6.1.0,<7" 71 | python-build = ">=1.2.2,<2" 72 | 73 | [tool.pixi.feature.publish.tasks] 74 | build = { cmd = "python -m build", description = "Build the package into the dist/ directory" } 75 | check-build = { cmd = "python -m twine check dist/*", depends-on = [ 76 | "build", 77 | ], description = "Check that the package can be uploaded" } 78 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # Recommend matching the black line length (default 88), 3 | # rather than using the flake8 default of 79: 4 | max-line-length = 106 5 | extend-ignore = 6 | # See https://github.com/PyCQA/pycodestyle/issues/373 7 | E203, -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster, Vanessa Sochat" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/_common.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster, Vanessa Sochat" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | 7 | executor_plugin_prefix = "snakemake-executor-plugin-" 8 | executor_plugin_module_prefix = executor_plugin_prefix.replace("-", "_") 9 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/cli.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Mapping 3 | 4 | from snakemake_interface_executor_plugins.settings import CommonSettings 5 | 6 | 7 | class SpawnedJobArgsFactoryExecutorInterface(ABC): 8 | @abstractmethod 9 | def general_args( 10 | self, 11 | executor_common_settings: CommonSettings, 12 | ) -> str: ... 13 | 14 | @abstractmethod 15 | def precommand(self, executor_common_settings: CommonSettings) -> str: ... 16 | 17 | @abstractmethod 18 | def envvars(self) -> Mapping[str, str]: ... 19 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/dag.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | from typing import Iterable, Optional 8 | 9 | from snakemake_interface_executor_plugins.jobs import JobExecutorInterface 10 | 11 | 12 | class DAGExecutorInterface(ABC): 13 | @abstractmethod 14 | def incomplete_external_jobid(self, job: JobExecutorInterface) -> Optional[str]: ... 15 | 16 | @abstractmethod 17 | def get_sources(self) -> Iterable[str]: ... 18 | 19 | @abstractmethod 20 | def get_unneeded_temp_files(self, job: JobExecutorInterface) -> Iterable[str]: ... 21 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/executors/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/executors/base.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | from dataclasses import dataclass 8 | from typing import Any, Dict, List, Optional 9 | 10 | from snakemake_interface_executor_plugins.jobs import JobExecutorInterface 11 | from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface 12 | from snakemake_interface_executor_plugins.utils import format_cli_arg 13 | from snakemake_interface_executor_plugins.workflow import WorkflowExecutorInterface 14 | 15 | 16 | @dataclass 17 | class SubmittedJobInfo: 18 | job: JobExecutorInterface 19 | external_jobid: Optional[str] = None 20 | aux: Optional[Dict[Any, Any]] = None 21 | 22 | 23 | class AbstractExecutor(ABC): 24 | def __init__( 25 | self, 26 | workflow: WorkflowExecutorInterface, 27 | logger: LoggerExecutorInterface, 28 | ): 29 | self.workflow = workflow 30 | self.dag = workflow.dag 31 | self.logger = logger 32 | 33 | def get_resource_declarations_dict(self, job: JobExecutorInterface): 34 | def isdigit(i): 35 | s = str(i) 36 | # Adapted from https://stackoverflow.com/a/1265696 37 | if s[0] in ("-", "+"): 38 | return s[1:].isdigit() 39 | return s.isdigit() 40 | 41 | excluded_resources = self.workflow.resource_scopes.excluded.union( 42 | {"_nodes", "_cores"} 43 | ) 44 | return { 45 | resource: value 46 | for resource, value in job.resources.items() 47 | if isinstance(value, int) 48 | # need to check bool seperately because bool is a subclass of int 49 | and isdigit(value) 50 | and resource not in excluded_resources 51 | } 52 | 53 | def get_resource_declarations(self, job: JobExecutorInterface): 54 | resources = [ 55 | f"{resource}={value}" 56 | for resource, value in self.get_resource_declarations_dict(job).items() 57 | ] 58 | return format_cli_arg("--resources", resources) 59 | 60 | def run_jobs( 61 | self, 62 | jobs: List[JobExecutorInterface], 63 | ): 64 | """Run a list of jobs that is ready at a given point in time. 65 | 66 | By default, this method just runs each job individually. 67 | This method can be overwritten to submit many jobs in a more efficient 68 | way than one-by-one. Note that in any case, for each job, the callback 69 | functions have to be called individually! 70 | """ 71 | for job in jobs: 72 | self.run_job_pre(job) 73 | self.run_job(job) 74 | 75 | @abstractmethod 76 | def run_job( 77 | self, 78 | job: JobExecutorInterface, 79 | ): 80 | """Run a specific job or group job. 81 | 82 | After successfull submission, you have to call self.report_job_submission(job). 83 | """ 84 | ... 85 | 86 | def run_job_pre(self, job: JobExecutorInterface): 87 | self.printjob(job) 88 | 89 | def report_job_success(self, job_info: SubmittedJobInfo): 90 | self.workflow.scheduler.finish_callback(job_info.job) 91 | 92 | def report_job_error(self, job_info: SubmittedJobInfo, msg=None, **kwargs): 93 | self.print_job_error(job_info, msg, **kwargs) 94 | self.workflow.scheduler.error_callback(job_info.job) 95 | 96 | def report_job_submission(self, job_info: SubmittedJobInfo): 97 | self.workflow.scheduler.submit_callback(job_info.job) 98 | 99 | @abstractmethod 100 | def shutdown(self): ... 101 | 102 | @abstractmethod 103 | def cancel(self): ... 104 | 105 | def rule_prefix(self, job: JobExecutorInterface): 106 | return "local " if job.is_local else "" 107 | 108 | def printjob(self, job: JobExecutorInterface): 109 | job.log_info() 110 | 111 | def print_job_error(self, job_info: SubmittedJobInfo, msg=None, **kwargs): 112 | job_info.job.log_error(msg, **kwargs) 113 | 114 | @abstractmethod 115 | def handle_job_success(self, job: JobExecutorInterface): ... 116 | 117 | @abstractmethod 118 | def handle_job_error(self, job: JobExecutorInterface): ... 119 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/executors/jobscript.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # properties = {properties} 3 | {exec_job} 4 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/executors/real.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import abstractmethod 7 | from typing import Dict 8 | from snakemake_interface_executor_plugins.executors.base import ( 9 | AbstractExecutor, 10 | SubmittedJobInfo, 11 | ) 12 | from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface 13 | from snakemake_interface_executor_plugins.settings import ExecMode 14 | from snakemake_interface_executor_plugins.utils import ( 15 | encode_target_jobs_cli_args, 16 | format_cli_arg, 17 | join_cli_args, 18 | ) 19 | from snakemake_interface_executor_plugins.jobs import JobExecutorInterface 20 | from snakemake_interface_executor_plugins.workflow import WorkflowExecutorInterface 21 | 22 | 23 | class RealExecutor(AbstractExecutor): 24 | def __init__( 25 | self, 26 | workflow: WorkflowExecutorInterface, 27 | logger: LoggerExecutorInterface, 28 | post_init: bool = True, 29 | ): 30 | super().__init__( 31 | workflow, 32 | logger, 33 | ) 34 | self.executor_settings = self.workflow.executor_settings 35 | self.snakefile = workflow.main_snakefile 36 | if post_init: 37 | self.__post_init__() 38 | 39 | def __post_init__(self): 40 | """This method is called after the constructor. By default, it does nothing.""" 41 | pass 42 | 43 | @property 44 | @abstractmethod 45 | def cores(self): 46 | # return "all" in case of remote executors, 47 | # otherwise self.workflow.resource_settings.cores 48 | ... 49 | 50 | def report_job_submission( 51 | self, job_info: SubmittedJobInfo, register_job: bool = True 52 | ): 53 | super().report_job_submission(job_info) 54 | 55 | if register_job: 56 | try: 57 | job_info.job.register(external_jobid=job_info.external_jobid) 58 | except IOError as e: 59 | self.logger.info( 60 | f"Failed to set marker file for job started ({e}). " 61 | "Snakemake will work, but cannot ensure that output files " 62 | "are complete in case of a kill signal or power loss. " 63 | "Please ensure write permissions for the " 64 | "directory {self.workflow.persistence.path}." 65 | ) 66 | 67 | def handle_job_success(self, job: JobExecutorInterface): 68 | pass 69 | 70 | def handle_job_error(self, job: JobExecutorInterface): 71 | pass 72 | 73 | def additional_general_args(self): 74 | """Inherit this method to add stuff to the general args. 75 | 76 | A list must be returned. 77 | """ 78 | return [] 79 | 80 | def get_job_args(self, job: JobExecutorInterface, **kwargs): 81 | unneeded_temp_files = list(self.workflow.dag.get_unneeded_temp_files(job)) 82 | return join_cli_args( 83 | [ 84 | format_cli_arg( 85 | "--target-jobs", encode_target_jobs_cli_args(job.get_target_spec()) 86 | ), 87 | # Restrict considered rules for faster DAG computation. 88 | # This does not work for updated jobs because they need 89 | # to be updated in the spawned process as well. 90 | format_cli_arg( 91 | "--allowed-rules", 92 | job.rules, 93 | quote=False, 94 | skip=job.is_updated, 95 | ), 96 | # Ensure that a group uses its proper local groupid. 97 | format_cli_arg("--local-groupid", job.jobid, skip=not job.is_group()), 98 | format_cli_arg("--cores", kwargs.get("cores", self.cores)), 99 | format_cli_arg("--attempt", job.attempt), 100 | format_cli_arg("--force-use-threads", not job.is_group()), 101 | format_cli_arg( 102 | "--unneeded-temp-files", 103 | unneeded_temp_files, 104 | skip=not unneeded_temp_files, 105 | ), 106 | self.get_resource_declarations(job), 107 | ] 108 | ) 109 | 110 | @property 111 | def job_specific_local_groupid(self): 112 | return True 113 | 114 | def get_snakefile(self): 115 | return self.snakefile 116 | 117 | @abstractmethod 118 | def get_python_executable(self): ... 119 | 120 | @abstractmethod 121 | def get_exec_mode(self) -> ExecMode: ... 122 | 123 | @property 124 | def common_settings(self): 125 | return self.workflow.executor_plugin.common_settings 126 | 127 | def get_envvar_declarations(self): 128 | declaration = "" 129 | envars = self.envvars() 130 | if self.common_settings.pass_envvar_declarations_to_cmd and envars: 131 | defs = " ".join(f"{var}={value!r}" for var, value in envars.items()) 132 | declaration = f"export {defs} &&" 133 | return declaration 134 | 135 | def get_job_exec_prefix(self, job: JobExecutorInterface): 136 | return "" 137 | 138 | def get_job_exec_suffix(self, job: JobExecutorInterface): 139 | return "" 140 | 141 | def format_job_exec(self, job: JobExecutorInterface) -> str: 142 | prefix = self.get_job_exec_prefix(job) 143 | if prefix: 144 | prefix += " &&" 145 | suffix = self.get_job_exec_suffix(job) 146 | if suffix: 147 | suffix = f"&& {suffix}" 148 | general_args = self.workflow.spawned_job_args_factory.general_args( 149 | executor_common_settings=self.common_settings 150 | ) 151 | precommand = self.workflow.spawned_job_args_factory.precommand( 152 | executor_common_settings=self.common_settings 153 | ) 154 | if precommand: 155 | precommand += " &&" 156 | 157 | args = join_cli_args( 158 | [ 159 | prefix, 160 | self.get_envvar_declarations(), 161 | precommand, 162 | self.get_python_executable(), 163 | "-m snakemake", 164 | format_cli_arg("--snakefile", self.get_snakefile()), 165 | self.get_job_args(job), 166 | general_args, 167 | self.additional_general_args(), 168 | format_cli_arg("--mode", self.get_exec_mode().item_to_choice()), 169 | format_cli_arg( 170 | "--local-groupid", 171 | self.workflow.group_settings.local_groupid, 172 | skip=self.job_specific_local_groupid, 173 | ), 174 | suffix, 175 | ] 176 | ) 177 | return args 178 | 179 | def envvars(self) -> Dict[str, str]: 180 | return self.workflow.spawned_job_args_factory.envvars() 181 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/executors/remote.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | import asyncio 8 | from fractions import Fraction 9 | import os 10 | import shutil 11 | import stat 12 | import sys 13 | import tempfile 14 | import threading 15 | from typing import Generator, List 16 | from snakemake_interface_common.exceptions import WorkflowError 17 | from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo 18 | from snakemake_interface_executor_plugins.executors.real import RealExecutor 19 | from snakemake_interface_executor_plugins.jobs import JobExecutorInterface 20 | from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface 21 | from snakemake_interface_executor_plugins.settings import ExecMode, SharedFSUsage 22 | from snakemake_interface_executor_plugins.utils import async_lock, format_cli_arg 23 | from snakemake_interface_executor_plugins.workflow import WorkflowExecutorInterface 24 | 25 | from throttler import Throttler 26 | 27 | 28 | class RemoteExecutor(RealExecutor, ABC): 29 | """Backend for distributed execution. 30 | 31 | The key idea is that a job is converted into a script that invokes 32 | Snakemake again, in whatever environment is targeted. The script 33 | is submitted to some job management platform (e.g. a cluster scheduler 34 | like slurm). 35 | This class can be specialized to generate more specific backends, 36 | also for the cloud. 37 | """ 38 | 39 | default_jobscript = "jobscript.sh" 40 | 41 | def __init__( 42 | self, 43 | workflow: WorkflowExecutorInterface, 44 | logger: LoggerExecutorInterface, 45 | ): 46 | super().__init__( 47 | workflow, 48 | logger, 49 | post_init=False, # we call __post_init__ ourselves 50 | ) 51 | self._next_seconds_between_status_checks = None 52 | self.max_status_checks_per_second = ( 53 | self.workflow.remote_execution_settings.max_status_checks_per_second 54 | ) 55 | self.jobname = self.workflow.remote_execution_settings.jobname 56 | 57 | if SharedFSUsage.SOURCES not in self.workflow.storage_settings.shared_fs_usage: 58 | # use relative path to Snakefile 59 | self.snakefile = os.path.relpath(workflow.main_snakefile) 60 | 61 | self.is_default_jobscript = False 62 | jobscript = workflow.remote_execution_settings.jobscript 63 | if jobscript is None: 64 | jobscript = os.path.join(os.path.dirname(__file__), self.default_jobscript) 65 | self.is_default_jobscript = True 66 | try: 67 | with open(jobscript) as f: 68 | self.jobscript = f.read() 69 | except IOError as e: 70 | raise WorkflowError(e) 71 | 72 | if "{jobid}" not in self.jobname: 73 | raise WorkflowError( 74 | f'Defined jobname ("{self.jobname}") has ' 75 | f"to contain the wildcard {{jobid}}." 76 | ) 77 | 78 | self._tmpdir = None 79 | 80 | self.active_jobs = list() 81 | self.lock = threading.Lock() 82 | self.wait = True 83 | self.wait_thread = threading.Thread(target=self._wait_thread) 84 | self.wait_thread.daemon = True 85 | self.wait_thread.start() 86 | 87 | max_status_checks_frac = Fraction( 88 | self.max_status_checks_per_second 89 | ).limit_denominator() 90 | self.status_rate_limiter = Throttler( 91 | rate_limit=max_status_checks_frac.numerator, 92 | period=max_status_checks_frac.denominator, 93 | ) 94 | 95 | self.__post_init__() 96 | 97 | @property 98 | def cores(self): 99 | cores = self.workflow.resource_settings.cores 100 | # if constrained, pass this info to the job 101 | if cores is not None and cores != sys.maxsize: 102 | return cores 103 | # otherwise, use whatever the node provides 104 | return "all" 105 | 106 | def cancel(self): 107 | with self.lock: 108 | active_jobs = list(self.active_jobs) 109 | self.cancel_jobs(active_jobs) 110 | self.shutdown() 111 | 112 | @abstractmethod 113 | def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): 114 | """Cancel the given jobs. 115 | 116 | This method is called when the workflow is cancelled. 117 | """ 118 | ... 119 | 120 | def get_exec_mode(self) -> ExecMode: 121 | return ExecMode.REMOTE 122 | 123 | def get_python_executable(self): 124 | return ( 125 | sys.executable 126 | if SharedFSUsage.SOFTWARE_DEPLOYMENT 127 | in self.workflow.storage_settings.shared_fs_usage 128 | else "python" 129 | ) 130 | 131 | def get_job_args(self, job: JobExecutorInterface): 132 | waitfiles_parameter = "" 133 | if SharedFSUsage.INPUT_OUTPUT in self.workflow.storage_settings.shared_fs_usage: 134 | wait_for_files = [] 135 | wait_for_files.append(self.tmpdir) 136 | wait_for_files.extend(job.get_wait_for_files()) 137 | 138 | # Only create extra file if we have more than 20 input files. 139 | # This should not require the file creation in most cases. 140 | if len(wait_for_files) > 20: 141 | wait_for_files_file = self.get_jobscript(job) + ".waitforfilesfile.txt" 142 | with open(wait_for_files_file, "w") as fd: 143 | print(*wait_for_files, sep="\n", file=fd) 144 | 145 | waitfiles_parameter = format_cli_arg( 146 | "--wait-for-files-file", wait_for_files_file 147 | ) 148 | else: 149 | waitfiles_parameter = format_cli_arg("--wait-for-files", wait_for_files) 150 | 151 | return f"{super().get_job_args(job)} {waitfiles_parameter}" 152 | 153 | def report_job_submission( 154 | self, job_info: SubmittedJobInfo, register_job: bool = True 155 | ): 156 | super().report_job_submission(job_info, register_job=register_job) 157 | with self.lock: 158 | self.active_jobs.append(job_info) 159 | 160 | @abstractmethod 161 | async def check_active_jobs( 162 | self, active_jobs: List[SubmittedJobInfo] 163 | ) -> Generator[SubmittedJobInfo, None, None]: 164 | """Check the status of active jobs. 165 | 166 | You have to iterate over the given list active_jobs. 167 | For jobs that have finished successfully, you have to call 168 | self.report_job_success(job). 169 | For jobs that have errored, you have to call 170 | self.report_job_error(job). 171 | Jobs that are still running have to be yielded. 172 | """ 173 | ... 174 | 175 | async def _wait_for_jobs(self): 176 | await asyncio.sleep( 177 | self.workflow.executor_plugin.common_settings.init_seconds_before_status_checks 178 | ) 179 | while True: 180 | async with async_lock(self.lock): 181 | if not self.wait: 182 | return 183 | active_jobs = list(self.active_jobs) 184 | self.active_jobs.clear() 185 | still_active_jobs = [ 186 | job_info async for job_info in self.check_active_jobs(active_jobs) 187 | ] 188 | async with async_lock(self.lock): 189 | # re-add the remaining jobs to active_jobs 190 | self.active_jobs.extend(still_active_jobs) 191 | await self.sleep() 192 | 193 | def _wait_thread(self): 194 | try: 195 | asyncio.run(self._wait_for_jobs()) 196 | except Exception as e: 197 | print(e) 198 | if self.workflow.scheduler is not None: 199 | self.workflow.scheduler.executor_error_callback(e) 200 | 201 | def shutdown(self): 202 | with self.lock: 203 | self.wait = False 204 | self.wait_thread.join() 205 | if not self.workflow.remote_execution_settings.immediate_submit: 206 | # Only delete tmpdir (containing jobscripts) if not using 207 | # immediate_submit. With immediate_submit, jobs can be scheduled 208 | # after this method is completed. Hence we have to keep the 209 | # directory. 210 | shutil.rmtree(self.tmpdir, ignore_errors=True) 211 | 212 | @property 213 | def tmpdir(self): 214 | if self._tmpdir is None: 215 | self._tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="tmp.") 216 | return os.path.abspath(self._tmpdir) 217 | 218 | def get_jobname(self, job: JobExecutorInterface): 219 | return job.format_wildcards(self.jobname) 220 | 221 | def get_jobscript(self, job: JobExecutorInterface): 222 | f = self.get_jobname(job) 223 | 224 | if os.path.sep in f: 225 | raise WorkflowError( 226 | "Path separator ({}) found in job name {}. " 227 | "This is not supported.".format(os.path.sep, f) 228 | ) 229 | 230 | return os.path.join(self.tmpdir, f) 231 | 232 | def write_jobscript(self, job: JobExecutorInterface, jobscript): 233 | exec_job = self.format_job_exec(job) 234 | 235 | try: 236 | content = self.jobscript.format( 237 | properties=job.properties(), 238 | exec_job=exec_job, 239 | ) 240 | except KeyError as e: 241 | if self.is_default_jobscript: 242 | raise e 243 | else: 244 | raise WorkflowError( 245 | "Error formatting custom jobscript " 246 | f"{self.jobscript}: value for {e} not found.\n" 247 | "Make sure that your custom jobscript is defined as " 248 | "expected." 249 | ) 250 | 251 | self.logger.debug(f"Jobscript:\n{content}") 252 | with open(jobscript, "w") as f: 253 | print(content, file=f) 254 | os.chmod(jobscript, os.stat(jobscript).st_mode | stat.S_IXUSR | stat.S_IRUSR) 255 | 256 | def handle_job_success(self, job: JobExecutorInterface): 257 | super().handle_job_success(job) 258 | 259 | def print_job_error(self, job_info: SubmittedJobInfo, msg=None, **kwargs): 260 | msg = msg or "" 261 | msg += ( 262 | "For further error details see the cluster/cloud " 263 | "log and the log files of the involved rule(s)." 264 | ) 265 | if job_info.external_jobid is not None: 266 | kwargs["external_jobid"] = job_info.external_jobid 267 | super().print_job_error(job_info, msg=msg, **kwargs) 268 | 269 | async def sleep(self): 270 | duration = ( 271 | self.workflow.remote_execution_settings.seconds_between_status_checks 272 | if self.next_seconds_between_status_checks is None 273 | else self.next_seconds_between_status_checks 274 | ) 275 | await asyncio.sleep(duration) 276 | 277 | @property 278 | def next_seconds_between_status_checks(self): 279 | if self._next_seconds_between_status_checks is None: 280 | return self.workflow.remote_execution_settings.seconds_between_status_checks 281 | else: 282 | return self._next_seconds_between_status_checks 283 | 284 | @next_seconds_between_status_checks.setter 285 | def next_seconds_between_status_checks(self, value): 286 | self._next_seconds_between_status_checks = value 287 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/jobs.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | import sys 8 | from typing import Any, Iterable, Mapping, Optional, Sequence, Union 9 | 10 | from snakemake_interface_common.rules import RuleInterface 11 | 12 | 13 | class JobExecutorInterface(ABC): 14 | HIGHEST_PRIORITY = sys.maxsize 15 | 16 | @property 17 | @abstractmethod 18 | def name(self) -> str: ... 19 | 20 | @property 21 | @abstractmethod 22 | def jobid(self) -> int: ... 23 | 24 | @abstractmethod 25 | def logfile_suggestion(self, prefix: str) -> str: ... 26 | 27 | @abstractmethod 28 | def is_group(self) -> bool: ... 29 | 30 | @abstractmethod 31 | def log_info(self, skip_dynamic: bool = False) -> None: ... 32 | 33 | @abstractmethod 34 | def log_error(self, msg: Optional[str] = None, **kwargs) -> None: ... 35 | 36 | @abstractmethod 37 | def properties( 38 | self, omit_resources: Sequence[str] = ("_cores", "_nodes"), **aux_properties 39 | ) -> Mapping[str, Any]: ... 40 | 41 | @property 42 | @abstractmethod 43 | def resources(self) -> Mapping[str, Union[int, str]]: ... 44 | 45 | @property 46 | @abstractmethod 47 | def is_local(self) -> bool: ... 48 | 49 | @property 50 | @abstractmethod 51 | def is_updated(self) -> bool: ... 52 | 53 | @property 54 | @abstractmethod 55 | def output(self) -> Iterable[str]: ... 56 | 57 | @abstractmethod 58 | def register(self, external_jobid: Optional[str] = None) -> None: ... 59 | 60 | @abstractmethod 61 | def get_target_spec(self) -> str: ... 62 | 63 | @abstractmethod 64 | def rules(self) -> Iterable[RuleInterface]: ... 65 | 66 | @property 67 | @abstractmethod 68 | def attempt(self) -> int: ... 69 | 70 | @property 71 | @abstractmethod 72 | def input(self) -> Iterable[str]: ... 73 | 74 | @property 75 | @abstractmethod 76 | def threads(self) -> int: ... 77 | 78 | @property 79 | @abstractmethod 80 | def log(self) -> Iterable[str]: ... 81 | 82 | @abstractmethod 83 | def get_wait_for_files(self) -> Iterable[str]: ... 84 | 85 | @abstractmethod 86 | def format_wildcards(self, string, **variables) -> str: ... 87 | 88 | @property 89 | @abstractmethod 90 | def is_containerized(self) -> bool: ... 91 | 92 | 93 | class SingleJobExecutorInterface(ABC): 94 | @property 95 | @abstractmethod 96 | def rule(self) -> RuleInterface: ... 97 | 98 | @property 99 | @abstractmethod 100 | def benchmark(self) -> Optional[str]: ... 101 | 102 | @property 103 | @abstractmethod 104 | def message(self): ... 105 | 106 | 107 | class GroupJobExecutorInterface(ABC): 108 | @property 109 | @abstractmethod 110 | def jobs(self): ... 111 | 112 | @property 113 | @abstractmethod 114 | def groupid(self): ... 115 | 116 | @property 117 | @abstractmethod 118 | def toposorted(self): ... 119 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/logging.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | 8 | 9 | class LoggerExecutorInterface(ABC): 10 | @abstractmethod 11 | def info(self, msg: str) -> None: ... 12 | 13 | @abstractmethod 14 | def error(self, msg: str) -> None: ... 15 | 16 | @abstractmethod 17 | def debug(self, msg: str) -> None: ... 18 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/persistence.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | from pathlib import Path 8 | 9 | 10 | class PersistenceExecutorInterface(ABC): 11 | @property 12 | @abstractmethod 13 | def path(self) -> Path: ... 14 | 15 | @property 16 | @abstractmethod 17 | def aux_path(self) -> Path: ... 18 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/registry/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster, Vanessa Sochat" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | import types 7 | from typing import Mapping 8 | from snakemake_interface_executor_plugins.settings import ( 9 | CommonSettings, 10 | ExecutorSettingsBase, 11 | ) 12 | 13 | from snakemake_interface_common.plugin_registry.attribute_types import ( 14 | AttributeKind, 15 | AttributeMode, 16 | AttributeType, 17 | ) 18 | from snakemake_interface_executor_plugins.registry.plugin import Plugin 19 | from snakemake_interface_common.plugin_registry import PluginRegistryBase 20 | from snakemake_interface_executor_plugins import _common as common 21 | 22 | 23 | class ExecutorPluginRegistry(PluginRegistryBase): 24 | """This class is a singleton that holds all registered executor plugins.""" 25 | 26 | @property 27 | def module_prefix(self) -> str: 28 | return common.executor_plugin_module_prefix 29 | 30 | def load_plugin(self, name: str, module: types.ModuleType) -> Plugin: 31 | """Load a plugin by name.""" 32 | return Plugin( 33 | _name=name, 34 | executor=module.Executor, 35 | common_settings=module.common_settings, 36 | _executor_settings_cls=getattr(module, "ExecutorSettings", None), 37 | ) 38 | 39 | def expected_attributes(self) -> Mapping[str, AttributeType]: 40 | # break otherwise circular import 41 | from snakemake_interface_executor_plugins.executors.base import AbstractExecutor 42 | 43 | return { 44 | "common_settings": AttributeType( 45 | cls=CommonSettings, 46 | mode=AttributeMode.REQUIRED, 47 | kind=AttributeKind.OBJECT, 48 | ), 49 | "ExecutorSettings": AttributeType( 50 | cls=ExecutorSettingsBase, 51 | mode=AttributeMode.OPTIONAL, 52 | kind=AttributeKind.CLASS, 53 | ), 54 | "Executor": AttributeType( 55 | cls=AbstractExecutor, 56 | mode=AttributeMode.REQUIRED, 57 | kind=AttributeKind.CLASS, 58 | ), 59 | } 60 | 61 | def collect_plugins(self): 62 | """Collect plugins and call register_plugin for each.""" 63 | super().collect_plugins() 64 | 65 | try: 66 | from snakemake.executors import local as local_executor 67 | from snakemake.executors import dryrun as dryrun_executor 68 | from snakemake.executors import touch as touch_executor 69 | except ImportError: 70 | # snakemake not present, proceed without adding these plugins 71 | return 72 | 73 | self.register_plugin("local", local_executor) 74 | self.register_plugin("dryrun", dryrun_executor) 75 | self.register_plugin("touch", touch_executor) 76 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/registry/plugin.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2022, Johannes Köster, Vanessa Sochat" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from dataclasses import dataclass 7 | from typing import Optional, Type 8 | from snakemake_interface_executor_plugins.settings import ( 9 | CommonSettings, 10 | ExecutorSettingsBase, 11 | ) 12 | import snakemake_interface_executor_plugins._common as common 13 | 14 | from snakemake_interface_common.plugin_registry.plugin import PluginBase 15 | 16 | 17 | @dataclass 18 | class Plugin(PluginBase): 19 | executor: object 20 | common_settings: CommonSettings 21 | _executor_settings_cls: Optional[Type[ExecutorSettingsBase]] 22 | _name: str 23 | 24 | @property 25 | def name(self): 26 | return self._name 27 | 28 | @property 29 | def cli_prefix(self): 30 | return self.name.replace(common.executor_plugin_module_prefix, "") 31 | 32 | @property 33 | def settings_cls(self): 34 | return self._executor_settings_cls 35 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/scheduler.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | 8 | 9 | class JobSchedulerExecutorInterface(ABC): 10 | @abstractmethod 11 | def executor_error_callback(self, exception: Exception) -> None: ... 12 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/settings.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from dataclasses import dataclass 3 | from typing import List, Sequence, Set 4 | 5 | from snakemake_interface_common.settings import SettingsEnumBase, TSettingsEnumBase 6 | 7 | 8 | import snakemake_interface_common.plugin_registry.plugin 9 | 10 | 11 | @dataclass 12 | class CommonSettings: 13 | """Common Snakemake settings shared between executors that can be specified 14 | by executor plugins. 15 | 16 | The plugin has to specify an instance of this class as the value of the 17 | common_settings attribute. 18 | 19 | Attributes 20 | ---------- 21 | non_local_exec : bool 22 | Whether to execute jobs locally or on a cluster. 23 | implies_no_shared_fs : bool 24 | Whether the executor implies to not have a shared file system. 25 | dryrun_exec : bool 26 | Whether jobs will be executed in dry-run mode. 27 | job_deploy_sources : bool 28 | Whether to deploy workflow sources before job execution. This is e.g. 29 | needed when remote jobs are guaranteed to be not executed on a shared 30 | filesystem. For example, this is True in the kubernetes executor plugin. 31 | touch_exec : bool 32 | Whether job outputs will be touched only. 33 | use_threads : bool 34 | Whether to use threads instead of processes. 35 | pass_default_storage_provider_args : bool 36 | Whether to pass default storage provider arguments to spawned jobs. 37 | pass_default_resources_args : bool 38 | Whether to pass default resources arguments to spawned jobs. 39 | pass_envvar_declarations_to_cmd : bool 40 | Whether envvars shall be declared in the job command. If false, envvars 41 | have to be declared in a different way by the executor, e.g. by passing 42 | them as secrets (see snakemake-executor-plugin-kubernetes). 43 | auto_deploy_default_storage_provider : bool 44 | Whether to automatically deploy the default storage provider in the spawned 45 | job via pip. This is usually needed in case the executor does not have a 46 | shared file system. 47 | init_seconds_before_status_checks : int 48 | Number of seconds to wait before starting to check the status of spawned jobs. 49 | pass_group_args : bool 50 | Whether to pass group arguments to spawned jobs. 51 | spawned_jobs_assume_shared_fs: bool 52 | Whether spawned jobs in the executor should always assume a shared FS regardless 53 | of the user provided settings. This should be True if the executor spawns 54 | another non-local executor that runs jobs on the same node. 55 | For example, it is used in snakemake-executor-plugin-slurm-jobstep. 56 | can_transfer_local_files: bool 57 | Indicates whether the plugin can transfer local files to the remote executor when 58 | run without a shared FS. If true, it's the plugin's responsibility and not 59 | Snakemake's to manage file transfers. 60 | """ 61 | 62 | non_local_exec: bool 63 | implies_no_shared_fs: bool 64 | job_deploy_sources: bool 65 | dryrun_exec: bool = False 66 | touch_exec: bool = False 67 | use_threads: bool = False 68 | pass_default_storage_provider_args: bool = True 69 | pass_default_resources_args: bool = True 70 | pass_envvar_declarations_to_cmd: bool = True 71 | auto_deploy_default_storage_provider: bool = True 72 | init_seconds_before_status_checks: int = 0 73 | pass_group_args: bool = False 74 | spawned_jobs_assume_shared_fs: bool = False 75 | can_transfer_local_files: bool = False 76 | 77 | @property 78 | def local_exec(self): 79 | return not self.non_local_exec 80 | 81 | 82 | @dataclass 83 | class ExecutorSettingsBase( 84 | snakemake_interface_common.plugin_registry.plugin.SettingsBase 85 | ): 86 | """Base class for executor settings. 87 | 88 | Executor plugins can define a subclass of this class, 89 | named 'ExecutorSettings'. 90 | """ 91 | 92 | pass 93 | 94 | 95 | class RemoteExecutionSettingsExecutorInterface(ABC): 96 | @property 97 | @abstractmethod 98 | def jobname(self) -> str: ... 99 | 100 | @property 101 | @abstractmethod 102 | def jobscript(self) -> str: ... 103 | 104 | @property 105 | @abstractmethod 106 | def immediate_submit(self) -> bool: ... 107 | 108 | @property 109 | @abstractmethod 110 | def envvars(self) -> Sequence[str]: ... 111 | 112 | @property 113 | @abstractmethod 114 | def max_status_checks_per_second(self) -> float: ... 115 | 116 | @property 117 | @abstractmethod 118 | def seconds_between_status_checks(self) -> int: ... 119 | 120 | 121 | class ExecMode(SettingsEnumBase): 122 | """ 123 | Enum for execution mode of Snakemake. 124 | This handles the behavior of e.g. the logger. 125 | """ 126 | 127 | DEFAULT = 0 128 | SUBPROCESS = 1 129 | REMOTE = 2 130 | 131 | 132 | class ExecutionSettingsExecutorInterface(ABC): 133 | @property 134 | @abstractmethod 135 | def keep_incomplete(self) -> bool: ... 136 | 137 | 138 | class SharedFSUsage(SettingsEnumBase): 139 | PERSISTENCE = 0 140 | INPUT_OUTPUT = 1 141 | SOFTWARE_DEPLOYMENT = 2 142 | SOURCES = 3 143 | STORAGE_LOCAL_COPIES = 4 144 | SOURCE_CACHE = 5 145 | 146 | @classmethod 147 | def choices(cls) -> List[str]: 148 | return super().choices() + ["none"] 149 | 150 | @classmethod 151 | def _parse_choices_into(cls, choices: str, container) -> List[TSettingsEnumBase]: 152 | if "none" in choices: 153 | if len(choices) > 1: 154 | raise ValueError( 155 | "Cannot specify 'none' together with other shared filesystem usages." 156 | ) 157 | return container([]) 158 | else: 159 | return container(cls.parse_choice(choice) for choice in choices) 160 | 161 | 162 | class StorageSettingsExecutorInterface(ABC): 163 | @property 164 | @abstractmethod 165 | def shared_fs_usage(self) -> Set[SharedFSUsage]: ... 166 | 167 | @property 168 | def assume_common_workdir(self) -> bool: 169 | return any( 170 | usage in self.shared_fs_usage 171 | for usage in ( 172 | SharedFSUsage.PERSISTENCE, 173 | SharedFSUsage.INPUT_OUTPUT, 174 | SharedFSUsage.SOFTWARE_DEPLOYMENT, 175 | ) 176 | ) 177 | 178 | 179 | class DeploymentMethod(SettingsEnumBase): 180 | CONDA = 0 181 | APPTAINER = 1 182 | ENV_MODULES = 2 183 | 184 | 185 | class DeploymentSettingsExecutorInterface(ABC): 186 | @property 187 | @abstractmethod 188 | def deployment_method(self) -> Set[DeploymentMethod]: ... 189 | 190 | 191 | class GroupSettingsExecutorInterface(ABC): 192 | @property 193 | @abstractmethod 194 | def local_groupid(self) -> str: ... 195 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/utils.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | import asyncio 7 | import base64 8 | from collections import UserDict 9 | from pathlib import Path 10 | import re 11 | import shlex 12 | import threading 13 | from typing import Any, List 14 | from urllib.parse import urlparse 15 | from collections import namedtuple 16 | import concurrent.futures 17 | import contextlib 18 | 19 | from snakemake_interface_common.settings import SettingsEnumBase 20 | from snakemake_interface_common.utils import not_iterable 21 | 22 | 23 | TargetSpec = namedtuple("TargetSpec", ["rulename", "wildcards_dict"]) 24 | 25 | 26 | def format_cli_arg(flag, value, quote=True, skip=False, base64_encode: bool = False): 27 | if not skip and value: 28 | if isinstance(value, bool): 29 | value = "" 30 | else: 31 | value = format_cli_pos_arg(value, quote=quote, base64_encode=base64_encode) 32 | return f"{flag} {value}" 33 | return "" 34 | 35 | 36 | def format_cli_pos_arg(value, quote=True, base64_encode: bool = False): 37 | if isinstance(value, (dict, UserDict)): 38 | 39 | def fmt_item(key, value): 40 | expr = f"{key}={format_cli_value(value)}" 41 | return encode_as_base64(expr) if base64_encode else repr(expr) 42 | 43 | return join_cli_args(fmt_item(key, val) for key, val in value.items()) 44 | elif not_iterable(value): 45 | return format_cli_value(value, quote=quote, base64_encode=base64_encode) 46 | else: 47 | return join_cli_args( 48 | format_cli_value(v, quote=quote, base64_encode=base64_encode) for v in value 49 | ) 50 | 51 | 52 | def format_cli_value( 53 | value: Any, quote: bool = False, base64_encode: bool = False 54 | ) -> str: 55 | """Format a given value for passing it to CLI. 56 | 57 | If base64_encode is True, str values are encoded and flagged as being base64 encoded. 58 | """ 59 | 60 | def maybe_encode(value): 61 | return encode_as_base64(value) if base64_encode else value 62 | 63 | if isinstance(value, SettingsEnumBase): 64 | return value.item_to_choice() 65 | elif isinstance(value, Path): 66 | if base64_encode: 67 | return encode_as_base64(str(value)) 68 | else: 69 | return shlex.quote(str(value)) 70 | elif isinstance(value, str): 71 | if is_quoted(value) and not base64_encode: 72 | # the value is already quoted, do not quote again 73 | return maybe_encode(value) 74 | elif quote and not base64_encode: 75 | return maybe_encode(repr(value)) 76 | else: 77 | return maybe_encode(value) 78 | else: 79 | return repr(value) 80 | 81 | 82 | def join_cli_args(args): 83 | try: 84 | return " ".join(arg for arg in args if arg) 85 | except TypeError as e: 86 | raise TypeError( 87 | f"bug: join_cli_args expects iterable of strings. Given: {args}" 88 | ) from e 89 | 90 | 91 | def url_can_parse(url: str) -> bool: 92 | """ 93 | returns true if urllib.parse.urlparse can parse 94 | scheme and netloc 95 | """ 96 | return all(list(urlparse(url))[:2]) 97 | 98 | 99 | def encode_target_jobs_cli_args( 100 | target_jobs: List[TargetSpec], 101 | ) -> List[str]: 102 | items = [] 103 | 104 | def add_quotes_if_contains_comma(s): 105 | if isinstance(s, str): 106 | if "," in s: 107 | return f'"{s}"' 108 | return s 109 | 110 | for spec in target_jobs: 111 | wildcards = ",".join( 112 | f"{key}={add_quotes_if_contains_comma(value)}" 113 | for key, value in spec.wildcards_dict.items() 114 | ) 115 | items.append(f"{spec.rulename}:{wildcards}") 116 | return items 117 | 118 | 119 | _pool = concurrent.futures.ThreadPoolExecutor() 120 | 121 | 122 | @contextlib.asynccontextmanager 123 | async def async_lock(_lock: threading.Lock): 124 | """Use a threaded lock form threading.Lock in an async context 125 | 126 | Necessary because asycio.Lock is not threadsafe, so only one thread can safely use 127 | it at a time. 128 | Source: https://stackoverflow.com/a/63425191 129 | """ 130 | loop = asyncio.get_event_loop() 131 | await loop.run_in_executor(_pool, _lock.acquire) 132 | try: 133 | yield # the lock is held 134 | finally: 135 | _lock.release() 136 | 137 | 138 | _is_quoted_re = re.compile(r"^['\"].+['\"]") 139 | 140 | 141 | def is_quoted(value: str) -> bool: 142 | return _is_quoted_re.match(value) is not None 143 | 144 | 145 | base64_prefix = "base64//" 146 | 147 | 148 | def maybe_base64(parser_func): 149 | """Parse optionally base64 encoded CLI args, applying parser_func if not None.""" 150 | 151 | def inner(args): 152 | def is_base64(arg): 153 | return arg.startswith(base64_prefix) 154 | 155 | def decode(arg): 156 | if is_base64(arg): 157 | return base64.b64decode(arg[len(base64_prefix) :]).decode() 158 | else: 159 | return arg 160 | 161 | def apply_parser(args): 162 | if parser_func is not None: 163 | return parser_func(args) 164 | else: 165 | return args 166 | 167 | if isinstance(args, str): 168 | return apply_parser(decode(args)) 169 | elif isinstance(args, list): 170 | decoded = [decode(arg) for arg in args] 171 | return apply_parser(decoded) 172 | else: 173 | raise NotImplementedError() 174 | 175 | return inner 176 | 177 | 178 | def encode_as_base64(arg: str): 179 | return f"{base64_prefix}{base64.b64encode(arg.encode()).decode()}" 180 | -------------------------------------------------------------------------------- /snakemake_interface_executor_plugins/workflow.py: -------------------------------------------------------------------------------- 1 | __author__ = "Johannes Köster" 2 | __copyright__ = "Copyright 2023, Johannes Köster" 3 | __email__ = "johannes.koester@uni-due.de" 4 | __license__ = "MIT" 5 | 6 | from abc import ABC, abstractmethod 7 | from typing import Optional 8 | 9 | from snakemake_interface_executor_plugins.cli import ( 10 | SpawnedJobArgsFactoryExecutorInterface, 11 | ) 12 | from snakemake_interface_executor_plugins.persistence import ( 13 | PersistenceExecutorInterface, 14 | ) 15 | from snakemake_interface_executor_plugins.registry.plugin import Plugin 16 | 17 | from snakemake_interface_executor_plugins.scheduler import JobSchedulerExecutorInterface 18 | from snakemake_interface_executor_plugins.settings import ( 19 | DeploymentSettingsExecutorInterface, 20 | ExecutionSettingsExecutorInterface, 21 | GroupSettingsExecutorInterface, 22 | RemoteExecutionSettingsExecutorInterface, 23 | StorageSettingsExecutorInterface, 24 | ) 25 | 26 | 27 | class WorkflowExecutorInterface(ABC): 28 | @property 29 | @abstractmethod 30 | def spawned_job_args_factory(self) -> SpawnedJobArgsFactoryExecutorInterface: ... 31 | 32 | @property 33 | @abstractmethod 34 | def execution_settings(self) -> ExecutionSettingsExecutorInterface: ... 35 | 36 | @property 37 | @abstractmethod 38 | def remote_execution_settings(self) -> RemoteExecutionSettingsExecutorInterface: ... 39 | 40 | @property 41 | @abstractmethod 42 | def storage_settings(self) -> StorageSettingsExecutorInterface: ... 43 | 44 | @property 45 | @abstractmethod 46 | def deployment_settings(self) -> DeploymentSettingsExecutorInterface: ... 47 | 48 | @property 49 | @abstractmethod 50 | def group_settings(self) -> GroupSettingsExecutorInterface: ... 51 | 52 | @property 53 | @abstractmethod 54 | def executor_plugin(self) -> Optional[Plugin]: ... 55 | 56 | @property 57 | @abstractmethod 58 | def resource_scopes(self): ... 59 | 60 | @property 61 | @abstractmethod 62 | def main_snakefile(self): ... 63 | 64 | @property 65 | @abstractmethod 66 | def persistence(self) -> PersistenceExecutorInterface: ... 67 | 68 | @property 69 | @abstractmethod 70 | def workdir_init(self): ... 71 | 72 | @property 73 | @abstractmethod 74 | def scheduler(self) -> JobSchedulerExecutorInterface: ... 75 | -------------------------------------------------------------------------------- /tests/test_py37.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def test_imports_from_py37(): 5 | assert sys.version_info >= (3, 7) and sys.version_info < (3, 8) 6 | from snakemake_interface_executor_plugins import ( # noqa: F401 7 | settings, 8 | jobs, 9 | logging, 10 | persistence, 11 | resources, 12 | scheduler, 13 | utils, 14 | workflow, 15 | dag, 16 | exceptions, 17 | ) 18 | -------------------------------------------------------------------------------- /tests/tests.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from snakemake_interface_executor_plugins.registry import ExecutorPluginRegistry 3 | from snakemake_interface_common.plugin_registry.tests import TestRegistryBase 4 | from snakemake_interface_common.plugin_registry.plugin import PluginBase, SettingsBase 5 | from snakemake_interface_common.plugin_registry import PluginRegistryBase 6 | from snakemake_interface_executor_plugins.utils import format_cli_arg 7 | 8 | 9 | class TestRegistry(TestRegistryBase): 10 | __test__ = True 11 | 12 | def get_registry(self) -> PluginRegistryBase: 13 | # ensure that the singleton is reset 14 | ExecutorPluginRegistry._instance = None 15 | return ExecutorPluginRegistry() 16 | 17 | def get_test_plugin_name(self) -> str: 18 | return "cluster-generic" 19 | 20 | def validate_plugin(self, plugin: PluginBase): 21 | assert plugin._executor_settings_cls is not None 22 | assert plugin.common_settings.non_local_exec is True 23 | assert plugin.executor is not None 24 | 25 | def validate_settings(self, settings: SettingsBase, plugin: PluginBase): 26 | assert isinstance(settings, plugin._executor_settings_cls) 27 | 28 | def get_example_args(self) -> List[str]: 29 | return ["--cluster-generic-submit-cmd", "qsub"] 30 | 31 | 32 | def test_format_cli_arg_single_quote(): 33 | fmt = format_cli_arg("--default-resources", {"slurm_extra": "'--gres=gpu:1'"}) 34 | assert fmt == "--default-resources \"slurm_extra='--gres=gpu:1'\"" 35 | 36 | 37 | def test_format_cli_arg_double_quote(): 38 | fmt = format_cli_arg("--default-resources", {"slurm_extra": '"--gres=gpu:1"'}) 39 | assert fmt == "--default-resources 'slurm_extra=\"--gres=gpu:1\"'" 40 | 41 | 42 | def test_format_cli_arg_int(): 43 | fmt = format_cli_arg("--default-resources", {"mem_mb": 200}) 44 | assert fmt == "--default-resources 'mem_mb=200'" 45 | 46 | 47 | def test_format_cli_arg_expr(): 48 | fmt = format_cli_arg( 49 | "--default-resources", {"mem_mb": "min(2 * input.size_mb, 2000)"} 50 | ) 51 | assert fmt == "--default-resources 'mem_mb=min(2 * input.size_mb, 2000)'" 52 | 53 | 54 | def test_format_cli_arg_list(): 55 | fmt = format_cli_arg("--config", ["foo={'bar': 1}"]) 56 | assert fmt == "--config \"foo={'bar': 1}\"" 57 | --------------------------------------------------------------------------------