├── .flake8 ├── .github ├── codeql │ └── config.yml ├── dependabot.yml └── workflows │ ├── ci-tests.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── bandit-requirements.txt ├── codecov.yml ├── docs ├── .wci.yml ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── _static │ └── theme_overrides.css │ ├── conf.py │ ├── index.rst │ └── reference.rst ├── examples ├── 1000-genome │ ├── README.md │ ├── docker │ │ ├── 1000-genome-notebook │ │ │ ├── Dockerfile │ │ │ └── build │ │ └── 1000-genome │ │ │ ├── Dockerfile │ │ │ └── build │ ├── k8s │ │ └── manifest.yaml │ ├── run │ ├── times │ │ ├── bar_chart.plt │ │ ├── plot.sh │ │ └── speedup.dat │ └── work │ │ ├── 1000-genome.ipynb │ │ ├── data │ │ └── download_data.sh │ │ └── environment │ │ └── k8s │ │ └── 1000-genome │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ └── deployment.yaml │ │ └── values.yaml ├── claire-covid │ ├── README.md │ ├── docker │ │ ├── Dockerfile │ │ └── build │ ├── node-details.txt │ ├── run │ ├── singularity │ │ └── ppcle64 │ │ │ ├── build │ │ │ ├── claire-covid.def │ │ │ └── pytorch.def │ ├── times │ │ ├── bar_chart.plt │ │ ├── plot.sh │ │ └── times.dat │ └── work │ │ ├── claire-covid.ipynb │ │ └── environment │ │ └── cineca-marconi100 │ │ └── slurm_template.jinja2 ├── quantum-espresso │ ├── README.md │ ├── docker │ │ ├── Dockerfile │ │ └── build │ ├── node-details.txt │ ├── run │ └── work │ │ ├── car-parrinello │ │ ├── INPDIR │ │ │ ├── H.pbe-van_bm.UPF │ │ │ ├── O.pbe-van_bm.UPF │ │ │ ├── cp-oneapi.x │ │ │ ├── h2o.in.00 │ │ │ ├── h2o.in.01 │ │ │ ├── h2o.in.02 │ │ │ ├── h2o.in.03.b0 │ │ │ ├── h2o.in.03.b1 │ │ │ └── h2o.in.03.b2 │ │ ├── car-parrinello.ipynb │ │ └── environment │ │ │ └── pbs_espresso │ │ └── primordial-soup │ │ ├── INPDIR │ │ ├── C.blyp-mt.UPF │ │ ├── H.blyp-vbc.UPF │ │ ├── N.blyp-mt.UPF │ │ ├── O.blyp-mt.UPF │ │ ├── chno.in.00 │ │ ├── chno.in.01 │ │ ├── chno.in.02 │ │ ├── chno.in.03 │ │ ├── chno.in.04.t1 │ │ ├── chno.in.04.t2 │ │ ├── chno.in.05.t1.p1 │ │ ├── chno.in.05.t1.p2 │ │ ├── chno.in.05.t2.p1 │ │ ├── chno.in.05.t2.p2 │ │ └── cp-oneapi.x │ │ ├── environment │ │ └── pbs_espresso │ │ └── primordial-soup.ipynb └── tensorflow │ ├── README.md │ ├── docker │ ├── tensorflow-notebook │ │ ├── Dockerfile │ │ └── build │ └── tensorflow-serving │ │ ├── Dockerfile │ │ └── build │ ├── run │ ├── serving-container-details.txt │ ├── training-node-details.txt │ └── work │ ├── environment │ ├── hpc │ │ └── ssh_template.jinja2 │ └── k8s │ │ └── tensorflow-serving │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── deployment.yaml │ │ ├── pvc.yaml │ │ ├── service.yaml │ │ └── tests │ │ │ └── test-connection.yaml │ │ └── values.yaml │ └── tf-serve.ipynb ├── jupyter_workflow ├── __init__.py ├── client │ ├── __init__.py │ ├── cli.py │ └── client.py ├── config │ ├── __init__.py │ ├── config.py │ ├── schema.py │ ├── schemas │ │ └── v1.0 │ │ │ └── config_schema.json │ └── validator.py ├── ipython │ ├── __init__.py │ ├── __main__.py │ ├── displayhook.py │ ├── install.py │ ├── iostream.py │ ├── ipkernel.py │ ├── kernelspec │ │ └── kernel.js │ └── shell.py ├── streamflow │ ├── __init__.py │ ├── command.py │ ├── executor.py │ ├── port.py │ ├── processor.py │ ├── step.py │ ├── token.py │ ├── transformer.py │ ├── translator.py │ ├── utils.py │ └── workflow.py └── version.py ├── lint-requirements.txt ├── pyproject.toml ├── requirements.txt ├── test-requirements.txt ├── tests ├── __init__.py ├── conftest.py ├── test_notebook.py ├── test_serializer.py ├── test_single_cell.py └── testdata │ ├── file_deps.ipynb │ ├── hello.txt │ ├── name_deps.ipynb │ ├── param_overwrite.ipynb │ ├── scatter_and_non_scatter_sequences.ipynb │ ├── scatter_deps.ipynb │ ├── serialization.ipynb │ ├── simple_scatter_sequence.ipynb │ └── two_steps_single_dep.ipynb └── tox.ini /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203,E501 -------------------------------------------------------------------------------- /.github/codeql/config.yml: -------------------------------------------------------------------------------- 1 | name: "Jupyter Workflow CodeQL configuration" 2 | queries: 3 | - uses: security-and-quality 4 | paths-ignore: 5 | - tests 6 | query-filters: 7 | # Reason: false positive on function body ellipsis (issue 11351) 8 | - exclude: 9 | id: py/ineffectual-statement 10 | # Reason: false positive on HasTraits class hierarchy 11 | - exclude: 12 | id: py/missing-call-to-init 13 | # Reason: no support for the TYPE_CHECKING directive (issue 4258) 14 | - exclude: 15 | id: py/unsafe-cyclic-import 16 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | - package-ecosystem: "pip" 8 | directory: "/" 9 | schedule: 10 | interval: "daily" -------------------------------------------------------------------------------- /.github/workflows/ci-tests.yml: -------------------------------------------------------------------------------- 1 | name: "CI Tests" 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | concurrency: 10 | group: build-${{ github.event.pull_request.number || github.ref }} 11 | cancel-in-progress: true 12 | jobs: 13 | code-ql-check: 14 | name: "Jupyter Workflow CodeQL check" 15 | runs-on: ubuntu-22.04 16 | permissions: 17 | security-events: write 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: github/codeql-action/init@v3 21 | with: 22 | config-file: .github/codeql/config.yml 23 | languages: python 24 | - uses: github/codeql-action/analyze@v3 25 | static-checks: 26 | name: "Jupyter Workflow static checks" 27 | runs-on: ubuntu-22.04 28 | strategy: 29 | matrix: 30 | step: [ "bandit", "lint" ] 31 | env: 32 | TOXENV: ${{ matrix.step }} 33 | steps: 34 | - uses: actions/checkout@v4 35 | - uses: actions/setup-python@v5 36 | with: 37 | python-version: "3.13" 38 | cache: pip 39 | cache-dependency-path: | 40 | requirements.txt 41 | tox.ini 42 | - name: "Install Python Dependencies and Jupyter Workflow" 43 | run: | 44 | python -m pip install tox --user 45 | python -m pip install . --user 46 | - name: "Run Jupyter Workflow static analysis via Tox" 47 | run: tox 48 | unit-tests: 49 | name: "Jupyter Workflow unit tests" 50 | runs-on: ubuntu-22.04 51 | strategy: 52 | matrix: 53 | python: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] 54 | env: 55 | TOXENV: ${{ format('py{0}-unit', matrix.python) }} 56 | steps: 57 | - uses: actions/checkout@v4 58 | - uses: actions/setup-python@v5 59 | with: 60 | python-version: ${{ matrix.python }} 61 | cache: pip 62 | cache-dependency-path: | 63 | requirements.txt 64 | tox.ini 65 | - uses: actions/setup-node@v4 66 | with: 67 | node-version: "20" 68 | - uses: docker/setup-qemu-action@v3 69 | - name: "Install Python Dependencies and Jupyter Workflow" 70 | run: | 71 | python -m pip install tox --user 72 | python -m pip install . --user 73 | - name: "Run Jupyter Workflow tests via Tox" 74 | run: tox 75 | - name: "Upload coverage report for unit tests" 76 | uses: actions/upload-artifact@v4 77 | with: 78 | name: ${{ format('py{0}-unit-tests', matrix.python) }} 79 | path: ./coverage.xml 80 | retention-days: 1 81 | if-no-files-found: error 82 | upload-to-codecov: 83 | name: "Codecov report upload" 84 | needs: [ "unit-tests" ] 85 | runs-on: ubuntu-22.04 86 | steps: 87 | - uses: actions/checkout@v4 88 | - name: "Download artifacts" 89 | uses: actions/download-artifact@v4 90 | - name: "Upload coverage to Codecov" 91 | uses: codecov/codecov-action@v5 92 | with: 93 | fail_ci_if_error: true 94 | token: ${{ secrets.CODECOV_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: "Release new version" 2 | on: 3 | workflow_run: 4 | workflows: 5 | - "CI Tests" 6 | branches: 7 | - master 8 | types: 9 | - completed 10 | jobs: 11 | github: 12 | name: "Create GitHub Release" 13 | runs-on: ubuntu-22.04 14 | permissions: 15 | contents: write 16 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: "Get Jupyter Workflow version" 20 | run: echo "JF_VERSION=$(cat jupyter_workflow/version.py | grep -oP '(?<=VERSION = \")(.*)(?=\")')" >> $GITHUB_ENV 21 | - name: "Check tag existence" 22 | uses: mukunku/tag-exists-action@v1.6.0 23 | id: check-tag 24 | with: 25 | tag: ${{ env.JF_VERSION }} 26 | - name: "Create Release" 27 | id: create-release 28 | uses: actions/create-release@v1 29 | if: ${{ steps.check-tag.outputs.exists == 'false' }} 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | with: 33 | tag_name: ${{ env.JF_VERSION }} 34 | release_name: ${{ env.JF_VERSION }} 35 | draft: false 36 | prerelease: false 37 | pypi: 38 | name: "Publish on PyPI" 39 | runs-on: ubuntu-22.04 40 | environment: 41 | name: pypi 42 | url: https://pypi.org/project/jupyter-workflow 43 | permissions: 44 | id-token: write 45 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 46 | steps: 47 | - uses: actions/checkout@v4 48 | - uses: actions/setup-python@v5 49 | with: 50 | python-version: "3.13" 51 | - name: "Get Jupyter Workflow version" 52 | run: echo "JF_VERSION=$(cat jupyter_workflow/version.py | grep -oP '(?<=VERSION = \")(.*)(?=\")')" >> $GITHUB_ENV 53 | - name: "Get PyPI version" 54 | run: echo "PYPI_VERSION=$(pip index versions --pre jupyter_workflow | grep jupyter_workflow | sed 's/.*(\(.*\))/\1/')" >> $GITHUB_ENV 55 | - name: "Build Python packages" 56 | if: ${{ env.JF_VERSION != env.PYPI_VERSION }} 57 | run: | 58 | python -m pip install build --user 59 | python -m build --sdist --wheel --outdir dist/ . 60 | - name: "Publish package to PyPI" 61 | uses: pypa/gh-action-pypi-publish@release/v1 62 | if: ${{ env.JF_VERSION != env.PYPI_VERSION }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # pytype static type analyzer 135 | .pytype/ 136 | 137 | # Cython debug symbols 138 | cython_debug/ 139 | 140 | # JetBrains 141 | .idea/ 142 | 143 | # Singularity 144 | *.sif 145 | 146 | # Streamflow 147 | .streamflow/ 148 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include requirements.txt 4 | include docs/requirements.txt 5 | include bandit-requirements.txt 6 | include lint-requirements.txt 7 | include test-requirements.txt -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | codespell: 2 | codespell -w $(shell git ls-files) 3 | 4 | codespell-check: 5 | codespell $(shell git ls-files) 6 | 7 | coverage.xml: testcov 8 | coverage xml 9 | 10 | coverage-report: testcov 11 | coverage report 12 | 13 | flake8: 14 | flake8 jupyter_workflow tests 15 | 16 | format: 17 | isort jupyter_workflow tests 18 | black jupyter_workflow tests 19 | 20 | format-check: 21 | isort --check-only jupyter_workflow tests 22 | black --diff --check jupyter_workflow tests 23 | 24 | pyupgrade: 25 | pyupgrade --py3-only --py39-plus $(shell git ls-files | grep .py) 26 | 27 | test: 28 | python -m pytest -rs ${PYTEST_EXTRA} 29 | 30 | testcov: 31 | coverage run -m pytest -rs ${PYTEST_EXTRA} 32 | coverage combine --quiet --append 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CI Tests](https://github.com/alpha-unito/jupyter-workflow/actions/workflows/ci-tests.yml/badge.svg?branch=master)](https://github.com/alpha-unito/jupyter-workflow/actions/workflows/ci-tests.yml) 2 | [![coverage](https://codecov.io/gh/alpha-unito/jupyter-workflow/branch/master/graph/badge.svg?token=2024K42B7O)](https://codecov.io/gh/alpha-unito/jupyter-workflow) 3 | 4 | # Jupyter Workflow 5 | 6 | The Jupyter Workflow framework enables Jupyter Notebooks to describe complex workflows and to execute them in a distributed fashion on hybrid HPC-Cloud infrastructures. Jupyter Workflow relies on the [StreamFlow](https://github.com/alpha-unito/streamflow) WMS as its underlying runtime support. 7 | 8 | ## Install Jupyter Workflow 9 | 10 | The Jupyter Workflow IPython kernel is available on [PyPI](https://pypi.org/project/jupyter-workflow/), so you can install it using pip. 11 | 12 | ```bash 13 | pip install jupyter-workflow 14 | ``` 15 | 16 | Then, you can install it on a Jupyter Notebook server by running the following command. 17 | 18 | ```bash 19 | python -m jupyter_workflow.ipython.install 20 | ``` 21 | 22 | Please note that Jupyter Workflow requires `python >= 3.9`. Then you can associate your Jupyter Notebooks with the newly installed kernel. Some examples can be found under the `examples` folder in the [GitHub repository](https://github.com/alpha-unito/jupyter-workflow). 23 | 24 | ## Jupyter Workflow Team 25 | 26 | Iacopo Colonnelli (creator and maintainer) 27 | Alberto Mulone (maintainer) 28 | Sergio Rabellino (maintainer) 29 | Barbara Cantalupo (maintainer) 30 | Marco Aldinucci (maintainer) 31 | -------------------------------------------------------------------------------- /bandit-requirements.txt: -------------------------------------------------------------------------------- 1 | bandit==1.8.3 -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: true 3 | coverage: 4 | status: 5 | project: off 6 | patch: off -------------------------------------------------------------------------------- /docs/.wci.yml: -------------------------------------------------------------------------------- 1 | name: Jupyter Workflow 2 | icon: https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/master/docs/logo.png 3 | headline: "Literate Distributed Computing" 4 | description: "Jupyter Workflow is an extension of the IPython kernel designed to support distributed literate workflows, developed and maintained by the Alpha research group at Università di Torino (UniTO). The Jupyter Workflow kernel enables Jupyter Notebooks to describe complex workflows and to execute them in a distributed fashion on hybrid cloud/HPC infrastructures." 5 | language: Python 6 | documentation: 7 | general: https://jupyter-workflow.di.unito.it/documentation/latest/ 8 | installation: https://jupyter-workflow.di.unito.it/documentation/latest/install.html 9 | tutorial: https://jupyter-workflow.di.unito.it/documentation/latest/operations.html 10 | execution_environment: 11 | interfaces: 12 | - Jupyter Notebook 13 | resource_managers: 14 | - Local 15 | - SSH 16 | - Kubernetes 17 | - Docker 18 | - Docker Compose 19 | - Singularity 20 | - SLURM 21 | - PBS 22 | transfer_protocols: 23 | - SCP 24 | - WebSocket (Kubernetes) 25 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==8.2.3 2 | sphinx-jsonschema==1.19.1 3 | sphinx-rtd-theme==3.0.2 -------------------------------------------------------------------------------- /docs/source/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | .wy-table-responsive table td, .wy-table-responsive table th { 2 | white-space: normal !important; 3 | } 4 | 5 | .wy-table-responsive { 6 | overflow: visible !important; 7 | } 8 | 9 | .jsonschema-table { 10 | border-left: none !important; 11 | border-right: none !important; 12 | } 13 | 14 | .jsonschema-table tr td { 15 | background-color: #fcfcfc !important; 16 | border-top: 1px solid #e1e4e5 !important; 17 | border-bottom: 1px solid #e1e4e5 !important; 18 | border-left: none !important; 19 | border-right: none !important; 20 | } 21 | 22 | .jsonschema-table tr:first-child td { 23 | text-align: center !important; 24 | text-transform: capitalize; 25 | font-weight: bold; 26 | } -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | import importlib 20 | 21 | import streamflow 22 | 23 | project = 'Jupyter Workflow' 24 | copyright = '2022, Alpha Research Group, Computer Science Dept., University of Torino' 25 | author = 'Iacopo Colonnelli' 26 | version = '0.1' 27 | release = '0.1.0' 28 | 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = [ 36 | 'sphinx.ext.autodoc', 37 | 'sphinx.ext.autosectionlabel', 38 | 'sphinx.ext.extlinks', 39 | 'sphinx-jsonschema', 40 | 'sphinx_rtd_theme' 41 | ] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ['_templates'] 45 | 46 | # List of patterns, relative to source directory, that match files and 47 | # directories to ignore when looking for source files. 48 | # This pattern also affects html_static_path and html_extra_path. 49 | exclude_patterns = [] 50 | 51 | 52 | # -- Options for HTML output ------------------------------------------------- 53 | 54 | # The theme to use for HTML and HTML Help pages. See the documentation for 55 | # a list of builtin themes. 56 | # 57 | html_theme = 'sphinx_rtd_theme' 58 | 59 | # Add any paths that contain custom static files (such as style sheets) here, 60 | # relative to this directory. They are copied after the builtin static files, 61 | # so a file named "default.css" will overwrite the builtin "default.css". 62 | html_static_path = ['_static'] 63 | 64 | 65 | def setup(app): 66 | app.add_css_file('theme_overrides.css') 67 | 68 | 69 | # Theme options are theme-specific and customize the look and feel of a theme 70 | # further. For a list of options available for each theme, see the 71 | # documentation. 72 | html_theme_options = { 73 | "logo_only": True 74 | } 75 | 76 | 77 | extlinks = { 78 | 'config-schema': ('https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/' + release + 79 | '/jupyter_workflow/config/schemas/v1.0/%s', 'GH#'), 80 | 'repo': ('https://github.com/alpha-unito/jupyter-workflow/tree/' + release + '/%s', 'GH#'), 81 | } 82 | 83 | # JSONSchema extensions 84 | sjs_wide_format = importlib.import_module("sphinx-jsonschema.wide_format") 85 | 86 | 87 | def _patched_simpletype(self, schema): 88 | rows = [] 89 | if 'title' in schema and (not self.options['lift_title'] or self.nesting > 1): 90 | rows.append(self._line(self._cell('*' + schema['title'] + '*'))) 91 | del schema['title'] 92 | self._check_description(schema, rows) 93 | if 'type' in schema: 94 | if '$ref' in schema: 95 | ref = self._reference(schema) 96 | rows.extend(self._prepend(self._cell('type'), ref)) 97 | del schema['type'] 98 | elif type(schema['type']) == list: 99 | cells = [self._line(self._decodetype(t)) for t in schema['type']] 100 | rows.extend(self._prepend(self._cell('type'), cells)) 101 | del schema['type'] 102 | rows.extend(_original_simpletype(self, schema)) 103 | return rows 104 | 105 | 106 | _original_simpletype = sjs_wide_format.WideFormat._simpletype 107 | sjs_wide_format.WideFormat._simpletype = _patched_simpletype 108 | 109 | 110 | def _patched_arraytype(self, schema): 111 | if 'items' in schema: 112 | if type(schema['items']) == list: 113 | return _original_arraytype(self, schema) 114 | else: 115 | schema['unique'] = 'uniqueItems' in schema['items'] 116 | if 'type' in schema['items']: 117 | schema['type'] = schema['items']['type'] + '[]' 118 | rows = self._simpletype(schema) 119 | return rows 120 | else: 121 | rows = _original_arraytype(self, schema) 122 | rows.extend(self._bool_or_object(schema, 'unique')) 123 | return rows 124 | 125 | 126 | _original_arraytype = sjs_wide_format.WideFormat._arraytype 127 | sjs_wide_format.WideFormat._arraytype = _patched_arraytype 128 | 129 | 130 | def _patched_objectproperties(self, schema, key): 131 | rows = [] 132 | if key in schema: 133 | rows.append(self._line(self._cell(key))) 134 | 135 | for prop in schema[key].keys(): 136 | # insert spaces around the regexp OR operator 137 | # allowing the regexp to be split over multiple lines. 138 | proplist = prop.split('|') 139 | dispprop = self._escape(' | '.join(proplist)) 140 | if 'required' in schema: 141 | if prop in schema['required']: 142 | dispprop = f'**{dispprop}**\n(required)' 143 | label = self._cell(dispprop) 144 | 145 | if isinstance(schema[key][prop], dict): 146 | obj = schema[key][prop] 147 | rows.extend(self._dispatch(obj, label)[0]) 148 | else: 149 | rows.append(self._line(label, self._cell(schema[key][prop]))) 150 | del schema[key] 151 | return rows 152 | 153 | 154 | _original_objectproperties = sjs_wide_format.WideFormat._objectproperties 155 | sjs_wide_format.WideFormat._objectproperties = _patched_objectproperties 156 | 157 | 158 | def _patched_complexstructures(self, schema): 159 | rows = [] 160 | if 'oneOf' in schema: 161 | types = [] 162 | for obj in schema['oneOf']: 163 | if 'type' in obj: 164 | if obj['type'] == 'object' and '$ref' in obj: 165 | types.extend(self._reference(obj)) 166 | else: 167 | types.append(self._line(self._decodetype(obj['type']))) 168 | del obj['type'] 169 | if not list(filter(bool, schema['oneOf'])): 170 | del schema['oneOf'] 171 | rows.extend(self._prepend(self._cell('type'), types)) 172 | rows.extend(_original_complexstructures(self, schema)) 173 | return rows 174 | 175 | 176 | _original_complexstructures = sjs_wide_format.WideFormat._complexstructures 177 | sjs_wide_format.WideFormat._complexstructures = _patched_complexstructures 178 | 179 | 180 | def patched_transform(self, schema): 181 | table, definitions = original_transform(self, schema) 182 | table['classes'] += ['jsonschema-table'] 183 | return table, definitions 184 | 185 | 186 | original_transform = sjs_wide_format.WideFormat.transform 187 | sjs_wide_format.WideFormat.transform = patched_transform 188 | 189 | 190 | def patched_run(self, schema, pointer=''): 191 | if 'id' in schema: 192 | del schema['id'] 193 | elif '$id' in schema: 194 | del schema['$id'] 195 | if 'type' in schema: 196 | del schema['type'] 197 | if 'additionalProperties' in schema: 198 | del schema['additionalProperties'] 199 | if 'required' in schema and 'properties' in schema: 200 | props = {} 201 | for prop in schema['required']: 202 | if prop in schema['properties']: 203 | props[prop] = schema['properties'][prop] 204 | del schema['properties'][prop] 205 | schema['properties'] = props | schema['properties'] 206 | return original_run(self, schema, pointer) 207 | 208 | 209 | original_run = sjs_wide_format.WideFormat.run 210 | sjs_wide_format.WideFormat.run = patched_run 211 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Jupyter Workflow 3 | ================ 4 | 5 | .. toctree:: 6 | :caption: Getting Started: 7 | :hidden: 8 | 9 | install.rst 10 | architecture.rst 11 | operations.rst 12 | 13 | .. toctree:: 14 | :caption: Workflow metadata 15 | :hidden: 16 | 17 | reference.rst 18 | .. toctree:: 19 | :caption: Connectors 20 | :hidden: 21 | 22 | connector/docker.rst 23 | connector/docker-compose.rst 24 | connector/helm3.rst 25 | connector/occam.rst 26 | connector/pbs.rst 27 | connector/singularity.rst 28 | connector/slurm.rst 29 | connector/ssh.rst 30 | 31 | 32 | 33 | Indices and tables 34 | ================== 35 | 36 | * :ref:`genindex` 37 | * :ref:`modindex` 38 | * :ref:`search` 39 | -------------------------------------------------------------------------------- /docs/source/reference.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Reference 3 | ========= 4 | 5 | 6 | .. jsonschema:: ../../jupyter_workflow/config/schemas/v1.0/config_schema.json 7 | :lift_description: true 8 | :lift_definitions: true 9 | :auto_reference: true 10 | :auto_target: true -------------------------------------------------------------------------------- /examples/1000-genome/README.md: -------------------------------------------------------------------------------- 1 | # Running the 1000-genome workflow interactively on Kubernetes 2 | 3 | To investigate *Jupyter-Workflow* strong scalability on a distribured infrastructure, we execute an 8-chromosomes instance of the [1000-genome workflow](https://github.com/pegasus-isi/1000genome-workflow) on up to 500 concurrent Kubernetes Pods. We selected the 1000-genome workflow for three main reasons: 4 | * Pegasus is a state-of-the-art representative of HPC-oriented WMSs supporting execution environments without shared data spaces (via HTCondor); 5 | * The host code of each step is written in either Bash or Python, both supported by the standard IPython kernel; 6 | * The critical portion of the workflow id a highly-parallel step, composed of 2000 independent short tasks (~120s each on our infrastructure), which are critical for batch workload managers, but that can be executed at scale on on-demand Cloud resources (e.g. Kubernetes) 7 | 8 | ## Preliminary steps 9 | 10 | In order to replicate the experiment, you need a Kubernetes cluster with 3 control plane VMs (4 cores, 8GB RAM each) and 16 large worker VMs (40 cores, 120GB RAM each). In our infrastructure, each Kubernetes worker node has been manually placed on top of a different physical node, managed by an OpenStack Cloud controller. Nodes were interconnected by a 10Gbps Ethernet. 11 | 12 | Each Pod requests 1 core and 2GB RAM and mounts a 1GB tmpfs under the `/tmp/streamflow` path, in order to avoid I/O bottlenecks. The description of such deployment is described in the `helm-1000-genome` model, which is managed by the StreamFlow Helm connector. 13 | 14 | The Dockerfile of the worker container can be found under the `docker/1000-genome/` folder. It can be recompiled locally (using the `build` script in the same folder) and published to a custom Docker registry. Alternatively, the original `alphaunito/1000-genome` container, published on Docker Hub, can be used. 15 | 16 | Initial inputs of the workflow should be downloaded (through the provided `work/data/download_data.sh` script) before lanching the pipeline steps. 17 | 18 | ## Run the notebook 19 | 20 | Also the Jupyter Nortebook driver has been placed inside the Kubernetes cluster, in order to avoid network bottlenecks caused by the external OpenStack loadbalancer in front of the kube-apiserver listeners. 21 | 22 | The Dockerfile of the Jupyter Notebook can be found under the `docker/1000-genome-notebook/` folder. As stated for the worker container, it can be either recompiled locally (using the `build` script in the same folder) and published to a custom Docker registry, or downloaded from Docker Hub at `alphaunito/1000-genome-notebook`. 23 | 24 | The Kubernetes deployment is described in the `k8s/manifest.yaml` file. It can be deployed by running the following command: 25 | ```bash 26 | kubectl apply -f k8s/manifest.yaml 27 | ``` 28 | Please note that the Jupyter Notebook Pod requires two persistent volumes, which are provided by the `cdk-cinder` StorageClass. To reproduce the experiment on your infrastructure, modify the name of the StorageClass accordingly. 29 | 30 | Documentation related to the single Notebook cells is reported directly in the `.ipynb` Notebook. Please be sure to select `Jupyter Workflow` as the Notebook kernel when running the example. -------------------------------------------------------------------------------- /examples/1000-genome/docker/1000-genome-notebook/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/minimal-notebook 2 | LABEL maintainer="Iacopo Colonnelli " 3 | 4 | # Install kernel 5 | RUN pip install --no-cache-dir \ 6 | bcrypt==3.2.0 \ 7 | dill==0.3.3 \ 8 | jupyter-workflow==0.0.37 \ 9 | matplotlib==3.4.2 \ 10 | numpy==1.20.3 \ 11 | && python -m jupyter_workflow.ipython.install 12 | 13 | USER root 14 | 15 | # Install required packages 16 | RUN apt-get update \ 17 | && apt-get install -y \ 18 | curl \ 19 | gawk \ 20 | gzip \ 21 | && apt-get clean \ 22 | && rm -rf /var/lib/apt/lists/* \ 23 | && wget https://git.io/get_helm.sh -O /tmp/get_helm.sh \ 24 | && chmod +x /tmp/get_helm.sh \ 25 | && /tmp/get_helm.sh --version v3.5.4 26 | 27 | USER "${NB_USER}" 28 | -------------------------------------------------------------------------------- /examples/1000-genome/docker/1000-genome-notebook/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker build \ 5 | -t alphaunito/1000-genome-notebook \ 6 | ${SCRIPT_DIRECTORY} 7 | -------------------------------------------------------------------------------- /examples/1000-genome/docker/1000-genome/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | LABEL maintainer="Iacopo Colonnelli " 3 | 4 | RUN apt-get update \ 5 | && apt-get install -y --no-install-recommends \ 6 | curl \ 7 | gawk \ 8 | gzip \ 9 | openssl \ 10 | procps \ 11 | && useradd -ms /bin/bash -u 1000 -g 100 jovyan \ 12 | && pip3 install --no-cache-dir \ 13 | bcrypt==3.2.0 \ 14 | dill==0.3.3 \ 15 | ipython==7.23.1 \ 16 | matplotlib==3.4.2 \ 17 | numpy==1.20.3 \ 18 | && curl -fsSL https://git.io/get_helm.sh -o /tmp/get_helm.sh \ 19 | && chmod +x /tmp/get_helm.sh \ 20 | && /tmp/get_helm.sh --version v3.5.4 21 | 22 | USER jovyan 23 | -------------------------------------------------------------------------------- /examples/1000-genome/docker/1000-genome/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker build \ 5 | -t alphaunito/1000-genome \ 6 | ${SCRIPT_DIRECTORY} 7 | -------------------------------------------------------------------------------- /examples/1000-genome/k8s/manifest.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: 1000-genome 5 | 6 | --- 7 | 8 | kind: PersistentVolumeClaim 9 | apiVersion: v1 10 | metadata: 11 | name: 1000-genome-work 12 | namespace: 1000-genome 13 | spec: 14 | accessModes: 15 | - ReadWriteOnce 16 | resources: 17 | requests: 18 | storage: 100Gi 19 | storageClassName: cdk-cinder 20 | 21 | --- 22 | 23 | kind: PersistentVolumeClaim 24 | apiVersion: v1 25 | metadata: 26 | name: 1000-genome-streamflow-tmp 27 | namespace: 1000-genome 28 | spec: 29 | accessModes: 30 | - ReadWriteOnce 31 | resources: 32 | requests: 33 | storage: 100Gi 34 | storageClassName: cdk-cinder 35 | 36 | --- 37 | 38 | kind: Deployment 39 | apiVersion: apps/v1 40 | metadata: 41 | name: 1000-genome-notebook 42 | namespace: 1000-genome 43 | labels: 44 | app: 1000-genome-notebook 45 | spec: 46 | replicas: 1 47 | selector: 48 | matchLabels: 49 | app: 1000-genome-notebook 50 | template: 51 | metadata: 52 | labels: 53 | app: 1000-genome-notebook 54 | spec: 55 | securityContext: 56 | runAsUser: 1000 57 | runAsGroup: 1000 58 | fsGroup: 1000 59 | serviceAccountName: 1000-genome-service-account 60 | containers: 61 | - name: 1000-genome-notebook 62 | image: alphaunito/1000-genome-notebook 63 | imagePullPolicy: Always 64 | ports: 65 | - name: jupyter 66 | containerPort: 8888 67 | stdin: true 68 | volumeMounts: 69 | - mountPath: /home/jovyan/work 70 | name: 1000-genome-work 71 | - mountPath: /tmp/streamflow 72 | name: 1000-genome-streamflow-tmp 73 | volumes: 74 | - name: 1000-genome-work 75 | persistentVolumeClaim: 76 | claimName: 1000-genome-work 77 | - name: 1000-genome-streamflow-tmp 78 | persistentVolumeClaim: 79 | claimName: 1000-genome-streamflow-tmp 80 | 81 | --- 82 | 83 | kind: Service 84 | apiVersion: v1 85 | metadata: 86 | name: svc-1000-genome-notebook 87 | namespace: 1000-genome 88 | spec: 89 | selector: 90 | app: 1000-genome-notebook 91 | ports: 92 | - protocol: TCP 93 | name: jupyter 94 | port: 8888 95 | targetPort: jupyter 96 | 97 | --- 98 | 99 | apiVersion: v1 100 | kind: ServiceAccount 101 | metadata: 102 | namespace: 1000-genome 103 | name: 1000-genome-service-account 104 | 105 | --- 106 | 107 | kind: RoleBinding 108 | apiVersion: rbac.authorization.k8s.io/v1 109 | metadata: 110 | name: 1000-genome-admin 111 | namespace: 1000-genome 112 | subjects: 113 | - kind: ServiceAccount 114 | name: 1000-genome-service-account 115 | apiGroup: "" 116 | roleRef: 117 | kind: ClusterRole 118 | name: admin 119 | apiGroup: "" 120 | -------------------------------------------------------------------------------- /examples/1000-genome/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker run \ 5 | -it \ 6 | --rm \ 7 | -p 8888:8888 \ 8 | -v ${SCRIPT_DIRECTORY}/work:/home/jovyan/work \ 9 | -v ${HOME}/.ssh:/home/jovyan/.ssh \ 10 | -v ${HOME}/.kube:/home/jovyan/.kube \ 11 | alphaunito/1000-genome-notebook 12 | -------------------------------------------------------------------------------- /examples/1000-genome/times/bar_chart.plt: -------------------------------------------------------------------------------- 1 | # Print to EPS 2 | set terminal eps font "libertine,14" 3 | 4 | # Set borders 5 | set boxwidth 0.75 absolute 6 | set border 3 front lt black linewidth 1.000 dashtype solid 7 | set style fill solid noborder 8 | 9 | # Set grid 10 | set grid nopolar 11 | set grid noxtics nomxtics ytics nomytics noztics nomztics nortics nomrtics nox2tics nomx2tics noy2tics nomy2tics nocbtics nomcbtics 12 | set grid layerdefault lt 0 linecolor 0 linewidth 0.500, lt 0 linecolor 0 linewidth 0.500 13 | 14 | # Set legend 15 | set key noinvert box 16 | set key left top vertical Left reverse noenhanced autotitle columnhead box lt black linewidth 1.000 dashtype solid 17 | set key spacing 1.5 18 | 19 | # Set X axis 20 | set xtics border in scale 0,0 mirror norotate autojustify 21 | set xrange [0:500] 22 | set xlabel "Cores" 23 | 24 | # Set Y axis 25 | set ytics border in scale 0,0 mirror norotate autojustify 26 | set yrange [0:*] 27 | set ylabel "Strong scalability" 28 | 29 | # Set line 30 | set style line 1 linecolor rgb '#008FD0' linetype 1 linewidth 2 pointtype 5 pointsize 1.2 31 | set style line 2 linecolor rgb '#FFC97D' linetype 1 linewidth 2 pointtype 5 pointsize 1.2 32 | 33 | # Plot data 34 | plot 'speedup.dat' using 1:2 with linespoints linestyle 1, '' using 1:3 with linespoints linestyle 2, x with linespoint lt 2 dt 2 pt 0 lc rgb '#828788' 35 | -------------------------------------------------------------------------------- /examples/1000-genome/times/plot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gnuplot -p bar_chart.plt > 1000-genome-speedup.eps 4 | -------------------------------------------------------------------------------- /examples/1000-genome/times/speedup.dat: -------------------------------------------------------------------------------- 1 | Cores Execution DryRun 2 | 50 49 50 3 | 100 94 98 4 | 125 114 121 5 | 200 180 192 6 | 250 217 228 7 | 400 251 343 8 | 500 267 425 9 | -------------------------------------------------------------------------------- /examples/1000-genome/work/data/download_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 4 | 5 | mkdir -p ${SCRIPT_DIRECTORY}/20130502/sifting ${SCRIPT_DIRECTORY}/populations 6 | 7 | wget -O ${SCRIPT_DIRECTORY}/20130502/columns.txt \ 8 | https://raw.githubusercontent.com/pegasus-isi/1000genome-workflow/master/data/20130502/columns.txt 9 | 10 | for i in {1..8}; do 11 | wget -O ${SCRIPT_DIRECTORY}/20130502/ALL.chr${i}.250000.vcf.gz \ 12 | https://github.com/pegasus-isi/1000genome-workflow/raw/master/data/20130502/ALL.chr${i}.250000.vcf.gz 13 | wget -O ${SCRIPT_DIRECTORY}/20130502/sifting/ALL.chr${i}.phase3_shapeit2_mvncall_integrated_v5.20130502.sites.annotation.vcf.gz \ 14 | ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/supporting/functional_annotation/filtered/ALL.chr${i}.phase3_shapeit2_mvncall_integrated_v5.20130502.sites.annotation.vcf.gz 15 | done 16 | 17 | populations=('AFR' 'ALL' 'AMR' 'EAS' 'EUR' 'GBR' 'SAS') 18 | for p in ${populations[*]}; do 19 | wget -O ${SCRIPT_DIRECTORY}/populations/${p} \ 20 | https://raw.githubusercontent.com/pegasus-isi/1000genome-workflow/master/data/populations/${p} 21 | done 22 | -------------------------------------------------------------------------------- /examples/1000-genome/work/environment/k8s/1000-genome/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /examples/1000-genome/work/environment/k8s/1000-genome/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: 1000-genome 3 | description: A Helm chart for the LodSeq workflow 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.2.0" 25 | -------------------------------------------------------------------------------- /examples/1000-genome/work/environment/k8s/1000-genome/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "1000-genome.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 3 | export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[1].containerPort}") 4 | echo "Visit http://127.0.0.1:8080 to use your application" 5 | kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT 6 | -------------------------------------------------------------------------------- /examples/1000-genome/work/environment/k8s/1000-genome/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "1000-genome.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "1000-genome.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "1000-genome.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "1000-genome.labels" -}} 37 | helm.sh/chart: {{ include "1000-genome.chart" . }} 38 | {{ include "1000-genome.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "1000-genome.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "1000-genome.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "1000-genome.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "1000-genome.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /examples/1000-genome/work/environment/k8s/1000-genome/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "1000-genome.fullname" . }} 5 | labels: 6 | {{- include "1000-genome.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | {{- include "1000-genome.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | {{- with .Values.podAnnotations }} 15 | annotations: 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | labels: 19 | {{- include "1000-genome.selectorLabels" . | nindent 8 }} 20 | spec: 21 | {{- with .Values.imagePullSecrets }} 22 | imagePullSecrets: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | securityContext: 26 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 27 | containers: 28 | - name: {{ .Chart.Name }} 29 | stdin: true 30 | securityContext: 31 | {{- toYaml .Values.securityContext | nindent 12 }} 32 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 33 | imagePullPolicy: {{ .Values.image.pullPolicy }} 34 | {{- if .Values.resources }} 35 | resources: 36 | {{- toYaml .Values.resources | nindent 12 }} 37 | {{- end }} 38 | volumeMounts: 39 | - name: {{ include "1000-genome.fullname" . }} 40 | mountPath: /tmp/streamflow 41 | {{- with .Values.nodeSelector }} 42 | nodeSelector: 43 | {{- toYaml . | nindent 8 }} 44 | {{- end }} 45 | {{- with .Values.affinity }} 46 | affinity: 47 | {{- toYaml . | nindent 8 }} 48 | {{- end }} 49 | {{- with .Values.tolerations }} 50 | tolerations: 51 | {{- toYaml . | nindent 8 }} 52 | {{- end }} 53 | volumes: 54 | - name: {{ include "1000-genome.fullname" . }} 55 | emptyDir: 56 | medium: Memory 57 | sizeLimit: 1Gi -------------------------------------------------------------------------------- /examples/1000-genome/work/environment/k8s/1000-genome/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for 1000-genome. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 500 6 | 7 | image: 8 | pullPolicy: Always 9 | repository: alphaunito/1000-genome 10 | tag: latest 11 | 12 | imagePullSecrets: [] 13 | nameOverride: "" 14 | fullnameOverride: "" 15 | 16 | podSecurityContext: 17 | runAsUser: 1000 18 | runAsGroup: 100 19 | fsGroup: 100 20 | 21 | securityContext: {} 22 | # capabilities: 23 | # drop: 24 | # - ALL 25 | # readOnlyRootFilesystem: true 26 | # runAsNonRoot: true 27 | # runAsUser: 1000 28 | 29 | resources: 30 | requests: 31 | cpu: 1 32 | memory: 2Gi 33 | 34 | nodeSelector: {} 35 | 36 | tolerations: [] 37 | 38 | affinity: {} 39 | -------------------------------------------------------------------------------- /examples/claire-covid/README.md: -------------------------------------------------------------------------------- 1 | # CLAIRE-COVID19 universal pipeline 2 | 3 | The CLAIRE-COVID19 universal pipeline has been designed to compare different training algorithms for the AI-assisted diagnosis of COVID-19, in order to define a baseline for such techniques and to allow the community to quantitatively measure AI’s progress in this field. For more information, please read [this post](https://streamflow.di.unito.it/2021/04/12/ai-assisted-covid-19-diagnosis-with-the-claire-universal-pipeline/). 4 | 5 | This notebook showcases how the classification-related portion of the pipeline can be successfully described and documented in Jupyter, using the *Jupyter-workflow* to execute it at scale on an HPC facility. 6 | 7 | The first preprocessing portion of the pipeline is left outside the notebook. It could clearly (and effectively) be included, but since this example wants to serve as an introductory demonstration, we wanted to keep it as simple as possible. 8 | 9 | ## Preliminary steps 10 | 11 | In order to successfully run this notebook, you first need to produce a pre-processed and filtered version of the [BIMCV-COVID19+](https://bimcv.cipf.es/bimcv-projects/bimcv-covid19/) dataset (Iteration 1). Instructions on how to do this can be found [here](https://github.com/CLAIRE-COVID/AI-Covid19-preprocessing) and [here](https://github.com/CLAIRE-COVID/AI-Covid19-pipelines). 12 | 13 | This procedure will produce three different versions of the dataset. For this experiment, we need the `final3_BB` folder, which can be either manually pre-transferred on the remote HPC facility (as we did) or left on the local machine, letting *Jupyter-workflow* automatically perform the data transfer when needed. 14 | 15 | Next, if you plan to run on an air-gapped architecture (as HPC facilities normally are), you will need to manually download all the pre-trained neural network models and place them into a `weights` folder, in a portion of file-system shared among all the worker nodes in the data centre. This can be done by running [this script](https://raw.githubusercontent.com/CLAIRE-COVID/AI-Covid19-benchmarking/master/nnframework/init_models.py). 16 | 17 | Finally, you need to transfer dependencies (i.e., PyTorch and the CLAIRE-COVID19 benchmarking [code](https://github.com/CLAIRE-COVID/AI-Covid19-benchmarking)) on the remote facility. To enhance portability, we used a Singularity container with everything inside, but you have to manually transfer it on the remote HPC facility, in a shared portion of the file-system, and to change the `environment/cineca-marconi100/slurm_template.jinja2` file accordingly. We plan to make this last transfer automatically managed by *Jupyter-workflow* in the very next releases. 18 | 19 | If you plan to run on an x86_64 architecture, creating the Singularity is as easy as trandforming the Docker image for this experiment in a Singularity image. Neverhteless, the [MARCONI 100](https://www.hpc.cineca.it/hardware/marconi100) HPC facility comes with an IBM POWER9 architecture. Therefore, we built a Singularity container on a `ppc64le` architecture using the `build` script in the `singularity` folder of this repository. 20 | 21 | ## Run the notebook 22 | 23 | In order to run the Notebook locally, you can use the `run` script in this folder. It automatically pulls the related container from [DockerHub](https://hub.docker.com/r/alphaunito/claire-covid-notebook). Conversely, if you want to produce your own local version of the container, you can run the `build` script in the `docker` folder of this repo prior to launch the `run` script. 24 | 25 | Documentation related to the single Notebook cells is reported directly in the Notebook. Please be sure to select `Jupyter Workflow` as the Notebook kernel when running the example. -------------------------------------------------------------------------------- /examples/claire-covid/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/scipy-notebook 2 | LABEL maintainer="Iacopo Colonnelli " 3 | 4 | RUN conda install --quiet --yes -c pytorch -c conda-forge \ 5 | dill==0.3.3 \ 6 | pytorch=1.8.1 \ 7 | torchvision=0.9.1 \ 8 | cudatoolkit=11.1 \ 9 | && conda clean --all -f -y 10 | 11 | RUN pip install --no-cache-dir \ 12 | bcrypt==3.2.0 \ 13 | jupyter-workflow==0.0.19 \ 14 | kiwisolver==1.3.0 \ 15 | matplotlib==3.3.2 \ 16 | nibabel==3.2.0 \ 17 | pandas==1.1.3 \ 18 | pydicom==2.1.1 \ 19 | pyprg==0.1.1b7 \ 20 | pyyaml==5.3.1 \ 21 | scikit-learn==0.23.2 \ 22 | scipy==1.5.2 \ 23 | tensorboard==2.3.0 \ 24 | typing==3.7.4.3 \ 25 | && python -m jupyter_workflow.ipython.install 26 | 27 | USER root 28 | 29 | RUN git clone \ 30 | https://github.com/CLAIRE-COVID/AI-Covid19-benchmarking.git \ 31 | /opt/claire-covid \ 32 | && mkdir -p ${HOME}/claire-covid/ \ 33 | && cp -r /opt/claire-covid/interpretability ${HOME}/claire-covid/ \ 34 | && cp -r /opt/claire-covid/nnframework ${HOME}/claire-covid/ \ 35 | && cp -r /opt/claire-covid/metrics ${HOME}/claire-covid/ \ 36 | && rm -rf /opt/claire-covid 37 | 38 | ENV PYTHONPATH="${PYTHONPATH}:${HOME}/claire-covid" 39 | 40 | RUN apt-get update \ 41 | && apt-get install -y openssh-client \ 42 | && apt-get clean \ 43 | && rm -rf /var/lib/apt/lists/* \ 44 | && fix-permissions ${HOME}/claire-covid 45 | 46 | USER ${NB_USER} 47 | -------------------------------------------------------------------------------- /examples/claire-covid/docker/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | # Rebuild Jupyter stack with CUDA support 5 | git clone https://github.com/jupyter/docker-stacks ${SCRIPT_DIRECTORY}/docker-stacks 6 | docker build \ 7 | --build-arg=ROOT_CONTAINER=nvcr.io/nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu20.04 \ 8 | -t jupyter/base-notebook \ 9 | ${SCRIPT_DIRECTORY}/docker-stacks/base-notebook 10 | docker build \ 11 | -t jupyter/minimal-notebook \ 12 | ${SCRIPT_DIRECTORY}/docker-stacks/minimal-notebook 13 | docker build \ 14 | -t jupyter/scipy-notebook \ 15 | ${SCRIPT_DIRECTORY}/docker-stacks/scipy-notebook 16 | rm -rf ${SCRIPT_DIRECTORY}/docker-stacks 17 | 18 | # Build Claire-Covid notebook 19 | docker build \ 20 | -t alphaunito/claire-covid-notebook \ 21 | ${SCRIPT_DIRECTORY} 22 | -------------------------------------------------------------------------------- /examples/claire-covid/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker run \ 5 | -it \ 6 | --rm \ 7 | -p 8888:8888 \ 8 | -v ${SCRIPT_DIRECTORY}/work:/home/jovyan/work \ 9 | -v ${HOME}/.ssh:/home/jovyan/.ssh \ 10 | alphaunito/claire-covid-notebook 11 | -------------------------------------------------------------------------------- /examples/claire-covid/singularity/ppcle64/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | # Build PyTorch image 5 | if [ ! -f "${SCRIPT_DIRECTORY}/pytorch.sif" ]; then 6 | sudo singularity build ${SCRIPT_DIRECTORY}/pytorch.sif ${SCRIPT_DIRECTORY}/pytorch.def 7 | fi 8 | 9 | # Build CLAIRE-COVID19 image 10 | sudo singularity build ${SCRIPT_DIRECTORY}/claire-covid.sif ${SCRIPT_DIRECTORY}/claire-covid.def 11 | -------------------------------------------------------------------------------- /examples/claire-covid/singularity/ppcle64/claire-covid.def: -------------------------------------------------------------------------------- 1 | Bootstrap: localimage 2 | From: pytorch.sif 3 | Stage: github 4 | 5 | %post 6 | apt-get update 7 | apt-get install -y git 8 | git clone https://github.com/CLAIRE-COVID/AI-Covid19-benchmarking.git /opt/claire-covid 9 | 10 | Bootstrap: localimage 11 | From: pytorch.sif 12 | Stage: final 13 | 14 | %post 15 | pip install --no-cache-dir dill 16 | 17 | %files from github 18 | /opt/claire-covid/interpretability/ /opt/claire-covid/interpretability/ 19 | /opt/claire-covid/nnframework/ /opt/claire-covid/nnframework/ 20 | /opt/claire-covid/metrics/ /opt/claire-covid/metrics/ 21 | 22 | %environment 23 | export PYTHONPATH="${PYTHONPATH}:/opt/claire-covid" 24 | 25 | %labels 26 | org.label-schema.name "CLAIRE COVID19 DNN Benchmark" 27 | org.label-schema.vcs-url "https://github.com/CLAIRE-COVID/AI-Covid19-benchmarking" 28 | 29 | %help 30 | Container for CLAIRE benchmarking of Deep Neural Network models for COVID19 diagnosis (https://github.com/CLAIRE-COVID/AI-Covid19-benchmarking) 31 | -------------------------------------------------------------------------------- /examples/claire-covid/singularity/ppcle64/pytorch.def: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: nvcr.io/nvidia/cuda-ppc64le:11.1.1-cudnn8-devel-ubuntu18.04 3 | Stage: build 4 | 5 | %environment 6 | export LD_LIBRARY_PATH="/usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}" 7 | export PATH="/opt/conda/bin:${PATH}" 8 | 9 | %post 10 | # Download apt packages 11 | export DEBIAN_FRONTEND=noninteractive 12 | apt-get update 13 | apt-get install -y --no-install-recommends \ 14 | build-essential \ 15 | cmake \ 16 | curl \ 17 | ca-certificates \ 18 | gfortran \ 19 | git \ 20 | libgomp1 \ 21 | libjpeg-dev \ 22 | libnuma-dev \ 23 | libopenblas-dev \ 24 | libopenmpi-dev \ 25 | libpng-dev 26 | apt-get clean 27 | rm -rf /var/lib/apt/lists/* 28 | 29 | # Download miniconda 30 | curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-ppc64le.sh 31 | chmod +x ~/miniconda.sh 32 | ~/miniconda.sh -b -p /opt/conda 33 | rm ~/miniconda.sh 34 | 35 | # Download conda dependencies 36 | /opt/conda/bin/conda install -y python=3.8 \ 37 | cython==0.29.21 \ 38 | grpcio==1.31.0 \ 39 | kiwisolver==1.3.0 \ 40 | matplotlib==3.3.2 \ 41 | ninja==1.10.1 \ 42 | numpy==1.19.2 \ 43 | pandas==1.1.3 \ 44 | pyyaml==5.3.1 \ 45 | scikit-learn==0.23.2 \ 46 | scipy==1.5.2 \ 47 | typing==3.7.4.3 48 | /opt/conda/bin/pip install --no-cache-dir \ 49 | nibabel==3.2.0 \ 50 | pydicom==2.1.1 \ 51 | pyprg==0.1.1b7 \ 52 | tensorboard==2.3.0 \ 53 | typing-extensions==3.7.4.3 54 | 55 | # Build Magma 56 | curl -fsSL -v -o /opt/magma-2.5.4.tar.gz -O http://icl.utk.edu/projectsfiles/magma/downloads/magma-2.5.4.tar.gz 57 | tar -xzf /opt/magma-2.5.4.tar.gz -C /opt 58 | rm -f /opt/magma-2.5.4.tar.gz 59 | cp /opt/magma-2.5.4/make.inc-examples/make.inc.openblas /opt/magma-2.5.4/make.inc 60 | cd /opt/magma-2.5.4 61 | export CUDADIR="/usr/local/cuda" 62 | export GPU_TARGET="sm_35 sm_52 sm_60 sm_61 sm_70 sm_75" 63 | export OPENBLASDIR="/usr/lib/powerpc64le-linux-gnu" 64 | make -j $(nproc) 65 | make install 66 | 67 | # Build PyTorch 68 | git clone --depth 1 --branch v1.8.1 https://github.com/pytorch/pytorch /opt/pytorch 69 | cd /opt/pytorch 70 | git submodule sync 71 | git submodule update --init --recursive 72 | curl -L \ 73 | --output /opt/pytorch/aten/src/ATen/cpu/vec256/vsx/vec256_complex_float_vsx.h \ 74 | https://raw.githubusercontent.com/pytorch/pytorch/1c64f862f6e03ce09152d45289dd37577c277aaf/aten/src/ATen/cpu/vec256/vsx/vec256_complex_float_vsx.h 75 | export USE_MKLDNN=0 76 | export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} 77 | export TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" 78 | export TORCH_NVCC_FLAGS="-Xfatbin -compress-all" 79 | /opt/conda/bin/python setup.py install 80 | 81 | # Build TorchVision 82 | git clone --depth 1 --branch v0.9.1 https://github.com/pytorch/vision /opt/torchvision 83 | cd /opt/torchvision 84 | /opt/conda/bin/python setup.py install 85 | 86 | 87 | Bootstrap: docker 88 | From: nvcr.io/nvidia/cuda-ppc64le:11.1.1-cudnn8-runtime-ubuntu18.04 89 | Stage: final 90 | 91 | %files from build 92 | /opt/conda /opt/conda 93 | /usr/local/magma/lib /usr/local/magma/lib 94 | 95 | %environment 96 | export LD_LIBRARY_PATH="/usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}" 97 | export NVIDIA_DRIVER_CAPABILITIES="compute,utility" 98 | export NVIDIA_VISIBLE_DEVICES="all" 99 | export PATH="/opt/conda/bin:${PATH}" 100 | 101 | %post 102 | # Download apt packages 103 | apt-get update 104 | apt-get install -y --no-install-recommends \ 105 | ca-certificates \ 106 | libgomp1 \ 107 | libjpeg8 \ 108 | libnuma1 \ 109 | libopenblas-base \ 110 | openmpi-bin \ 111 | libpng16-16 112 | apt-get clean 113 | rm -rf /var/lib/apt/lists/* 114 | 115 | -------------------------------------------------------------------------------- /examples/claire-covid/times/bar_chart.plt: -------------------------------------------------------------------------------- 1 | # Print to EPS 2 | set terminal eps font "libertine,14" 3 | 4 | # Set borders 5 | set boxwidth 0.75 absolute 6 | set border 3 front lt black linewidth 1.000 dashtype solid 7 | set style fill solid noborder 8 | 9 | # Set grid 10 | set grid nopolar 11 | set grid noxtics nomxtics ytics nomytics noztics nomztics nortics nomrtics nox2tics nomx2tics noy2tics nomy2tics nocbtics nomcbtics 12 | set grid layerdefault lt 0 linecolor 0 linewidth 0.500, lt 0 linecolor 0 linewidth 0.500 13 | 14 | # Set histogram chart 15 | set style histogram rowstacked title textcolor lt -1 16 | set datafile missing '-' 17 | set style data histograms 18 | 19 | # Set legend 20 | set key noinvert box 21 | set key right top vertical Left reverse noenhanced autotitle columnhead box lt black linewidth 1.000 dashtype solid 22 | set key spacing 1.5 23 | 24 | # Set X axis 25 | unset xtics 26 | set xlabel "Slurm jobs" 27 | 28 | # Set Y axis 29 | set ytics border in scale 0,0 mirror norotate autojustify 30 | set yrange [0:*] 31 | set ylabel "Time (min)" 32 | 33 | # Plot data 34 | plot 'times.dat' using 2 ti col lc rgb '#FFC97D', '' using 3 ti col lc rgb '#008FD0' 35 | -------------------------------------------------------------------------------- /examples/claire-covid/times/plot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gnuplot -p bar_chart.plt > claire-covid-execution-times.eps 4 | -------------------------------------------------------------------------------- /examples/claire-covid/times/times.dat: -------------------------------------------------------------------------------- 1 | JobID Pending Running 2 | 2806226 2 63 3 | 2806227 2 45 4 | 2806228 2 60 5 | 2806229 2 51 6 | 2806230 2 45 7 | 2806231 2 47 8 | 2806232 2 50 9 | 2806233 2 52 10 | 2806234 2 49 11 | 2806235 2 52 12 | 13 | 2806236 2 42 14 | 2806237 2 39 15 | 2806238 2 43 16 | 2806239 2 41 17 | 2806240 2 38 18 | 2806241 2 46 19 | 2806242 2 43 20 | 2806243 2 45 21 | 2806244 2 40 22 | 2806245 2 79 23 | 24 | 2806246 4 45 25 | 2806247 4 43 26 | 2806248 4 45 27 | 2806249 4 43 28 | 2806250 4 70 29 | 2806251 4 50 30 | 2806252 4 42 31 | 2806253 4 58 32 | 2806254 4 72 33 | 2806255 4 42 34 | 35 | 2806256 4 42 36 | 2806257 4 78 37 | 2806258 4 38 38 | 2806259 4 38 39 | 2806260 4 41 40 | 2806261 4 45 41 | 2806262 4 41 42 | 2806263 4 38 43 | 2806264 4 39 44 | 2806265 4 39 45 | 46 | 2806266 6 39 47 | 2806267 6 41 48 | 2806268 6 38 49 | 2806269 6 39 50 | 2806270 6 42 51 | 2806271 6 43 52 | 2806273 6 38 53 | 2806274 6 38 54 | 2806275 6 42 55 | 2806276 6 43 56 | 57 | 2806277 6 42 58 | 2806278 6 43 59 | 2806279 6 43 60 | 2806280 6 38 61 | 2806281 15 41 62 | 2806282 15 42 63 | 2806283 20 46 64 | 2806284 20 43 65 | 2806285 22 47 66 | 2806286 24 41 -------------------------------------------------------------------------------- /examples/claire-covid/work/environment/cineca-marconi100/slurm_template.jinja2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #SBATCH --job-name="claire-covid" 4 | #SBATCH --partition=m100_usr_prod 5 | #SBATCH --account="account-name" 6 | #SBATCH --time=24:00:00 7 | #SBATCH -D {{ streamflow_workdir }} 8 | #SBATCH --cpus-per-task=8 9 | #SBATCH --ntasks=1 10 | #SBATCH --ntasks-per-node=1 11 | #SBATCH --gres=gpu:1 12 | 13 | # Load modules 14 | module load singularity 15 | 16 | singularity exec --nv /path/to/claire-covid.sif {{ streamflow_command }} 17 | -------------------------------------------------------------------------------- /examples/quantum-espresso/README.md: -------------------------------------------------------------------------------- 1 | # Interactive simulation on Quantum ESPRESSO 2 | 3 | In order to assess the *Jupyter-workflow* capabilities to enable interactive simulations of realistic, large-scale systems, we implement two Notebooks describing multi-step simulation workflows in Quantum ESPRESSO. 4 | 5 | In particular, the analyzed workflows implement: 6 | 7 | * A Car-Parrinello simulation of 32 Water Molecules, with the aim to sample the water at different temperatures using a Nose-hover thermostat together with a reference microcanonical trajectory. 8 | * A Car-Parrinello simulation of a mixture of water, ammonia and methane molecules to represent the basic ingredients of life (the so-called primordial soup). 9 | 10 | Both workflows are composed of six steps, which were executed on top of the PBS-managed *davinci-1* facility, the HPC centre of the Leonard S.p.A. company. The first, simpler pipeline was included in an early version of the manuscript. The second workflow, included in the last version of the article, contains a nested scatter pattern which has been scaled up to 32 nodes. 11 | 12 | What follows is valid for both workflows. Specific instructions can be found in each `.ipynb` file, located in the `work//` folder. 13 | 14 | ## Preliminary steps 15 | 16 | This time we did not use a Singularity container, but ran the steps directly on bare metal to fully exploit compile-time optimisations of a Quantum ESPRESSO executable compiled with Intel OneAPI. Nevertheless, we packed both the executable (named `cp-oneapi.x`) and all the input files inside the `INPDIR` directory on this repo. 17 | 18 | In order to replicate the experiment in a fully-optimised execution environment, you need to compile the Quantum ESPRESSO `cp` executable directly on your facility. The [node-details.txt file](https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/master/examples/quantum-espresso/node-details.txt) provides some information on the libraries used to compile Quantum ESPRESSO. 19 | 20 | ## Run the notebook 21 | 22 | In order to run the Notebook locally, you can use the `run` script in this folder. It automatically pulls the related container from [DockerHub](https://hub.docker.com/r/alphaunito/quantum-espresso-notebook). Conversely, if you want to produce your own local version of the container, you can run the `build` script in the `docker` folder of this repo prior to launch the `run` script. 23 | 24 | Documentation related to the single Notebook cells is reported directly in the `.ipynb` Notebook. Please be sure to select `Jupyter Workflow` as the Notebook kernel when running the example. 25 | -------------------------------------------------------------------------------- /examples/quantum-espresso/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/minimal-notebook AS qe_builder 2 | LABEL maintainer="Iacopo Colonnelli " 3 | 4 | ARG QE_VERSION="6.6" 5 | 6 | ENV DEBIAN_FRONTEND="noninteractive" 7 | ENV LC_ALL="C" 8 | 9 | USER root 10 | 11 | # Install build dependencies 12 | RUN apt-get update \ 13 | && apt-get install -y \ 14 | build-essential \ 15 | libopenblas-openmp-dev \ 16 | libfftw3-dev \ 17 | libhdf5-dev \ 18 | libmpich-dev \ 19 | wget 20 | 21 | # Compile QuantumEspresso 22 | RUN wget https://github.com/QEF/q-e/archive/qe-${QE_VERSION}.tar.gz \ 23 | && tar -xzf qe-${QE_VERSION}.tar.gz \ 24 | && rm -f qe-${QE_VERSION}.tar.gz \ 25 | && cd q-e-qe-${QE_VERSION} \ 26 | && ./configure --with-default-prefix="/usr/local" \ 27 | && make pw cp \ 28 | && make install 29 | 30 | 31 | FROM jupyter/minimal-notebook 32 | 33 | ENV DEBIAN_FRONTEND="noninteractive" 34 | 35 | COPY --from=qe_builder "/usr/local/bin" "/usr/local/bin" 36 | 37 | USER root 38 | 39 | RUN apt-get update \ 40 | && apt-get install -y \ 41 | libopenblas0-openmp \ 42 | libfftw3-bin \ 43 | libhdf5-103 \ 44 | libgfortran5 \ 45 | mpich \ 46 | openssh-client \ 47 | wget \ 48 | && apt-get clean \ 49 | && rm -rf /var/lib/apt/lists/* 50 | 51 | RUN pip install --no-cache-dir \ 52 | bcrypt==3.2.0 \ 53 | dill==0.3.3 \ 54 | jupyter-workflow==0.0.37 \ 55 | && python -m jupyter_workflow.ipython.install \ 56 | && fix-permissions /usr/local/bin \ 57 | && fix-permissions "${CONDA_DIR}" 58 | 59 | USER ${NB_USER} 60 | -------------------------------------------------------------------------------- /examples/quantum-espresso/docker/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker build \ 5 | -t alphaunito/quantum-espresso-notebook \ 6 | ${SCRIPT_DIRECTORY} 7 | -------------------------------------------------------------------------------- /examples/quantum-espresso/node-details.txt: -------------------------------------------------------------------------------- 1 | The davinci-1 HPC facility is a private infrastructure, so not all the details can be disclosed for security reasons. 2 | The relevant details about compilation and linking of the QuantumEspresso CP executable, together with the MPI version 3 | used to perform the parallel run and some information on the hardware equipment, are reported below: 4 | 5 | Version: CP v.6.7GPU 6 | 7 | Compilers: 8 | - ifort version 2021.1 9 | - icc version 2021.1 10 | 11 | Linked libraries (ldd output): 12 | - linux-vdso.so.1 (0x0000155555551000) 13 | - libmkl_scalapack_lp64.so.1 => /archive/apps/INTEL/OneAPI/mkl/2021.1.1/env/../lib/intel64/libmkl_scalapack_lp64.so.1 (0x0000155554a00000) 14 | - libmkl_blacs_intelmpi_lp64.so.1 => /archive/apps/INTEL/OneAPI/mkl/2021.1.1/env/../lib/intel64/libmkl_blacs_intelmpi_lp64.so.1 (0x00001555547ba000) 15 | - libmkl_intel_lp64.so.1 => /archive/apps/INTEL/OneAPI/mkl/2021.1.1/env/../lib/intel64/libmkl_intel_lp64.so.1 (0x0000155553a7f000) 16 | - libmkl_intel_thread.so.1 => /archive/apps/INTEL/OneAPI/mkl/2021.1.1/env/../lib/intel64/libmkl_intel_thread.so.1 (0x0000155550186000) 17 | - libmkl_core.so.1 => /archive/apps/INTEL/OneAPI/mkl/2021.1.1/env/../lib/intel64/libmkl_core.so.1 (0x00001555481c0000) 18 | - libmpifort.so.12 => /archive/apps/INTEL/OneAPI/mpi/2021.1.1//lib/libmpifort.so.12 (0x0000155547e02000) 19 | - libmpi.so.12 => /archive/apps/INTEL/OneAPI/mpi/2021.1.1//lib/release/libmpi.so.12 (0x0000155546a89000) 20 | - libdl.so.2 => /lib64/libdl.so.2 (0x0000155546885000) 21 | - librt.so.1 => /lib64/librt.so.1 (0x000015554667c000) 22 | - libpthread.so.0 => /lib64/libpthread.so.0 (0x000015554645c000) 23 | - libm.so.6 => /lib64/libm.so.6 (0x00001555460da000) 24 | - libiomp5.so => /archive/apps/INTEL/OneAPI/compiler/2021.1.1/linux/compiler/lib/intel64_lin/libiomp5.so (0x0000155545cd3000) 25 | - libc.so.6 => /lib64/libc.so.6 (0x0000155545911000) 26 | - libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00001555456f9000) 27 | - libfabric.so.1 => /archive/apps/INTEL/OneAPI/mpi/2021.1.1//libfabric/lib/libfabric.so.1 (0x00001555454b3000) 28 | - /lib64/ld-linux-x86-64.so.2 (0x000015555532b000) 29 | 30 | MPI Version: 31 | mpigcc for the Intel(R) MPI Library 2021.1 for Linux* 32 | Copyright 2003-2020, Intel Corporation. 33 | Using built-in specs. 34 | COLLECT_GCC=gcc 35 | COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper 36 | OFFLOAD_TARGET_NAMES=nvptx-none 37 | OFFLOAD_TARGET_DEFAULT=1 38 | Target: x86_64-redhat-linux 39 | Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --disable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux 40 | Thread model: posix 41 | gcc version 8.3.1 20191121 (Red Hat 8.3.1-5) (GCC) 42 | 43 | Node HW: 44 | - 2 Intel Xeon Platinum 8260 sockets (24 cores, 2.40 GHz each) 45 | - 1 TB RAM 46 | -------------------------------------------------------------------------------- /examples/quantum-espresso/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker run \ 5 | -it \ 6 | --rm \ 7 | -p 8888:8888 \ 8 | -v ${SCRIPT_DIRECTORY}/work:/home/jovyan/work \ 9 | -v ${HOME}/.ssh:/home/jovyan/.ssh \ 10 | alphaunito/quantum-espresso-notebook 11 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/cp-oneapi.x: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/examples/quantum-espresso/work/car-parrinello/INPDIR/cp-oneapi.x -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/h2o.in.00: -------------------------------------------------------------------------------- 1 | &CONTROL 2 | title = ' Water 32 molecules ', 3 | calculation = 'cp', 4 | restart_mode = 'from_scratch', ! 'restart', 5 | ndr = 50, 6 | ndw = 50, 7 | nstep = 1000, 8 | iprint = 100, 9 | isave = 100, 10 | tstress = .TRUE., 11 | tprnfor = .TRUE., 12 | dt = 5.0d0, 13 | etot_conv_thr = 1.d-6, 14 | prefix = 'h2o' 15 | outdir = './OUTDIR' 16 | pseudo_dir = './INPDIR' 17 | / 18 | 19 | &SYSTEM 20 | ibrav = 14, 21 | celldm(1) = 18.65, 22 | celldm(2) = 1.0, 23 | celldm(3) = 1.0, 24 | celldm(4) = 0.0, 25 | celldm(5) = 0.0, 26 | celldm(6) = 0.0, 27 | nat = 96, 28 | ntyp = 2, 29 | ecutwfc = 25.0, 30 | nr1b= 24, nr2b = 24, nr3b = 24, 31 | / 32 | 33 | &ELECTRONS 34 | emass = 400.d0, 35 | emass_cutoff = 2.5d0, 36 | orthogonalization = 'ortho', 37 | ortho_eps = 2.d-7, 38 | ortho_max = 100, 39 | electron_dynamics = 'damp', ! 'damp', 40 | electron_damping = 0.2, 41 | / 42 | 43 | &IONS 44 | ion_dynamics = 'none', 45 | ion_radius(1) = 0.8d0, 46 | ion_radius(2) = 0.5d0, 47 | ! ion_velocities = 'zero', 48 | ion_temperature = 'not_controlled' 49 | / 50 | 51 | &CELL 52 | cell_dynamics = 'none', 53 | cell_velocities = 'zero', 54 | press = 0.0d0, 55 | wmass = 70000.0d0 56 | / 57 | 58 | ATOMIC_SPECIES 59 | O 16.0d0 O.pbe-van_bm.UPF 60 | H 1.0079d0 H.pbe-van_bm.UPF 61 | 62 | ATOMIC_POSITIONS (crystal) 63 | O 0.3342 0.3858 0.1702 64 | O 1.0634 0.4197 0.2665 65 | O 1.0088 0.1409 0.5073 66 | O 0.6297 -0.3261 -0.5303 67 | O 1.4465 -0.1611 0.2161 68 | O 1.2637 -0.2524 -0.2121 69 | O 0.6889 -0.5572 -0.1900 70 | O 0.4894 0.2752 -0.0336 71 | O 1.8042 0.4375 0.4942 72 | O 1.6981 0.1893 0.5833 73 | O 0.9273 -0.1141 -0.7252 74 | O 0.6681 0.0772 0.0996 75 | O 0.8374 0.0165 -0.1115 76 | O 0.4164 0.1406 -0.4626 77 | O 0.9298 -0.3241 0.0546 78 | O 1.1835 0.3971 -0.2192 79 | O 1.5203 -0.3671 -0.2786 80 | O 0.7260 -0.0428 -0.5486 81 | O 0.9200 0.2746 -0.2521 82 | O 1.2450 0.2024 -0.6526 83 | O 1.1931 -0.4262 0.0049 84 | O 1.1879 -0.0335 0.1899 85 | O 1.3714 -0.1237 -0.5101 86 | O 1.7915 -0.1710 -0.2946 87 | O 0.5197 -0.4227 0.2470 88 | O 1.0876 -0.3333 0.4085 89 | O 1.2908 0.5198 0.5234 90 | O 0.8453 -0.7692 0.2531 91 | O 0.9539 -0.3703 -0.3696 92 | O 1.1436 -0.0101 -0.0703 93 | O 0.7080 -0.5488 0.1102 94 | O 1.3062 0.1574 -0.2005 95 | H 0.3742 0.3360 0.0929 96 | H 0.3150 0.3226 0.2472 97 | H 1.1146 0.3475 0.3129 98 | H 1.1177 0.4592 0.1936 99 | H 0.9405 0.1804 0.4516 100 | H 1.0984 0.1864 0.4941 101 | H 0.7242 -0.3512 -0.5486 102 | H 0.5812 -0.3844 -0.5967 103 | H 1.3477 -0.1441 0.1901 104 | H 1.4738 -0.2582 0.2103 105 | H 1.2728 -0.3005 -0.1238 106 | H 1.3551 -0.2521 -0.2483 107 | H 0.7610 -0.6009 -0.2414 108 | H 0.7253 -0.5451 -0.0988 109 | H 0.5460 0.2002 -0.0116 110 | H 0.5502 0.3269 -0.0971 111 | H 1.8732 0.4903 0.5432 112 | H 1.8466 0.4047 0.4105 113 | H 1.5983 0.1849 0.5758 114 | H 1.7255 0.2866 0.5619 115 | H 0.9805 -0.1882 -0.6842 116 | H 0.8774 -0.0766 -0.6492 117 | H 0.6062 0.0086 0.1372 118 | H 0.7231 0.0354 0.0264 119 | H 0.8537 0.1049 -0.1520 120 | H 0.8266 -0.0568 -0.1815 121 | H 0.4251 0.0432 -0.4790 122 | H 0.3722 0.1543 -0.3725 123 | H 1.0297 -0.3374 0.0605 124 | H 0.9083 -0.2661 0.1314 125 | H 1.2218 0.4478 -0.3016 126 | H 1.2272 0.3027 -0.2264 127 | H 1.5812 -0.4424 -0.2421 128 | H 1.5605 -0.3383 -0.3651 129 | H 0.6439 -0.0946 -0.5342 130 | H 0.7041 0.0486 -0.5091 131 | H 0.9378 0.2191 -0.3380 132 | H 1.0089 0.3172 -0.2251 133 | H 1.3206 0.1756 -0.5874 134 | H 1.2308 0.1250 -0.7200 135 | H 1.1741 -0.4861 -0.0716 136 | H 1.2668 -0.4640 0.0599 137 | H 1.1691 -0.0117 0.0958 138 | H 1.0952 -0.0622 0.2237 139 | H 1.3026 -0.1942 -0.4916 140 | H 1.4067 -0.1427 -0.6060 141 | H 1.7040 -0.2035 -0.2635 142 | H 1.7716 -0.1392 -0.3903 143 | H 0.5940 -0.4597 0.1919 144 | H 0.4413 -0.4860 0.2303 145 | H 1.0161 -0.3512 0.4768 146 | H 1.0940 -0.4261 0.3729 147 | H 1.2468 0.6010 0.4948 148 | H 1.3655 0.5512 0.5766 149 | H 0.7975 -0.8377 0.1939 150 | H 0.9351 -0.7510 0.2169 151 | H 0.9099 -0.2918 -0.3261 152 | H 1.0148 -0.3957 -0.2936 153 | H 1.1834 -0.0888 -0.1338 154 | H 1.0492 -0.0106 -0.0793 155 | H 0.7495 -0.6206 0.1764 156 | H 0.7769 -0.4739 0.0949 157 | H 1.2420 0.1027 -0.1450 158 | H 1.3752 0.1963 -0.1297 159 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/h2o.in.01: -------------------------------------------------------------------------------- 1 | &CONTROL 2 | title = ' Water 32 molecules ', 3 | calculation = 'cp', 4 | restart_mode = 'restart', 5 | ndr = 50, 6 | ndw = 50, 7 | nstep = 1000, 8 | iprint = 100, 9 | isave = 100, 10 | tstress = .TRUE., 11 | tprnfor = .TRUE., 12 | dt = 5.0d0, 13 | etot_conv_thr = 1.d-6, 14 | prefix = 'h2o' 15 | outdir = './OUTDIR' 16 | pseudo_dir = './INPDIR' 17 | / 18 | 19 | &SYSTEM 20 | ibrav = 14, 21 | celldm(1) = 18.65, 22 | celldm(2) = 1.0, 23 | celldm(3) = 1.0, 24 | celldm(4) = 0.0, 25 | celldm(5) = 0.0, 26 | celldm(6) = 0.0, 27 | nat = 96, 28 | ntyp = 2, 29 | ecutwfc = 25.0, 30 | nr1b= 24, nr2b = 24, nr3b = 24, 31 | / 32 | 33 | &ELECTRONS 34 | emass = 400.d0, 35 | emass_cutoff = 2.5d0, 36 | orthogonalization = 'ortho', 37 | ortho_eps = 2.d-7, 38 | ortho_max = 100, 39 | electron_dynamics = 'damp', ! 'damp', 40 | electron_damping = 0.2, 41 | / 42 | 43 | &IONS 44 | ion_dynamics = 'none', 45 | ion_radius(1) = 0.8d0, 46 | ion_radius(2) = 0.5d0, 47 | tranp(1) = .true. 48 | tranp(2) = .true. 49 | amprp(1) = 0.05 50 | amprp(2) = 0.10 51 | ion_velocities = 'zero', 52 | ion_temperature = 'not_controlled' 53 | / 54 | 55 | &CELL 56 | cell_dynamics = 'none', 57 | cell_velocities = 'zero', 58 | press = 0.0d0, 59 | wmass = 70000.0d0 60 | / 61 | 62 | ATOMIC_SPECIES 63 | O 16.0d0 O.pbe-van_bm.UPF 64 | H 1.0079d0 H.pbe-van_bm.UPF 65 | 66 | ATOMIC_POSITIONS (crystal) 67 | O 0.3342 0.3858 0.1702 68 | O 1.0634 0.4197 0.2665 69 | O 1.0088 0.1409 0.5073 70 | O 0.6297 -0.3261 -0.5303 71 | O 1.4465 -0.1611 0.2161 72 | O 1.2637 -0.2524 -0.2121 73 | O 0.6889 -0.5572 -0.1900 74 | O 0.4894 0.2752 -0.0336 75 | O 1.8042 0.4375 0.4942 76 | O 1.6981 0.1893 0.5833 77 | O 0.9273 -0.1141 -0.7252 78 | O 0.6681 0.0772 0.0996 79 | O 0.8374 0.0165 -0.1115 80 | O 0.4164 0.1406 -0.4626 81 | O 0.9298 -0.3241 0.0546 82 | O 1.1835 0.3971 -0.2192 83 | O 1.5203 -0.3671 -0.2786 84 | O 0.7260 -0.0428 -0.5486 85 | O 0.9200 0.2746 -0.2521 86 | O 1.2450 0.2024 -0.6526 87 | O 1.1931 -0.4262 0.0049 88 | O 1.1879 -0.0335 0.1899 89 | O 1.3714 -0.1237 -0.5101 90 | O 1.7915 -0.1710 -0.2946 91 | O 0.5197 -0.4227 0.2470 92 | O 1.0876 -0.3333 0.4085 93 | O 1.2908 0.5198 0.5234 94 | O 0.8453 -0.7692 0.2531 95 | O 0.9539 -0.3703 -0.3696 96 | O 1.1436 -0.0101 -0.0703 97 | O 0.7080 -0.5488 0.1102 98 | O 1.3062 0.1574 -0.2005 99 | H 0.3742 0.3360 0.0929 100 | H 0.3150 0.3226 0.2472 101 | H 1.1146 0.3475 0.3129 102 | H 1.1177 0.4592 0.1936 103 | H 0.9405 0.1804 0.4516 104 | H 1.0984 0.1864 0.4941 105 | H 0.7242 -0.3512 -0.5486 106 | H 0.5812 -0.3844 -0.5967 107 | H 1.3477 -0.1441 0.1901 108 | H 1.4738 -0.2582 0.2103 109 | H 1.2728 -0.3005 -0.1238 110 | H 1.3551 -0.2521 -0.2483 111 | H 0.7610 -0.6009 -0.2414 112 | H 0.7253 -0.5451 -0.0988 113 | H 0.5460 0.2002 -0.0116 114 | H 0.5502 0.3269 -0.0971 115 | H 1.8732 0.4903 0.5432 116 | H 1.8466 0.4047 0.4105 117 | H 1.5983 0.1849 0.5758 118 | H 1.7255 0.2866 0.5619 119 | H 0.9805 -0.1882 -0.6842 120 | H 0.8774 -0.0766 -0.6492 121 | H 0.6062 0.0086 0.1372 122 | H 0.7231 0.0354 0.0264 123 | H 0.8537 0.1049 -0.1520 124 | H 0.8266 -0.0568 -0.1815 125 | H 0.4251 0.0432 -0.4790 126 | H 0.3722 0.1543 -0.3725 127 | H 1.0297 -0.3374 0.0605 128 | H 0.9083 -0.2661 0.1314 129 | H 1.2218 0.4478 -0.3016 130 | H 1.2272 0.3027 -0.2264 131 | H 1.5812 -0.4424 -0.2421 132 | H 1.5605 -0.3383 -0.3651 133 | H 0.6439 -0.0946 -0.5342 134 | H 0.7041 0.0486 -0.5091 135 | H 0.9378 0.2191 -0.3380 136 | H 1.0089 0.3172 -0.2251 137 | H 1.3206 0.1756 -0.5874 138 | H 1.2308 0.1250 -0.7200 139 | H 1.1741 -0.4861 -0.0716 140 | H 1.2668 -0.4640 0.0599 141 | H 1.1691 -0.0117 0.0958 142 | H 1.0952 -0.0622 0.2237 143 | H 1.3026 -0.1942 -0.4916 144 | H 1.4067 -0.1427 -0.6060 145 | H 1.7040 -0.2035 -0.2635 146 | H 1.7716 -0.1392 -0.3903 147 | H 0.5940 -0.4597 0.1919 148 | H 0.4413 -0.4860 0.2303 149 | H 1.0161 -0.3512 0.4768 150 | H 1.0940 -0.4261 0.3729 151 | H 1.2468 0.6010 0.4948 152 | H 1.3655 0.5512 0.5766 153 | H 0.7975 -0.8377 0.1939 154 | H 0.9351 -0.7510 0.2169 155 | H 0.9099 -0.2918 -0.3261 156 | H 1.0148 -0.3957 -0.2936 157 | H 1.1834 -0.0888 -0.1338 158 | H 1.0492 -0.0106 -0.0793 159 | H 0.7495 -0.6206 0.1764 160 | H 0.7769 -0.4739 0.0949 161 | H 1.2420 0.1027 -0.1450 162 | H 1.3752 0.1963 -0.1297 163 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/h2o.in.02: -------------------------------------------------------------------------------- 1 | &CONTROL 2 | title = ' Water 32 molecules ', 3 | calculation = 'cp', 4 | restart_mode = 'restart', 5 | ndr = 50, 6 | ndw = 51, 7 | nstep = 1000, 8 | iprint = 100, 9 | isave = 100, 10 | tstress = .TRUE., 11 | tprnfor = .TRUE., 12 | dt = 5.0d0, 13 | etot_conv_thr = 1.d-6, 14 | prefix = 'h2o' 15 | outdir = './OUTDIR' 16 | pseudo_dir = './INPDIR' 17 | / 18 | 19 | &SYSTEM 20 | ibrav = 14, 21 | celldm(1) = 18.65, 22 | celldm(2) = 1.0, 23 | celldm(3) = 1.0, 24 | celldm(4) = 0.0, 25 | celldm(5) = 0.0, 26 | celldm(6) = 0.0, 27 | nat = 96, 28 | ntyp = 2, 29 | ecutwfc = 25.0, 30 | nr1b= 24, nr2b = 24, nr3b = 24, 31 | / 32 | 33 | &ELECTRONS 34 | emass = 400.d0, 35 | emass_cutoff = 2.5d0, 36 | orthogonalization = 'ortho', 37 | ortho_eps = 2.d-7, 38 | ortho_max = 100, 39 | electron_dynamics = 'verlet', 40 | electron_velocities = 'zero' 41 | / 42 | 43 | &IONS 44 | ion_dynamics = 'verlet', 45 | ion_radius(1) = 0.8d0, 46 | ion_radius(2) = 0.5d0, 47 | ion_velocities = 'zero', 48 | ion_temperature = 'not_controlled' 49 | / 50 | 51 | &CELL 52 | cell_dynamics = 'none', 53 | cell_velocities = 'zero', 54 | press = 0.0d0, 55 | wmass = 70000.0d0 56 | / 57 | 58 | ATOMIC_SPECIES 59 | O 16.0d0 O.pbe-van_bm.UPF 60 | H 1.0079d0 H.pbe-van_bm.UPF 61 | 62 | ATOMIC_POSITIONS (crystal) 63 | O 0.3342 0.3858 0.1702 64 | O 1.0634 0.4197 0.2665 65 | O 1.0088 0.1409 0.5073 66 | O 0.6297 -0.3261 -0.5303 67 | O 1.4465 -0.1611 0.2161 68 | O 1.2637 -0.2524 -0.2121 69 | O 0.6889 -0.5572 -0.1900 70 | O 0.4894 0.2752 -0.0336 71 | O 1.8042 0.4375 0.4942 72 | O 1.6981 0.1893 0.5833 73 | O 0.9273 -0.1141 -0.7252 74 | O 0.6681 0.0772 0.0996 75 | O 0.8374 0.0165 -0.1115 76 | O 0.4164 0.1406 -0.4626 77 | O 0.9298 -0.3241 0.0546 78 | O 1.1835 0.3971 -0.2192 79 | O 1.5203 -0.3671 -0.2786 80 | O 0.7260 -0.0428 -0.5486 81 | O 0.9200 0.2746 -0.2521 82 | O 1.2450 0.2024 -0.6526 83 | O 1.1931 -0.4262 0.0049 84 | O 1.1879 -0.0335 0.1899 85 | O 1.3714 -0.1237 -0.5101 86 | O 1.7915 -0.1710 -0.2946 87 | O 0.5197 -0.4227 0.2470 88 | O 1.0876 -0.3333 0.4085 89 | O 1.2908 0.5198 0.5234 90 | O 0.8453 -0.7692 0.2531 91 | O 0.9539 -0.3703 -0.3696 92 | O 1.1436 -0.0101 -0.0703 93 | O 0.7080 -0.5488 0.1102 94 | O 1.3062 0.1574 -0.2005 95 | H 0.3742 0.3360 0.0929 96 | H 0.3150 0.3226 0.2472 97 | H 1.1146 0.3475 0.3129 98 | H 1.1177 0.4592 0.1936 99 | H 0.9405 0.1804 0.4516 100 | H 1.0984 0.1864 0.4941 101 | H 0.7242 -0.3512 -0.5486 102 | H 0.5812 -0.3844 -0.5967 103 | H 1.3477 -0.1441 0.1901 104 | H 1.4738 -0.2582 0.2103 105 | H 1.2728 -0.3005 -0.1238 106 | H 1.3551 -0.2521 -0.2483 107 | H 0.7610 -0.6009 -0.2414 108 | H 0.7253 -0.5451 -0.0988 109 | H 0.5460 0.2002 -0.0116 110 | H 0.5502 0.3269 -0.0971 111 | H 1.8732 0.4903 0.5432 112 | H 1.8466 0.4047 0.4105 113 | H 1.5983 0.1849 0.5758 114 | H 1.7255 0.2866 0.5619 115 | H 0.9805 -0.1882 -0.6842 116 | H 0.8774 -0.0766 -0.6492 117 | H 0.6062 0.0086 0.1372 118 | H 0.7231 0.0354 0.0264 119 | H 0.8537 0.1049 -0.1520 120 | H 0.8266 -0.0568 -0.1815 121 | H 0.4251 0.0432 -0.4790 122 | H 0.3722 0.1543 -0.3725 123 | H 1.0297 -0.3374 0.0605 124 | H 0.9083 -0.2661 0.1314 125 | H 1.2218 0.4478 -0.3016 126 | H 1.2272 0.3027 -0.2264 127 | H 1.5812 -0.4424 -0.2421 128 | H 1.5605 -0.3383 -0.3651 129 | H 0.6439 -0.0946 -0.5342 130 | H 0.7041 0.0486 -0.5091 131 | H 0.9378 0.2191 -0.3380 132 | H 1.0089 0.3172 -0.2251 133 | H 1.3206 0.1756 -0.5874 134 | H 1.2308 0.1250 -0.7200 135 | H 1.1741 -0.4861 -0.0716 136 | H 1.2668 -0.4640 0.0599 137 | H 1.1691 -0.0117 0.0958 138 | H 1.0952 -0.0622 0.2237 139 | H 1.3026 -0.1942 -0.4916 140 | H 1.4067 -0.1427 -0.6060 141 | H 1.7040 -0.2035 -0.2635 142 | H 1.7716 -0.1392 -0.3903 143 | H 0.5940 -0.4597 0.1919 144 | H 0.4413 -0.4860 0.2303 145 | H 1.0161 -0.3512 0.4768 146 | H 1.0940 -0.4261 0.3729 147 | H 1.2468 0.6010 0.4948 148 | H 1.3655 0.5512 0.5766 149 | H 0.7975 -0.8377 0.1939 150 | H 0.9351 -0.7510 0.2169 151 | H 0.9099 -0.2918 -0.3261 152 | H 1.0148 -0.3957 -0.2936 153 | H 1.1834 -0.0888 -0.1338 154 | H 1.0492 -0.0106 -0.0793 155 | H 0.7495 -0.6206 0.1764 156 | H 0.7769 -0.4739 0.0949 157 | H 1.2420 0.1027 -0.1450 158 | H 1.3752 0.1963 -0.1297 159 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/h2o.in.03.b0: -------------------------------------------------------------------------------- 1 | &CONTROL 2 | title = ' Water 32 molecules ', 3 | calculation = 'cp', 4 | restart_mode = 'restart', 5 | ndr = 51, 6 | ndw = 52, 7 | nstep = 1000, 8 | iprint = 100, 9 | isave = 100, 10 | tstress = .TRUE., 11 | tprnfor = .TRUE., 12 | dt = 5.0d0, 13 | etot_conv_thr = 1.d-6, 14 | prefix = 'h2o' 15 | outdir = './OUTDIR.b0' 16 | pseudo_dir = './INPDIR' 17 | / 18 | 19 | &SYSTEM 20 | ibrav = 14, 21 | celldm(1) = 18.65, 22 | celldm(2) = 1.0, 23 | celldm(3) = 1.0, 24 | celldm(4) = 0.0, 25 | celldm(5) = 0.0, 26 | celldm(6) = 0.0, 27 | nat = 96, 28 | ntyp = 2, 29 | ecutwfc = 25.0, 30 | nr1b= 24, nr2b = 24, nr3b = 24, 31 | / 32 | 33 | &ELECTRONS 34 | emass = 400.d0, 35 | emass_cutoff = 2.5d0, 36 | orthogonalization = 'ortho', 37 | ortho_eps = 2.d-7, 38 | ortho_max = 100, 39 | electron_dynamics = 'verlet', 40 | / 41 | 42 | &IONS 43 | ion_dynamics = 'verlet', 44 | ion_radius(1) = 0.8d0, 45 | ion_radius(2) = 0.5d0, 46 | ion_temperature = 'not_controlled', 47 | / 48 | 49 | &CELL 50 | cell_dynamics = 'none', 51 | cell_velocities = 'zero', 52 | press = 0.0d0, 53 | wmass = 70000.0d0 54 | / 55 | 56 | ATOMIC_SPECIES 57 | O 16.0d0 O.pbe-van_bm.UPF 58 | H 1.0079d0 H.pbe-van_bm.UPF 59 | 60 | ATOMIC_POSITIONS (crystal) 61 | O 0.3342 0.3858 0.1702 62 | O 1.0634 0.4197 0.2665 63 | O 1.0088 0.1409 0.5073 64 | O 0.6297 -0.3261 -0.5303 65 | O 1.4465 -0.1611 0.2161 66 | O 1.2637 -0.2524 -0.2121 67 | O 0.6889 -0.5572 -0.1900 68 | O 0.4894 0.2752 -0.0336 69 | O 1.8042 0.4375 0.4942 70 | O 1.6981 0.1893 0.5833 71 | O 0.9273 -0.1141 -0.7252 72 | O 0.6681 0.0772 0.0996 73 | O 0.8374 0.0165 -0.1115 74 | O 0.4164 0.1406 -0.4626 75 | O 0.9298 -0.3241 0.0546 76 | O 1.1835 0.3971 -0.2192 77 | O 1.5203 -0.3671 -0.2786 78 | O 0.7260 -0.0428 -0.5486 79 | O 0.9200 0.2746 -0.2521 80 | O 1.2450 0.2024 -0.6526 81 | O 1.1931 -0.4262 0.0049 82 | O 1.1879 -0.0335 0.1899 83 | O 1.3714 -0.1237 -0.5101 84 | O 1.7915 -0.1710 -0.2946 85 | O 0.5197 -0.4227 0.2470 86 | O 1.0876 -0.3333 0.4085 87 | O 1.2908 0.5198 0.5234 88 | O 0.8453 -0.7692 0.2531 89 | O 0.9539 -0.3703 -0.3696 90 | O 1.1436 -0.0101 -0.0703 91 | O 0.7080 -0.5488 0.1102 92 | O 1.3062 0.1574 -0.2005 93 | H 0.3742 0.3360 0.0929 94 | H 0.3150 0.3226 0.2472 95 | H 1.1146 0.3475 0.3129 96 | H 1.1177 0.4592 0.1936 97 | H 0.9405 0.1804 0.4516 98 | H 1.0984 0.1864 0.4941 99 | H 0.7242 -0.3512 -0.5486 100 | H 0.5812 -0.3844 -0.5967 101 | H 1.3477 -0.1441 0.1901 102 | H 1.4738 -0.2582 0.2103 103 | H 1.2728 -0.3005 -0.1238 104 | H 1.3551 -0.2521 -0.2483 105 | H 0.7610 -0.6009 -0.2414 106 | H 0.7253 -0.5451 -0.0988 107 | H 0.5460 0.2002 -0.0116 108 | H 0.5502 0.3269 -0.0971 109 | H 1.8732 0.4903 0.5432 110 | H 1.8466 0.4047 0.4105 111 | H 1.5983 0.1849 0.5758 112 | H 1.7255 0.2866 0.5619 113 | H 0.9805 -0.1882 -0.6842 114 | H 0.8774 -0.0766 -0.6492 115 | H 0.6062 0.0086 0.1372 116 | H 0.7231 0.0354 0.0264 117 | H 0.8537 0.1049 -0.1520 118 | H 0.8266 -0.0568 -0.1815 119 | H 0.4251 0.0432 -0.4790 120 | H 0.3722 0.1543 -0.3725 121 | H 1.0297 -0.3374 0.0605 122 | H 0.9083 -0.2661 0.1314 123 | H 1.2218 0.4478 -0.3016 124 | H 1.2272 0.3027 -0.2264 125 | H 1.5812 -0.4424 -0.2421 126 | H 1.5605 -0.3383 -0.3651 127 | H 0.6439 -0.0946 -0.5342 128 | H 0.7041 0.0486 -0.5091 129 | H 0.9378 0.2191 -0.3380 130 | H 1.0089 0.3172 -0.2251 131 | H 1.3206 0.1756 -0.5874 132 | H 1.2308 0.1250 -0.7200 133 | H 1.1741 -0.4861 -0.0716 134 | H 1.2668 -0.4640 0.0599 135 | H 1.1691 -0.0117 0.0958 136 | H 1.0952 -0.0622 0.2237 137 | H 1.3026 -0.1942 -0.4916 138 | H 1.4067 -0.1427 -0.6060 139 | H 1.7040 -0.2035 -0.2635 140 | H 1.7716 -0.1392 -0.3903 141 | H 0.5940 -0.4597 0.1919 142 | H 0.4413 -0.4860 0.2303 143 | H 1.0161 -0.3512 0.4768 144 | H 1.0940 -0.4261 0.3729 145 | H 1.2468 0.6010 0.4948 146 | H 1.3655 0.5512 0.5766 147 | H 0.7975 -0.8377 0.1939 148 | H 0.9351 -0.7510 0.2169 149 | H 0.9099 -0.2918 -0.3261 150 | H 1.0148 -0.3957 -0.2936 151 | H 1.1834 -0.0888 -0.1338 152 | H 1.0492 -0.0106 -0.0793 153 | H 0.7495 -0.6206 0.1764 154 | H 0.7769 -0.4739 0.0949 155 | H 1.2420 0.1027 -0.1450 156 | H 1.3752 0.1963 -0.1297 157 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/h2o.in.03.b1: -------------------------------------------------------------------------------- 1 | &CONTROL 2 | title = ' Water 32 molecules ', 3 | calculation = 'cp', 4 | restart_mode = 'restart', 5 | ndr = 51, 6 | ndw = 62, 7 | nstep = 1000, 8 | iprint = 100, 9 | isave = 100, 10 | tstress = .TRUE., 11 | tprnfor = .TRUE., 12 | dt = 5.0d0, 13 | etot_conv_thr = 1.d-6, 14 | prefix = 'h2o' 15 | outdir = './OUTDIR.b1' 16 | pseudo_dir = './INPDIR' 17 | / 18 | 19 | &SYSTEM 20 | ibrav = 14, 21 | celldm(1) = 18.65, 22 | celldm(2) = 1.0, 23 | celldm(3) = 1.0, 24 | celldm(4) = 0.0, 25 | celldm(5) = 0.0, 26 | celldm(6) = 0.0, 27 | nat = 96, 28 | ntyp = 2, 29 | ecutwfc = 25.0, 30 | nr1b= 24, nr2b = 24, nr3b = 24, 31 | / 32 | 33 | &ELECTRONS 34 | emass = 400.d0, 35 | emass_cutoff = 2.5d0, 36 | orthogonalization = 'ortho', 37 | ortho_eps = 2.d-7, 38 | ortho_max = 100, 39 | electron_dynamics = 'verlet', 40 | / 41 | 42 | &IONS 43 | ion_dynamics = 'verlet', 44 | ion_radius(1) = 0.8d0, 45 | ion_radius(2) = 0.5d0, 46 | ion_temperature = 'nose', 47 | tempw = 300. 48 | fnosep = 500. 49 | / 50 | 51 | &CELL 52 | cell_dynamics = 'none', 53 | cell_velocities = 'zero', 54 | press = 0.0d0, 55 | wmass = 70000.0d0 56 | / 57 | 58 | ATOMIC_SPECIES 59 | O 16.0d0 O.pbe-van_bm.UPF 60 | H 1.0079d0 H.pbe-van_bm.UPF 61 | 62 | ATOMIC_POSITIONS (crystal) 63 | O 0.3342 0.3858 0.1702 64 | O 1.0634 0.4197 0.2665 65 | O 1.0088 0.1409 0.5073 66 | O 0.6297 -0.3261 -0.5303 67 | O 1.4465 -0.1611 0.2161 68 | O 1.2637 -0.2524 -0.2121 69 | O 0.6889 -0.5572 -0.1900 70 | O 0.4894 0.2752 -0.0336 71 | O 1.8042 0.4375 0.4942 72 | O 1.6981 0.1893 0.5833 73 | O 0.9273 -0.1141 -0.7252 74 | O 0.6681 0.0772 0.0996 75 | O 0.8374 0.0165 -0.1115 76 | O 0.4164 0.1406 -0.4626 77 | O 0.9298 -0.3241 0.0546 78 | O 1.1835 0.3971 -0.2192 79 | O 1.5203 -0.3671 -0.2786 80 | O 0.7260 -0.0428 -0.5486 81 | O 0.9200 0.2746 -0.2521 82 | O 1.2450 0.2024 -0.6526 83 | O 1.1931 -0.4262 0.0049 84 | O 1.1879 -0.0335 0.1899 85 | O 1.3714 -0.1237 -0.5101 86 | O 1.7915 -0.1710 -0.2946 87 | O 0.5197 -0.4227 0.2470 88 | O 1.0876 -0.3333 0.4085 89 | O 1.2908 0.5198 0.5234 90 | O 0.8453 -0.7692 0.2531 91 | O 0.9539 -0.3703 -0.3696 92 | O 1.1436 -0.0101 -0.0703 93 | O 0.7080 -0.5488 0.1102 94 | O 1.3062 0.1574 -0.2005 95 | H 0.3742 0.3360 0.0929 96 | H 0.3150 0.3226 0.2472 97 | H 1.1146 0.3475 0.3129 98 | H 1.1177 0.4592 0.1936 99 | H 0.9405 0.1804 0.4516 100 | H 1.0984 0.1864 0.4941 101 | H 0.7242 -0.3512 -0.5486 102 | H 0.5812 -0.3844 -0.5967 103 | H 1.3477 -0.1441 0.1901 104 | H 1.4738 -0.2582 0.2103 105 | H 1.2728 -0.3005 -0.1238 106 | H 1.3551 -0.2521 -0.2483 107 | H 0.7610 -0.6009 -0.2414 108 | H 0.7253 -0.5451 -0.0988 109 | H 0.5460 0.2002 -0.0116 110 | H 0.5502 0.3269 -0.0971 111 | H 1.8732 0.4903 0.5432 112 | H 1.8466 0.4047 0.4105 113 | H 1.5983 0.1849 0.5758 114 | H 1.7255 0.2866 0.5619 115 | H 0.9805 -0.1882 -0.6842 116 | H 0.8774 -0.0766 -0.6492 117 | H 0.6062 0.0086 0.1372 118 | H 0.7231 0.0354 0.0264 119 | H 0.8537 0.1049 -0.1520 120 | H 0.8266 -0.0568 -0.1815 121 | H 0.4251 0.0432 -0.4790 122 | H 0.3722 0.1543 -0.3725 123 | H 1.0297 -0.3374 0.0605 124 | H 0.9083 -0.2661 0.1314 125 | H 1.2218 0.4478 -0.3016 126 | H 1.2272 0.3027 -0.2264 127 | H 1.5812 -0.4424 -0.2421 128 | H 1.5605 -0.3383 -0.3651 129 | H 0.6439 -0.0946 -0.5342 130 | H 0.7041 0.0486 -0.5091 131 | H 0.9378 0.2191 -0.3380 132 | H 1.0089 0.3172 -0.2251 133 | H 1.3206 0.1756 -0.5874 134 | H 1.2308 0.1250 -0.7200 135 | H 1.1741 -0.4861 -0.0716 136 | H 1.2668 -0.4640 0.0599 137 | H 1.1691 -0.0117 0.0958 138 | H 1.0952 -0.0622 0.2237 139 | H 1.3026 -0.1942 -0.4916 140 | H 1.4067 -0.1427 -0.6060 141 | H 1.7040 -0.2035 -0.2635 142 | H 1.7716 -0.1392 -0.3903 143 | H 0.5940 -0.4597 0.1919 144 | H 0.4413 -0.4860 0.2303 145 | H 1.0161 -0.3512 0.4768 146 | H 1.0940 -0.4261 0.3729 147 | H 1.2468 0.6010 0.4948 148 | H 1.3655 0.5512 0.5766 149 | H 0.7975 -0.8377 0.1939 150 | H 0.9351 -0.7510 0.2169 151 | H 0.9099 -0.2918 -0.3261 152 | H 1.0148 -0.3957 -0.2936 153 | H 1.1834 -0.0888 -0.1338 154 | H 1.0492 -0.0106 -0.0793 155 | H 0.7495 -0.6206 0.1764 156 | H 0.7769 -0.4739 0.0949 157 | H 1.2420 0.1027 -0.1450 158 | H 1.3752 0.1963 -0.1297 159 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/INPDIR/h2o.in.03.b2: -------------------------------------------------------------------------------- 1 | &CONTROL 2 | title = ' Water 32 molecules ', 3 | calculation = 'cp', 4 | restart_mode = 'restart', 5 | ndr = 51, 6 | ndw = 72, 7 | nstep = 1000, 8 | iprint = 100, 9 | isave = 100, 10 | tstress = .TRUE., 11 | tprnfor = .TRUE., 12 | dt = 5.0d0, 13 | etot_conv_thr = 1.d-6, 14 | prefix = 'h2o' 15 | outdir = './OUTDIR.b2' 16 | pseudo_dir = './INPDIR' 17 | / 18 | 19 | &SYSTEM 20 | ibrav = 14, 21 | celldm(1) = 18.65, 22 | celldm(2) = 1.0, 23 | celldm(3) = 1.0, 24 | celldm(4) = 0.0, 25 | celldm(5) = 0.0, 26 | celldm(6) = 0.0, 27 | nat = 96, 28 | ntyp = 2, 29 | ecutwfc = 25.0, 30 | nr1b= 24, nr2b = 24, nr3b = 24, 31 | / 32 | 33 | &ELECTRONS 34 | emass = 400.d0, 35 | emass_cutoff = 2.5d0, 36 | orthogonalization = 'ortho', 37 | ortho_eps = 2.d-7, 38 | ortho_max = 100, 39 | electron_dynamics = 'verlet', 40 | / 41 | 42 | &IONS 43 | ion_dynamics = 'verlet', 44 | ion_radius(1) = 0.8d0, 45 | ion_radius(2) = 0.5d0, 46 | ion_temperature = 'nose', 47 | tempw = 400. 48 | fnosep = 500. 49 | / 50 | 51 | &CELL 52 | cell_dynamics = 'none', 53 | cell_velocities = 'zero', 54 | press = 0.0d0, 55 | wmass = 70000.0d0 56 | / 57 | 58 | ATOMIC_SPECIES 59 | O 16.0d0 O.pbe-van_bm.UPF 60 | H 1.0079d0 H.pbe-van_bm.UPF 61 | 62 | ATOMIC_POSITIONS (crystal) 63 | O 0.3342 0.3858 0.1702 64 | O 1.0634 0.4197 0.2665 65 | O 1.0088 0.1409 0.5073 66 | O 0.6297 -0.3261 -0.5303 67 | O 1.4465 -0.1611 0.2161 68 | O 1.2637 -0.2524 -0.2121 69 | O 0.6889 -0.5572 -0.1900 70 | O 0.4894 0.2752 -0.0336 71 | O 1.8042 0.4375 0.4942 72 | O 1.6981 0.1893 0.5833 73 | O 0.9273 -0.1141 -0.7252 74 | O 0.6681 0.0772 0.0996 75 | O 0.8374 0.0165 -0.1115 76 | O 0.4164 0.1406 -0.4626 77 | O 0.9298 -0.3241 0.0546 78 | O 1.1835 0.3971 -0.2192 79 | O 1.5203 -0.3671 -0.2786 80 | O 0.7260 -0.0428 -0.5486 81 | O 0.9200 0.2746 -0.2521 82 | O 1.2450 0.2024 -0.6526 83 | O 1.1931 -0.4262 0.0049 84 | O 1.1879 -0.0335 0.1899 85 | O 1.3714 -0.1237 -0.5101 86 | O 1.7915 -0.1710 -0.2946 87 | O 0.5197 -0.4227 0.2470 88 | O 1.0876 -0.3333 0.4085 89 | O 1.2908 0.5198 0.5234 90 | O 0.8453 -0.7692 0.2531 91 | O 0.9539 -0.3703 -0.3696 92 | O 1.1436 -0.0101 -0.0703 93 | O 0.7080 -0.5488 0.1102 94 | O 1.3062 0.1574 -0.2005 95 | H 0.3742 0.3360 0.0929 96 | H 0.3150 0.3226 0.2472 97 | H 1.1146 0.3475 0.3129 98 | H 1.1177 0.4592 0.1936 99 | H 0.9405 0.1804 0.4516 100 | H 1.0984 0.1864 0.4941 101 | H 0.7242 -0.3512 -0.5486 102 | H 0.5812 -0.3844 -0.5967 103 | H 1.3477 -0.1441 0.1901 104 | H 1.4738 -0.2582 0.2103 105 | H 1.2728 -0.3005 -0.1238 106 | H 1.3551 -0.2521 -0.2483 107 | H 0.7610 -0.6009 -0.2414 108 | H 0.7253 -0.5451 -0.0988 109 | H 0.5460 0.2002 -0.0116 110 | H 0.5502 0.3269 -0.0971 111 | H 1.8732 0.4903 0.5432 112 | H 1.8466 0.4047 0.4105 113 | H 1.5983 0.1849 0.5758 114 | H 1.7255 0.2866 0.5619 115 | H 0.9805 -0.1882 -0.6842 116 | H 0.8774 -0.0766 -0.6492 117 | H 0.6062 0.0086 0.1372 118 | H 0.7231 0.0354 0.0264 119 | H 0.8537 0.1049 -0.1520 120 | H 0.8266 -0.0568 -0.1815 121 | H 0.4251 0.0432 -0.4790 122 | H 0.3722 0.1543 -0.3725 123 | H 1.0297 -0.3374 0.0605 124 | H 0.9083 -0.2661 0.1314 125 | H 1.2218 0.4478 -0.3016 126 | H 1.2272 0.3027 -0.2264 127 | H 1.5812 -0.4424 -0.2421 128 | H 1.5605 -0.3383 -0.3651 129 | H 0.6439 -0.0946 -0.5342 130 | H 0.7041 0.0486 -0.5091 131 | H 0.9378 0.2191 -0.3380 132 | H 1.0089 0.3172 -0.2251 133 | H 1.3206 0.1756 -0.5874 134 | H 1.2308 0.1250 -0.7200 135 | H 1.1741 -0.4861 -0.0716 136 | H 1.2668 -0.4640 0.0599 137 | H 1.1691 -0.0117 0.0958 138 | H 1.0952 -0.0622 0.2237 139 | H 1.3026 -0.1942 -0.4916 140 | H 1.4067 -0.1427 -0.6060 141 | H 1.7040 -0.2035 -0.2635 142 | H 1.7716 -0.1392 -0.3903 143 | H 0.5940 -0.4597 0.1919 144 | H 0.4413 -0.4860 0.2303 145 | H 1.0161 -0.3512 0.4768 146 | H 1.0940 -0.4261 0.3729 147 | H 1.2468 0.6010 0.4948 148 | H 1.3655 0.5512 0.5766 149 | H 0.7975 -0.8377 0.1939 150 | H 0.9351 -0.7510 0.2169 151 | H 0.9099 -0.2918 -0.3261 152 | H 1.0148 -0.3957 -0.2936 153 | H 1.1834 -0.0888 -0.1338 154 | H 1.0492 -0.0106 -0.0793 155 | H 0.7495 -0.6206 0.1764 156 | H 0.7769 -0.4739 0.0949 157 | H 1.2420 0.1027 -0.1450 158 | H 1.3752 0.1963 -0.1297 159 | -------------------------------------------------------------------------------- /examples/quantum-espresso/work/car-parrinello/environment/pbs_espresso: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #PBS -N espresso_work 4 | #PBS -o espresso_work.txt 5 | #PBS -q cpu 6 | #PBS -e espresso_work_error.txt 7 | #PBS -k oe 8 | #PBS -m e 9 | #PBS -l select=1:ncpus=48:mpiprocs=16:mem=128gb 10 | 11 | module load ucx 12 | source /apps/INTEL/OneAPI/compiler/latest/env/vars.sh 13 | source /apps/INTEL/OneAPI/mpi/latest/env/vars.sh 14 | source /apps/INTEL/OneAPI/mkl/latest/env/vars.sh 15 | 16 | export OMP_NUM_THREADS=3 17 | export FI_PROVIDER=mlx 18 | 19 | {{ streamflow_command }} -------------------------------------------------------------------------------- /examples/quantum-espresso/work/primordial-soup/INPDIR/cp-oneapi.x: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/examples/quantum-espresso/work/primordial-soup/INPDIR/cp-oneapi.x -------------------------------------------------------------------------------- /examples/quantum-espresso/work/primordial-soup/environment/pbs_espresso: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #PBS -N espresso_work 4 | #PBS -o espresso_work.txt 5 | #PBS -q cpu 6 | #PBS -e espresso_work_error.txt 7 | #PBS -k oe 8 | #PBS -m e 9 | #PBS -l select=2:ncpus=48:mpiprocs=16:mem=128gb 10 | 11 | module load ucx 12 | source /apps/INTEL/OneAPI/compiler/latest/env/vars.sh 13 | source /apps/INTEL/OneAPI/mpi/latest/env/vars.sh 14 | source /apps/INTEL/OneAPI/mkl/latest/env/vars.sh 15 | 16 | export OMP_NUM_THREADS=3 17 | export FI_PROVIDER=mlx 18 | 19 | {{ streamflow_command }} -------------------------------------------------------------------------------- /examples/tensorflow/README.md: -------------------------------------------------------------------------------- 1 | # Train and serve a TensorFlow model with TensorFlow Serving 2 | 3 | This Notebook showcases how Deep Learning can benefit from hybrid Cloud-HPC scenarios, in particular to offload the computationally demanding training phase to an HPC facility, and offering the resulting network as-a-Service on a Cloud architecture to answer time-constrained inference queries. 4 | 5 | The notebook is a modified version of [this one](https://github.com/tensorflow/tfx/blob/master/docs/tutorials/serving/rest_simple.ipynb), which focuses mainly on Google Colab. Since we are not interested in pure performance for this use case, we use the small [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset and a very small custom Convolutional Neural Network (CNN). 6 | 7 | The training phase has been offloaded to an HPC node on the [HPC4AI facility](https://hpc4ai.unito.it/), equipped with four NVIDIA Tesla V100 and two Intel Xeon Gols 6230 sockets. Then, the trained model is sent to a pre-existing TensorFlow Serving Pod on a Kubernetes cluster, hosted on the OpenStack-based HPC4AI cloud. 8 | 9 | ## Run the notebook 10 | 11 | In order to run the Notebook locally, you can use the `run` script in this folder. It automatically pulls the related container from [DockerHub](https://hub.docker.com/r/alphaunito/tensorflow-notebook). Conversely, if you want to produce your own local version of the container, you can run the `build` script in the `docker/tensorflow-notebook` folder of this repo prior to launch the `run` script. 12 | 13 | Documentation related to the single Notebook cells is reported directly in the Notebook. Please be sure to select `Jupyter Workflow` as the Notebook kernel when running the example. 14 | -------------------------------------------------------------------------------- /examples/tensorflow/docker/tensorflow-notebook/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/scipy-notebook 2 | LABEL maintainer="Iacopo Colonnelli " 3 | 4 | RUN pip install --no-cache-dir \ 5 | bcrypt==3.2.0 \ 6 | dill==0.3.3 \ 7 | jupyter-workflow==0.0.19 \ 8 | tensorflow==2.4.1 \ 9 | && python -m jupyter_workflow.ipython.install 10 | 11 | USER root 12 | 13 | RUN apt-get update \ 14 | && apt-get install -y \ 15 | curl \ 16 | openssh-client \ 17 | && apt-get clean \ 18 | && rm -rf /var/lib/apt/lists/* \ 19 | && fix-permissions "${CONDA_DIR}" \ 20 | && ln -s /usr/local/cuda/lib64/libcusolver.so.11 /usr/local/cuda/lib64/libcusolver.so.10 \ 21 | && curl -L https://git.io/get_helm.sh -o /tmp/get_helm.sh \ 22 | && chmod +x /tmp/get_helm.sh \ 23 | && /tmp/get_helm.sh --version v3.5.4 24 | 25 | ENV LD_LIBRARY_PATH="/usr/local/cuda/lib:/usr/local/cuda/lib64" 26 | 27 | USER ${NB_USER} 28 | -------------------------------------------------------------------------------- /examples/tensorflow/docker/tensorflow-notebook/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | # Rebuild Jupyter stack with CUDA support 5 | git clone https://github.com/jupyter/docker-stacks ${SCRIPT_DIRECTORY}/docker-stacks 6 | docker build \ 7 | --build-arg=ROOT_CONTAINER=nvcr.io/nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu20.04 \ 8 | -t jupyter/base-notebook \ 9 | ${SCRIPT_DIRECTORY}/docker-stacks/base-notebook 10 | docker build \ 11 | -t jupyter/minimal-notebook \ 12 | ${SCRIPT_DIRECTORY}/docker-stacks/minimal-notebook 13 | docker build \ 14 | -t jupyter/scipy-notebook \ 15 | ${SCRIPT_DIRECTORY}/docker-stacks/scipy-notebook 16 | rm -rf ${SCRIPT_DIRECTORY}/docker-stacks 17 | 18 | # Build TensorFlow GPU notebook 19 | docker build \ 20 | -t alphaunito/tensorflow-notebook \ 21 | ${SCRIPT_DIRECTORY} 22 | -------------------------------------------------------------------------------- /examples/tensorflow/docker/tensorflow-serving/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving 2 | LABEL maintainer="Iacopo Colonnelli " 3 | 4 | RUN apt-get update \ 5 | && apt-get install -y \ 6 | python3-pip \ 7 | && apt-get clean \ 8 | && rm -rf /var/lib/apt/lists/* \ 9 | && pip3 install --no-cache-dir \ 10 | dill==0.3.3 \ 11 | ipython==7.16.1 \ 12 | numpy==1.19.5 \ 13 | requests==2.25.1 14 | -------------------------------------------------------------------------------- /examples/tensorflow/docker/tensorflow-serving/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker build \ 5 | -t alphaunito/tensorflow-serving \ 6 | ${SCRIPT_DIRECTORY} 7 | -------------------------------------------------------------------------------- /examples/tensorflow/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | docker run \ 5 | -it \ 6 | --rm \ 7 | -p 8888:8888 \ 8 | -v ${SCRIPT_DIRECTORY}/work:/home/jovyan/work \ 9 | -v ${HOME}/.kube:/home/jovyan/.kube \ 10 | -v ${HOME}/.ssh:/home/jovyan/.ssh \ 11 | alphaunito/tensorflow-notebook 12 | -------------------------------------------------------------------------------- /examples/tensorflow/serving-container-details.txt: -------------------------------------------------------------------------------- 1 | + uname -a 2 | Linux tf-serving-tensorflow-serving-59495c9974-7dkr5 4.15.0-121-generic #123-Ubuntu SMP Mon Oct 5 16:16:40 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux 3 | + lscpu 4 | Architecture: x86_64 5 | CPU op-mode(s): 32-bit, 64-bit 6 | Byte Order: Little Endian 7 | CPU(s): 4 8 | On-line CPU(s) list: 0-3 9 | Thread(s) per core: 1 10 | Core(s) per socket: 1 11 | Socket(s): 4 12 | NUMA node(s): 1 13 | Vendor ID: GenuineIntel 14 | CPU family: 6 15 | Model: 85 16 | Model name: Intel Xeon Processor (Skylake, IBRS) 17 | Stepping: 4 18 | CPU MHz: 2100.000 19 | BogoMIPS: 4200.00 20 | Virtualization: VT-x 21 | Hypervisor vendor: KVM 22 | Virtualization type: full 23 | L1d cache: 32K 24 | L1i cache: 32K 25 | L2 cache: 4096K 26 | L3 cache: 16384K 27 | NUMA node0 CPU(s): 0-3 28 | Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single pti ssbd ibrs ibpb tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat pku ospke avx512_vnni md_clear arch_capabilities 29 | + cat /proc/meminfo 30 | MemTotal: 8167548 kB 31 | MemFree: 2558208 kB 32 | MemAvailable: 6901288 kB 33 | Buffers: 157288 kB 34 | Cached: 3961684 kB 35 | SwapCached: 0 kB 36 | Active: 2164624 kB 37 | Inactive: 2812980 kB 38 | Active(anon): 722092 kB 39 | Inactive(anon): 408 kB 40 | Active(file): 1442532 kB 41 | Inactive(file): 2812572 kB 42 | Unevictable: 0 kB 43 | Mlocked: 0 kB 44 | SwapTotal: 0 kB 45 | SwapFree: 0 kB 46 | Dirty: 148 kB 47 | Writeback: 0 kB 48 | AnonPages: 829196 kB 49 | Mapped: 830300 kB 50 | Shmem: 2420 kB 51 | Slab: 548440 kB 52 | SReclaimable: 389424 kB 53 | SUnreclaim: 159016 kB 54 | KernelStack: 14256 kB 55 | PageTables: 12968 kB 56 | NFS_Unstable: 0 kB 57 | Bounce: 0 kB 58 | WritebackTmp: 0 kB 59 | CommitLimit: 4083772 kB 60 | Committed_AS: 5010460 kB 61 | VmallocTotal: 34359738367 kB 62 | VmallocUsed: 0 kB 63 | VmallocChunk: 0 kB 64 | HardwareCorrupted: 0 kB 65 | AnonHugePages: 55296 kB 66 | ShmemHugePages: 0 kB 67 | ShmemPmdMapped: 0 kB 68 | CmaTotal: 0 kB 69 | CmaFree: 0 kB 70 | HugePages_Total: 0 71 | HugePages_Free: 0 72 | HugePages_Rsvd: 0 73 | HugePages_Surp: 0 74 | Hugepagesize: 2048 kB 75 | DirectMap4k: 337776 kB 76 | DirectMap2M: 8050688 kB 77 | DirectMap1G: 2097152 kB 78 | + lsblk -a 79 | NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT 80 | loop0 7:0 0 1 loop 81 | loop1 7:1 0 99.2M 1 loop 82 | loop2 7:2 0 21.3M 1 loop 83 | loop3 7:3 0 8.9M 1 loop 84 | loop4 7:4 0 9.1M 1 loop 85 | loop5 7:5 0 55.5M 1 loop 86 | loop6 7:6 0 98.4M 1 loop 87 | loop7 7:7 0 8.9M 1 loop 88 | loop8 7:8 0 21.3M 1 loop 89 | loop9 7:9 0 55.5M 1 loop 90 | loop10 7:10 0 9.1M 1 loop 91 | loop11 7:11 0 0 loop 92 | vda 252:0 0 30G 0 disk 93 | |-vda1 252:1 0 29.9G 0 part /etc/resolv.conf 94 | |-vda14 252:14 0 4M 0 part 95 | `-vda15 252:15 0 106M 0 part 96 | vdb 252:16 0 8G 0 disk /models 97 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/hpc/ssh_template.jinja2: -------------------------------------------------------------------------------- 1 | cd {{ streamflow_workdir }} && docker run \ 2 | -v /tmp:/tmp \ 3 | --user root \ 4 | --gpus all \ 5 | alphaunito/tensorflow-notebook \ 6 | {{ streamflow_command }} 7 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: tensorflow-serving 3 | description: A Helm chart for TensorFlow Serving 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "2.4.1" 25 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "tensorflow-serving.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 3 | export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[1].containerPort}") 4 | echo "Visit http://127.0.0.1:8080 to use your application" 5 | kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT 6 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "tensorflow-serving.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "tensorflow-serving.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "tensorflow-serving.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "tensorflow-serving.labels" -}} 37 | helm.sh/chart: {{ include "tensorflow-serving.chart" . }} 38 | {{ include "tensorflow-serving.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "tensorflow-serving.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "tensorflow-serving.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "tensorflow-serving.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "tensorflow-serving.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "tensorflow-serving.fullname" . }} 5 | labels: 6 | {{- include "tensorflow-serving.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | {{- include "tensorflow-serving.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | {{- with .Values.podAnnotations }} 15 | annotations: 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | labels: 19 | {{- include "tensorflow-serving.selectorLabels" . | nindent 8 }} 20 | spec: 21 | {{- with .Values.imagePullSecrets }} 22 | imagePullSecrets: 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | securityContext: 26 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 27 | containers: 28 | - name: {{ .Chart.Name }} 29 | securityContext: 30 | {{- toYaml .Values.securityContext | nindent 12 }} 31 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 32 | imagePullPolicy: {{ .Values.image.pullPolicy }} 33 | ports: 34 | - name: tf-serving 35 | containerPort: 8500 36 | - name: tf-serving-api 37 | containerPort: 8501 38 | {{- if .Values.resources }} 39 | resources: 40 | {{- toYaml .Values.resources | nindent 12 }} 41 | {{- end }} 42 | {{- if .Values.persistence.enabled }} 43 | volumeMounts: 44 | - name: {{ include "tensorflow-serving.fullname" . }}-volume 45 | mountPath: /models 46 | {{- end }} 47 | {{- with .Values.nodeSelector }} 48 | nodeSelector: 49 | {{- toYaml . | nindent 8 }} 50 | {{- end }} 51 | {{- with .Values.affinity }} 52 | affinity: 53 | {{- toYaml . | nindent 8 }} 54 | {{- end }} 55 | {{- with .Values.tolerations }} 56 | tolerations: 57 | {{- toYaml . | nindent 8 }} 58 | {{- end }} 59 | {{- if .Values.persistence.enabled }} 60 | volumes: 61 | - name: {{ include "tensorflow-serving.fullname" . }}-volume 62 | persistentVolumeClaim: 63 | claimName: {{ include "tensorflow-serving.fullname" . }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/templates/pvc.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.persistence.enabled }} 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: {{ include "tensorflow-serving.fullname" . }} 6 | labels: {{- include "tensorflow-serving.labels" . | nindent 4 }} 7 | spec: 8 | accessModes: 9 | {{- range .Values.persistence.accessModes }} 10 | - {{ . | quote }} 11 | {{- end }} 12 | resources: 13 | requests: 14 | storage: {{ .Values.persistence.size | quote }} 15 | {{- if .Values.persistence.storageClass }} 16 | storageClassName: {{ .Values.persistence.storageClass }} 17 | {{- end }} 18 | {{- end }} -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "tensorflow-serving.fullname" . }} 5 | labels: 6 | {{- include "tensorflow-serving.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: http 12 | protocol: TCP 13 | name: http 14 | selector: 15 | {{- include "tensorflow-serving.selectorLabels" . | nindent 4 }} 16 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/templates/tests/test-connection.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: "{{ include "tensorflow-serving.fullname" . }}-test-connection" 5 | labels: 6 | {{- include "tensorflow-serving.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": test 9 | spec: 10 | containers: 11 | - name: wget 12 | image: busybox 13 | command: ['wget'] 14 | args: ['{{ include "tensorflow-serving.fullname" . }}:{{ .Values.service.port }}'] 15 | restartPolicy: Never 16 | -------------------------------------------------------------------------------- /examples/tensorflow/work/environment/k8s/tensorflow-serving/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for tensorflow-serving. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | image: 8 | pullPolicy: Always 9 | repository: alphaunito/tensorflow-serving 10 | tag: latest 11 | 12 | imagePullSecrets: [] 13 | nameOverride: "" 14 | fullnameOverride: "" 15 | 16 | persistence: 17 | enabled: true 18 | accessModes: 19 | - ReadWriteOnce 20 | size: 8Gi 21 | storageClass: cdk-cinder 22 | 23 | podAnnotations: {} 24 | 25 | podSecurityContext: {} 26 | 27 | securityContext: {} 28 | # capabilities: 29 | # drop: 30 | # - ALL 31 | # readOnlyRootFilesystem: true 32 | # runAsNonRoot: true 33 | # runAsUser: 1000 34 | 35 | service: 36 | type: ClusterIP 37 | port: 8501 38 | 39 | resources: {} 40 | # We usually recommend not to specify default resources and to leave this as a conscious 41 | # choice for the user. This also increases chances charts run on environments with little 42 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 43 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 44 | # limits: 45 | # cpu: 100m 46 | # memory: 128Mi 47 | # requests: 48 | # cpu: 100m 49 | # memory: 128Mi 50 | 51 | nodeSelector: {} 52 | 53 | tolerations: [] 54 | 55 | affinity: {} 56 | -------------------------------------------------------------------------------- /jupyter_workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/jupyter_workflow/__init__.py -------------------------------------------------------------------------------- /jupyter_workflow/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/jupyter_workflow/client/__init__.py -------------------------------------------------------------------------------- /jupyter_workflow/client/cli.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import pathlib 5 | import sys 6 | from textwrap import dedent 7 | 8 | import nbformat 9 | import traitlets.config 10 | from jupyter_core.application import JupyterApp 11 | 12 | from jupyter_workflow.client.client import WorkflowClient 13 | from jupyter_workflow.version import VERSION 14 | 15 | aliases = { 16 | "timeout": "JupyterWorkflowApp.timeout", 17 | "startup_timeout": "JupyterWorkflowApp.startup_timeout", 18 | } 19 | 20 | 21 | class JupyterWorkflowApp(JupyterApp): 22 | """ 23 | An application used to execute notebook files (``*.ipynb``) as distributed workflows 24 | """ 25 | 26 | version = traitlets.Unicode(VERSION) 27 | name = "jupyter-workflow" 28 | aliases = aliases 29 | 30 | description = "An application used to execute notebook files (``*.ipynb``) as distributed workflows" 31 | notebooks = traitlets.List([], help="Path of notebooks to convert").tag(config=True) 32 | timeout = traitlets.Integer( 33 | None, 34 | allow_none=True, 35 | help=dedent( 36 | """ 37 | The time to wait (in seconds) for output from executions. 38 | If workflow execution takes longer, a TimeoutError is raised. 39 | ``-1`` will disable the timeout. 40 | """ 41 | ), 42 | ).tag(config=True) 43 | startup_timeout = traitlets.Integer( 44 | 60, 45 | help=dedent( 46 | """ 47 | The time to wait (in seconds) for the kernel to start. 48 | If kernel startup takes longer, a RuntimeError is 49 | raised. 50 | """ 51 | ), 52 | ).tag(config=True) 53 | interactive = traitlets.Bool( 54 | False, 55 | help=dedent( 56 | """ 57 | Simulates an interactive notebook execution, executing 58 | cells sequentially from beginning to end instead of 59 | constructing a DAG. Mainly used for testing purposes. 60 | """ 61 | ), 62 | ).tag(config=True) 63 | 64 | @traitlets.default("log_level") 65 | def _log_level_default(self): 66 | return logging.INFO 67 | 68 | @traitlets.config.catch_config_error 69 | def initialize(self, argv=None): 70 | super().initialize(argv) 71 | self.notebooks = self.extra_args or self.notebooks 72 | if not self.notebooks: 73 | sys.exit(-1) 74 | [self.run_notebook(path) for path in self.notebooks] 75 | 76 | def run_notebook(self, notebook_path): 77 | self.log.info(f"Executing {notebook_path}") 78 | name = notebook_path.replace(".ipynb", "") 79 | path = pathlib.Path(notebook_path).parent.absolute() 80 | input_path = f"{name}.ipynb" 81 | with open(input_path) as f: 82 | nb = nbformat.read(f, as_version=4) 83 | client = WorkflowClient( 84 | nb, 85 | timeout=self.timeout, 86 | startup_timeout=self.startup_timeout, 87 | resources={"metadata": {"path": path}}, 88 | ) 89 | if self.interactive: 90 | client.execute() 91 | else: 92 | client.execute_workflow() 93 | 94 | 95 | main = JupyterWorkflowApp.launch_instance 96 | -------------------------------------------------------------------------------- /jupyter_workflow/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/jupyter_workflow/config/__init__.py -------------------------------------------------------------------------------- /jupyter_workflow/config/config.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class SplitType(Enum): 5 | size = 0 6 | num = 1 7 | -------------------------------------------------------------------------------- /jupyter_workflow/config/schema.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from importlib.resources import files 4 | 5 | from streamflow.config import ext_schemas 6 | from streamflow.core.config import Schema 7 | from streamflow.deployment.connector import connector_classes 8 | 9 | 10 | class JfSchema(Schema): 11 | def __init__(self): 12 | super().__init__( 13 | { 14 | "v1.0": "https://jupyter-workflow.di.unito.it/config/schemas/v1.0/config_schema.json" 15 | } 16 | ) 17 | for version in self.configs.keys(): 18 | self.add_schema( 19 | files(__package__) 20 | .joinpath("schemas") 21 | .joinpath(version) 22 | .joinpath("config_schema.json") 23 | .read_text("utf-8") 24 | ) 25 | self.inject_ext(connector_classes, "deployment") 26 | for schema in ext_schemas: 27 | self.add_schema(schema.read_text("utf-8")) 28 | self._registry = self.registry.crawl() 29 | -------------------------------------------------------------------------------- /jupyter_workflow/config/validator.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import MutableMapping 4 | from typing import Any 5 | 6 | from jsonschema.validators import validator_for 7 | from streamflow.core.exception import WorkflowDefinitionException 8 | 9 | from jupyter_workflow.config.schema import JfSchema 10 | 11 | 12 | def handle_errors(errors): 13 | errors = list(sorted(errors, key=str)) 14 | if not errors: 15 | return 16 | raise WorkflowDefinitionException(f"Invalid metadata:\n{errors[0]}") 17 | 18 | 19 | def validate(workflow_config: MutableMapping[str, Any]) -> None: 20 | if "version" not in workflow_config: 21 | raise WorkflowDefinitionException( 22 | "Required field `version` not found in workflow configuration." 23 | ) 24 | schema = JfSchema() 25 | config = schema.get_config(workflow_config["version"]).contents 26 | cls = validator_for(config) 27 | validator = cls(config, registry=schema.registry) 28 | handle_errors(validator.iter_errors(workflow_config)) 29 | -------------------------------------------------------------------------------- /jupyter_workflow/ipython/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/jupyter_workflow/ipython/__init__.py -------------------------------------------------------------------------------- /jupyter_workflow/ipython/__main__.py: -------------------------------------------------------------------------------- 1 | from ipykernel.kernelapp import IPKernelApp 2 | 3 | from jupyter_workflow.ipython.ipkernel import WorkflowIPythonKernel 4 | 5 | IPKernelApp.launch_instance( 6 | kernel_class=WorkflowIPythonKernel, 7 | outstream_class="jupyter_workflow.ipython.iostream.WorkflowOutStream", 8 | ) 9 | -------------------------------------------------------------------------------- /jupyter_workflow/ipython/displayhook.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import MutableMapping 4 | from contextvars import ContextVar 5 | from functools import partial 6 | 7 | from ipykernel.displayhook import ZMQShellDisplayHook 8 | from ipykernel.zmqshell import ZMQDisplayPublisher 9 | 10 | 11 | def add_cell_id_hook(msg: MutableMapping, cell_id: ContextVar): 12 | msg["content"]["metadata"]["cell_id"] = cell_id.get() 13 | return msg 14 | 15 | 16 | class StreamFlowDisplayPublisher(ZMQDisplayPublisher): 17 | def __init__(self, *args, **kwargs): 18 | super().__init__(*args, **kwargs) 19 | self._parent_headers: MutableMapping[str, MutableMapping] = {} 20 | self.cell_id: ContextVar[str] = ContextVar("cell_id", default="") 21 | self.register_hook(partial(add_cell_id_hook, cell_id=self.cell_id)) 22 | 23 | def delete_parent(self, parent): 24 | if "workflow" in parent["content"]: 25 | self.set_cell_id(parent["content"]["workflow"].get("cell_id", "")) 26 | del self.parent_header 27 | 28 | @property 29 | def parent_header(self): 30 | return self._parent_headers.get(self.cell_id.get(), {}) 31 | 32 | @parent_header.setter 33 | def parent_header(self, header: MutableMapping): 34 | self._parent_headers[self.cell_id.get()] = header 35 | 36 | @parent_header.deleter 37 | def parent_header(self): 38 | self._parent_headers.pop(self.cell_id.get()) 39 | 40 | def set_cell_id(self, cell_id: str): 41 | self.cell_id.set(cell_id) 42 | 43 | def set_parent(self, parent): 44 | if "workflow" in parent["content"]: 45 | self.set_cell_id(parent["content"]["workflow"].get("cell_id", "")) 46 | super().set_parent(parent) 47 | 48 | 49 | class StreamFlowShellDisplayHook(ZMQShellDisplayHook): 50 | def __init__(self, **kwargs): 51 | super().__init__(**kwargs) 52 | self._parent_headers: MutableMapping[str, MutableMapping] = {} 53 | self.cell_id: ContextVar[str] = ContextVar("cell_id", default="") 54 | 55 | def delete_parent(self, parent): 56 | if "workflow" in parent["content"]: 57 | self.set_cell_id(parent["content"]["workflow"].get("cell_id", "")) 58 | del self.parent_header 59 | 60 | @property 61 | def parent_header(self): 62 | return self._parent_headers[self.cell_id.get()] 63 | 64 | @parent_header.setter 65 | def parent_header(self, header: MutableMapping): 66 | self._parent_headers[self.cell_id.get()] = header 67 | 68 | @parent_header.deleter 69 | def parent_header(self): 70 | del self._parent_headers[self.cell_id.get()] 71 | 72 | def set_cell_id(self, cell_id: str): 73 | self.cell_id.set(cell_id) 74 | 75 | def set_parent(self, parent): 76 | if "workflow" in parent["content"]: 77 | self.set_cell_id(parent["content"]["workflow"].get("cell_id", "")) 78 | super().set_parent(parent) 79 | 80 | def write_format_data(self, format_dict, md_dict=None): 81 | if not md_dict: 82 | md_dict = {} 83 | md_dict["cell_id"] = self.cell_id.get() 84 | super().write_format_data(format_dict, md_dict) 85 | -------------------------------------------------------------------------------- /jupyter_workflow/ipython/install.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import argparse 4 | import json 5 | import os 6 | import shutil 7 | import sys 8 | from tempfile import TemporaryDirectory 9 | 10 | from jupyter_client.kernelspec import KernelSpecManager 11 | 12 | kernel_json = { 13 | "argv": [ 14 | sys.executable, 15 | "-m", 16 | "jupyter_workflow.ipython", 17 | "-f", 18 | "{connection_file}", 19 | ], 20 | "display_name": "Jupyter Workflow", 21 | "language": "python", 22 | } 23 | 24 | 25 | def install_kernel_spec(user=True, prefix=None): 26 | with TemporaryDirectory() as td: 27 | with open(os.path.join(td, "kernel.json"), "w") as f: 28 | json.dump(kernel_json, f, sort_keys=True) 29 | kernel_js = os.path.join(os.path.dirname(__file__), "kernelspec", "kernel.js") 30 | shutil.copy(kernel_js, td) 31 | 32 | print("Installing Jupyter kernel spec") 33 | KernelSpecManager().install_kernel_spec( 34 | td, "jupyter-workflow", user=user, prefix=prefix 35 | ) 36 | 37 | 38 | def _is_root(): 39 | try: 40 | return os.geteuid() == 0 41 | except AttributeError: 42 | return False # assume not an admin on non-Unix platforms 43 | 44 | 45 | def main(argv=None): 46 | ap = argparse.ArgumentParser() 47 | ap.add_argument( 48 | "--user", 49 | action="store_true", 50 | help="Install to the per-user kernels registry. Default if not root.", 51 | ) 52 | ap.add_argument( 53 | "--sys-prefix", 54 | action="store_true", 55 | help="Install to sys.prefix (e.g. a virtualenv or conda env)", 56 | ) 57 | ap.add_argument( 58 | "--prefix", 59 | help="Install to the given prefix. " 60 | "Kernelspec will be installed in {PREFIX}/share/jupyter/kernels/", 61 | ) 62 | args = ap.parse_args(argv) 63 | 64 | if args.sys_prefix: 65 | args.prefix = sys.prefix 66 | if not args.prefix and not _is_root(): 67 | args.user = True 68 | 69 | install_kernel_spec(user=args.user, prefix=args.prefix) 70 | 71 | 72 | if __name__ == "__main__": 73 | main() 74 | -------------------------------------------------------------------------------- /jupyter_workflow/ipython/iostream.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import MutableMapping 4 | from contextvars import ContextVar 5 | from functools import partial 6 | from io import StringIO 7 | from typing import cast 8 | 9 | from ipykernel.iostream import IOPubThread, OutStream 10 | 11 | 12 | def add_cell_id_hook(msg: MutableMapping, cell_id: ContextVar): 13 | msg["content"]["metadata"]["cell_id"] = cell_id.get() 14 | return msg 15 | 16 | 17 | class CellAwareIOPubThreadWrapper: 18 | def __init__(self, pub_thread: IOPubThread, cell_id: ContextVar[str]): 19 | self.pub_thread: IOPubThread = pub_thread 20 | self.cell_id: ContextVar[str] = cell_id 21 | 22 | def schedule(self, f): 23 | if f.__name__ == "_flush": 24 | self.pub_thread.schedule(partial(f, cell_id=self.cell_id.get())) 25 | else: 26 | self.pub_thread.schedule(f) 27 | 28 | def __getattr__(self, item): 29 | return getattr(self.pub_thread, item) 30 | 31 | 32 | class WorkflowOutStream(OutStream): 33 | def __init__(self, *args, **kwargs): 34 | self._buffer_dict: MutableMapping[str, StringIO] = {} 35 | self._parent_headers: MutableMapping[str, MutableMapping] = {} 36 | self.cell_id: ContextVar[str] = ContextVar("cell_id", default="") 37 | super().__init__(*args, **kwargs) 38 | self.pub_thread = CellAwareIOPubThreadWrapper( 39 | cast(IOPubThread, self.pub_thread), self.cell_id 40 | ) 41 | self.register_hook(partial(add_cell_id_hook, cell_id=self.cell_id)) 42 | 43 | @property 44 | def _buffer(self): 45 | return self._buffer_dict[self.cell_id.get()] 46 | 47 | @_buffer.setter 48 | def _buffer(self, stream: StringIO): 49 | self._buffer_dict[self.cell_id.get()] = stream 50 | 51 | @_buffer.deleter 52 | def _buffer(self): 53 | del self._buffer_dict[self.cell_id.get()] 54 | 55 | def _flush(self, cell_id: str = None): 56 | if cell_id: 57 | self.set_cell_id(cell_id) 58 | super()._flush() 59 | 60 | def delete_parent(self, parent): 61 | if "workflow" in parent["content"]: 62 | self.set_cell_id(parent["content"]["workflow"].get("cell_id", "")) 63 | del self.parent_header 64 | 65 | @property 66 | def parent_header(self): 67 | return self._parent_headers.get(self.cell_id.get(), {}) 68 | 69 | @parent_header.setter 70 | def parent_header(self, header: MutableMapping): 71 | self._parent_headers[self.cell_id.get()] = header 72 | 73 | @parent_header.deleter 74 | def parent_header(self): 75 | self._parent_headers.pop(self.cell_id.get()) 76 | 77 | def set_cell_id(self, cell_id: str): 78 | self.cell_id.set(cell_id) 79 | if cell_id not in self._buffer_dict: 80 | self._buffer_dict[cell_id] = StringIO() 81 | 82 | def set_parent(self, parent): 83 | if "workflow" in parent["content"]: 84 | self.set_cell_id(parent["content"]["workflow"].get("cell_id", "")) 85 | super().set_parent(parent) 86 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/jupyter_workflow/streamflow/__init__.py -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/executor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import argparse 4 | import ast 5 | import asyncio 6 | import builtins 7 | import codeop 8 | import inspect 9 | import io 10 | import json 11 | import sys 12 | import traceback 13 | from collections.abc import MutableMapping 14 | from contextlib import contextmanager, redirect_stderr, redirect_stdout 15 | from tempfile import NamedTemporaryFile 16 | from typing import Any, cast 17 | 18 | CELL_OUTPUT = "__JF_CELL_OUTPUT__" 19 | CELL_STATUS = "__JF_CELL_STATUS__" 20 | CELL_LOCAL_NS = "__JF_CELL_LOCAL_NS__" 21 | 22 | try: 23 | import cloudpickle as pickle 24 | except ImportError: 25 | import pickle # nosec 26 | 27 | # Define args parser 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument("code_file", metavar="JF_CELL_CODE") 30 | parser.add_argument("output_file", metavar="JF_CELL_OUTPUT") 31 | parser.add_argument("--autoawait", action="store_true") 32 | parser.add_argument("--local-ns-file", nargs="?") 33 | parser.add_argument("--postload-input-serializers", nargs="?") 34 | parser.add_argument("--predump-output-serializers", nargs="?") 35 | parser.add_argument("--output-name", action="append") 36 | parser.add_argument("--tmpdir", nargs="?") 37 | 38 | 39 | def _deserialize(path, default=None): 40 | if path is not None: 41 | with open(path, "rb") as f: 42 | return pickle.load(f) 43 | else: 44 | return default 45 | 46 | 47 | def _serialize(compiler, namespace, args): 48 | if args.predump_output_serializers: 49 | predump_output_serializers = _deserialize(args.predump_output_serializers) 50 | namespace = { 51 | k: predump( 52 | compiler=compiler, 53 | name=k, 54 | value=v, 55 | serializer=predump_output_serializers.get(k), 56 | ) 57 | for k, v in namespace.items() 58 | if k in args.output_name 59 | } 60 | else: 61 | namespace = {k: v for k, v in namespace.items() if k in args.output_name} 62 | with NamedTemporaryFile(dir=args.tmpdir, delete=False) as f: 63 | pickle.dump(namespace, f) 64 | return f.name 65 | 66 | 67 | def compare(code_obj): 68 | is_async = inspect.CO_COROUTINE & code_obj.co_flags == inspect.CO_COROUTINE 69 | return is_async 70 | 71 | 72 | def postload(compiler, name, value, serializer): 73 | if serializer is not None and "postload" in serializer: 74 | serialization_context = prepare_ns({"x": value}) 75 | postload_module = compiler.ast_parse( 76 | source=serializer["postload"], filename=f"{name}.postload" 77 | ) 78 | for node in postload_module.body: 79 | mod = ast.Module([node], []) 80 | code_obj = compiler(mod, "", "exec") # nosec 81 | exec(code_obj, {}, serialization_context) # nosec 82 | return serialization_context["y"] 83 | else: 84 | return value 85 | 86 | 87 | def predump(compiler, name, value, serializer): 88 | if serializer is not None and "predump" in serializer: 89 | serialization_context = prepare_ns({"x": value}) 90 | predump_module = compiler.ast_parse( 91 | source=serializer["predump"], filename=f"{name}.predump" 92 | ) 93 | for node in predump_module.body: 94 | mod = ast.Module([node], []) 95 | code_obj = compiler(mod, "", "exec") # nosec 96 | exec(code_obj, {}, serialization_context) # nosec 97 | return serialization_context["y"] 98 | else: 99 | return value 100 | 101 | 102 | class RemoteCompiler(codeop.Compile): 103 | def ast_parse(self, source, filename="", symbol="exec"): 104 | return compile(source, filename, symbol, self.flags | ast.PyCF_ONLY_AST, 1) 105 | 106 | @contextmanager 107 | def extra_flags(self, flags): 108 | turn_on_bits = ~self.flags & flags 109 | self.flags = self.flags | flags 110 | try: 111 | yield 112 | finally: 113 | self.flags &= ~turn_on_bits 114 | 115 | 116 | class RemoteDisplayHook: 117 | def __init__(self, displayhook): 118 | self.displayhook = displayhook 119 | self.out_var = io.StringIO() 120 | 121 | def __call__(self, obj): 122 | with redirect_stdout(self.out_var), redirect_stderr(self.out_var): 123 | self.displayhook(obj) 124 | 125 | 126 | def prepare_ns(namespace: dict) -> dict: 127 | namespace.setdefault("__name__", "__main__") 128 | namespace.setdefault("__builtin__", builtins) 129 | namespace.setdefault("__builtins__", builtins) 130 | if "get_ipython" in locals(): 131 | namespace.setdefault("get_ipython", locals()["get_ipython"]) 132 | elif "get_ipython" in globals(): 133 | namespace.setdefault("get_ipython", globals()["get_ipython"]) 134 | return namespace 135 | 136 | 137 | async def run_ast_nodes( 138 | ast_nodes: list[tuple[ast.AST, str]], 139 | autoawait: bool, 140 | compiler: codeop.Compile, 141 | user_ns: MutableMapping[str, Any], 142 | ): 143 | for node, mode in ast_nodes: 144 | if mode == "exec": 145 | mod = ast.Module([node], []) 146 | elif mode == "single": 147 | mod = ast.Interactive([node]) 148 | with compiler.extra_flags( 149 | getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0) if autoawait else 0x0 150 | ): 151 | code_obj = compiler(mod, "", mode) 152 | asynchronous = compare(code_obj) 153 | if asynchronous: 154 | await eval(code_obj, cast(dict[str, Any], user_ns), user_ns) # nosec 155 | else: 156 | exec(code_obj, cast(dict[str, Any], user_ns), user_ns) # nosec 157 | 158 | 159 | async def run_code(args): 160 | output = {CELL_LOCAL_NS: {}} 161 | command_output = io.StringIO() 162 | try: 163 | # Instantiate compiler 164 | compiler = RemoteCompiler() 165 | # Deserialize elements 166 | ast_nodes = _deserialize(args.code_file) 167 | user_ns = prepare_ns(_deserialize(args.local_ns_file, {})) 168 | # Apply postload serialization hooks if present 169 | if args.postload_input_serializers: 170 | postload_input_serializers = _deserialize(args.postload_input_serializers) 171 | user_ns = { 172 | k: postload( 173 | compiler=compiler, 174 | name=k, 175 | value=v, 176 | serializer=postload_input_serializers.get(k), 177 | ) 178 | for k, v in user_ns.items() 179 | } 180 | if "get_ipython" in user_ns: 181 | user_ns["get_ipython"]().user_ns = user_ns 182 | # Exec cell code 183 | with redirect_stdout(command_output), redirect_stderr(command_output): 184 | sys.displayhook = RemoteDisplayHook(sys.displayhook) 185 | await run_ast_nodes(ast_nodes, args.autoawait, compiler, user_ns) 186 | # Populate output object 187 | output[CELL_OUTPUT] = command_output.getvalue().strip() 188 | if "Out" not in user_ns: 189 | user_ns["Out"] = {} 190 | user_ns["Out"] = sys.displayhook.out_var.getvalue().strip() 191 | if args.output_name: 192 | output[CELL_LOCAL_NS] = _serialize(compiler, user_ns, args) 193 | else: 194 | output[CELL_LOCAL_NS] = "" 195 | output[CELL_STATUS] = "COMPLETED" 196 | except Exception: 197 | # Populate output object 198 | output[CELL_OUTPUT] = command_output.getvalue().strip() 199 | if output[CELL_OUTPUT]: 200 | output[CELL_OUTPUT] = ( 201 | output[CELL_OUTPUT] + "\n" + traceback.format_exc().strip() 202 | ) 203 | else: 204 | output[CELL_OUTPUT] = traceback.format_exc().strip() 205 | output[CELL_STATUS] = "FAILED" 206 | finally: 207 | # Save output json to file 208 | with open(args.output_file, "w") as f: 209 | f.write(json.dumps(output)) 210 | 211 | 212 | def main(args): 213 | # Load arguments 214 | args = parser.parse_args(args) 215 | # Run code asynchronously 216 | asyncio.run(run_code(args)) 217 | 218 | 219 | if __name__ == "__main__": 220 | main(sys.argv[1:]) 221 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/port.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import MutableMapping 4 | from typing import Any 5 | 6 | from streamflow.core.workflow import Port 7 | from streamflow.workflow.token import TerminationToken 8 | 9 | from jupyter_workflow.streamflow.token import ProgramContextToken 10 | 11 | 12 | class ProgramContextPort(Port): 13 | async def get_context(self, consumer: str) -> MutableMapping[str, Any] | None: 14 | token = await self.get(consumer) 15 | if isinstance(token, TerminationToken): 16 | return None 17 | else: 18 | return token.value 19 | 20 | def put_context(self, context: MutableMapping[str, Any]): 21 | self.put(ProgramContextToken(value=context)) 22 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/processor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from streamflow.core.command import CommandOutputProcessor 4 | from streamflow.core.deployment import Connector, Target 5 | from streamflow.core.workflow import Job, Token, Workflow 6 | 7 | from jupyter_workflow.streamflow import utils 8 | from jupyter_workflow.streamflow.command import JupyterCommandOutput 9 | 10 | 11 | class JupyterFileCommandOutputProcessor(CommandOutputProcessor): 12 | def __init__( 13 | self, 14 | name: str, 15 | workflow: Workflow, 16 | target: Target | None = None, 17 | value: str | None = None, 18 | value_from: str | None = None, 19 | ): 20 | super().__init__(name, workflow, target) 21 | self.value: str | None = value 22 | self.value_from: str | None = value_from 23 | 24 | async def process( 25 | self, 26 | job: Job, 27 | command_output: JupyterCommandOutput, 28 | connector: Connector | None = None, 29 | ) -> Token | None: 30 | return await utils.get_file_token_from_ns( 31 | context=self.workflow.context, 32 | connector=self._get_connector(connector, job), 33 | job=job, 34 | locations=await self._get_locations(connector, job), 35 | output_directory=( 36 | self.target.workdir if self.target else job.output_directory 37 | ), 38 | user_ns=command_output.user_ns, 39 | value=self.value, 40 | value_from=self.value_from, 41 | ) 42 | 43 | 44 | class JupyterNameCommandOutputProcessor(CommandOutputProcessor): 45 | def __init__( 46 | self, 47 | name: str, 48 | workflow: Workflow, 49 | value: str | None = None, 50 | value_from: str | None = None, 51 | ): 52 | super().__init__(name, workflow) 53 | self.value: str | None = value 54 | self.value_from: str | None = value_from 55 | 56 | async def process( 57 | self, 58 | job: Job, 59 | command_output: JupyterCommandOutput, 60 | connector: Connector | None = None, 61 | ) -> Token | None: 62 | return utils.get_token_from_ns( 63 | job=job, 64 | user_ns=command_output.user_ns, 65 | value=self.value, 66 | value_from=self.value_from, 67 | ) 68 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/token.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | from collections.abc import MutableMapping, MutableSequence 5 | from typing import Any 6 | 7 | from cloudpickle import dumps, loads 8 | from streamflow.core.context import StreamFlowContext 9 | from streamflow.core.data import DataType 10 | from streamflow.core.exception import UnrecoverableTokenException 11 | from streamflow.core.persistence import DatabaseLoadingContext 12 | from streamflow.core.workflow import Token 13 | from streamflow.data.remotepath import StreamFlowPath 14 | from streamflow.workflow.token import FileToken 15 | from streamflow.workflow.utils import get_token_value 16 | 17 | 18 | async def _get_file_token_weight( 19 | context: StreamFlowContext, paths: MutableSequence[str] 20 | ): 21 | weight = 0 22 | for path in paths: 23 | data_locations = context.data_manager.get_data_locations( 24 | path=path, data_type=DataType.PRIMARY 25 | ) 26 | if data_locations: 27 | sf_path = StreamFlowPath( 28 | path, context=context, location=next(iter(data_locations)).location 29 | ) 30 | weight += await (await sf_path.resolve()).size() 31 | return weight 32 | 33 | 34 | class JupyterFileToken(FileToken): 35 | async def get_paths(self, context: StreamFlowContext) -> MutableSequence[str]: 36 | value = get_token_value(self) 37 | return [value] 38 | 39 | async def get_weight(self, context): 40 | return await _get_file_token_weight(context, await self.get_paths(context)) 41 | 42 | 43 | class JupyterToken(Token): 44 | @classmethod 45 | async def _load( 46 | cls, 47 | context: StreamFlowContext, 48 | row: MutableMapping[str, Any], 49 | loading_context: DatabaseLoadingContext, 50 | ) -> Token: 51 | value = json.loads(row["value"]) 52 | if isinstance(value, MutableMapping) and "token" in value: 53 | value = await loading_context.load_token(context, value["token"]) 54 | else: 55 | value = loads(bytes.fromhex(value)) 56 | return cls(tag=row["tag"], value=value) 57 | 58 | async def _save_value(self, context: StreamFlowContext): 59 | return ( 60 | {"token": self.value.persistent_id} 61 | if isinstance(self.value, Token) 62 | else dumps(self.value).hex() 63 | ) 64 | 65 | 66 | class ProgramContextToken(Token): 67 | @classmethod 68 | async def _load( 69 | cls, 70 | context: StreamFlowContext, 71 | row: MutableMapping[str, Any], 72 | loading_context: DatabaseLoadingContext, 73 | ) -> Token: 74 | value = json.loads(row["value"]) 75 | if isinstance(value, MutableMapping) and "token" in value: 76 | value = await loading_context.load_token(context, value["token"]) 77 | else: 78 | names = globals() 79 | for name in value: 80 | if name in names: 81 | value = names[name] 82 | else: 83 | raise UnrecoverableTokenException( 84 | f"Variable {name} cannot be restored from the current program context." 85 | ) 86 | return cls(tag=row["tag"], value=value) 87 | 88 | async def _save_value(self, context: StreamFlowContext): 89 | return list(self.value.keys()) 90 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/transformer.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import ast 4 | import json 5 | from collections.abc import MutableMapping, MutableSequence 6 | from typing import Any, cast 7 | 8 | from streamflow.core.context import StreamFlowContext 9 | from streamflow.core.exception import WorkflowExecutionException 10 | from streamflow.core.persistence import DatabaseLoadingContext 11 | from streamflow.core.workflow import Token, Workflow 12 | from streamflow.workflow.token import ListToken 13 | from streamflow.workflow.transformer import OneToOneTransformer 14 | from streamflow.workflow.utils import get_token_value 15 | 16 | from jupyter_workflow.config.config import SplitType 17 | from jupyter_workflow.streamflow.token import JupyterToken 18 | 19 | 20 | def _flatten_list( 21 | token_list: MutableSequence[Token], buffer: MutableSequence[Token] 22 | ) -> MutableSequence[Token]: 23 | for token in token_list: 24 | if isinstance(token, ListToken): 25 | return _flatten_list(token.value, buffer) 26 | else: 27 | buffer.append(token) 28 | return buffer 29 | 30 | 31 | class ListJoinTransformer(OneToOneTransformer): 32 | def _transform(self, token: Token): 33 | if isinstance(token, ListToken): 34 | value = [] 35 | for t in token.value: 36 | value.extend(t.value) 37 | return token.update(value) 38 | else: 39 | raise WorkflowExecutionException( 40 | self.name + " step must receive a list input." 41 | ) 42 | 43 | async def transform( 44 | self, inputs: MutableMapping[str, Token] 45 | ) -> MutableMapping[str, Token]: 46 | return {self.get_output_name(): self._transform(next(iter(inputs.values())))} 47 | 48 | 49 | class MakeListTransformer(OneToOneTransformer): 50 | def __init__( 51 | self, name: str, split_size: int, split_type: SplitType, workflow: Workflow 52 | ): 53 | super().__init__(name, workflow) 54 | self.split_type: SplitType = split_type 55 | self.split_size: int = split_size 56 | 57 | @classmethod 58 | async def _load( 59 | cls, 60 | context: StreamFlowContext, 61 | row: MutableMapping[str, Any], 62 | loading_context: DatabaseLoadingContext, 63 | ): 64 | params = json.loads(row["params"]) 65 | return cls( 66 | name=row["name"], 67 | workflow=await loading_context.load_workflow(context, row["workflow"]), 68 | split_type=SplitType[params["split_type"]], 69 | split_size=params["split_size"], 70 | ) 71 | 72 | async def _save_additional_params( 73 | self, context: StreamFlowContext 74 | ) -> MutableMapping[str, Any]: 75 | return cast(dict, await super()._save_additional_params(context)) | { 76 | "split_type": self.split_type.name, 77 | "split_size": self.split_size, 78 | } 79 | 80 | def _transform(self, token: Token): 81 | token_list = [] 82 | token_value = get_token_value(token) 83 | size = len(token_value) 84 | if self.split_type == SplitType.size: 85 | for i in range(0, size, self.split_size): 86 | token_list.append( 87 | JupyterToken( 88 | tag=token.tag, 89 | value=token_value[i : min(i + self.split_size, size)], 90 | ) 91 | ) 92 | else: 93 | d, r = divmod(size, self.split_size) 94 | for i in range(self.split_size): 95 | si = (d + 1) * (i if i < r else r) + d * (0 if i < r else i - r) 96 | token_list.append( 97 | JupyterToken( 98 | tag=token.tag, 99 | value=token_value[si : si + (d + 1 if i < r else d)], 100 | ) 101 | ) 102 | return ListToken(tag=token.tag, value=token_list) 103 | 104 | async def transform( 105 | self, inputs: MutableMapping[str, Token] 106 | ) -> MutableMapping[str, Token]: 107 | return {self.get_output_name(): self._transform(next(iter(inputs.values())))} 108 | 109 | 110 | class OutputJoinTransformer(OneToOneTransformer): 111 | def _transform(self, token: Token): 112 | if isinstance(token, ListToken): 113 | output = [] 114 | is_list = False 115 | for value in [t.value for t in token.value]: 116 | try: 117 | value = ast.literal_eval(value) 118 | except (SyntaxError, ValueError): 119 | pass 120 | if isinstance(value, MutableSequence): 121 | is_list = True 122 | output.extend(value) 123 | else: 124 | output.append(value) 125 | if is_list: 126 | return Token( 127 | tag=token.tag, 128 | value="[{}]".format(", ".join([str(v) for v in output])), 129 | ) 130 | else: 131 | return Token( 132 | tag=token.tag, 133 | value="\n".join([str(v) for v in output if str(v) != ""]), 134 | ) 135 | else: 136 | raise WorkflowExecutionException( 137 | self.name + " step must receive a list input." 138 | ) 139 | 140 | async def transform( 141 | self, inputs: MutableMapping[str, Token] 142 | ) -> MutableMapping[str, Token]: 143 | return {self.get_output_name(): self._transform(next(iter(inputs.values())))} 144 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import builtins 5 | import posixpath 6 | from collections.abc import MutableMapping, MutableSequence 7 | from typing import Any 8 | 9 | from streamflow.core import utils 10 | from streamflow.core.context import StreamFlowContext 11 | from streamflow.core.deployment import Connector, ExecutionLocation, Target 12 | from streamflow.core.workflow import Job, Token, Workflow 13 | from streamflow.data.remotepath import StreamFlowPath 14 | from streamflow.deployment.utils import get_path_processor 15 | from streamflow.workflow.step import DeployStep 16 | from streamflow.workflow.token import ListToken 17 | 18 | from jupyter_workflow.streamflow.token import JupyterFileToken, JupyterToken 19 | 20 | 21 | async def _register_path(context: StreamFlowContext, job: Job, path: str): 22 | connector = context.scheduler.get_connector(job.name) 23 | path_processor = get_path_processor(connector) 24 | locations = context.scheduler.get_locations(job.name) 25 | relpath = ( 26 | path_processor.relpath(path, job.output_directory) 27 | if path.startswith(job.output_directory) 28 | else path_processor.basename(path) 29 | ) 30 | for location in locations: 31 | if await StreamFlowPath(path, context=context, location=location).exists(): 32 | context.data_manager.register_path( 33 | location=location, path=path, relpath=relpath 34 | ) 35 | 36 | 37 | def get_deploy_step( 38 | deployment_map: MutableMapping[str, DeployStep], target: Target, workflow: Workflow 39 | ): 40 | if target.deployment.name not in deployment_map: 41 | deployment_map[target.deployment.name] = workflow.create_step( 42 | cls=DeployStep, 43 | name=posixpath.join("__deploy__", target.deployment.name), 44 | deployment_config=target.deployment, 45 | ) 46 | return deployment_map[target.deployment.name] 47 | 48 | 49 | async def get_file_token_from_ns( 50 | context: StreamFlowContext, 51 | connector: Connector, 52 | job: Job, 53 | locations: MutableSequence[ExecutionLocation], 54 | output_directory: str, 55 | user_ns: MutableMapping[str, Any], 56 | value: Any, 57 | value_from: str, 58 | ) -> Token: 59 | path_processor = get_path_processor(connector) 60 | if value is not None: 61 | paths = [] 62 | for location in locations: 63 | outdir = StreamFlowPath( 64 | output_directory, context=context, location=location 65 | ) 66 | paths.extend([str(p) async for p in outdir.glob(value)]) 67 | await asyncio.gather( 68 | *( 69 | asyncio.create_task(_register_path(context=context, job=job, path=p)) 70 | for p in paths 71 | ) 72 | ) 73 | value = paths[0] if len(paths) == 1 else paths 74 | else: 75 | value = user_ns.get(value_from) 76 | if not path_processor.isabs(value): 77 | value = path_processor.join(output_directory, value) 78 | await _register_path(context=context, job=job, path=value) 79 | if isinstance(value, MutableSequence): 80 | return ListToken( 81 | tag=utils.get_tag(job.inputs.values()), 82 | value=[JupyterFileToken(value=v) for v in value], 83 | ) 84 | else: 85 | return JupyterFileToken(tag=utils.get_tag(job.inputs.values()), value=value) 86 | 87 | 88 | def get_token_from_ns( 89 | job: Job, user_ns: MutableMapping[str, Any], value: Any, value_from: str 90 | ) -> Token: 91 | value = ( 92 | value 93 | if value is not None 94 | else user_ns.get(value_from, builtins.__dict__.get(value_from)) 95 | ) 96 | if isinstance(value, MutableSequence): 97 | return ListToken( 98 | tag=utils.get_tag(job.inputs.values()), 99 | value=[JupyterToken(value=v) for v in value], 100 | ) 101 | else: 102 | return JupyterToken(tag=utils.get_tag(job.inputs.values()), value=value) 103 | -------------------------------------------------------------------------------- /jupyter_workflow/streamflow/workflow.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import MutableMapping 4 | from typing import Any 5 | 6 | from streamflow.core.context import StreamFlowContext 7 | from streamflow.core.workflow import Workflow 8 | 9 | 10 | class JupyterWorkflow(Workflow): 11 | def __init__( 12 | self, 13 | context: StreamFlowContext, 14 | config: MutableMapping[str, Any], 15 | name: str = None, 16 | ): 17 | super().__init__(context, config, name) 18 | self.type: str = "jupyter" 19 | -------------------------------------------------------------------------------- /jupyter_workflow/version.py: -------------------------------------------------------------------------------- 1 | VERSION = "0.1.0.dev4" 2 | -------------------------------------------------------------------------------- /lint-requirements.txt: -------------------------------------------------------------------------------- 1 | black==25.1.0 2 | codespell==2.4.1 3 | flake8-bugbear==24.12.12 4 | isort==6.0.1 5 | pyupgrade==3.20.0 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "jupyter-workflow" 7 | authors = [ 8 | {name = "Iacopo Colonnelli", email = "iacopo.colonnelli@unito.it"} 9 | ] 10 | description = "Jupyter Workflow Kernel" 11 | readme = "README.md" 12 | requires-python = ">=3.9" 13 | license = {text = "LGPL-3.0-or-later"} 14 | classifiers = [ 15 | "Development Status :: 3 - Alpha", 16 | "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", 17 | "Intended Audience :: Developers", 18 | "Intended Audience :: Science/Research", 19 | "Operating System :: POSIX", 20 | "Operating System :: MacOS", 21 | "Programming Language :: Python", 22 | "Programming Language :: Python :: 3 :: Only", 23 | "Programming Language :: Python :: 3.9", 24 | "Programming Language :: Python :: 3.10", 25 | "Programming Language :: Python :: 3.11", 26 | "Programming Language :: Python :: 3.12", 27 | "Programming Language :: Python :: 3.13", 28 | "Topic :: Scientific/Engineering", 29 | "Topic :: System :: Distributed Computing" 30 | ] 31 | dynamic = ["dependencies", "optional-dependencies", "version"] 32 | 33 | [project.scripts] 34 | jupyter-workflow = "jupyter_workflow.client.cli:main" 35 | 36 | [project.urls] 37 | Homepage = "https://jupyter-workflow.di.unito.it" 38 | Package = "https://pypi.org/project/jupyter-workflow" 39 | Repository = "https://github.com/alpha-unito/jupyter-workflow" 40 | 41 | [tool.setuptools] 42 | packages = [ 43 | "jupyter_workflow", 44 | "jupyter_workflow.client", 45 | "jupyter_workflow.config", 46 | "jupyter_workflow.ipython", 47 | "jupyter_workflow.streamflow", 48 | ] 49 | zip-safe = true 50 | 51 | [tool.setuptools.package-data] 52 | "jupyter_workflow.config" = ["schemas/v1.0/*.json"] 53 | "jupyter_workflow.ipython" = ["kernelspec/kernel.js"] 54 | 55 | [tool.setuptools.dynamic] 56 | dependencies = {file = "requirements.txt"} 57 | version = {attr = "jupyter_workflow.version.VERSION"} 58 | 59 | [tool.setuptools.dynamic.optional-dependencies] 60 | bandit = {file = "bandit-requirements.txt"} 61 | docs = {file = "docs/requirements.txt"} 62 | lint = {file = "lint-requirements.txt"} 63 | test = {file = "test-requirements.txt"} 64 | 65 | [tool.codespell] 66 | ignore-words-list = "inout" 67 | 68 | [tool.coverage.paths] 69 | executor = ["jupyter_workflow/streamflow/executor.py", "/tmp/streamflow/*/executor.py"] 70 | 71 | [tool.coverage.run] 72 | branch = true 73 | source_pkgs = ["jupyter_workflow"] 74 | 75 | [tool.coverage.report] 76 | exclude_lines = [ 77 | # Exclude not implemented methods 78 | "raise NotImplementedError", 79 | # Exclude abstract methods 80 | "@(abc\\.)?abstractmethod", 81 | # Exclude Python script entrypoints 82 | "if __name__ == .__main__.:", 83 | # Exclude type checking lines 84 | "if TYPE_CHECKING:", 85 | # Exclude log messages 86 | "if logger.isEnabledFor" 87 | ] 88 | ignore_errors = true 89 | omit = [ 90 | "jupyter_workflow/ipython/__main__.py", 91 | "tests/*" 92 | ] 93 | 94 | [tool.isort] 95 | profile = "black" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cloudpickle==3.1.1 2 | ipython_genutils==0.2.0 3 | ipykernel==6.29.5 4 | jsonref==1.1.0 5 | jupyter_client==8.6.3 6 | nbclient==0.10.2 7 | nbformat==5.10.4 8 | streamflow==0.2.0.dev12 9 | traitlets==5.14.3 -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | coverage[toml]==7.8.2 2 | pytest==8.4.0 3 | pytest-asyncio==0.26.0 4 | pytest-xdist==3.7.0 5 | testbook==0.4.2 6 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alpha-unito/jupyter-workflow/d4e14769778125b34ef012dd8f8667976f80e7f0/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | from functools import partial 5 | from pathlib import Path 6 | 7 | import nbformat 8 | from testbook import testbook 9 | 10 | from jupyter_workflow.client.client import ( 11 | WorkflowClient, 12 | WorkflowKernelManager, 13 | on_cell_execute, 14 | ) 15 | 16 | 17 | def get_file(filename): 18 | return os.path.join(Path(__file__).parent, "testdata", filename) 19 | 20 | 21 | class CoverageWorkflowKernelManager(WorkflowKernelManager): 22 | def format_kernel_cmd(self, extra_arguments: list[str] | None = None) -> list[str]: 23 | cmd = super().format_kernel_cmd(extra_arguments) 24 | if "COVERAGE_RUN" in os.environ: 25 | cmd = ( 26 | cmd[0:1] 27 | + ["-m", "coverage", "run", "--concurrency=multiprocessing", "-p", "-m"] 28 | + cmd[2:] 29 | ) 30 | return cmd 31 | 32 | 33 | class testflow(testbook): 34 | def __init__( 35 | self, 36 | nb, 37 | execute=None, 38 | workflow=None, 39 | timeout=60, 40 | kernel_name="jupyter-workflow", 41 | allow_errors=False, 42 | **kwargs, 43 | ): 44 | super().__init__(nb, execute, timeout, kernel_name, allow_errors, **kwargs) 45 | self.client.on_cell_execute = partial(on_cell_execute, client=self.client) 46 | self.client.kernel_manager_class = CoverageWorkflowKernelManager 47 | self.workflow = workflow 48 | self.workflow_client = WorkflowClient( 49 | nb=( 50 | nbformat.read(nb, as_version=4) 51 | if not isinstance(nb, nbformat.NotebookNode) 52 | else nb 53 | ), 54 | timeout=timeout, 55 | allow_errors=allow_errors, 56 | kernel_name=kernel_name, 57 | **kwargs, 58 | ) 59 | self.workflow_client.kernel_manager_class = CoverageWorkflowKernelManager 60 | 61 | def _prepare(self): 62 | if self.workflow is True: 63 | self.workflow_client.execute_workflow() 64 | self.client.nb = self.workflow_client.nb 65 | else: 66 | super()._prepare() 67 | -------------------------------------------------------------------------------- /tests/test_notebook.py: -------------------------------------------------------------------------------- 1 | from tests.conftest import ( 2 | get_file, 3 | testflow, 4 | ) 5 | 6 | 7 | @testflow( 8 | get_file("two_steps_single_dep.ipynb"), 9 | execute=True, 10 | ) 11 | def test_two_steps_single_dep(tb): 12 | assert tb.cell_output_text(1) == "2" 13 | 14 | 15 | @testflow( 16 | get_file("two_steps_single_dep.ipynb"), 17 | workflow=True, 18 | ) 19 | def test_two_steps_single_dep_workflow(tb): 20 | assert tb.cell_output_text(1) == "2" 21 | 22 | 23 | @testflow( 24 | get_file("param_overwrite.ipynb"), 25 | execute=True, 26 | ) 27 | def test_param_overwrite(tb): 28 | assert tb.cell_output_text(2) == "3" 29 | 30 | 31 | @testflow( 32 | get_file("param_overwrite.ipynb"), 33 | workflow=True, 34 | ) 35 | def test_param_overwrite_workflow(tb): 36 | assert tb.cell_output_text(2) == "3" 37 | 38 | 39 | @testflow( 40 | get_file("simple_scatter_sequence.ipynb"), 41 | execute=True, 42 | ) 43 | def test_simple_scatter_sequence(tb): 44 | assert tb.cell_execute_result(3) == [{"text/plain": "[4, 5, 6, 7]"}] 45 | 46 | 47 | @testflow( 48 | get_file("simple_scatter_sequence.ipynb"), 49 | workflow=True, 50 | ) 51 | def test_simple_scatter_sequence_workflow(tb): 52 | assert tb.cell_execute_result(3) == [{"text/plain": "[4, 5, 6, 7]"}] 53 | 54 | 55 | @testflow( 56 | get_file("scatter_and_non_scatter_sequences.ipynb"), 57 | execute=True, 58 | ) 59 | def test_scatter_and_non_scatter_sequences(tb): 60 | assert tb.cell_execute_result(3) == [{"text/plain": "[4, 5, 6, 7]"}] 61 | assert tb.cell_execute_result(4) == [{"text/plain": "[4, 5, 6, 7]"}] 62 | 63 | 64 | @testflow( 65 | get_file("scatter_and_non_scatter_sequences.ipynb"), 66 | workflow=True, 67 | ) 68 | def test_scatter_and_non_scatter_sequences_workflow(tb): 69 | assert tb.cell_execute_result(3) == [{"text/plain": "[4, 5, 6, 7]"}] 70 | assert tb.cell_execute_result(4) == [{"text/plain": "[4, 5, 6, 7]"}] 71 | -------------------------------------------------------------------------------- /tests/test_serializer.py: -------------------------------------------------------------------------------- 1 | from tests.conftest import get_file, testflow 2 | 3 | 4 | @testflow(get_file("serialization.ipynb")) 5 | def test_input_predump_serialization(tb): 6 | tb.inject("a = 'test'") 7 | tb.execute_cell("input_predump_serializer") 8 | assert tb.cell_execute_result("input_predump_serializer") == [ 9 | {"text/plain": "'Predumped test'"} 10 | ] 11 | assert tb.value("a") == "Predumped test" 12 | 13 | 14 | @testflow(get_file("serialization.ipynb")) 15 | def test_input_postload_serialization(tb): 16 | tb.inject("a = 'test'") 17 | tb.execute_cell("input_postload_serializer") 18 | assert tb.cell_execute_result("input_postload_serializer") == [ 19 | {"text/plain": "'Postloaded test'"} 20 | ] 21 | assert tb.value("a") == "Postloaded test" 22 | 23 | 24 | @testflow(get_file("serialization.ipynb")) 25 | def test_output_predump_serialization(tb): 26 | tb.inject("a = 'test'") 27 | tb.execute_cell("output_predump_serializer") 28 | assert tb.cell_execute_result("output_predump_serializer") == [ 29 | {"text/plain": "'test'"} 30 | ] 31 | assert tb.value("a") == "Predumped test" 32 | 33 | 34 | @testflow(get_file("serialization.ipynb")) 35 | def test_output_postload_serialization(tb): 36 | tb.inject("a = 'test'") 37 | tb.execute_cell("output_postload_serializer") 38 | assert tb.cell_execute_result("output_postload_serializer") == [ 39 | {"text/plain": "'test'"} 40 | ] 41 | assert tb.value("a") == "Postloaded test" 42 | -------------------------------------------------------------------------------- /tests/test_single_cell.py: -------------------------------------------------------------------------------- 1 | from tests.conftest import get_file, testflow 2 | 3 | 4 | @testflow(get_file("name_deps.ipynb")) 5 | def test_single_explicit_input_dep(tb): 6 | tb.inject("a = 1") 7 | tb.execute_cell("single_explicit_input_dep") 8 | assert tb.cell_output_text("single_explicit_input_dep") == "2" 9 | 10 | 11 | @testflow(get_file("name_deps.ipynb")) 12 | def test_single_implicit_input_dep(tb): 13 | tb.inject("a = 1") 14 | tb.execute_cell("single_implicit_input_dep") 15 | assert tb.cell_output_text("single_implicit_input_dep") == "2" 16 | 17 | 18 | @testflow(get_file("name_deps.ipynb")) 19 | def test_interactive_execution(tb): 20 | tb.inject("a = 1") 21 | tb.execute_cell("interactive_execution") 22 | assert tb.cell_execute_result("interactive_execution") == [{"text/plain": "2"}] 23 | 24 | 25 | @testflow(get_file("name_deps.ipynb")) 26 | def test_bash_execution(tb): 27 | tb.inject("a = 1") 28 | tb.execute_cell("bash_execution") 29 | assert tb.cell_output_text("bash_execution") == "1" 30 | 31 | 32 | @testflow(get_file("name_deps.ipynb")) 33 | def test_list_input_dep(tb): 34 | tb.inject("a = [1, 2, 3, 4]") 35 | tb.execute_cell("list_input_dep") 36 | assert tb.cell_execute_result("list_input_dep") == [{"text/plain": "[2, 3, 4, 5]"}] 37 | 38 | 39 | @testflow(get_file("name_deps.ipynb")) 40 | def test_multiple_input_deps(tb): 41 | tb.inject("a = [1, 2, 3, 4]") 42 | tb.inject("b = [1, 2, 3, 4]") 43 | tb.execute_cell("multiple_input_deps") 44 | assert ( 45 | tb.cell_output_text("multiple_input_deps") 46 | == "2\n3\n4\n5\n3\n4\n5\n6\n4\n5\n6\n7\n5\n6\n7\n8" 47 | ) 48 | 49 | 50 | @testflow(get_file("file_deps.ipynb")) 51 | def test_file_input(tb): 52 | input_file = get_file("hello.txt") 53 | tb.inject(f'input_file = "{input_file}"') 54 | tb.execute_cell("file_input") 55 | with open(input_file) as f: 56 | content = f.read().encode("unicode_escape").decode("utf-8") 57 | assert tb.cell_execute_result("file_input") == [{"text/plain": f"'{content}'"}] 58 | 59 | 60 | @testflow(get_file("file_deps.ipynb")) 61 | def test_file_input_bash(tb): 62 | input_file = get_file("hello.txt") 63 | tb.inject(f'input_file = "{input_file}"') 64 | tb.execute_cell("file_input_bash") 65 | with open(input_file) as f: 66 | content = f.read().encode("unicode_escape").decode("utf-8") 67 | assert tb.cell_output_text("file_input_bash") == content.rstrip("\\n") 68 | 69 | 70 | @testflow(get_file("file_deps.ipynb")) 71 | def test_file_output_value(tb): 72 | input_file = get_file("hello.txt") 73 | tb.inject(f'input_file = "{input_file}"') 74 | tb.execute_cell("file_output_value") 75 | with open(input_file) as f: 76 | content = f.read() 77 | with open(tb.ref("output_file")) as f: 78 | assert content == f.read() 79 | 80 | 81 | @testflow(get_file("file_deps.ipynb")) 82 | def test_file_output_value_from(tb): 83 | input_file = get_file("hello.txt") 84 | tb.inject(f'input_file = "{input_file}"') 85 | tb.execute_cell("file_output_value_from") 86 | with open(input_file) as f: 87 | content = f.read() 88 | with open(tb.ref("output_file")) as f: 89 | assert content == f.read() 90 | 91 | 92 | @testflow(get_file("scatter_deps.ipynb")) 93 | def test_scatter_input_dep(tb): 94 | tb.inject("a = [1, 2, 3, 4]") 95 | tb.execute_cell("scatter_input_dep") 96 | assert tb.cell_execute_result("scatter_input_dep") == [ 97 | {"text/plain": "[2, 3, 4, 5]"} 98 | ] 99 | 100 | 101 | @testflow(get_file("scatter_deps.ipynb")) 102 | def test_scatter_input_dep_size(tb): 103 | tb.inject("a = [1, 2, 3, 4]") 104 | tb.execute_cell("scatter_input_dep_size") 105 | assert tb.cell_execute_result("scatter_input_dep_size") == [ 106 | {"text/plain": "[2, 3, 4, 5]"} 107 | ] 108 | 109 | 110 | @testflow(get_file("scatter_deps.ipynb")) 111 | def test_scatter_input_dep_num(tb): 112 | tb.inject("a = [1, 2, 3, 4]") 113 | tb.execute_cell("scatter_input_dep_num") 114 | assert tb.cell_execute_result("scatter_input_dep_num") == [ 115 | {"text/plain": "[2, 3, 4, 5]"} 116 | ] 117 | 118 | 119 | @testflow(get_file("scatter_deps.ipynb")) 120 | def test_scatter_input_deps_default(tb): 121 | tb.inject("a = [1, 2, 3, 4]") 122 | tb.inject("b = [1, 2, 3, 4]") 123 | tb.execute_cell("scatter_input_deps_default") 124 | assert ( 125 | tb.cell_output_text("scatter_input_deps_default") 126 | == "2\n3\n4\n5\n3\n4\n5\n6\n4\n5\n6\n7\n5\n6\n7\n8" 127 | ) 128 | 129 | 130 | @testflow(get_file("scatter_deps.ipynb")) 131 | def test_mixed_input_deps(tb): 132 | tb.inject("a = [1, 2, 3, 4]") 133 | tb.inject("b = [1, 2, 3, 4]") 134 | tb.execute_cell("mixed_input_deps") 135 | assert ( 136 | tb.cell_output_text("mixed_input_deps") 137 | == "2\n3\n4\n5\n3\n4\n5\n6\n4\n5\n6\n7\n5\n6\n7\n8" 138 | ) 139 | 140 | 141 | @testflow(get_file("scatter_deps.ipynb")) 142 | def test_scatter_input_deps_cartesian(tb): 143 | tb.inject("a = [1, 2, 3, 4]") 144 | tb.inject("b = [1, 2, 3, 4]") 145 | tb.execute_cell("scatter_input_deps_cartesian") 146 | assert ( 147 | tb.cell_output_text("scatter_input_deps_cartesian") 148 | == "2\n3\n4\n5\n3\n4\n5\n6\n4\n5\n6\n7\n5\n6\n7\n8" 149 | ) 150 | 151 | 152 | @testflow(get_file("scatter_deps.ipynb")) 153 | def test_scatter_input_deps_dotproduct(tb): 154 | tb.inject("a = [1, 2, 3, 4]") 155 | tb.inject("b = [1, 2, 3, 4]") 156 | tb.execute_cell("scatter_input_deps_dotproduct") 157 | assert tb.cell_output_text("scatter_input_deps_dotproduct") == "2\n4\n6\n8" 158 | -------------------------------------------------------------------------------- /tests/testdata/file_deps.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "78336dfa", 7 | "metadata": { 8 | "slideshow": { 9 | "slide_type": "" 10 | }, 11 | "tags": [ 12 | "file_input" 13 | ], 14 | "workflow": { 15 | "step": { 16 | "autoin": true, 17 | "background": false, 18 | "in": [ 19 | { 20 | "name": "input_file", 21 | "type": "file", 22 | "valueFrom": "input_file" 23 | } 24 | ], 25 | "out": [] 26 | }, 27 | "version": "v1.0" 28 | } 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "# file_input\n", 33 | "with open(input_file) as f:\n", 34 | " content = f.read()\n", 35 | "content" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "id": "c460067b", 42 | "metadata": { 43 | "slideshow": { 44 | "slide_type": "" 45 | }, 46 | "tags": [ 47 | "file_input_bash" 48 | ], 49 | "workflow": { 50 | "step": { 51 | "autoin": true, 52 | "background": false, 53 | "in": [ 54 | { 55 | "name": "input_file", 56 | "type": "file", 57 | "valueFrom": "input_file" 58 | } 59 | ], 60 | "out": [] 61 | }, 62 | "version": "v1.0" 63 | } 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "%%bash -s \"$input_file\"\n", 68 | "\n", 69 | "# file_input in bash command\n", 70 | "cat $1" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "id": "b5fa5254", 77 | "metadata": { 78 | "slideshow": { 79 | "slide_type": "" 80 | }, 81 | "tags": [ 82 | "file_output_value" 83 | ], 84 | "workflow": { 85 | "step": { 86 | "autoin": true, 87 | "background": false, 88 | "in": [ 89 | { 90 | "name": "input_file", 91 | "type": "file", 92 | "valueFrom": "input_file" 93 | } 94 | ], 95 | "out": [ 96 | { 97 | "name": "output_file", 98 | "type": "file", 99 | "value": "out.txt" 100 | } 101 | ] 102 | }, 103 | "version": "v1.0" 104 | } 105 | }, 106 | "outputs": [], 107 | "source": [ 108 | "# file_output_value\n", 109 | "with open(input_file) as f:\n", 110 | " content = f.read()\n", 111 | "output_file = \"out.txt\"\n", 112 | "with open(output_file, \"w\") as f:\n", 113 | " f.write(content)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "id": "c68243e1", 120 | "metadata": { 121 | "slideshow": { 122 | "slide_type": "" 123 | }, 124 | "tags": [ 125 | "file_output_value_from" 126 | ], 127 | "workflow": { 128 | "step": { 129 | "autoin": true, 130 | "background": false, 131 | "in": [ 132 | { 133 | "name": "input_file", 134 | "type": "file", 135 | "valueFrom": "input_file" 136 | } 137 | ], 138 | "out": [ 139 | { 140 | "name": "output_file", 141 | "type": "file", 142 | "valueFrom": "output_file" 143 | } 144 | ] 145 | }, 146 | "version": "v1.0" 147 | } 148 | }, 149 | "outputs": [], 150 | "source": [ 151 | "# file_output_value_from\n", 152 | "with open(input_file) as f:\n", 153 | " content = f.read()\n", 154 | "output_file = \"out.txt\"\n", 155 | "with open(output_file, \"w\") as f:\n", 156 | " f.write(content)" 157 | ] 158 | } 159 | ], 160 | "metadata": { 161 | "celltoolbar": "Edit Workflow Step", 162 | "kernelspec": { 163 | "display_name": "Jupyter Workflow", 164 | "language": "python", 165 | "name": "jupyter-workflow" 166 | }, 167 | "language_info": { 168 | "codemirror_mode": { 169 | "name": "ipython", 170 | "version": 3 171 | }, 172 | "file_extension": ".py", 173 | "mimetype": "text/x-python", 174 | "name": "python", 175 | "nbconvert_exporter": "python", 176 | "pygments_lexer": "ipython3", 177 | "version": "3.10.12" 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 5 182 | } 183 | -------------------------------------------------------------------------------- /tests/testdata/hello.txt: -------------------------------------------------------------------------------- 1 | Hello; World! 2 | -------------------------------------------------------------------------------- /tests/testdata/name_deps.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "18df079d", 7 | "metadata": { 8 | "tags": [ 9 | "single_explicit_input_dep" 10 | ], 11 | "workflow": { 12 | "step": { 13 | "autoin": true, 14 | "background": false, 15 | "in": [ 16 | { 17 | "name": "a", 18 | "type": "name", 19 | "valueFrom": "a" 20 | } 21 | ], 22 | "out": [] 23 | }, 24 | "version": "v1.0" 25 | } 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "# single_explicit_input_dep\n", 30 | "print(a + 1)" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "id": "2333b160", 37 | "metadata": { 38 | "tags": [ 39 | "single_implicit_input_dep" 40 | ], 41 | "workflow": { 42 | "step": { 43 | "autoin": true, 44 | "background": false, 45 | "in": [], 46 | "out": [] 47 | }, 48 | "version": "v1.0" 49 | } 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "# single_implicit_input_dep\n", 54 | "print(a + 1)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "id": "8284b7fc", 61 | "metadata": { 62 | "tags": [ 63 | "interactive_execution" 64 | ], 65 | "workflow": { 66 | "step": { 67 | "autoin": true, 68 | "background": false, 69 | "in": [], 70 | "out": [] 71 | }, 72 | "version": "v1.0" 73 | } 74 | }, 75 | "outputs": [], 76 | "source": [ 77 | "# interactive_execution\n", 78 | "a + 1" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "id": "b0831eea", 85 | "metadata": { 86 | "slideshow": { 87 | "slide_type": "" 88 | }, 89 | "tags": [ 90 | "bash_execution" 91 | ], 92 | "workflow": { 93 | "step": { 94 | "autoin": true, 95 | "background": false, 96 | "in": [], 97 | "out": [] 98 | }, 99 | "version": "v1.0" 100 | } 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "%%bash -s \"$a\"\n", 105 | "\n", 106 | "# bash execution\n", 107 | "echo \"$1\"" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "id": "359352c9", 114 | "metadata": { 115 | "tags": [ 116 | "list_input_dep" 117 | ], 118 | "workflow": { 119 | "step": { 120 | "autoin": true, 121 | "background": false, 122 | "in": [], 123 | "out": [] 124 | }, 125 | "version": "v1.0" 126 | } 127 | }, 128 | "outputs": [], 129 | "source": [ 130 | "# list_input_dep\n", 131 | "[i + 1 for i in a]" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "id": "4d6d2a1e", 138 | "metadata": { 139 | "tags": [ 140 | "multiple_input_deps" 141 | ], 142 | "workflow": { 143 | "step": { 144 | "autoin": true, 145 | "background": false, 146 | "in": [ 147 | { 148 | "name": "a", 149 | "type": "name", 150 | "valueFrom": "a" 151 | } 152 | ], 153 | "out": [] 154 | }, 155 | "version": "v1.0" 156 | } 157 | }, 158 | "outputs": [], 159 | "source": [ 160 | "# multiple_input_deps\n", 161 | "for i in a:\n", 162 | " for j in b:\n", 163 | " print(i + j)" 164 | ] 165 | } 166 | ], 167 | "metadata": { 168 | "celltoolbar": "Edit Workflow Step", 169 | "kernelspec": { 170 | "display_name": "Jupyter Workflow", 171 | "language": "python", 172 | "name": "jupyter-workflow" 173 | }, 174 | "language_info": { 175 | "codemirror_mode": { 176 | "name": "ipython", 177 | "version": 3 178 | }, 179 | "file_extension": ".py", 180 | "mimetype": "text/x-python", 181 | "name": "python", 182 | "nbconvert_exporter": "python", 183 | "pygments_lexer": "ipython3", 184 | "version": "3.10.12" 185 | } 186 | }, 187 | "nbformat": 4, 188 | "nbformat_minor": 5 189 | } 190 | -------------------------------------------------------------------------------- /tests/testdata/param_overwrite.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "636f0fcd", 7 | "metadata": { 8 | "workflow": { 9 | "step": { 10 | "autoin": true, 11 | "background": false, 12 | "in": [], 13 | "out": [ 14 | { 15 | "name": "a", 16 | "type": "name", 17 | "valueFrom": "a" 18 | } 19 | ] 20 | }, 21 | "version": "v1.0" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "a = 1" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "df252cb4", 33 | "metadata": { 34 | "workflow": { 35 | "step": { 36 | "autoin": true, 37 | "background": false, 38 | "in": [], 39 | "out": [ 40 | { 41 | "name": "a", 42 | "type": "name", 43 | "valueFrom": "a" 44 | } 45 | ] 46 | }, 47 | "version": "v1.0" 48 | } 49 | }, 50 | "outputs": [], 51 | "source": [ 52 | "a = a + 1" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "id": "a947f0f3", 59 | "metadata": { 60 | "workflow": { 61 | "step": { 62 | "autoin": true, 63 | "background": false, 64 | "in": [], 65 | "out": [] 66 | }, 67 | "version": "v1.0" 68 | } 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "print(a + 1)" 73 | ] 74 | } 75 | ], 76 | "metadata": { 77 | "kernelspec": { 78 | "display_name": "Jupyter Workflow", 79 | "language": "python", 80 | "name": "jupyter-workflow" 81 | }, 82 | "language_info": { 83 | "codemirror_mode": { 84 | "name": "ipython", 85 | "version": 3 86 | }, 87 | "file_extension": ".py", 88 | "mimetype": "text/x-python", 89 | "name": "python", 90 | "nbconvert_exporter": "python", 91 | "pygments_lexer": "ipython3", 92 | "version": "3.10.6" 93 | } 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 5 97 | } 98 | -------------------------------------------------------------------------------- /tests/testdata/scatter_and_non_scatter_sequences.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "dc843a3a", 7 | "metadata": { 8 | "workflow": { 9 | "step": { 10 | "autoin": true, 11 | "background": false, 12 | "in": [], 13 | "out": [ 14 | { 15 | "name": "a", 16 | "type": "name", 17 | "valueFrom": "a" 18 | } 19 | ] 20 | }, 21 | "version": "v1.0" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "a = [1, 2, 3, 4]" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "636f0fcd", 33 | "metadata": { 34 | "workflow": { 35 | "step": { 36 | "autoin": true, 37 | "background": false, 38 | "in": [], 39 | "out": [ 40 | { 41 | "name": "a", 42 | "type": "name", 43 | "valueFrom": "a" 44 | } 45 | ], 46 | "scatter": { 47 | "items": [ 48 | "a" 49 | ] 50 | } 51 | }, 52 | "version": "v1.0" 53 | } 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "a = [i + 1 for i in a]" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "df252cb4", 64 | "metadata": { 65 | "workflow": { 66 | "step": { 67 | "autoin": true, 68 | "background": false, 69 | "in": [], 70 | "out": [ 71 | { 72 | "name": "a", 73 | "type": "name", 74 | "valueFrom": "a" 75 | } 76 | ], 77 | "scatter": { 78 | "items": [ 79 | "a" 80 | ] 81 | } 82 | }, 83 | "version": "v1.0" 84 | } 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "a = [i + 1 for i in a]" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "id": "c7d627fc", 95 | "metadata": { 96 | "workflow": { 97 | "step": { 98 | "autoin": true, 99 | "background": false, 100 | "in": [], 101 | "out": [], 102 | "scatter": { 103 | "items": [ 104 | "a" 105 | ] 106 | } 107 | }, 108 | "version": "v1.0" 109 | } 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "[i + 1 for i in a]" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "id": "c4929443", 120 | "metadata": { 121 | "workflow": { 122 | "step": { 123 | "autoin": true, 124 | "background": false, 125 | "in": [], 126 | "out": [] 127 | }, 128 | "version": "v1.0" 129 | } 130 | }, 131 | "outputs": [], 132 | "source": [ 133 | "[i + 1 for i in a]" 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Jupyter Workflow", 140 | "language": "python", 141 | "name": "jupyter-workflow" 142 | }, 143 | "language_info": { 144 | "codemirror_mode": { 145 | "name": "ipython", 146 | "version": 3 147 | }, 148 | "file_extension": ".py", 149 | "mimetype": "text/x-python", 150 | "name": "python", 151 | "nbconvert_exporter": "python", 152 | "pygments_lexer": "ipython3", 153 | "version": "3.10.6" 154 | } 155 | }, 156 | "nbformat": 4, 157 | "nbformat_minor": 5 158 | } 159 | -------------------------------------------------------------------------------- /tests/testdata/scatter_deps.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "a3971256", 7 | "metadata": { 8 | "tags": [ 9 | "scatter_input_dep" 10 | ], 11 | "workflow": { 12 | "step": { 13 | "autoin": true, 14 | "background": false, 15 | "in": [], 16 | "out": [], 17 | "scatter": { 18 | "items": [ 19 | "a" 20 | ] 21 | } 22 | }, 23 | "version": "v1.0" 24 | } 25 | }, 26 | "outputs": [], 27 | "source": [ 28 | "# scatter_input_dep\n", 29 | "[i + 1 for i in a]" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "id": "4aacc900", 36 | "metadata": { 37 | "tags": [ 38 | "scatter_input_dep_size" 39 | ], 40 | "workflow": { 41 | "step": { 42 | "autoin": true, 43 | "background": false, 44 | "in": [], 45 | "out": [], 46 | "scatter": { 47 | "items": [ 48 | { 49 | "name": "a", 50 | "size": 3 51 | } 52 | ] 53 | } 54 | }, 55 | "version": "v1.0" 56 | } 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "# scatter_input_dep_size\n", 61 | "[i + 1 for i in a]" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "id": "8061229a", 68 | "metadata": { 69 | "tags": [ 70 | "scatter_input_dep_num" 71 | ], 72 | "workflow": { 73 | "step": { 74 | "autoin": true, 75 | "background": false, 76 | "in": [], 77 | "out": [], 78 | "scatter": { 79 | "items": [ 80 | { 81 | "name": "a", 82 | "num": 3 83 | } 84 | ] 85 | } 86 | }, 87 | "version": "v1.0" 88 | } 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "# scatter_input_dep_num\n", 93 | "[i + 1 for i in a]" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "id": "86f10e90", 100 | "metadata": { 101 | "tags": [ 102 | "scatter_input_deps_default" 103 | ], 104 | "workflow": { 105 | "step": { 106 | "autoin": true, 107 | "background": false, 108 | "in": [], 109 | "out": [], 110 | "scatter": { 111 | "items": [ 112 | "a", 113 | "b" 114 | ] 115 | } 116 | }, 117 | "version": "v1.0" 118 | } 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "# scatter_input_deps_default\n", 123 | "for i in a:\n", 124 | " for j in b:\n", 125 | " print(i + j)" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "id": "192bcf57", 132 | "metadata": { 133 | "tags": [ 134 | "mixed_input_deps" 135 | ], 136 | "workflow": { 137 | "step": { 138 | "autoin": true, 139 | "background": false, 140 | "in": [], 141 | "out": [], 142 | "scatter": { 143 | "items": [ 144 | "a" 145 | ] 146 | } 147 | }, 148 | "version": "v1.0" 149 | } 150 | }, 151 | "outputs": [], 152 | "source": [ 153 | "# mixed_input_deps\n", 154 | "for i in a:\n", 155 | " for j in b:\n", 156 | " print(i + j)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "id": "55da4c75", 163 | "metadata": { 164 | "tags": [ 165 | "scatter_input_deps_cartesian" 166 | ], 167 | "workflow": { 168 | "step": { 169 | "autoin": true, 170 | "background": false, 171 | "in": [], 172 | "out": [], 173 | "scatter": { 174 | "items": [ 175 | "a", 176 | "b" 177 | ], 178 | "method": "cartesian" 179 | } 180 | }, 181 | "version": "v1.0" 182 | } 183 | }, 184 | "outputs": [], 185 | "source": [ 186 | "# scatter_input_deps_cartesian\n", 187 | "for i in a:\n", 188 | " for j in b:\n", 189 | " print(i + j)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "id": "eaa48bad", 196 | "metadata": { 197 | "tags": [ 198 | "scatter_input_deps_dotproduct" 199 | ], 200 | "workflow": { 201 | "step": { 202 | "autoin": true, 203 | "background": false, 204 | "in": [], 205 | "out": [], 206 | "scatter": { 207 | "items": [ 208 | "a", 209 | "b" 210 | ], 211 | "method": "dotproduct" 212 | } 213 | }, 214 | "version": "v1.0" 215 | } 216 | }, 217 | "outputs": [], 218 | "source": [ 219 | "# scatter_input_deps_dotproduct\n", 220 | "for i in a:\n", 221 | " for j in b:\n", 222 | " print(i + j)" 223 | ] 224 | } 225 | ], 226 | "metadata": { 227 | "celltoolbar": "Edit Workflow Step", 228 | "kernelspec": { 229 | "display_name": "Jupyter Workflow", 230 | "language": "python", 231 | "name": "jupyter-workflow" 232 | }, 233 | "language_info": { 234 | "codemirror_mode": { 235 | "name": "ipython", 236 | "version": 3 237 | }, 238 | "file_extension": ".py", 239 | "mimetype": "text/x-python", 240 | "name": "python", 241 | "nbconvert_exporter": "python", 242 | "pygments_lexer": "ipython3", 243 | "version": "3.10.6" 244 | } 245 | }, 246 | "nbformat": 4, 247 | "nbformat_minor": 5 248 | } 249 | -------------------------------------------------------------------------------- /tests/testdata/serialization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "c4c26d05-0e55-474e-a345-698e0af7dfee", 7 | "metadata": { 8 | "tags": [ 9 | "input_predump_serializer" 10 | ], 11 | "workflow": { 12 | "serializers": { 13 | "predump": { 14 | "predump": "y = f'Predumped {x}'" 15 | } 16 | }, 17 | "step": { 18 | "in": [ 19 | { 20 | "name": "a", 21 | "serializer": "predump", 22 | "type": "name" 23 | } 24 | ], 25 | "out": [ 26 | { 27 | "name": "a", 28 | "type": "name" 29 | } 30 | ] 31 | }, 32 | "version": "v1.0" 33 | } 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "a" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "id": "7dc6b908-3a8c-4a8f-a066-af8f8fc044c0", 44 | "metadata": { 45 | "tags": [ 46 | "input_postload_serializer" 47 | ], 48 | "workflow": { 49 | "serializers": { 50 | "postload": { 51 | "postload": "y = f'Postloaded {x}'" 52 | } 53 | }, 54 | "step": { 55 | "in": [ 56 | { 57 | "name": "a", 58 | "serializer": "postload", 59 | "type": "name" 60 | } 61 | ], 62 | "out": [ 63 | { 64 | "name": "a", 65 | "type": "name" 66 | } 67 | ] 68 | }, 69 | "version": "v1.0" 70 | } 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "a" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "id": "9410a2bd-463b-41a0-91cb-7c9e326223f8", 81 | "metadata": { 82 | "tags": [ 83 | "output_predump_serializer" 84 | ], 85 | "workflow": { 86 | "serializers": { 87 | "predump": { 88 | "predump": "y = f'Predumped {x}'" 89 | } 90 | }, 91 | "step": { 92 | "in": [ 93 | { 94 | "name": "a", 95 | "type": "name" 96 | } 97 | ], 98 | "out": [ 99 | { 100 | "name": "a", 101 | "serializer": "predump", 102 | "type": "name" 103 | } 104 | ] 105 | }, 106 | "version": "v1.0" 107 | } 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "a" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "id": "7c12169e-01f5-4cc5-9d3b-15a965783dd6", 118 | "metadata": { 119 | "tags": [ 120 | "output_postload_serializer" 121 | ], 122 | "workflow": { 123 | "serializers": { 124 | "postload": { 125 | "postload": "y = f'Postloaded {x}'" 126 | } 127 | }, 128 | "step": { 129 | "in": [ 130 | { 131 | "name": "a", 132 | "type": "name" 133 | } 134 | ], 135 | "out": [ 136 | { 137 | "name": "a", 138 | "serializer": "postload", 139 | "type": "name" 140 | } 141 | ] 142 | }, 143 | "version": "v1.0" 144 | } 145 | }, 146 | "outputs": [], 147 | "source": [ 148 | "a" 149 | ] 150 | } 151 | ], 152 | "metadata": { 153 | "kernelspec": { 154 | "display_name": "Jupyter Workflow", 155 | "language": "python", 156 | "name": "jupyter-workflow" 157 | }, 158 | "language_info": { 159 | "codemirror_mode": { 160 | "name": "ipython", 161 | "version": 3 162 | }, 163 | "file_extension": ".py", 164 | "mimetype": "text/x-python", 165 | "name": "python", 166 | "nbconvert_exporter": "python", 167 | "pygments_lexer": "ipython3", 168 | "version": "3.12.10" 169 | } 170 | }, 171 | "nbformat": 4, 172 | "nbformat_minor": 5 173 | } 174 | -------------------------------------------------------------------------------- /tests/testdata/simple_scatter_sequence.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "af9b2b18", 7 | "metadata": { 8 | "workflow": { 9 | "step": { 10 | "autoin": true, 11 | "background": false, 12 | "in": [], 13 | "out": [ 14 | { 15 | "name": "a", 16 | "type": "name", 17 | "valueFrom": "a" 18 | } 19 | ] 20 | }, 21 | "version": "v1.0" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "a = [1, 2, 3, 4]" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "636f0fcd", 33 | "metadata": { 34 | "workflow": { 35 | "step": { 36 | "autoin": true, 37 | "background": false, 38 | "in": [], 39 | "out": [ 40 | { 41 | "name": "a", 42 | "type": "name", 43 | "valueFrom": "a" 44 | } 45 | ], 46 | "scatter": { 47 | "items": [ 48 | "a" 49 | ] 50 | } 51 | }, 52 | "version": "v1.0" 53 | } 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "a = [i + 1 for i in a]" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "df252cb4", 64 | "metadata": { 65 | "workflow": { 66 | "step": { 67 | "autoin": true, 68 | "background": false, 69 | "in": [], 70 | "out": [ 71 | { 72 | "name": "a", 73 | "type": "name", 74 | "valueFrom": "a" 75 | } 76 | ], 77 | "scatter": { 78 | "items": [ 79 | "a" 80 | ] 81 | } 82 | }, 83 | "version": "v1.0" 84 | } 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "a = [i + 1 for i in a]" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "id": "a947f0f3", 95 | "metadata": { 96 | "workflow": { 97 | "step": { 98 | "autoin": true, 99 | "background": false, 100 | "in": [], 101 | "out": [], 102 | "scatter": { 103 | "items": [ 104 | "a" 105 | ] 106 | } 107 | }, 108 | "version": "v1.0" 109 | } 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "[i + 1 for i in a]" 114 | ] 115 | } 116 | ], 117 | "metadata": { 118 | "kernelspec": { 119 | "display_name": "Jupyter Workflow", 120 | "language": "python", 121 | "name": "jupyter-workflow" 122 | }, 123 | "language_info": { 124 | "codemirror_mode": { 125 | "name": "ipython", 126 | "version": 3 127 | }, 128 | "file_extension": ".py", 129 | "mimetype": "text/x-python", 130 | "name": "python", 131 | "nbconvert_exporter": "python", 132 | "pygments_lexer": "ipython3", 133 | "version": "3.10.6" 134 | } 135 | }, 136 | "nbformat": 4, 137 | "nbformat_minor": 5 138 | } 139 | -------------------------------------------------------------------------------- /tests/testdata/two_steps_single_dep.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "636f0fcd", 7 | "metadata": { 8 | "workflow": { 9 | "step": { 10 | "autoin": true, 11 | "background": false, 12 | "in": [], 13 | "out": [ 14 | { 15 | "name": "a", 16 | "type": "name", 17 | "valueFrom": "a" 18 | } 19 | ] 20 | }, 21 | "version": "v1.0" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "a = 1" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "df252cb4", 33 | "metadata": { 34 | "workflow": { 35 | "step": { 36 | "autoin": true, 37 | "background": false, 38 | "in": [], 39 | "out": [] 40 | }, 41 | "version": "v1.0" 42 | } 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "print(a + 1)" 47 | ] 48 | } 49 | ], 50 | "metadata": { 51 | "kernelspec": { 52 | "display_name": "Jupyter Workflow", 53 | "language": "python", 54 | "name": "jupyter-workflow" 55 | }, 56 | "language_info": { 57 | "codemirror_mode": { 58 | "name": "ipython", 59 | "version": 3 60 | }, 61 | "file_extension": ".py", 62 | "mimetype": "text/x-python", 63 | "name": "python", 64 | "nbconvert_exporter": "python", 65 | "pygments_lexer": "ipython3", 66 | "version": "3.10.6" 67 | } 68 | }, 69 | "nbformat": 4, 70 | "nbformat_minor": 5 71 | } 72 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | bandit 4 | lint 5 | py3.{8,9,10,11}-unit 6 | skip_missing_interpreters = True 7 | 8 | [pytest] 9 | asyncio_default_fixture_loop_scope = module 10 | asyncio_mode = strict 11 | testpaths = tests 12 | 13 | [testenv] 14 | allowlist_externals = make 15 | commands_pre = 16 | py3.{9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel 17 | py3.{9,10,11,12,13}-unit: python -m jupyter_workflow.ipython.install 18 | commands = 19 | py3.{9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} 20 | deps = 21 | py3.{9,10,11,12,13}-unit: -rrequirements.txt 22 | py3.{9,10,11,12,13}-unit: -rtest-requirements.txt 23 | description = 24 | py3.{9,10,11,12,13}-unit: Run the unit tests 25 | passenv = 26 | CI 27 | GITHUB_* 28 | setenv = 29 | py3.{9,10,11,12,13}-unit: LC_ALL = C.UTF-8 30 | 31 | [testenv:bandit] 32 | commands = bandit -r jupyter_workflow 33 | deps = 34 | -rrequirements.txt 35 | -rbandit-requirements.txt 36 | description = Search for common security issues 37 | passenv = 38 | CI 39 | GITHUB_* 40 | 41 | [testenv:lint] 42 | allowlist_externals = make 43 | commands = make flake8 format-check codespell-check pyupgrade 44 | deps = 45 | -rrequirements.txt 46 | -rlint-requirements.txt 47 | description = Lint the Python code --------------------------------------------------------------------------------