├── .bumpversion.cfg ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── codecov.yaml ├── dependabot.yml ├── stale.yml └── workflows │ ├── ci.yml │ └── codeql-analysis.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPMENT_GUIDE.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── binder ├── cli-simple │ ├── README.md │ ├── cli_example.ipynb │ ├── pm_example.ipynb │ ├── simple_input.ipynb │ └── simple_output.ipynb ├── postBuild └── requirements.txt ├── docs ├── Makefile ├── UPDATE.md ├── _static │ ├── custom.css │ └── images │ │ └── papermill.png ├── changelog.md ├── conf.py ├── extending-developing.rst ├── extending-entry-points.rst ├── extending-overview.rst ├── img │ ├── custom_execution_engine.png │ ├── enable_parameters.gif │ ├── lab3_parameters.png │ ├── lab_parameters.png │ ├── matplotlib_hist.png │ ├── nb_dataframe.png │ ├── nbs_dataframe.png │ └── parameters.png ├── index.rst ├── installation.rst ├── make.bat ├── reference │ ├── index.rst │ ├── papermill-cli.rst │ ├── papermill-io.rst │ ├── papermill-storage.rst │ ├── papermill-translators.rst │ ├── papermill-utilities.rst │ └── papermill-workflow.rst ├── troubleshooting.rst ├── usage-cli.rst ├── usage-execute.rst ├── usage-inspect.rst ├── usage-parameterize.rst ├── usage-store.rst └── usage-workflow.rst ├── papermill ├── __init__.py ├── __main__.py ├── abs.py ├── adl.py ├── cli.py ├── clientwrap.py ├── engines.py ├── exceptions.py ├── execute.py ├── inspection.py ├── iorw.py ├── log.py ├── models.py ├── parameterize.py ├── s3.py ├── tests │ ├── __init__.py │ ├── fixtures │ │ └── rock.txt │ ├── notebooks │ │ ├── binder.ipynb │ │ ├── blank-vscode.ipynb │ │ ├── broken.ipynb │ │ ├── broken1.ipynb │ │ ├── broken2.ipynb │ │ ├── complex_parameters.ipynb │ │ ├── gcs │ │ │ └── gcs_in │ │ │ │ └── gcs-simple_notebook.ipynb │ │ ├── keyboard_interrupt.ipynb │ │ ├── line_magic_error.ipynb │ │ ├── nb_version_4.4.ipynb │ │ ├── no_parameters.ipynb │ │ ├── notimplemented_translator.ipynb │ │ ├── read_check.ipynb │ │ ├── report_mode_test.ipynb │ │ ├── result_no_exec.ipynb │ │ ├── s3 │ │ │ ├── s3_in │ │ │ │ └── s3-simple_notebook.ipynb │ │ │ └── s3_out │ │ │ │ └── .keep │ │ ├── simple_execute.ipynb │ │ ├── sysexit.ipynb │ │ ├── sysexit0.ipynb │ │ ├── sysexit1.ipynb │ │ ├── systemexit.ipynb │ │ ├── test_autosave.ipynb │ │ ├── test_logging.ipynb │ │ └── test_notebooknode_io.ipynb │ ├── parameters │ │ ├── example.json │ │ └── example.yaml │ ├── test_abs.py │ ├── test_adl.py │ ├── test_autosave.py │ ├── test_cli.py │ ├── test_clientwrap.py │ ├── test_conf.py │ ├── test_engines.py │ ├── test_exceptions.py │ ├── test_execute.py │ ├── test_gcs.py │ ├── test_hdfs.py │ ├── test_inspect.py │ ├── test_iorw.py │ ├── test_parameterize.py │ ├── test_s3.py │ ├── test_translators.py │ └── test_utils.py ├── translators.py ├── utils.py └── version.py ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── requirements ├── azure.txt ├── dev.txt ├── docs.txt ├── gcs.txt ├── github.txt ├── hdfs.txt └── s3.txt ├── setup.py ├── tox.ini └── tox_py_installer.sh /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 2.6.0 3 | commit = True 4 | tag = True 5 | tag_name = {new_version} 6 | 7 | [bumpversion:file:papermill/version.py] 8 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | papermill/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug, help wanted 6 | assignees: '' 7 | --- 8 | 9 | ## 🐛 Bug 10 | 11 | 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement, help wanted 6 | assignees: '' 7 | --- 8 | 9 | ## 🚀 Feature 10 | 11 | 12 | 13 | ### Motivation 14 | 15 | 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## What does this PR do? 2 | 3 | 8 | 9 | Fixes #\ 10 | -------------------------------------------------------------------------------- /.github/codecov.yaml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "70...100" 8 | threshold: 1% 9 | 10 | parsers: 11 | gcov: 12 | branch_detection: 13 | conditional: yes 14 | loop: yes 15 | method: no 16 | macro: no 17 | 18 | comment: 19 | layout: "reach,diff,flags,files,footer" 20 | behavior: default 21 | require_changes: no 22 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Basic dependabot.yml file with minimum configuration for two package managers 2 | 3 | version: 2 4 | updates: 5 | # Enable version updates for GitHub Actions 6 | - package-ecosystem: "github-actions" 7 | directory: "/" 8 | # Check for updates once a week 9 | schedule: 10 | interval: "monthly" 11 | # Labels on pull requests for version updates only 12 | labels: ["ci"] 13 | pull-request-branch-name: 14 | # Separate sections of the branch name with a hyphen for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1` 15 | separator: "-" 16 | # Allow up to 5 open pull requests for GitHub Actions 17 | open-pull-requests-limit: 5 18 | reviewers: 19 | - "borda" 20 | 21 | # Enable version updates for python 22 | - package-ecosystem: "pip" 23 | # Look for a `requirements` in the `root` directory 24 | directory: "/requirements" 25 | # Check for updates once a week 26 | schedule: 27 | interval: "weekly" 28 | # Labels on pull requests for version updates only 29 | labels: ["enhancement"] 30 | pull-request-branch-name: 31 | # Separate sections of the branch name with a hyphen 32 | # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1` 33 | separator: "-" 34 | # Allow up to 5 open pull requests for pip dependencies 35 | open-pull-requests-limit: 5 36 | reviewers: 37 | - "willingc" 38 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # https://github.com/marketplace/stale 2 | # https://github.com/probot/stale 3 | 4 | issues: 5 | # Number of days of inactivity before an issue becomes stale 6 | daysUntilStale: 365 7 | # Number of days of inactivity before a stale issue is closed 8 | daysUntilClose: 60 9 | # Issues with these labels will never be considered stale 10 | #exemptLabels: 11 | # - "priority: 0" 12 | # - "priority: 1" 13 | # Comment to post when marking an issue as stale. Set to `false` to disable 14 | markComment: > 15 | This issue has been automatically marked as stale because it has been inactive for 365 days. 16 | This issue will be automatically closed in 60 days if no further activity occurs. 17 | Thank you for your contributions and using Papermill! 18 | # Comment to post when closing a stale issue. Set to `false` to disable 19 | closeComment: false 20 | 21 | pulls: 22 | # Number of days of inactivity before an pulls becomes stale 23 | daysUntilStale: 180 24 | # Number of days of inactivity before a stale pull is closed 25 | daysUntilClose: 30 26 | # Label to use when marking an issue as stale 27 | staleLabel: "stale" 28 | # Comment to post when marking an issue as stale. Set to `false` to disable 29 | markComment: > 30 | This pull request has been automatically marked as stale because it has been inactive for 180 days. 31 | It will be closed in 30 days if no further activity occurs. If you need further help see our docs: 32 | https://github.com/nteract/papermill/blob/main/CONTRIBUTING.md 33 | or ask the assistance. Thank you for your contributions. 34 | # Comment to post when closing a stale issue. Set to `false` to disable 35 | closeComment: > 36 | This pull request is going to be closed. Please feel free to reopen it or create a new one based on top of the 37 | 'main' branch. 38 | 39 | # Label to use when marking an issue as stale 40 | staleLabel: "stale" 41 | # Limit the number of actions per hour, from 1-30. Default is 30 42 | limitPerRun: 2 43 | 44 | # Set to true to ignore issues in a project (defaults to false) 45 | exemptProjects: true 46 | # Set to true to ignore issues in a milestone (defaults to false) 47 | exemptMilestones: true 48 | # Set to true to ignore issues with an assignee (defaults to false) 49 | exemptAssignees: true 50 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["main"] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | tests: 12 | name: "Python ${{ matrix.python-version }}" 13 | runs-on: "ubuntu-latest" 14 | env: 15 | USING_COVERAGE: '3.8,3.9,3.10,3.11,3.12' 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] 21 | 22 | steps: 23 | - uses: "actions/checkout@v4" 24 | - uses: "actions/setup-python@v5" 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: "Install dependencies" 28 | run: | 29 | set -xe 30 | python -VV 31 | python -m site 32 | python -m pip install --upgrade pip setuptools wheel coverage[toml] virtualenv tox tox-gh-actions 33 | 34 | - name: "Run tox targets for ${{ matrix.python-version }}" 35 | run: python -m tox 36 | 37 | - name: "Generate coverage XML" 38 | if: "contains(env.USING_COVERAGE, matrix.python-version)" 39 | run: python -m coverage xml 40 | 41 | - name: "Upload coverage to Codecov" 42 | uses: codecov/codecov-action@v4 43 | 44 | others: 45 | runs-on: ubuntu-20.04 46 | strategy: 47 | fail-fast: false 48 | matrix: 49 | toxenv: ["manifest", "docs", "binder"] 50 | env: 51 | TOXENV: ${{ matrix.toxenv }} 52 | steps: 53 | - uses: actions/checkout@v4 54 | - uses: actions/setup-python@v5 55 | with: 56 | python-version: 3.11 57 | - name: "Install dependencies" 58 | run: | 59 | set -xe 60 | python -m pip install virtualenv tox 61 | - name: "Run tox targets for ${{ matrix.toxenv }}" 62 | run: python -m tox 63 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '23 13 * * 2' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v3 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v3 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v3 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # pyCharm 104 | .idea 105 | 106 | # VSCode 107 | .vscode/ 108 | 109 | # Visual Studio 110 | .vs/ 111 | 112 | # pytest 113 | .pytest_cache 114 | 115 | # scissor fixture 116 | papermill/tests/fixtures/scissor.txt 117 | 118 | # Binder example 119 | binder/*run*.ipynb 120 | 121 | # pip generated wheel metadata 122 | pip-wheel-metadata 123 | 124 | # VIM swap files 125 | *.swp 126 | 127 | 128 | # lint 129 | .ruff_cache/ 130 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: true 3 | autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions' 4 | autoupdate_schedule: quarterly 5 | # submodules: true 6 | 7 | repos: 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v4.5.0 10 | hooks: 11 | - id: end-of-file-fixer 12 | - id: trailing-whitespace 13 | - id: check-case-conflict 14 | - id: check-yaml 15 | - id: check-toml 16 | - id: check-json 17 | - id: check-added-large-files 18 | - id: check-docstring-first 19 | - id: detect-private-key 20 | 21 | #- repo: https://github.com/myint/docformatter 22 | # rev: v1.5.0 23 | # hooks: 24 | # - id: docformatter 25 | # args: [--in-place, --wrap-summaries=120, --wrap-descriptions=120] 26 | 27 | - repo: https://github.com/codespell-project/codespell 28 | rev: v2.2.6 29 | hooks: 30 | - id: codespell 31 | additional_dependencies: [tomli] 32 | #args: ["--write-changes"] # uncomment if you want to get automatic fixing 33 | 34 | - repo: https://github.com/psf/black 35 | rev: 23.12.1 36 | hooks: 37 | - id: black 38 | name: Black code 39 | 40 | - repo: https://github.com/executablebooks/mdformat 41 | rev: 0.7.17 42 | hooks: 43 | - id: mdformat 44 | additional_dependencies: 45 | - mdformat-gfm 46 | - mdformat_frontmatter 47 | 48 | - repo: https://github.com/astral-sh/ruff-pre-commit 49 | rev: v0.1.9 50 | hooks: 51 | - id: ruff 52 | args: ["--fix"] 53 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # Optionally build your docs in additional formats such as PDF 19 | formats: 20 | - pdf 21 | 22 | # Optionally set the version of Python and requirements required to build your docs 23 | python: 24 | install: 25 | - requirements: requirements/docs.txt 26 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | See the [papermill documentation](https://papermill.readthedocs.io/en/latest/changelog.html) 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Please read our entire [Code of Conduct](https://github.com/nteract/nteract/blob/main/CODE_OF_CONDUCT.md) 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # So You Want to Contribute to Papermill! 2 | 3 | We welcome all contributions to Papermill both large and small. We encourage you to join our community. 4 | 5 | ## Our Community Values 6 | 7 | We are an open and friendly community. Everybody is welcome. 8 | 9 | We encourage friendly discussions and respect for all. There are no exceptions. 10 | 11 | All contributions are equally important. Documentation, answering questions, and fixing bugs are equally as valuable as adding new features. 12 | 13 | Please read our entire [Code of Conduct](https://github.com/nteract/nteract/blob/main/CODE_OF_CONDUCT.md). 14 | 15 | ## Setting up Your Development Environment 16 | 17 | Following these instructions should give you an efficient path to opening your first pull-request. 18 | 19 | ### Cloning the Papermill Repository 20 | 21 | Fork the repository to your local Github account. Clone this repository to your local development machine. 22 | 23 | ```bash 24 | git clone https://github.com//papermill 25 | cd papermill 26 | ``` 27 | 28 | ### Install an Editable Version 29 | 30 | We prefer to use native venv to manage the development environment. 31 | 32 | ```bash 33 | python3 -m venv dev 34 | source dev/bin/activate 35 | ``` 36 | 37 | Install Papermill using: 38 | 39 | ```bash 40 | pip install -e '.[dev]' 41 | ``` 42 | 43 | or use conda if you prefer [conda](https://conda.io/docs/user-guide/tasks/manage-environments.html): 44 | 45 | ```bash 46 | conda create -n dev 47 | . activate dev 48 | ``` 49 | 50 | _Note: When you are finished you can use `source deactivate` to go back to your base environment._ 51 | 52 | ### Running Tests Locally 53 | 54 | If you are contributing with documentation please jump to [building documentation.](#Building-Documentation) 55 | 56 | We need to install the development package before we can run the tests. If anything is confusing below, always resort to the relevant documentation. 57 | 58 | For the most basic test runs against python 3.12 use this tox subset (callable after `pip install tox`): 59 | 60 | ```bash 61 | tox -e py312 62 | ``` 63 | 64 | This will just execute the unittests against python 3.8 in a new virtual env. The first run will take longer to setup the virtualenv, but will be fast after that point. 65 | 66 | For a full test suite of all envs and linting checks simply run tox without any arguments 67 | 68 | ```bash 69 | tox 70 | ``` 71 | 72 | This will require python3.8, 3.9, 3.10, 3.11, and 3.12 to be installed. 73 | 74 | Alternavitely pytest can be used if you have an environment already setup which works or has custom packages not present in the tox build. 75 | 76 | ```bash 77 | pytest 78 | ``` 79 | 80 | Now there should be a working and editable installation of Papermill to start making your own contributions. 81 | 82 | ### Building Documentation 83 | 84 | The documentation is built using the [Sphinx](http://www.sphinx-doc.org/en/master/) engine. To contribute, edit the [RestructuredText (`.rst`)](https://en.wikipedia.org/wiki/ReStructuredText) files in the docs directory to make changes and additions. 85 | 86 | Once you are done editing, to generate the documentation, use tox and the following command from the root directory of the repository: 87 | 88 | ```bash 89 | tox -e docs 90 | ``` 91 | 92 | This will generate `.html` files in the `/.tox/docs_out/` directory. Once you are satisfied, feel free to jump to the next section. 93 | 94 | ## So You're Ready to Pull Request 95 | 96 | The general workflow for this will be: 97 | 98 | 1. Run local tests 99 | 1. Pushed changes to your forked repository 100 | 1. Open pull request to main repository 101 | 102 | ### Run Tests Locally 103 | 104 | ```bash 105 | pytest --pyargs papermill 106 | ``` 107 | 108 | Run check manifest to ensure all files are accounted for in the repository. 109 | 110 | ```bash 111 | check-manifest 112 | ``` 113 | 114 | This commands read the `MANIFEST.in` file and explicitly specify the files to include in the source distribution. You can read more about how this works [here](https://docs.python.org/3/distutils/sourcedist.html). 115 | 116 | ### Push Changes to Forked Repo 117 | 118 | Your commits should be pushed to the forked repository. To verify this type 119 | 120 | ```bash 121 | git remote -v 122 | ``` 123 | 124 | and ensure the remotes point to your GitHub. Don't work on the main branch! 125 | 126 | 1. Commit changes to local repository: 127 | 128 | ```bash 129 | git checkout -b my-feature 130 | git add 131 | git commit 132 | ``` 133 | 134 | 1. Push changes to your remote repository: 135 | 136 | ```bash 137 | git push -u origin my-feature 138 | ``` 139 | 140 | ### Create Pull Request 141 | 142 | Follow [these](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) instrucutions to create a pull request from a forked repository. If you are submitting a bug-fix for a specific issue make sure to reference the issue in the pull request. 143 | 144 | There are good references to the [Git documentation](https://git-scm.com/doc) and [Git workflows](https://docs.scipy.org/doc/numpy/dev/gitwash/development_workflow.html) for more information if any of this is unfamiliar. 145 | 146 | _Note: You might want to set a reference to the main repository to fetch/merge from there instead of your forked repository. You can do that using:_ 147 | 148 | ```bash 149 | git remote add upstream https://github.com/nteract/papermill 150 | ``` 151 | 152 | It's possible you will have conflicts between your repository and main. Here, `main` is meant to be synchronized with the `upstream` repository. GitHub has some good [documentation](https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/) on merging pull requests from the command line. 153 | 154 | Happy hacking on Papermill! 155 | -------------------------------------------------------------------------------- /DEVELOPMENT_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Development Guide 2 | 3 | _Note: If you haven't read the CONTRIBUTING.md instructions, make sure to do so before continuing._ 4 | 5 | ## IO Sources / Sinks 6 | 7 | To add a new input source + sink to papermill look in the `iorw.py` file for a few examples. 8 | 9 | The `papermill_io` object at root context of the module holds the registered IO handlers. This maintains the LIFO queue of potential path consumers for any IO request. Each handler is registered to a prefix path which it uses as a prefix match against the input arguments. You'll notice `local` is the lowest order handler and is a special case we use to fall-back on if no other handler matches. 10 | 11 | To add a new handler, simply create a new class which implements the `read`, `listdir`, `write`, and `pretty_path` class methods. `listdir` is optional, and not used by the execute method / cli command. Then add `papermill_io.register("my-new-prefix", MyNewHandler)` and papermill will pick up the new IO handler for any paths it processes. 12 | 13 | Remember to add tests to prove your new handler works! 14 | 15 | _Note: You can also extend the registry in your own modules to enable domain specific io extensions without needing to commit to papermill._ 16 | 17 | ## Language Translations 18 | 19 | When you wish to add a new language or kernel to papermill, look in the `translators.py` file. 20 | 21 | Like with the iorw pattern, there is a `papermill_translators` object at the root of the file which holds all key-value mappings from kernel / language names to translators. Each Translator inherits from `Translator` which gives the basic JSON conversion structures. Then for each JSON type you'll need to add the relevant translate_type class method. Additionally, you'll want to implement the `comment` function for mapping single line comments. For languages which have a special format for assigning variables you can also override the assign method (see ScalaTranslator for an example). 22 | 23 | Finally, register the new handler to the `papermill_translators` object. The translator name must either match the kernel or language name being processed to be used for your notebook execution. This will enable any notebook using the named kernel to use your new parameter translations. 24 | 25 | Test additions are easy to create -- just copy the few language specific pytest methods in `test_translators.py` and swap to your new translator name / expected values. 26 | 27 | ## Engines 28 | 29 | By default papermill uses nbconvert to process notebooks. But it's setup as a plug-n-play system so any function that can process a notebook and return the output nbformat object can be registered into papermill. 30 | 31 | To enable a new engine, first look in `engines.py` at the `NBConvertEngine` as a working example. This class inherits from `Engine` and is required to implement the classmethod `execute_managed_notebook`. The first argument to this method is a `NotebookExecutionManager` -- which is built and passed in the Engine `execute_notebook` classmethod -- and is used to provide callback bindings for cell execution signals. 32 | 33 | The `NotebookExecutionManager` class tracks the notebook object in progress, which is copied from the input notebook to provide functional execution isolation. It also tracks metadata updates and execution timing. In general you don't need to worry about this class except to know it has a `nb` attribute and three callbacks you can call from your engine implementation. 34 | 35 | - `cell_start` takes a cell argument and sets the cell metadata up for execution. This triggers a notebook save. 36 | - `cell_exception` takes a cell argument and flags the cell as failed. This does **not** trigger a notebook save (as the notebook completion after cell failure will save). 37 | - `cell_complete` takes a cell argument and finalizes timing information in the cell metadata. This triggers a notebook save. 38 | 39 | These functions can be optionally called to better render and populate notebooks with appropriate metadata attributes to reflect their execution. Manually saving the notebook object is unnecessary as the base class wrapper will save the notebook on notebook start and completion on your behalf. If you wish to disable saving, overwrite the `wrap_and_execute_notebook` and prevent the `output_path` from propagating to the base method call. 40 | 41 | `papermill.execute_notebook` allows you to pass arbitrary arguments down to the engine. Make sure that engine handles keyword arguments properly. Use utility `merge_kwargs` and `remove_args` to merge and clean arguments. 42 | 43 | To update tests you'll need to add a new test class in `test_engines.py`. Copying the `TestNBConvertEngine` class and modifying it is recommended. 44 | 45 | ## CLI / Execute 46 | 47 | When adding an option to papermill, first look in `cli.py` and then `execute.py` for the two places to add your new configurable. 48 | 49 | In `cli.py` you'll want to add an `@click.option` above the `papermill` method and the option name as a positional argument in the `papermill` method. These are fairly straight forward to assign, and you can refer to click's documentation for how to do all the basic options. Then you'll need to pass the argument to `execute_notebook`. We treat the CLI layer as a very light wrapper on the execute method in an attempt to both obey DRY (don't repeat yourself) and to ensure that the imported python module has the same capabilities as the CLI. 50 | 51 | Now in `execute.py`'s `execute_notebook` we want to add the appropriate argument and default it to something sane. Add the argument to the docstring as well. Then pass or use that argument where it's needed to achieve the desired effect. Usually these options get passed to `_execute_parameterized_notebook`. 52 | 53 | To update the tests you'll need both `test_cli.py` and `test_execute.py` to include the new option. Though the CLI tests need only check that the appropriate values get passed to `execute_notebook`. 54 | 55 | # Releasing 56 | 57 | ## Prerequisites 58 | 59 | - First check that the CHANGELOG is up to date for the next release version 60 | - Ensure dev requirements are installed `pip install -r requirements/dev.txt` 61 | 62 | ## Push to GitHub 63 | 64 | Change from patch to minor or major for appropriate version updates. 65 | 66 | ```bash 67 | bumpversion patch 68 | git push upstream && git push upstream --tags 69 | ``` 70 | 71 | ## Push to PyPI 72 | 73 | ```bash 74 | rm -rf dist/* 75 | rm -rf build/* 76 | python setup.py sdist bdist_wheel 77 | twine upload dist/* 78 | ``` 79 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, nteract 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include papermill *.py 2 | recursive-include papermill *.ipynb 3 | recursive-include papermill *.json 4 | recursive-include papermill *.yaml 5 | recursive-include papermill *.yml 6 | recursive-include papermill *.keep 7 | recursive-include papermill *.txt 8 | 9 | include setup.py 10 | include requirements.txt 11 | include tox_py_installer.sh 12 | recursive-include requirements *.txt 13 | include tox.ini 14 | include pytest.ini 15 | include README.md 16 | include LICENSE 17 | include MANIFEST.in 18 | include *.md 19 | include *.toml 20 | 21 | include .bumpversion.cfg 22 | 23 | # Documentation 24 | prune docs 25 | 26 | exclude .pre-commit-config.yaml 27 | # exclude sample notebooks for binder 28 | prune binder 29 | # Scripts 30 | graft scripts 31 | # Test env 32 | prune .tox 33 | # Exclude notebooks checkpoints generated by testing 34 | recursive-exclude papermill/.ipynb_checkpoints *.ipynb 35 | recursive-exclude papermill/tests/notebooks/.ipynb_checkpoints *.ipynb 36 | 37 | # Build files 38 | exclude .github 39 | exclude .readthedocs.yaml 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 2 | 3 | 4 | 5 | [![CI](https://github.com/nteract/papermill/actions/workflows/ci.yml/badge.svg)](https://github.com/nteract/papermill/actions/workflows/ci.yml) 6 | [![CI](https://github.com/nteract/papermill/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/nteract/papermill/actions/workflows/ci.yml) 7 | [![image](https://codecov.io/github/nteract/papermill/coverage.svg?branch=main)](https://codecov.io/github/nteract/papermill?branch=main) 8 | [![Documentation Status](https://readthedocs.org/projects/papermill/badge/?version=latest)](http://papermill.readthedocs.io/en/latest/?badge=latest) 9 | [![badge](https://tinyurl.com/ybwovtw2)](https://mybinder.org/v2/gh/nteract/papermill/main?filepath=binder%2Fprocess_highlight_dates.ipynb) 10 | [![badge](https://tinyurl.com/y7uz2eh9)](https://mybinder.org/v2/gh/nteract/papermill/main?) 11 | [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/papermill)](https://pypi.org/project/papermill/) 12 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) 13 | [![papermill](https://snyk.io/advisor/python/papermill/badge.svg)](https://snyk.io/advisor/python/papermill) 14 | [![Anaconda-Server Badge](https://anaconda.org/conda-forge/papermill/badges/downloads.svg)](https://anaconda.org/conda-forge/papermill) 15 | [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/nteract/papermill/main.svg)](https://results.pre-commit.ci/latest/github/nteract/papermill/main) 16 | 17 | **papermill** is a tool for parameterizing, executing, and analyzing 18 | Jupyter Notebooks. 19 | 20 | Papermill lets you: 21 | 22 | - **parameterize** notebooks 23 | - **execute** notebooks 24 | 25 | This opens up new opportunities for how notebooks can be used. For 26 | example: 27 | 28 | - Perhaps you have a financial report that you wish to run with 29 | different values on the first or last day of a month or at the 30 | beginning or end of the year, **using parameters** makes this task 31 | easier. 32 | - Do you want to run a notebook and depending on its results, choose a 33 | particular notebook to run next? You can now programmatically 34 | **execute a workflow** without having to copy and paste from 35 | notebook to notebook manually. 36 | 37 | Papermill takes an *opinionated* approach to notebook parameterization and 38 | execution based on our experiences using notebooks at scale in data 39 | pipelines. 40 | 41 | ## Installation 42 | 43 | From the command line: 44 | 45 | ```{.sourceCode .bash} 46 | pip install papermill 47 | ``` 48 | 49 | For all optional io dependencies, you can specify individual bundles 50 | like `s3`, or `azure` -- or use `all`. To use Black to format parameters you can add as an extra requires \['black'\]. 51 | 52 | ```{.sourceCode .bash} 53 | pip install papermill[all] 54 | ``` 55 | 56 | ## Python Version Support 57 | 58 | This library currently supports Python 3.8+ versions. As minor Python 59 | versions are officially sunset by the Python org papermill will similarly 60 | drop support in the future. 61 | 62 | ## Usage 63 | 64 | ### Parameterizing a Notebook 65 | 66 | To parameterize your notebook designate a cell with the tag `parameters`. 67 | 68 | ![enable parameters in Jupyter](docs/img/enable_parameters.gif) 69 | 70 | Papermill looks for the `parameters` cell and treats this cell as defaults for the parameters passed in at execution time. Papermill will add a new cell tagged with `injected-parameters` with input parameters in order to overwrite the values in `parameters`. If no cell is tagged with `parameters` the injected cell will be inserted at the top of the notebook. 71 | 72 | Additionally, if you rerun notebooks through papermill and it will reuse the `injected-parameters` cell from the prior run. In this case Papermill will replace the old `injected-parameters` cell with the new run's inputs. 73 | 74 | ![image](docs/img/parameters.png) 75 | 76 | ### Executing a Notebook 77 | 78 | The two ways to execute the notebook with parameters are: (1) through 79 | the Python API and (2) through the command line interface. 80 | 81 | #### Execute via the Python API 82 | 83 | ```{.sourceCode .python} 84 | import papermill as pm 85 | 86 | pm.execute_notebook( 87 | 'path/to/input.ipynb', 88 | 'path/to/output.ipynb', 89 | parameters = dict(alpha=0.6, ratio=0.1) 90 | ) 91 | ``` 92 | 93 | #### Execute via CLI 94 | 95 | Here's an example of a local notebook being executed and output to an 96 | Amazon S3 account: 97 | 98 | ```{.sourceCode .bash} 99 | $ papermill local/input.ipynb s3://bkt/output.ipynb -p alpha 0.6 -p l1_ratio 0.1 100 | ``` 101 | 102 | **NOTE:** 103 | If you use multiple AWS accounts, and you have [properly configured your AWS credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html), then you can specify which account to use by setting the `AWS_PROFILE` environment variable at the command-line. For example: 104 | 105 | ```{.sourceCode .bash} 106 | $ AWS_PROFILE=dev_account papermill local/input.ipynb s3://bkt/output.ipynb -p alpha 0.6 -p l1_ratio 0.1 107 | ``` 108 | 109 | In the above example, two parameters are set: `alpha` and `l1_ratio` using `-p` (`--parameters` also works). Parameter values that look like booleans or numbers will be interpreted as such. Here are the different ways users may set parameters: 110 | 111 | ```{.sourceCode .bash} 112 | $ papermill local/input.ipynb s3://bkt/output.ipynb -r version 1.0 113 | ``` 114 | 115 | Using `-r` or `--parameters_raw`, users can set parameters one by one. However, unlike `-p`, the parameter will remain a string, even if it may be interpreted as a number or boolean. 116 | 117 | ```{.sourceCode .bash} 118 | $ papermill local/input.ipynb s3://bkt/output.ipynb -f parameters.yaml 119 | ``` 120 | 121 | Using `-f` or `--parameters_file`, users can provide a YAML file from which parameter values should be read. 122 | 123 | ```{.sourceCode .bash} 124 | $ papermill local/input.ipynb s3://bkt/output.ipynb -y " 125 | alpha: 0.6 126 | l1_ratio: 0.1" 127 | ``` 128 | 129 | Using `-y` or `--parameters_yaml`, users can directly provide a YAML string containing parameter values. 130 | 131 | ```{.sourceCode .bash} 132 | $ papermill local/input.ipynb s3://bkt/output.ipynb -b YWxwaGE6IDAuNgpsMV9yYXRpbzogMC4xCg== 133 | ``` 134 | 135 | Using `-b` or `--parameters_base64`, users can provide a YAML string, base64-encoded, containing parameter values. 136 | 137 | When using YAML to pass arguments, through `-y`, `-b` or `-f`, parameter values can be arrays or dictionaries: 138 | 139 | ```{.sourceCode .bash} 140 | $ papermill local/input.ipynb s3://bkt/output.ipynb -y " 141 | x: 142 | - 0.0 143 | - 1.0 144 | - 2.0 145 | - 3.0 146 | linear_function: 147 | slope: 3.0 148 | intercept: 1.0" 149 | ``` 150 | 151 | #### Supported Name Handlers 152 | 153 | Papermill supports the following name handlers for input and output paths during execution: 154 | 155 | - Local file system: `local` 156 | 157 | - HTTP, HTTPS protocol: `http://, https://` 158 | 159 | - Amazon Web Services: [AWS S3](https://aws.amazon.com/s3/) `s3://` 160 | 161 | - Azure: [Azure DataLake Store](https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-overview), [Azure Blob Store](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-overview) `adl://, abs://` 162 | 163 | - Google Cloud: [Google Cloud Storage](https://cloud.google.com/storage/) `gs://` 164 | 165 | ## Development Guide 166 | 167 | Read [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines on how to setup a local development environment and make code changes back to Papermill. 168 | 169 | For development guidelines look in the [DEVELOPMENT_GUIDE.md](./DEVELOPMENT_GUIDE.md) file. This should inform you on how to make particular additions to the code base. 170 | 171 | ## Documentation 172 | 173 | We host the [Papermill documentation](http://papermill.readthedocs.io) 174 | on ReadTheDocs. 175 | -------------------------------------------------------------------------------- /binder/cli-simple/README.md: -------------------------------------------------------------------------------- 1 | # Simple CLI Example 2 | 3 | This example uses three notebooks: 4 | 5 | - simple_input.ipynb 6 | - simple_output.ipynb 7 | - cli_example.ipynb 8 | 9 | 1. Open the cli_example.ipynb notebook. 10 | 11 | 1. Begin executing the cells. 12 | 13 | 1. One cell demonstrates running papermill from the command line. 14 | -------------------------------------------------------------------------------- /binder/cli-simple/cli_example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Example - Executing a notebook from the Command Line" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## How to add a parameter tag\n", 15 | "\n", 16 | "If you wanted to add your own parameter tag, you would go to the menu: View ➡️ Cell Toolbar ➡️ Tags\n" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Input notebook\n", 24 | "\n", 25 | "Let's take a look at the input notebook (`simple_input.ipynb`). This notebook has cells containing\n", 26 | "tags for the parameters. \n", 27 | "\n", 28 | "[See input notebook](simple_input.ipynb)" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "## Execute using the Command Line\n", 36 | "\n", 37 | "We are going to use the Jupyter magic `%%bash` to simulate executing the notebook from\n", 38 | "the command line from a bash terminal.\n", 39 | "\n", 40 | "The command takes the form:\n", 41 | "\n", 42 | "`papermill -p `" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": { 49 | "tags": [ 50 | "parameters" 51 | ] 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "binder_dir = '..'" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "!papermill {binder_dir}/cli-simple/simple_input.ipynb {binder_dir}/cli-simple/simple_output.ipynb -p msg 'Hello'" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "## Output notebook" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "Let's take a look at the output notebook.\n", 79 | "\n", 80 | "[See output notebook](simple_output.ipynb)" 81 | ] 82 | } 83 | ], 84 | "metadata": { 85 | "celltoolbar": "Tags", 86 | "kernelspec": { 87 | "display_name": "Python 3", 88 | "language": "python", 89 | "name": "python3" 90 | }, 91 | "language_info": { 92 | "codemirror_mode": { 93 | "name": "ipython", 94 | "version": 3 95 | }, 96 | "file_extension": ".py", 97 | "mimetype": "text/x-python", 98 | "name": "python", 99 | "nbconvert_exporter": "python", 100 | "pygments_lexer": "ipython3", 101 | "version": "3.6.7" 102 | } 103 | }, 104 | "nbformat": 4, 105 | "nbformat_minor": 2 106 | } 107 | -------------------------------------------------------------------------------- /binder/cli-simple/pm_example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Example - Executing a notebook from python" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## How to add a parameter tag\n", 15 | "\n", 16 | "If you wanted to add your own parameter tag, you would go to the menu: View ➡️ Cell Toolbar ➡️ Tags\n" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Input notebook\n", 24 | "\n", 25 | "Let's take a look at the input notebook (`simple_input.ipynb`). This notebook has cells containing\n", 26 | "tags for the parameters. \n", 27 | "\n", 28 | "[See input notebook](simple_input.ipynb)" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "## Execute using the Python\n", 36 | "\n", 37 | "We are going to use papermill's python interface to execute the notebook\n", 38 | "\n", 39 | "The command takes the form:\n", 40 | "\n", 41 | "`pm.execute_notebook(input, output, parameters={param_name: param_value}`" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 11, 47 | "metadata": { 48 | "tags": [ 49 | "parameters" 50 | ] 51 | }, 52 | "outputs": [ 53 | { 54 | "ename": "Exception", 55 | "evalue": "/home/mseal/Workspace/papermill/binder/cli-simple", 56 | "output_type": "error", 57 | "traceback": [ 58 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 59 | "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", 60 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mbinder_dir2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'..'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabspath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 61 | "\u001b[0;31mException\u001b[0m: /home/mseal/Workspace/papermill/binder/cli-simple" 62 | ] 63 | } 64 | ], 65 | "source": [ 66 | "import os\n", 67 | "import papermill as pm\n", 68 | "\n", 69 | "binder_dir = '..'" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "result = pm.execute_notebook(\n", 79 | " os.path.join(binder_dir, 'cli-simple', 'simple_input.ipynb'),\n", 80 | " os.path.join(binder_dir, 'cli-simple', 'simple_output.ipynb'),\n", 81 | " parameters={'msg': 'Hello'}\n", 82 | ")" 83 | ] 84 | } 85 | ], 86 | "metadata": { 87 | "celltoolbar": "Tags", 88 | "kernelspec": { 89 | "display_name": "Python 3", 90 | "language": "python", 91 | "name": "python3" 92 | }, 93 | "language_info": { 94 | "codemirror_mode": { 95 | "name": "ipython", 96 | "version": 3 97 | }, 98 | "file_extension": ".py", 99 | "mimetype": "text/x-python", 100 | "name": "python", 101 | "nbconvert_exporter": "python", 102 | "pygments_lexer": "ipython3", 103 | "version": "3.6.7" 104 | } 105 | }, 106 | "nbformat": 4, 107 | "nbformat_minor": 4 108 | } 109 | -------------------------------------------------------------------------------- /binder/cli-simple/simple_input.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Simple input notebook with parameters" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": { 14 | "tags": [ 15 | "parameters" 16 | ] 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "msg = None" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "print(msg)" 30 | ] 31 | } 32 | ], 33 | "metadata": { 34 | "celltoolbar": "Tags", 35 | "hide_input": false, 36 | "kernelspec": { 37 | "display_name": "Python 3", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.6.7" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 1 56 | } 57 | -------------------------------------------------------------------------------- /binder/cli-simple/simple_output.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "df2b0bde", 6 | "metadata": { 7 | "papermill": { 8 | "duration": 0.003083, 9 | "end_time": "2023-11-01T18:34:41.235066", 10 | "exception": false, 11 | "start_time": "2023-11-01T18:34:41.231983", 12 | "status": "completed" 13 | }, 14 | "tags": [] 15 | }, 16 | "source": [ 17 | "# Simple input notebook with parameters" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 1, 23 | "id": "0d7af2f3", 24 | "metadata": { 25 | "execution": { 26 | "iopub.execute_input": "2023-11-01T18:34:41.301099Z", 27 | "iopub.status.busy": "2023-11-01T18:34:41.300307Z", 28 | "iopub.status.idle": "2023-11-01T18:34:41.306171Z", 29 | "shell.execute_reply": "2023-11-01T18:34:41.305038Z" 30 | }, 31 | "papermill": { 32 | "duration": 0.012465, 33 | "end_time": "2023-11-01T18:34:41.308519", 34 | "exception": false, 35 | "start_time": "2023-11-01T18:34:41.296054", 36 | "status": "completed" 37 | }, 38 | "tags": [ 39 | "parameters" 40 | ] 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "msg = None" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "id": "a5d4ad35", 51 | "metadata": { 52 | "execution": { 53 | "iopub.execute_input": "2023-11-01T18:34:41.314681Z", 54 | "iopub.status.busy": "2023-11-01T18:34:41.314338Z", 55 | "iopub.status.idle": "2023-11-01T18:34:41.318662Z", 56 | "shell.execute_reply": "2023-11-01T18:34:41.317802Z" 57 | }, 58 | "papermill": { 59 | "duration": 0.010058, 60 | "end_time": "2023-11-01T18:34:41.320883", 61 | "exception": false, 62 | "start_time": "2023-11-01T18:34:41.310825", 63 | "status": "completed" 64 | }, 65 | "tags": [ 66 | "injected-parameters" 67 | ] 68 | }, 69 | "outputs": [], 70 | "source": [ 71 | "# Parameters\n", 72 | "msg = \"Hello\"\n" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 3, 78 | "id": "99512b79", 79 | "metadata": { 80 | "execution": { 81 | "iopub.execute_input": "2023-11-01T18:34:41.327359Z", 82 | "iopub.status.busy": "2023-11-01T18:34:41.326530Z", 83 | "iopub.status.idle": "2023-11-01T18:34:41.333035Z", 84 | "shell.execute_reply": "2023-11-01T18:34:41.331879Z" 85 | }, 86 | "papermill": { 87 | "duration": 0.012117, 88 | "end_time": "2023-11-01T18:34:41.335151", 89 | "exception": false, 90 | "start_time": "2023-11-01T18:34:41.323034", 91 | "status": "completed" 92 | }, 93 | "tags": [] 94 | }, 95 | "outputs": [ 96 | { 97 | "name": "stdout", 98 | "output_type": "stream", 99 | "text": [ 100 | "Hello\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "print(msg)" 106 | ] 107 | } 108 | ], 109 | "metadata": { 110 | "celltoolbar": "Tags", 111 | "hide_input": false, 112 | "kernelspec": { 113 | "display_name": "Python 3", 114 | "language": "python", 115 | "name": "python3" 116 | }, 117 | "language_info": { 118 | "codemirror_mode": { 119 | "name": "ipython", 120 | "version": 3 121 | }, 122 | "file_extension": ".py", 123 | "mimetype": "text/x-python", 124 | "name": "python", 125 | "nbconvert_exporter": "python", 126 | "pygments_lexer": "ipython3", 127 | "version": "3.12.0" 128 | }, 129 | "papermill": { 130 | "default_parameters": {}, 131 | "duration": 1.612177, 132 | "end_time": "2023-11-01T18:34:41.659543", 133 | "environment_variables": {}, 134 | "exception": null, 135 | "input_path": "binder/cli-simple/simple_input.ipynb", 136 | "output_path": "binder/cli-simple/simple_output.ipynb", 137 | "parameters": { 138 | "msg": "Hello" 139 | }, 140 | "start_time": "2023-11-01T18:34:40.047366", 141 | "version": "2.4.0" 142 | } 143 | }, 144 | "nbformat": 4, 145 | "nbformat_minor": 5 146 | } 147 | -------------------------------------------------------------------------------- /binder/postBuild: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | # Work-around until pip stops making this broken in requirement.txt 4 | pip install -e . --no-use-pep517 --no-cache-dir 5 | -------------------------------------------------------------------------------- /binder/requirements.txt: -------------------------------------------------------------------------------- 1 | nteract-scrapbook 2 | pandas 3 | matplotlib 4 | ipywidgets 5 | ipykernel 6 | ansicolors 7 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = papermill 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/UPDATE.md: -------------------------------------------------------------------------------- 1 | TODO: Figure out make options needed for non-api changes 2 | 3 | ``` 4 | sphinx-apidoc -f -o reference ../papermill 5 | ``` 6 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | img.logo { 2 | width:100% 3 | } 4 | 5 | .right-next { 6 | float: right; 7 | max-width: 45%; 8 | overflow: auto; 9 | text-overflow: ellipsis; 10 | white-space: nowrap; 11 | } 12 | 13 | .right-next::after{ 14 | content: ' »'; 15 | } 16 | 17 | .left-prev { 18 | float: left; 19 | max-width: 45%; 20 | overflow: auto; 21 | text-overflow: ellipsis; 22 | white-space: nowrap; 23 | } 24 | 25 | .left-prev::before{ 26 | content: '« '; 27 | } 28 | 29 | .prev-next-bottom { 30 | margin-top: 3em; 31 | } 32 | 33 | .prev-next-top { 34 | margin-bottom: 1em; 35 | } 36 | -------------------------------------------------------------------------------- /docs/_static/images/papermill.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/_static/images/papermill.png -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # papermill documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jan 15 09:50:01 2018. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | import os 20 | import sys 21 | 22 | sys.path.insert(0, os.path.abspath('..')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | needs_sphinx = '3.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.autodoc', 35 | 'sphinx.ext.intersphinx', 36 | 'sphinx.ext.mathjax', 37 | 'sphinx.ext.napoleon', 38 | 'myst_parser', 39 | 'sphinx_copybutton', 40 | ] 41 | 42 | # Add any paths that contain templates here, relative to this directory. 43 | templates_path = ['_templates'] 44 | 45 | # The suffix(es) of source filenames. 46 | # You can specify multiple suffix as a list of string: 47 | # 48 | source_suffix = ['.rst', '.md', '.ipynb'] 49 | 50 | # The master toctree document. 51 | master_doc = 'index' 52 | 53 | # General information about the project. 54 | project = 'papermill' 55 | copyright = '2018, nteract team' 56 | author = 'nteract team' 57 | 58 | # The version info for the project you're documenting, acts as replacement for 59 | # |version| and |release|, also used in various other places throughout the 60 | # built documents. 61 | # 62 | import papermill 63 | 64 | # The short X.Y version. 65 | version = '.'.join(papermill.__version__.split('.')[0:2]) 66 | 67 | # The full version, including alpha/beta/rc tags. 68 | release = papermill.__version__ 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | # 73 | # This is also used if you do content translation via gettext catalogs. 74 | # Usually you set "language" from the command line foexitr these cases. 75 | language = 'python' 76 | 77 | # List of patterns, relative to source directory, that match files and 78 | # directories to ignore when looking for source files. 79 | # This patterns also effect to html_static_path and html_extra_path 80 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'UPDATE.md'] 81 | 82 | # The name of the Pygments (syntax highlighting) style to use. 83 | pygments_style = "sphinx" 84 | 85 | # If true, `todo` and `todoList` produce output, else they produce nothing. 86 | todo_include_todos = False 87 | 88 | 89 | # -- Options for HTML output ---------------------------------------------- 90 | 91 | # The theme to use for HTML and HTML Help pages. See the documentation for 92 | # a list of builtin themes. 93 | html_theme = "furo" 94 | 95 | # Theme options are theme-specific and customize the look and feel of a theme 96 | # further. For a list of options available for each theme, see the 97 | # documentation. 98 | # 99 | html_theme_options = { 100 | "sidebar_hide_name": True, 101 | } 102 | 103 | # Add any paths that contain custom static files (such as style sheets) here, 104 | # relative to this directory. They are copied after the builtin static files, 105 | # so a file named "default.css" will overwrite the builtin "default.css". 106 | html_static_path = ['_static'] 107 | 108 | html_logo = "_static/images/papermill.png" 109 | 110 | # -- Options for HTMLHelp output ------------------------------------------ 111 | 112 | # Output file base name for HTML help builder. 113 | htmlhelp_basename = 'papermilldoc' 114 | 115 | # -- Options for LaTeX output --------------------------------------------- 116 | 117 | latex_elements = { 118 | # The paper size ('letterpaper' or 'a4paper'). 119 | # 120 | # 'papersize': 'letterpaper', 121 | # The font size ('10pt', '11pt' or '12pt'). 122 | # 123 | # 'pointsize': '10pt', 124 | # Additional stuff for the LaTeX preamble. 125 | # 126 | # 'preamble': '', 127 | # Latex figure (float) alignment 128 | # 129 | # 'figure_align': 'htbp', 130 | } 131 | 132 | # Grouping the document tree into LaTeX files. List of tuples 133 | # (source start file, target name, title, 134 | # author, documentclass [howto, manual, or own class]). 135 | latex_documents = [(master_doc, 'papermill.tex', 'papermill Documentation', 'nteract team', 'manual')] 136 | 137 | 138 | # -- Options for manual page output --------------------------------------- 139 | 140 | # One entry per manual page. List of tuples 141 | # (source start file, name, description, authors, manual section). 142 | man_pages = [(master_doc, 'papermill', 'papermill Documentation', [author], 1)] 143 | 144 | 145 | # -- Options for Texinfo output ------------------------------------------- 146 | 147 | # Grouping the document tree into Texinfo files. List of tuples 148 | # (source start file, target name, title, author, 149 | # dir menu entry, description, category) 150 | texinfo_documents = [ 151 | ( 152 | master_doc, 153 | 'papermill', 154 | 'papermill Documentation', 155 | author, 156 | 'papermill', 157 | 'One line description of project.', 158 | 'Miscellaneous', 159 | ) 160 | ] 161 | 162 | # Example configuration for intersphinx: refer to the Python standard library. 163 | intersphinx_mapping = {'python': ('https://docs.python.org/', None)} 164 | -------------------------------------------------------------------------------- /docs/extending-developing.rst: -------------------------------------------------------------------------------- 1 | .. _developing-papermill: 2 | 3 | Extending papermill by contributing to it 4 | ========================================= 5 | 6 | If you find that you'd like to not only add I/O and execution handlers, but 7 | think a fundamental aspect of the project could use some improvement, then you 8 | may want to contribute to it. 9 | 10 | Development of papermill happens on github, and a 11 | `detailed guide to contributing`_ to it can be found there. There is also a 12 | `code of conduct`_ there. Please read both documents before beginning! 13 | 14 | 15 | .. _`detailed guide to contributing`: https://github.com/nteract/papermill/blob/main/CONTRIBUTING.md 16 | .. _`code of conduct`: https://github.com/nteract/nteract/blob/main/CODE_OF_CONDUCT.md 17 | -------------------------------------------------------------------------------- /docs/extending-overview.rst: -------------------------------------------------------------------------------- 1 | Extending papermill 2 | =================== 3 | 4 | Papermill provides some interfaces with external services out of the box. 5 | However, you may find that you would like papermill to do more than it currently 6 | does. You could contribute to the papermill project yourself (see 7 | :ref:`developing-papermill`). However, an easier method might be to extend 8 | papermill using `entry points`_. 9 | 10 | In general, when you run a notebook with papermill, the following happens: 11 | 12 | 1. The notebook file is read in 13 | 2. The file content is converted to a notebook python object 14 | 3. The notebook is executed 15 | 4. The notebook is written to a file 16 | 17 | Through entry points, you can write your own tools to handle steps 1, 3, and 4. 18 | If you find that there's more you want to contribute to papermill, consider 19 | developing papermill itself. 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | extending-entry-points 25 | extending-developing 26 | 27 | 28 | .. _`entry points`: https://packaging.python.org/specifications/entry-points/ 29 | -------------------------------------------------------------------------------- /docs/img/custom_execution_engine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/custom_execution_engine.png -------------------------------------------------------------------------------- /docs/img/enable_parameters.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/enable_parameters.gif -------------------------------------------------------------------------------- /docs/img/lab3_parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/lab3_parameters.png -------------------------------------------------------------------------------- /docs/img/lab_parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/lab_parameters.png -------------------------------------------------------------------------------- /docs/img/matplotlib_hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/matplotlib_hist.png -------------------------------------------------------------------------------- /docs/img/nb_dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/nb_dataframe.png -------------------------------------------------------------------------------- /docs/img/nbs_dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/nbs_dataframe.png -------------------------------------------------------------------------------- /docs/img/parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/docs/img/parameters.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to papermill 2 | ==================== 3 | 4 | .. image:: https://github.com/nteract/papermill/actions/workflows/ci.yml/badge.svg 5 | :target: https://github.com/nteract/papermill/actions/workflows/ci.yml 6 | .. image:: https://codecov.io/github/nteract/papermill/coverage.svg?branch=main 7 | :target: https://codecov.io/github/nteract/papermill?branch=main 8 | .. image:: https://readthedocs.org/projects/papermill/badge/?version=latest 9 | :target: http://papermill.readthedocs.io/en/latest/?badge=latest 10 | .. image:: https://tinyurl.com/ybwovtw2 11 | :target: https://mybinder.org/v2/gh/nteract/papermill/main?filepath=binder%2Fprocess_highlight_dates.ipynb 12 | .. image:: https://tinyurl.com/y7uz2eh9 13 | :target: https://mybinder.org/v2/gh/nteract/papermill/main?filepath=binder%2Fcli-simple%2Fcli_example.ipynb 14 | .. image:: https://img.shields.io/badge/code%20style-black-000000.svg 15 | :target: https://github.com/ambv/black 16 | 17 | **Papermill** is a tool for parameterizing and executing Jupyter Notebooks. 18 | 19 | Papermill lets you: 20 | 21 | * **parameterize** notebooks 22 | * **execute** notebooks 23 | 24 | This opens up new opportunities for how notebooks can be used. For example: 25 | 26 | - Perhaps you have a financial report that you wish to run with different 27 | values on the first or last day of a month or at the beginning or end 28 | of the year, **using parameters** makes this task easier. 29 | - Do you want to run a notebook and depending on its results, 30 | choose a particular notebook to run next? You can now programmatically 31 | **execute a workflow** without having to copy and paste from notebook to 32 | notebook manually. 33 | 34 | Python Version Support 35 | ---------------------- 36 | 37 | This library currently supports python 3.8+ versions. As minor python 38 | versions are officially sunset by the python org papermill will similarly 39 | drop support in the future. 40 | 41 | Documentation 42 | ------------- 43 | 44 | These pages guide you through the installation and usage of papermill. 45 | 46 | .. toctree:: 47 | :maxdepth: 2 48 | 49 | installation 50 | usage-workflow 51 | usage-cli 52 | extending-overview 53 | troubleshooting 54 | changelog 55 | 56 | API Reference 57 | ------------- 58 | 59 | If you are looking for information about a specific function, class, or method, 60 | this documentation section will help you. 61 | 62 | .. toctree:: 63 | :maxdepth: 3 64 | 65 | reference/index.rst 66 | 67 | Indices and tables 68 | ------------------ 69 | 70 | * :ref:`genindex` 71 | * :ref:`modindex` 72 | * :ref:`search` 73 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | Installing papermill 5 | -------------------- 6 | 7 | From the command line: 8 | 9 | .. code-block:: bash 10 | 11 | python3 -m pip install papermill 12 | 13 | 14 | Installing In-Notebook language bindings 15 | ---------------------------------------- 16 | 17 | In-Notebook language bindings provide helpers and utilities for using Papermill 18 | with a programming language. 19 | 20 | Python bindings 21 | ~~~~~~~~~~~~~~~ 22 | 23 | No additional installation steps are required since python bindings are built 24 | into **papermill**. 25 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=papermill 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | This part of the documentation lists the full API reference of all public classes and functions. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | papermill-cli 10 | papermill-workflow 11 | papermill-translators 12 | papermill-io 13 | papermill-storage 14 | papermill-utilities 15 | -------------------------------------------------------------------------------- /docs/reference/papermill-cli.rst: -------------------------------------------------------------------------------- 1 | CLI 2 | === 3 | 4 | 5 | papermill.cli 6 | ------------- 7 | 8 | .. automodule:: papermill.cli 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | Command Line options 14 | -------------------- 15 | 16 | .. code-block:: bash 17 | 18 | Usage: papermill [OPTIONS] NOTEBOOK_PATH OUTPUT_PATH 19 | 20 | This utility executes a single notebook in a subprocess. 21 | 22 | Papermill takes a source notebook, applies parameters to the source 23 | notebook, executes the notebook with the specified kernel, and saves the 24 | output in the destination notebook. 25 | 26 | The NOTEBOOK_PATH and OUTPUT_PATH can now be replaced by `-` representing 27 | stdout and stderr, or by the presence of pipe inputs / outputs. Meaning 28 | that 29 | 30 | `... | papermill | ...` 31 | 32 | with `papermill - -` being implied by the pipes will read a notebook from 33 | stdin and write it out to stdout. 34 | 35 | Options: 36 | -p, --parameters TEXT... Parameters to pass to the parameters cell. 37 | -r, --parameters_raw TEXT... Parameters to be read as raw string. 38 | -f, --parameters_file TEXT Path to YAML file containing parameters. 39 | -y, --parameters_yaml TEXT YAML string to be used as parameters. 40 | -b, --parameters_base64 TEXT Base64 encoded YAML string as parameters. 41 | --inject-input-path Insert the path of the input notebook as 42 | PAPERMILL_INPUT_PATH as a notebook 43 | parameter. 44 | --inject-output-path Insert the path of the output notebook as 45 | PAPERMILL_OUTPUT_PATH as a notebook 46 | parameter. 47 | --inject-paths Insert the paths of input/output notebooks 48 | as 49 | PAPERMILL_INPUT_PATH/PAPERMILL_OUTPUT_PATH 50 | as notebook parameters. 51 | --engine TEXT The execution engine name to use in 52 | evaluating the notebook. 53 | --request-save-on-cell-execute / --no-request-save-on-cell-execute 54 | Request save notebook after each cell 55 | execution 56 | --prepare-only / --prepare-execute 57 | Flag for outputting the notebook without 58 | execution, but with parameters applied. 59 | -k, --kernel TEXT Name of kernel to run. 60 | --cwd TEXT Working directory to run notebook in. 61 | --progress-bar / --no-progress-bar 62 | Flag for turning on the progress bar. 63 | --log-output / --no-log-output Flag for writing notebook output to the 64 | configured logger. 65 | --stdout-file FILENAME File to write notebook stdout output to. 66 | --stderr-file FILENAME File to write notebook stderr output to. 67 | --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] 68 | Set log level 69 | --start_timeout INTEGER Time in seconds to wait for kernel to start. 70 | --execution_timeout INTEGER Time in seconds to wait for each cell before 71 | failing execution (default: forever) 72 | --report-mode / --no-report-mode 73 | Flag for hiding input. 74 | --version Flag for displaying the version. 75 | -h, --help Show this message and exit. 76 | -------------------------------------------------------------------------------- /docs/reference/papermill-io.rst: -------------------------------------------------------------------------------- 1 | Input / Output 2 | ============== 3 | 4 | 5 | papermill.iorw 6 | -------------- 7 | 8 | .. automodule:: papermill.iorw 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | -------------------------------------------------------------------------------- /docs/reference/papermill-storage.rst: -------------------------------------------------------------------------------- 1 | Storage 2 | ======= 3 | 4 | Azure 5 | ----- 6 | 7 | These modules outline how to interact with Azure data stores, specifically 8 | Azure Blob Storage and Azure Data Lakes. 9 | 10 | papermill.abs module 11 | ~~~~~~~~~~~~~~~~~~~~ 12 | 13 | .. automodule:: papermill.abs 14 | :members: 15 | :undoc-members: 16 | :show-inheritance: 17 | 18 | papermill.adl module 19 | ~~~~~~~~~~~~~~~~~~~~ 20 | 21 | .. automodule:: papermill.adl 22 | :members: 23 | :undoc-members: 24 | :show-inheritance: 25 | 26 | AWS 27 | --- 28 | 29 | This module shows how to interact with AWS S3 data stores. 30 | 31 | papermill.s3 module 32 | ~~~~~~~~~~~~~~~~~~~ 33 | 34 | .. automodule:: papermill.s3 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | -------------------------------------------------------------------------------- /docs/reference/papermill-translators.rst: -------------------------------------------------------------------------------- 1 | Language Translators 2 | ==================== 3 | 4 | .. currentmodule:: papermill.translators 5 | 6 | 7 | Translators 8 | ----------- 9 | 10 | Translator 11 | ~~~~~~~~~~ 12 | 13 | .. autoclass:: papermill.translators.Translator 14 | :members: 15 | :undoc-members: 16 | 17 | PapermillTranslators 18 | ~~~~~~~~~~~~~~~~~~~~ 19 | 20 | .. autoclass:: papermill.translators.PapermillTranslators 21 | :members: 22 | :undoc-members: 23 | 24 | Python 25 | ------ 26 | 27 | .. autoclass:: papermill.translators.PythonTranslator 28 | :members: 29 | :undoc-members: 30 | 31 | R 32 | - 33 | 34 | .. autoclass:: papermill.translators.RTranslator 35 | :members: 36 | :undoc-members: 37 | 38 | Julia 39 | ----- 40 | 41 | .. autoclass:: papermill.translators.JuliaTranslator 42 | :members: 43 | :undoc-members: 44 | 45 | Scala 46 | ----- 47 | 48 | .. autoclass:: papermill.translators.ScalaTranslator 49 | :members: 50 | :undoc-members: 51 | 52 | Functions 53 | --------- 54 | 55 | .. autofunction:: translate_parameters 56 | -------------------------------------------------------------------------------- /docs/reference/papermill-utilities.rst: -------------------------------------------------------------------------------- 1 | Utilities 2 | ========= 3 | 4 | 5 | Utils 6 | ----- 7 | 8 | .. automodule:: papermill.utils 9 | :members: 10 | :undoc-members: 11 | 12 | Exceptions 13 | ---------- 14 | 15 | .. automodule:: papermill.exceptions 16 | :members: 17 | :undoc-members: 18 | 19 | Log 20 | --- 21 | 22 | .. automodule:: papermill.log 23 | :members: 24 | :undoc-members: 25 | -------------------------------------------------------------------------------- /docs/reference/papermill-workflow.rst: -------------------------------------------------------------------------------- 1 | Workflow 2 | ======== 3 | 4 | 5 | papermill.engines 6 | ----------------- 7 | 8 | .. automodule:: papermill.engines 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | 14 | papermill.execute 15 | ----------------- 16 | 17 | .. automodule:: papermill.execute 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | papermill.clientwrap 23 | -------------------- 24 | 25 | .. automodule:: papermill.clientwrap 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | -------------------------------------------------------------------------------- /docs/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | Troubleshooting 2 | =============== 3 | 4 | NoSuchKernel Errors (using Conda) 5 | --------------------------------- 6 | 7 | ``NoSuchKernel`` Errors can appear when running papermill on jupyter notebooks whose 8 | kernel has been specified via conda (nb_conda). nb_conda is used to easily set 9 | conda environment per notebook from within jupyterlab. 10 | 11 | To illustrate, the following example demonstrates the creation of a new 12 | environment with all the dependencies necessary for an analysis. 13 | 14 | :: 15 | 16 | conda create -n analysis_1 python=2 ipykernel 17 | 18 | Once nb_conda is used within the jupyter server to set the kernel for a 19 | notebook to *analysis_1*, the notebook gets metadata similar to the following: 20 | 21 | .. code-block:: json 22 | 23 | { 24 | "kernelspec": { 25 | "display_name": "Python [conda env:analysis_1]", 26 | "language": "python", 27 | "name": "conda-env-analysis_1-py" 28 | } 29 | } 30 | 31 | Papermill cannot use this metadata to determine that it should use 32 | *analysis_1* to execute this notebook. Running papermill (from *analysis_1* or 33 | another environment) will raise the following error: 34 | 35 | :: 36 | 37 | jupyter_client.kernelspec.NoSuchKernel: No such kernel named conda-env-analysis_1-py 38 | 39 | This can be fixed by: 40 | 41 | * Installing jupyter 42 | (`or at least ipykernel `_) 43 | in *analysis_1* 44 | 45 | :: 46 | 47 | conda install -n analysis_1 jupyter 48 | 49 | * Expose the *analysis_1* environment as a jupyter kernel 50 | (`this is no longer automatic `_). 51 | 52 | :: 53 | 54 | conda activate analysis_1 55 | jupyter kernelspec install --user --name analysis_1 56 | 57 | * Run papermill (from any environment) specifying the correct kernel using the 58 | ``-k`` option 59 | 60 | :: 61 | 62 | papermill my_notebook.ipynb output_notebook.ipynb -k analysis_1 63 | -------------------------------------------------------------------------------- /docs/usage-cli.rst: -------------------------------------------------------------------------------- 1 | Command Line Interface 2 | ====================== 3 | 4 | papermill may be executed from the terminal. The following are the command 5 | options: 6 | 7 | .. code-block:: bash 8 | 9 | Usage: papermill [OPTIONS] NOTEBOOK_PATH [OUTPUT_PATH] 10 | 11 | This utility executes a single notebook in a subprocess. 12 | 13 | Papermill takes a source notebook, applies parameters to the source 14 | notebook, executes the notebook with the specified kernel, and saves the 15 | output in the destination notebook. 16 | 17 | The NOTEBOOK_PATH and OUTPUT_PATH can now be replaced by `-` representing 18 | stdout and stderr, or by the presence of pipe inputs / outputs. Meaning 19 | that 20 | 21 | `... | papermill | ...` 22 | 23 | with `papermill - -` being implied by the pipes will read a notebook from 24 | stdin and write it out to stdout. 25 | 26 | Options: 27 | --help-notebook Display parameters information for the given 28 | notebook path. 29 | 30 | -p, --parameters TEXT... Parameters to pass to the parameters cell. 31 | -r, --parameters_raw TEXT... Parameters to be read as raw string. 32 | -f, --parameters_file TEXT Path to YAML file containing parameters. 33 | -y, --parameters_yaml TEXT YAML string to be used as parameters. 34 | -b, --parameters_base64 TEXT Base64 encoded YAML string as parameters. 35 | --inject-input-path Insert the path of the input notebook as 36 | PAPERMILL_INPUT_PATH as a notebook 37 | parameter. 38 | 39 | --inject-output-path Insert the path of the output notebook as 40 | PAPERMILL_OUTPUT_PATH as a notebook 41 | parameter. 42 | 43 | --inject-paths Insert the paths of input/output notebooks 44 | as 45 | PAPERMILL_INPUT_PATH/PAPERMILL_OUTPUT_PATH 46 | as notebook parameters. 47 | 48 | --engine TEXT The execution engine name to use in 49 | evaluating the notebook. 50 | 51 | --request-save-on-cell-execute / --no-request-save-on-cell-execute 52 | Request save notebook after each cell 53 | execution 54 | 55 | --autosave-cell-every INTEGER How often in seconds to autosave the 56 | notebook during long cell executions (0 to 57 | disable) 58 | 59 | --prepare-only / --prepare-execute 60 | Flag for outputting the notebook without 61 | execution, but with parameters applied. 62 | 63 | -k, --kernel TEXT Name of kernel to run. 64 | --cwd TEXT Working directory to run notebook in. 65 | --progress-bar / --no-progress-bar 66 | Flag for turning on the progress bar. 67 | --log-output / --no-log-output Flag for writing notebook output to the 68 | configured logger. 69 | 70 | --stdout-file FILENAME File to write notebook stdout output to. 71 | --stderr-file FILENAME File to write notebook stderr output to. 72 | --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] 73 | Set log level 74 | --start-timeout, --start_timeout INTEGER 75 | Time in seconds to wait for kernel to start. 76 | --execution-timeout INTEGER Time in seconds to wait for each cell before 77 | failing execution (default: forever) 78 | 79 | --report-mode / --no-report-mode 80 | Flag for hiding input. 81 | --version Flag for displaying the version. 82 | -h, --help Show this message and exit. 83 | -------------------------------------------------------------------------------- /docs/usage-execute.rst: -------------------------------------------------------------------------------- 1 | Execute 2 | ======= 3 | 4 | The two ways to execute the notebook with parameters are: (1) through the 5 | Python API and (2) through the command line interface. 6 | 7 | Execute via the Python API 8 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 9 | 10 | The `execute_notebook` function can be called to execute an input notebook 11 | when passed a dictionary of parameters: 12 | 13 | .. code-block:: python 14 | 15 | execute_notebook(, , ) 16 | 17 | 18 | 19 | .. code-block:: python 20 | 21 | import papermill as pm 22 | 23 | pm.execute_notebook( 24 | 'path/to/input.ipynb', 25 | 'path/to/output.ipynb', 26 | parameters=dict(alpha=0.6, ratio=0.1) 27 | ) 28 | 29 | Execute via CLI 30 | ~~~~~~~~~~~~~~~ 31 | 32 | To execute a notebook using the CLI, enter the ``papermill`` command in the 33 | terminal with the input notebook, location for output notebook, and options. 34 | 35 | .. seealso:: 36 | 37 | :doc:`CLI reference <./usage-cli>` 38 | 39 | Execute a notebook with parameters 40 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 41 | 42 | Here's an example of a local notebook being executed and output to an 43 | Amazon S3 account: 44 | 45 | .. code-block:: bash 46 | 47 | $ papermill local/input.ipynb s3://bkt/output.ipynb -p alpha 0.6 -p l1_ratio 0.1 48 | 49 | In the above example, two parameters are set: ``alpha`` and ``l1_ratio`` using ``-p`` (``--parameters`` also works). 50 | Parameter values that look like booleans or numbers will be interpreted as such. 51 | 52 | Here are the different ways users may set parameters: 53 | 54 | Using raw strings as parameters 55 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 56 | 57 | Using ``-r`` or ``--parameters_raw``, users can set parameters one by one. However, unlike ``-p``, the parameter will 58 | remain a string, even if it may be interpreted as a number or boolean. 59 | 60 | .. code-block:: bash 61 | 62 | $ papermill local/input.ipynb s3://bkt/output.ipynb -r version 1.0 63 | 64 | Using a parameters file 65 | ^^^^^^^^^^^^^^^^^^^^^^^ 66 | 67 | Using ``-f`` or ``--parameters_file``, users can provide a YAML file from which parameter values should be read. 68 | 69 | .. code-block:: bash 70 | 71 | $ papermill local/input.ipynb s3://bkt/output.ipynb -f parameters.yaml 72 | 73 | Using a YAML string for parameters 74 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 75 | 76 | Using ``-y`` or ``--parameters_yaml``, users can directly provide a YAML string containing parameter values. 77 | 78 | .. code-block:: bash 79 | 80 | $ papermill local/input.ipynb s3://bkt/output.ipynb -y " 81 | x: 82 | - 0.0 83 | - 1.0 84 | - 2.0 85 | - 3.0 86 | linear_function: 87 | slope: 3.0 88 | intercept: 1.0" 89 | 90 | Using ``-b`` or ``--parameters_base64``, users can provide a YAML string, base64-encoded, containing parameter values. 91 | 92 | .. code-block:: bash 93 | 94 | $ papermill local/input.ipynb s3://bkt/output.ipynb -b YWxwaGE6IDAuNgpsMV9yYXRpbzogMC4xCg== 95 | 96 | Note about using YAML 97 | ^^^^^^^^^^^^^^^^^^^^^ 98 | 99 | When using YAML to pass arguments, through ``-y``, ``-b`` or ``-f``, parameter values can be arrays or dictionaries: 100 | 101 | .. code-block:: bash 102 | 103 | $ papermill local/input.ipynb s3://bkt/output.ipynb -y " 104 | x: 105 | - 0.0 106 | - 1.0 107 | - 2.0 108 | - 3.0 109 | linear_function: 110 | slope: 3.0 111 | intercept: 1.0" 112 | 113 | Note about using with multiple account credentials 114 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 115 | 116 | If you use multiple AWS accounts and are accessing S3 files, you can 117 | `configure your AWS credentials `__, 118 | to specify which account to use by setting the `AWS_PROFILE` environment 119 | variable at the command-line. For example: 120 | 121 | .. code-block:: bash 122 | 123 | $ AWS_PROFILE=dev_account papermill local/input.ipynb s3://bkt/output.ipynb -p alpha 0.6 -p l1_ratio 0.1 124 | 125 | A similar pattern may be needed for other types of remote storage accounts. 126 | 127 | Tracking the Execution with Cell Descriptions 128 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 129 | If you want to keep track of the execution in a closer way, you can add cell descriptions to the notebook being executed that will be injected to the TQDM progress bar. 130 | 131 | Here is an example of the TQDM output during the execution of a 4-cells notebook without cell descriptions: 132 | 133 | .. code-block:: bash 134 | 135 | 10:10 # papermill input_notebook.ipynb output_notebook.ipynb 136 | Input Notebook: input_notebook.ipynb 137 | Output Notebook: output_notebook.ipynb 138 | Executing: 0%| | 0/4 [00:00) 15 | 16 | 17 | 18 | .. code-block:: python 19 | 20 | import papermill as pm 21 | 22 | pm.inspect_notebook('path/to/input.ipynb') 23 | 24 | .. note:: 25 | If your path is parameterized, you can pass those parameters in a dictionary 26 | as second parameter: 27 | 28 | ``inspect_notebook('path/to/input_{month}.ipynb', parameters={month='Feb'})`` 29 | 30 | Inspect via CLI 31 | ~~~~~~~~~~~~~~~ 32 | 33 | To inspect a notebook using the CLI, enter the ``papermill --help-notebook`` command in the 34 | terminal with the notebook and optionally path parameters. 35 | 36 | .. seealso:: 37 | 38 | :doc:`CLI reference <./usage-cli>` 39 | 40 | Inspect a notebook 41 | ^^^^^^^^^^^^^^^^^^ 42 | 43 | Here's an example of a local notebook being inspected and an output example: 44 | 45 | .. code-block:: bash 46 | 47 | papermill --help-notebook ./papermill/tests/notebooks/complex_parameters.ipynb 48 | 49 | Usage: papermill [OPTIONS] NOTEBOOK_PATH [OUTPUT_PATH] 50 | 51 | Parameters inferred for notebook './papermill/tests/notebooks/complex_parameters.ipynb': 52 | msg: Unknown type (default None) 53 | a: float (default 2.25) Variable a 54 | b: List[str] (default ['Hello','World']) 55 | Nice list 56 | c: NoneType (default None) 57 | -------------------------------------------------------------------------------- /docs/usage-parameterize.rst: -------------------------------------------------------------------------------- 1 | Parameterize 2 | ============ 3 | 4 | .. seealso:: 5 | 6 | :doc:`Workflow reference <./reference/papermill-workflow>` 7 | 8 | Generally, the first workflow step when using papermill is to parameterize the 9 | notebook. 10 | 11 | To do this, tag notebook cells with ``parameters``. These ``parameters`` are 12 | later used when the notebook is executed or run. 13 | 14 | Designate parameters for a cell 15 | ------------------------------- 16 | 17 | To parameterize a notebook, designate a cell with the tag ``parameters``. 18 | 19 | .. image:: img/parameters.png 20 | 21 | Notebook 22 | ~~~~~~~~ 23 | 24 | If using the `Jupyter Notebook`_ interface 25 | 26 | 1. Activate the tagging toolbar by navigating to ``View``, ``Cell Toolbar``, and then ``Tags`` 27 | 2. Enter ``parameters`` into a textbox at the top right of a cell 28 | 3. Click ``Add tag`` 29 | 30 | JupyterLab 3.0+ 31 | ~~~~~~~~~~~~~~~ 32 | 33 | If using `JupyterLab`_ v3 or above: 34 | 35 | 1. Select the cell to parameterize 36 | 2. Click the property inspector in the right sidebar (double gear icon) 37 | 3. Type "parameters" in the "Add Tag" box and hit "Enter". 38 | 39 | .. image:: img/lab3_parameters.png 40 | 41 | JupyterLab 2.0 - 2.2.x 42 | ~~~~~~~~~~~~~~~~~~~~~~ 43 | 44 | If using the `JupyterLab`_ interface 45 | 46 | 1. Select the cell to parameterize 47 | 2. Click the property inspector on the left side (double gear icon) 48 | 3. Type "parameters" in the "Add Tag" box and hit "Enter". 49 | 50 | .. image:: img/lab_parameters.png 51 | 52 | JupyterLab < 2.0 53 | ~~~~~~~~~~~~~~~~ 54 | If using JupyterLab < 2.0, consider using the 55 | `jupyterlab-celltags`_ extension. 56 | 57 | 1. Select the cell to parameterize 58 | 2. Click the cell inspector (wrench icon) 59 | 3. Type "parameters" in the "Add Tag" box and hit "Enter". 60 | 61 | If the extension is not installed, you can manually add the tag by 62 | editing the Cell Metadata field in the cell inspector by adding "tags":["parameters"]. 63 | 64 | Learn more about the jupyter notebook format and metadata fields `here`_. 65 | 66 | 67 | How parameters work 68 | ------------------- 69 | 70 | The ``parameters`` cell is assumed to specify default values which may be 71 | overridden by values specified at execution time. 72 | 73 | - papermill inserts a new cell tagged ``injected-parameters`` immediately after 74 | the ``parameters`` cell 75 | - ``injected-parameters`` contains only the overridden parameters 76 | - subsequent cells are treated as normal cells, even if also tagged ``parameters`` 77 | - if no cell is tagged ``parameters``, the ``injected-parameters`` cell 78 | is inserted at the top of the notebook 79 | 80 | One caveat is that a ``parameters`` cell may not behave intuitively with 81 | inter-dependent parameters. Consider a notebook ``note.ipynb`` with two cells: 82 | 83 | .. code-block:: python 84 | 85 | #parameters 86 | a = 1 87 | twice = a * 2 88 | 89 | .. code-block:: python 90 | 91 | print("a =", a, "and twice =", twice) 92 | 93 | when executed with ``papermill note.ipynb -p a 9``, the output will be 94 | ``a = 9 and twice = 2`` (not ``twice = 18``). 95 | 96 | .. _`JupyterLab`: https://github.com/jupyterlab/jupyterlab 97 | .. _`Jupyter Notebook`: https://github.com/jupyter/notebook 98 | .. _`here`: https://ipython.org/ipython-doc/dev/notebook/nbformat.html#cell-metadata 99 | .. _`jupyterlab-celltags`: https://github.com/jupyterlab/jupyterlab-celltags 100 | -------------------------------------------------------------------------------- /docs/usage-store.rst: -------------------------------------------------------------------------------- 1 | Store 2 | ===== 3 | 4 | .. seealso:: 5 | 6 | :doc:`Reference - Storage <./reference/papermill-storage>` 7 | 8 | Papermill can store notebooks in a number of locations including 9 | AWS S3, Azure data blobs, and Azure data lakes. 10 | 11 | The modular architecture of papermill allows new data stores to be 12 | added over time. 13 | -------------------------------------------------------------------------------- /docs/usage-workflow.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | 5 | For an interactive example that demonstrates the usage of papermill, click the Binder link 6 | below: 7 | 8 | .. image:: https://mybinder.org/badge.svg 9 | :target: https://mybinder.org/v2/gh/nteract/papermill/main?filepath=binder%2Fprocess_highlight_dates.ipynb 10 | :alt: launch binder 11 | 12 | Using papermill 13 | --------------- 14 | 15 | The general workflow when using papermill is **parameterizing** a notebook, 16 | **executing** it, as well as **storing** the results. 17 | In addition to operating on a single notebook, papermill also works on 18 | a collection of notebooks. 19 | 20 | .. toctree:: 21 | :maxdepth: 2 22 | 23 | usage-parameterize 24 | usage-inspect 25 | usage-execute 26 | usage-store 27 | -------------------------------------------------------------------------------- /papermill/__init__.py: -------------------------------------------------------------------------------- 1 | from .exceptions import PapermillException, PapermillExecutionError # noqa: F401 2 | from .execute import execute_notebook # noqa: F401 3 | from .inspection import inspect_notebook # noqa: F401 4 | from .version import version as __version__ # noqa: F401 5 | -------------------------------------------------------------------------------- /papermill/__main__.py: -------------------------------------------------------------------------------- 1 | from papermill.cli import papermill 2 | 3 | if __name__ == '__main__': 4 | papermill() 5 | -------------------------------------------------------------------------------- /papermill/abs.py: -------------------------------------------------------------------------------- 1 | """Utilities for working with Azure blob storage""" 2 | import io 3 | import re 4 | 5 | from azure.identity import EnvironmentCredential 6 | from azure.storage.blob import BlobServiceClient 7 | 8 | 9 | class AzureBlobStore: 10 | """ 11 | Represents a Blob of storage on Azure 12 | 13 | Methods 14 | ------- 15 | The following are wrapped utilities for Azure storage: 16 | - read 17 | - listdir 18 | - write 19 | """ 20 | 21 | def _blob_service_client(self, account_name, sas_token=None): 22 | blob_service_client = BlobServiceClient( 23 | account_url=f"{account_name}.blob.core.windows.net", 24 | credential=sas_token or EnvironmentCredential(), 25 | ) 26 | 27 | return blob_service_client 28 | 29 | @classmethod 30 | def _split_url(self, url): 31 | """ 32 | see: https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1 33 | abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken 34 | """ 35 | match = re.match(r"abs://(.*)\.blob\.core\.windows\.net\/(.*?)\/([^\?]*)\??(.*)$", url) 36 | if not match: 37 | raise Exception(f"Invalid azure blob url '{url}'") 38 | else: 39 | params = { 40 | "account": match.group(1), 41 | "container": match.group(2), 42 | "blob": match.group(3), 43 | "sas_token": match.group(4), 44 | } 45 | return params 46 | 47 | def read(self, url): 48 | """Read storage at a given url""" 49 | params = self._split_url(url) 50 | output_stream = io.BytesIO() 51 | blob_service_client = self._blob_service_client(params["account"], params["sas_token"]) 52 | blob_client = blob_service_client.get_blob_client(params['container'], params['blob']) 53 | blob_client.download_blob().readinto(output_stream) 54 | output_stream.seek(0) 55 | return [line.decode("utf-8") for line in output_stream] 56 | 57 | def listdir(self, url): 58 | """Returns a list of the files under the specified path""" 59 | params = self._split_url(url) 60 | blob_service_client = self._blob_service_client(params["account"], params["sas_token"]) 61 | container_client = blob_service_client.get_container_client(params["container"]) 62 | return list(container_client.list_blobs(params["blob"])) 63 | 64 | def write(self, buf, url): 65 | """Write buffer to storage at a given url""" 66 | params = self._split_url(url) 67 | blob_service_client = self._blob_service_client(params["account"], params["sas_token"]) 68 | blob_client = blob_service_client.get_blob_client(params['container'], params['blob']) 69 | blob_client.upload_blob(data=buf, overwrite=True) 70 | -------------------------------------------------------------------------------- /papermill/adl.py: -------------------------------------------------------------------------------- 1 | """Utilities for working with Azure data lake storage""" 2 | import re 3 | 4 | from azure.datalake.store import core, lib 5 | 6 | 7 | class ADL: 8 | """ 9 | Represents an Azure Data Lake 10 | 11 | Methods 12 | ------- 13 | The following are wrapped utilities for Azure storage: 14 | - read 15 | - listdir 16 | - write 17 | """ 18 | 19 | def __init__(self): 20 | self.token = None 21 | 22 | @classmethod 23 | def _split_url(cls, url): 24 | match = re.match(r'adl://(.*)\.azuredatalakestore\.net\/(.*)$', url) 25 | if not match: 26 | raise Exception(f"Invalid ADL url '{url}'") 27 | else: 28 | return (match.group(1), match.group(2)) 29 | 30 | def _get_token(self): 31 | if self.token is None: 32 | self.token = lib.auth() 33 | return self.token 34 | 35 | def _create_adapter(self, store_name): 36 | return core.AzureDLFileSystem(self._get_token(), store_name=store_name) 37 | 38 | def listdir(self, url): 39 | """Returns a list of the files under the specified path""" 40 | (store_name, path) = self._split_url(url) 41 | adapter = self._create_adapter(store_name) 42 | return [f"adl://{store_name}.azuredatalakestore.net/{path_to_child}" for path_to_child in adapter.ls(path)] 43 | 44 | def read(self, url): 45 | """Read storage at a given url""" 46 | (store_name, path) = self._split_url(url) 47 | adapter = self._create_adapter(store_name) 48 | lines = [] 49 | with adapter.open(path) as f: 50 | for line in f: 51 | lines.append(line.decode()) 52 | return lines 53 | 54 | def write(self, buf, url): 55 | """Write buffer to storage at a given url""" 56 | (store_name, path) = self._split_url(url) 57 | adapter = self._create_adapter(store_name) 58 | with adapter.open(path, 'wb') as f: 59 | f.write(buf.encode()) 60 | -------------------------------------------------------------------------------- /papermill/clientwrap.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | from nbclient import NotebookClient 5 | from nbclient.exceptions import CellExecutionError 6 | from traitlets import Bool, Instance 7 | 8 | 9 | class PapermillNotebookClient(NotebookClient): 10 | """ 11 | Module containing a that executes the code cells 12 | and updates outputs 13 | """ 14 | 15 | log_output = Bool(False).tag(config=True) 16 | stdout_file = Instance(object, default_value=None).tag(config=True) 17 | stderr_file = Instance(object, default_value=None).tag(config=True) 18 | 19 | def __init__(self, nb_man, km=None, raise_on_iopub_timeout=True, **kw): 20 | """Initializes the execution manager. 21 | 22 | Parameters 23 | ---------- 24 | nb_man : NotebookExecutionManager 25 | Notebook execution manager wrapper being executed. 26 | km : KernerlManager (optional) 27 | Optional kernel manager. If none is provided, a kernel manager will 28 | be created. 29 | """ 30 | super().__init__(nb_man.nb, km=km, raise_on_iopub_timeout=raise_on_iopub_timeout, **kw) 31 | self.nb_man = nb_man 32 | 33 | def execute(self, **kwargs): 34 | """ 35 | Wraps the parent class process call slightly 36 | """ 37 | self.reset_execution_trackers() 38 | 39 | # See https://bugs.python.org/issue37373 :( 40 | if sys.version_info[0] == 3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'): 41 | asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) 42 | 43 | with self.setup_kernel(**kwargs): 44 | self.log.info(f"Executing notebook with kernel: {self.kernel_name}") 45 | self.papermill_execute_cells() 46 | info_msg = self.wait_for_reply(self.kc.kernel_info()) 47 | self.nb.metadata['language_info'] = info_msg['content']['language_info'] 48 | self.set_widgets_metadata() 49 | 50 | return self.nb 51 | 52 | def papermill_execute_cells(self): 53 | """ 54 | This function replaces cell execution with it's own wrapper. 55 | 56 | We are doing this for the following reasons: 57 | 58 | 1. Notebooks will stop executing when they encounter a failure but not 59 | raise a `CellException`. This allows us to save the notebook with the 60 | traceback even though a `CellExecutionError` was encountered. 61 | 62 | 2. We want to write the notebook as cells are executed. We inject our 63 | logic for that here. 64 | 65 | 3. We want to include timing and execution status information with the 66 | metadata of each cell. 67 | """ 68 | # Execute each cell and update the output in real time. 69 | for index, cell in enumerate(self.nb.cells): 70 | try: 71 | self.nb_man.cell_start(cell, index) 72 | self.execute_cell(cell, index) 73 | except CellExecutionError as ex: 74 | self.nb_man.cell_exception(self.nb.cells[index], cell_index=index, exception=ex) 75 | break 76 | finally: 77 | self.nb_man.cell_complete(self.nb.cells[index], cell_index=index) 78 | 79 | def log_output_message(self, output): 80 | """ 81 | Process a given output. May log it in the configured logger and/or write it into 82 | the configured stdout/stderr files. 83 | 84 | :param output: nbformat.notebooknode.NotebookNode 85 | :return: 86 | """ 87 | if output.output_type == "stream": 88 | content = "".join(output.text) 89 | if output.name == "stdout": 90 | if self.log_output: 91 | self.log.info(content) 92 | if self.stdout_file: 93 | self.stdout_file.write(content) 94 | self.stdout_file.flush() 95 | elif output.name == "stderr": 96 | if self.log_output: 97 | # In case users want to redirect stderr differently, pipe to warning 98 | self.log.warning(content) 99 | if self.stderr_file: 100 | self.stderr_file.write(content) 101 | self.stderr_file.flush() 102 | elif self.log_output and ("data" in output and "text/plain" in output.data): 103 | self.log.info("".join(output.data['text/plain'])) 104 | 105 | def process_message(self, *arg, **kwargs): 106 | output = super().process_message(*arg, **kwargs) 107 | self.nb_man.autosave_cell() 108 | if output and (self.log_output or self.stderr_file or self.stdout_file): 109 | self.log_output_message(output) 110 | return output 111 | -------------------------------------------------------------------------------- /papermill/exceptions.py: -------------------------------------------------------------------------------- 1 | from colors import strip_color 2 | 3 | 4 | class AwsError(Exception): 5 | """Raised when an AWS Exception is encountered.""" 6 | 7 | 8 | class FileExistsError(AwsError): 9 | """Raised when a File already exists on S3.""" 10 | 11 | 12 | class PapermillException(Exception): 13 | """Raised when an exception is encountered when operating on a notebook.""" 14 | 15 | 16 | class PapermillMissingParameterException(PapermillException): 17 | """Raised when a parameter without a value is required to operate on a notebook.""" 18 | 19 | 20 | class PapermillExecutionError(PapermillException): 21 | """Raised when an exception is encountered in a notebook.""" 22 | 23 | def __init__(self, cell_index, exec_count, source, ename, evalue, traceback): 24 | args = cell_index, exec_count, source, ename, evalue, traceback 25 | self.cell_index = cell_index 26 | self.exec_count = exec_count 27 | self.source = source 28 | self.ename = ename 29 | self.evalue = evalue 30 | self.traceback = traceback 31 | 32 | super().__init__(*args) 33 | 34 | def __str__(self): 35 | # Standard Behavior of an exception is to produce a string representation of its arguments 36 | # when called with str(). In order to maintain compatibility with previous versions which 37 | # passed only the message to the superclass constructor, __str__ method is implemented to 38 | # provide the same result as was produced in the past. 39 | message = f"\n{75 * '-'}\n" 40 | message += f'Exception encountered at "In [{self.exec_count}]":\n' 41 | message += strip_color("\n".join(self.traceback)) 42 | message += "\n" 43 | return message 44 | 45 | 46 | class PapermillRateLimitException(PapermillException): 47 | """Raised when an io request has been rate limited""" 48 | 49 | 50 | class PapermillOptionalDependencyException(PapermillException): 51 | """Raised when an exception is encountered when an optional plugin is missing.""" 52 | 53 | 54 | class PapermillWarning(Warning): 55 | """Base warning for papermill.""" 56 | 57 | 58 | class PapermillParameterOverwriteWarning(PapermillWarning): 59 | """Callee overwrites caller argument to pass down the stream.""" 60 | 61 | 62 | def missing_dependency_generator(package, dep): 63 | def missing_dep(): 64 | raise PapermillOptionalDependencyException( 65 | f"The {package} optional dependency is missing. " 66 | f"Please run pip install papermill[{dep}] to install this dependency" 67 | ) 68 | 69 | return missing_dep 70 | 71 | 72 | def missing_environment_variable_generator(package, env_key): 73 | def missing_dep(): 74 | raise PapermillOptionalDependencyException( 75 | f"The {package} optional dependency is present, but the environment " 76 | f"variable {env_key} is not set. Please set this variable as " 77 | f"required by {package} on your platform." 78 | ) 79 | 80 | return missing_dep 81 | -------------------------------------------------------------------------------- /papermill/inspection.py: -------------------------------------------------------------------------------- 1 | """Deduce parameters of a notebook from the parameters cell.""" 2 | from pathlib import Path 3 | 4 | import click 5 | 6 | from .iorw import get_pretty_path, load_notebook_node, local_file_io_cwd 7 | from .log import logger 8 | from .parameterize import add_builtin_parameters, parameterize_path 9 | from .translators import papermill_translators 10 | from .utils import any_tagged_cell, find_first_tagged_cell_index, nb_kernel_name, nb_language 11 | 12 | 13 | def _open_notebook(notebook_path, parameters): 14 | path_parameters = add_builtin_parameters(parameters) 15 | input_path = parameterize_path(notebook_path, path_parameters) 16 | logger.info(f"Input Notebook: {get_pretty_path(input_path)}") 17 | 18 | with local_file_io_cwd(): 19 | return load_notebook_node(input_path) 20 | 21 | 22 | def _infer_parameters(nb, name=None, language=None): 23 | """Infer the notebook parameters. 24 | 25 | Parameters 26 | ---------- 27 | nb : nbformat.NotebookNode 28 | Notebook 29 | 30 | Returns 31 | ------- 32 | List[Parameter] 33 | List of parameters (name, inferred_type_name, default, help) 34 | """ 35 | params = [] 36 | 37 | parameter_cell_idx = find_first_tagged_cell_index(nb, "parameters") 38 | if parameter_cell_idx < 0: 39 | return params 40 | parameter_cell = nb.cells[parameter_cell_idx] 41 | 42 | kernel_name = nb_kernel_name(nb, name) 43 | language = nb_language(nb, language) 44 | 45 | translator = papermill_translators.find_translator(kernel_name, language) 46 | try: 47 | params = translator.inspect(parameter_cell) 48 | except NotImplementedError: 49 | logger.warning(f"Translator for '{language}' language does not support parameter introspection.") 50 | 51 | return params 52 | 53 | 54 | def display_notebook_help(ctx, notebook_path, parameters): 55 | """Display help on notebook parameters. 56 | 57 | Parameters 58 | ---------- 59 | ctx : click.Context 60 | Click context 61 | notebook_path : str 62 | Path to the notebook to be inspected 63 | """ 64 | nb = _open_notebook(notebook_path, parameters) 65 | click.echo(ctx.command.get_usage(ctx)) 66 | pretty_path = get_pretty_path(notebook_path) 67 | click.echo(f"\nParameters inferred for notebook '{pretty_path}':") 68 | 69 | if not any_tagged_cell(nb, "parameters"): 70 | click.echo("\n No cell tagged 'parameters'") 71 | return 1 72 | 73 | params = _infer_parameters(nb) 74 | if params: 75 | for param in params: 76 | p = param._asdict() 77 | type_repr = p["inferred_type_name"] 78 | if type_repr == "None": 79 | type_repr = "Unknown type" 80 | 81 | definition = f" {p['name']}: {type_repr} (default {p['default']})" 82 | if len(definition) > 30: 83 | if len(p["help"]): 84 | param_help = f"{definition}\n{34 * ' '}{p['help']}" 85 | else: 86 | param_help = definition 87 | else: 88 | param_help = f"{definition:<34}{p['help']}" 89 | click.echo(param_help) 90 | else: 91 | click.echo( 92 | "\n Can't infer anything about this notebook's parameters. " "It may not have any parameter defined." 93 | ) 94 | 95 | return 0 96 | 97 | 98 | def inspect_notebook(notebook_path, parameters=None): 99 | """Return the inferred notebook parameters. 100 | 101 | Parameters 102 | ---------- 103 | notebook_path : str or Path 104 | Path to notebook 105 | parameters : dict, optional 106 | Arbitrary keyword arguments to pass to the notebook parameters 107 | 108 | Returns 109 | ------- 110 | Dict[str, Parameter] 111 | Mapping of (parameter name, {name, inferred_type_name, default, help}) 112 | """ 113 | if isinstance(notebook_path, Path): 114 | notebook_path = str(notebook_path) 115 | 116 | nb = _open_notebook(notebook_path, parameters) 117 | 118 | params = _infer_parameters(nb) 119 | return {p.name: p._asdict() for p in params} 120 | -------------------------------------------------------------------------------- /papermill/log.py: -------------------------------------------------------------------------------- 1 | """Sets up a logger""" 2 | import logging 3 | 4 | logger = logging.getLogger('papermill') 5 | -------------------------------------------------------------------------------- /papermill/models.py: -------------------------------------------------------------------------------- 1 | """Models used by papermill.""" 2 | from collections import namedtuple 3 | 4 | Parameter = namedtuple( 5 | 'Parameter', 6 | [ 7 | 'name', 8 | 'inferred_type_name', # string of type 9 | 'default', # string representing the default value 10 | 'help', 11 | ], 12 | ) 13 | -------------------------------------------------------------------------------- /papermill/parameterize.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from uuid import uuid4 3 | 4 | import nbformat 5 | 6 | from .engines import papermill_engines 7 | from .exceptions import PapermillMissingParameterException 8 | from .iorw import read_yaml_file 9 | from .log import logger 10 | from .translators import translate_parameters 11 | from .utils import find_first_tagged_cell_index 12 | 13 | 14 | def add_builtin_parameters(parameters): 15 | """Add built-in parameters to a dictionary of parameters 16 | 17 | Parameters 18 | ---------- 19 | parameters : dict 20 | Dictionary of parameters provided by the user 21 | """ 22 | with_builtin_parameters = { 23 | "pm": { 24 | "run_uuid": str(uuid4()), 25 | "current_datetime_local": datetime.now(), 26 | "current_datetime_utc": datetime.utcnow(), 27 | } 28 | } 29 | 30 | if parameters is not None: 31 | with_builtin_parameters.update(parameters) 32 | 33 | return with_builtin_parameters 34 | 35 | 36 | def parameterize_path(path, parameters): 37 | """Format a path with a provided dictionary of parameters 38 | 39 | Parameters 40 | ---------- 41 | path : string or nbformat.NotebookNode or None 42 | Path with optional parameters, as a python format string. If path is a NotebookNode 43 | or None, the path is returned without modification 44 | parameters : dict or None 45 | Arbitrary keyword arguments to fill in the path 46 | """ 47 | if path is None or isinstance(path, nbformat.NotebookNode): 48 | return path 49 | 50 | if parameters is None: 51 | parameters = {} 52 | 53 | try: 54 | return path.format(**parameters) 55 | except KeyError as key_error: 56 | raise PapermillMissingParameterException(f"Missing parameter {key_error}") 57 | 58 | 59 | def parameterize_notebook( 60 | nb, 61 | parameters, 62 | report_mode=False, 63 | comment='Parameters', 64 | kernel_name=None, 65 | language=None, 66 | engine_name=None, 67 | ): 68 | """Assigned parameters into the appropriate place in the input notebook 69 | 70 | Parameters 71 | ---------- 72 | nb : NotebookNode 73 | Executable notebook object 74 | parameters : dict 75 | Arbitrary keyword arguments to pass as notebook parameters 76 | report_mode : bool, optional 77 | Flag to set report mode 78 | comment : str, optional 79 | Comment added to the injected cell 80 | """ 81 | # Load from a file if 'parameters' is a string. 82 | if isinstance(parameters, str): 83 | parameters = read_yaml_file(parameters) 84 | 85 | # Fetch out the name and language from the notebook document by dropping-down into the engine's implementation 86 | kernel_name = papermill_engines.nb_kernel_name(engine_name, nb, kernel_name) 87 | language = papermill_engines.nb_language(engine_name, nb, language) 88 | 89 | # Generate parameter content based on the kernel_name 90 | param_content = translate_parameters(kernel_name, language, parameters, comment) 91 | 92 | # Upgrade the Notebook to the latest v4 before writing into it 93 | nb = nbformat.v4.upgrade(nb) 94 | 95 | newcell = nbformat.v4.new_code_cell(source=param_content) 96 | newcell.metadata['tags'] = ['injected-parameters'] 97 | 98 | if report_mode: 99 | newcell.metadata['jupyter'] = newcell.get('jupyter', {}) 100 | newcell.metadata['jupyter']['source_hidden'] = True 101 | 102 | param_cell_index = find_first_tagged_cell_index(nb, 'parameters') 103 | injected_cell_index = find_first_tagged_cell_index(nb, 'injected-parameters') 104 | if injected_cell_index >= 0: 105 | # Replace the injected cell with a new version 106 | before = nb.cells[:injected_cell_index] 107 | after = nb.cells[injected_cell_index + 1 :] 108 | elif param_cell_index >= 0: 109 | # Add an injected cell after the parameter cell 110 | before = nb.cells[: param_cell_index + 1] 111 | after = nb.cells[param_cell_index + 1 :] 112 | else: 113 | # Inject to the top of the notebook 114 | logger.warning("Input notebook does not contain a cell with tag 'parameters'") 115 | before = [] 116 | after = nb.cells 117 | 118 | nb.cells = before + [newcell] + after 119 | nb.metadata.papermill['parameters'] = parameters 120 | 121 | return nb 122 | -------------------------------------------------------------------------------- /papermill/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | kernel_name = 'python3' 4 | 5 | 6 | def get_notebook_path(*args): 7 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'notebooks', *args) 8 | 9 | 10 | def get_notebook_dir(*args): 11 | return os.path.dirname(get_notebook_path(*args)) 12 | -------------------------------------------------------------------------------- /papermill/tests/fixtures/rock.txt: -------------------------------------------------------------------------------- 1 | ✄ 2 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/binder.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[See input here!](simple_execute.ipynb)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "%%bash\n", 17 | "cd ../../../\n", 18 | "python -m papermill papermill/tests/notebooks/simple_execute.ipynb papermill/tests/notebooks/simple_output.ipynb -p msg 'Hello'" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "[See output here!](simple_output.ipynb)" 26 | ] 27 | } 28 | ], 29 | "metadata": { 30 | "kernelspec": { 31 | "display_name": "Python 3", 32 | "language": "python", 33 | "name": "python3" 34 | }, 35 | "language_info": { 36 | "codemirror_mode": { 37 | "name": "ipython", 38 | "version": 3 39 | }, 40 | "file_extension": ".py", 41 | "mimetype": "text/x-python", 42 | "name": "python", 43 | "nbconvert_exporter": "python", 44 | "pygments_lexer": "ipython3", 45 | "version": "3.6.7" 46 | } 47 | }, 48 | "nbformat": 4, 49 | "nbformat_minor": 2 50 | } 51 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/blank-vscode.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": 3 14 | }, 15 | "orig_nbformat": 2 16 | }, 17 | "nbformat": 4, 18 | "nbformat_minor": 2, 19 | "cells": [ 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [] 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/broken.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "papermill": {}, 8 | "tags": [] 9 | }, 10 | "outputs": [ 11 | { 12 | "data": {}, 13 | "metadata": {}, 14 | "output_type": "display_data" 15 | }, 16 | { 17 | "data": {}, 18 | "metadata": {}, 19 | "output_type": "display_data" 20 | }, 21 | { 22 | "data": {}, 23 | "metadata": {}, 24 | "output_type": "display_data" 25 | }, 26 | { 27 | "data": { 28 | "text/plain": [ 29 | "'Hello World!'" 30 | ] 31 | }, 32 | "metadata": { 33 | "papermill": { 34 | "name": "output" 35 | } 36 | }, 37 | "output_type": "display_data" 38 | } 39 | ], 40 | "source": [ 41 | "print \"We're good.\"" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "assert False" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "print \"Shouldn't get here.\"" 60 | ] 61 | } 62 | ], 63 | "metadata": { 64 | "celltoolbar": "Tags", 65 | "hide_input": false, 66 | "kernelspec": { 67 | "display_name": "Python 3", 68 | "language": "python", 69 | "name": "python3" 70 | }, 71 | "language_info": { 72 | "codemirror_mode": { 73 | "name": "ipython", 74 | "version": 3 75 | }, 76 | "file_extension": ".py", 77 | "mimetype": "text/x-python", 78 | "name": "python", 79 | "nbconvert_exporter": "python", 80 | "pygments_lexer": "ipython3", 81 | "version": "3.6.7" 82 | }, 83 | "papermill": { 84 | "environment_variables": {}, 85 | "metrics": { 86 | "duration": 3.6040260791778564 87 | }, 88 | "parameters": { 89 | "bar": "Hello World!", 90 | "foo": 1 91 | }, 92 | "version": "0.4+2.ge10f94c.dirty" 93 | } 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 1 97 | } 98 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/broken1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "tags": [ 7 | "papermill-error-cell-tag" 8 | ] 9 | }, 10 | "source": [ 11 | "An Exception was encountered at 'In [1]'." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "A markdown cell that makes the execution counts different to indices within the list of all cells." 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "tags": [ 25 | "papermill-error-cell-tag" 26 | ] 27 | }, 28 | "source": [ 29 | "Execution encountered an exception here and stopped:" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": { 36 | "papermill": {}, 37 | "tags": [] 38 | }, 39 | "outputs": [ 40 | { 41 | "data": {}, 42 | "metadata": {}, 43 | "output_type": "display_data" 44 | }, 45 | { 46 | "data": {}, 47 | "metadata": {}, 48 | "output_type": "display_data" 49 | }, 50 | { 51 | "data": {}, 52 | "metadata": {}, 53 | "output_type": "display_data" 54 | }, 55 | { 56 | "data": { 57 | "text/plain": [ 58 | "'Hello World!'" 59 | ] 60 | }, 61 | "metadata": { 62 | "papermill": { 63 | "name": "output" 64 | } 65 | }, 66 | "output_type": "display_data" 67 | } 68 | ], 69 | "source": [ 70 | "print(\"We're good.\")" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "Another markdown cell" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "A third one." 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "assert False" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "print \"Shouldn't get here.\"" 103 | ] 104 | } 105 | ], 106 | "metadata": { 107 | "celltoolbar": "Tags", 108 | "hide_input": false, 109 | "kernelspec": { 110 | "display_name": "Python 3", 111 | "language": "python", 112 | "name": "python3" 113 | }, 114 | "language_info": { 115 | "codemirror_mode": { 116 | "name": "ipython", 117 | "version": 3 118 | }, 119 | "file_extension": ".py", 120 | "mimetype": "text/x-python", 121 | "name": "python", 122 | "nbconvert_exporter": "python", 123 | "pygments_lexer": "ipython3", 124 | "version": "3.6.7" 125 | }, 126 | "papermill": { 127 | "environment_variables": {}, 128 | "metrics": { 129 | "duration": 3.6040260791778564 130 | }, 131 | "parameters": { 132 | "bar": "Hello World!", 133 | "foo": 1 134 | }, 135 | "version": "0.4+2.ge10f94c.dirty" 136 | } 137 | }, 138 | "nbformat": 4, 139 | "nbformat_minor": 1 140 | } 141 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/broken2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "papermill": {}, 8 | "tags": [] 9 | }, 10 | "outputs": [ 11 | { 12 | "data": {}, 13 | "metadata": {}, 14 | "output_type": "display_data" 15 | }, 16 | { 17 | "data": {}, 18 | "metadata": {}, 19 | "output_type": "display_data" 20 | }, 21 | { 22 | "data": {}, 23 | "metadata": {}, 24 | "output_type": "display_data" 25 | }, 26 | { 27 | "data": { 28 | "text/plain": [ 29 | "'Hello World!'" 30 | ] 31 | }, 32 | "metadata": { 33 | "papermill": { 34 | "name": "output" 35 | } 36 | }, 37 | "output_type": "display_data" 38 | } 39 | ], 40 | "source": [ 41 | "print(\"We're good.\")" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "# To suppress a DeprecationWarning from IPython.\n", 51 | "import warnings\n", 52 | "warnings.filterwarnings('ignore')\n", 53 | "\n", 54 | "# This causes multiple output elements to be created, one of type \"display_data\" and another of type \"error.\"\n", 55 | "from IPython.core.display import display, HTML\n", 56 | "display(HTML(\"\"))\n", 57 | "\n", 58 | "assert False" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "print \"Shouldn't get here.\"" 68 | ] 69 | } 70 | ], 71 | "metadata": { 72 | "celltoolbar": "Tags", 73 | "hide_input": false, 74 | "kernelspec": { 75 | "display_name": "Python 3", 76 | "language": "python", 77 | "name": "python3" 78 | }, 79 | "language_info": { 80 | "codemirror_mode": { 81 | "name": "ipython", 82 | "version": 3 83 | }, 84 | "file_extension": ".py", 85 | "mimetype": "text/x-python", 86 | "name": "python", 87 | "nbconvert_exporter": "python", 88 | "pygments_lexer": "ipython3", 89 | "version": "3.6.7" 90 | }, 91 | "papermill": { 92 | "environment_variables": {}, 93 | "metrics": { 94 | "duration": 3.6040260791778564 95 | }, 96 | "parameters": { 97 | "bar": "Hello World!", 98 | "foo": 1 99 | }, 100 | "version": "0.4+2.ge10f94c.dirty" 101 | } 102 | }, 103 | "nbformat": 4, 104 | "nbformat_minor": 1 105 | } 106 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/complex_parameters.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "msg = None\n", 14 | "a: float = 2.25 # Variable a\n", 15 | "b = [\n", 16 | " 'Hello', # First element\n", 17 | "# Dummy comment\n", 18 | " 'World'\n", 19 | "] # type: List[str] Nice list\n", 20 | "\n", 21 | "# Interesting c variable\n", 22 | "c: \"NoneType\" = None\n", 23 | "# Not introspectable line\n", 24 | "d = a == 3\n", 25 | "# Broken name definition\n", 26 | "= 2" 27 | ] 28 | } 29 | ], 30 | "metadata": { 31 | "celltoolbar": "Tags", 32 | "hide_input": false, 33 | "kernelspec": { 34 | "display_name": "Python 3", 35 | "language": "python", 36 | "name": "python3" 37 | }, 38 | "language_info": { 39 | "codemirror_mode": { 40 | "name": "ipython", 41 | "version": 3 42 | }, 43 | "file_extension": ".py", 44 | "mimetype": "text/x-python", 45 | "name": "python", 46 | "nbconvert_exporter": "python", 47 | "pygments_lexer": "ipython3", 48 | "version": "3.7.6" 49 | } 50 | }, 51 | "nbformat": 4, 52 | "nbformat_minor": 4 53 | } 54 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/keyboard_interrupt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "ename": "KeyboardInterrupt", 10 | "evalue": "", 11 | "output_type": "error", 12 | "traceback": [ 13 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 14 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 15 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msleep\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m60\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 16 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 17 | ] 18 | } 19 | ], 20 | "source": [ 21 | "from time import sleep\n", 22 | "sleep(60)" 23 | ] 24 | } 25 | ], 26 | "metadata": { 27 | "kernelspec": { 28 | "display_name": "Python 3", 29 | "language": "python", 30 | "name": "python3" 31 | }, 32 | "language_info": { 33 | "codemirror_mode": { 34 | "name": "ipython", 35 | "version": 3 36 | }, 37 | "file_extension": ".py", 38 | "mimetype": "text/x-python", 39 | "name": "python", 40 | "nbconvert_exporter": "python", 41 | "pygments_lexer": "ipython3", 42 | "version": "3.6.7" 43 | } 44 | }, 45 | "nbformat": 4, 46 | "nbformat_minor": 2 47 | } 48 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/line_magic_error.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%undefined-line-magic" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "print(\"Cell should not execute.\")" 19 | ] 20 | } 21 | ], 22 | "metadata": { 23 | "kernelspec": { 24 | "display_name": "Python 3", 25 | "language": "python", 26 | "name": "python3" 27 | }, 28 | "language_info": { 29 | "codemirror_mode": { 30 | "name": "ipython", 31 | "version": 3 32 | }, 33 | "file_extension": ".py", 34 | "mimetype": "text/x-python", 35 | "name": "python", 36 | "nbconvert_exporter": "python", 37 | "pygments_lexer": "ipython3", 38 | "version": "3.6.7" 39 | } 40 | }, 41 | "nbformat": 4, 42 | "nbformat_minor": 2 43 | } 44 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/nb_version_4.4.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "# Parameters\n", 14 | "var = 1" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 5, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "1\n" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "print(var)" 32 | ] 33 | } 34 | ], 35 | "metadata": { 36 | "kernelspec": { 37 | "display_name": "Python 3", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.7.6" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 4 56 | } 57 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/no_parameters.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "msg = None" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "print(msg)" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [] 27 | } 28 | ], 29 | "metadata": { 30 | "celltoolbar": "Tags", 31 | "hide_input": false, 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 3 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython3", 47 | "version": "3.6.7" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 1 52 | } 53 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/notimplemented_translator.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "msg = None" 14 | ] 15 | } 16 | ], 17 | "metadata": { 18 | "kernelspec": { 19 | "display_name": "r", 20 | "language": "R", 21 | "name": "r" 22 | }, 23 | "language_info": { 24 | "name": "" 25 | } 26 | }, 27 | "nbformat": 4, 28 | "nbformat_minor": 4 29 | } 30 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/read_check.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "print(\"NB_CWD\", os.getcwd())\n", 19 | "assert(os.path.isfile('check.txt'))" 20 | ] 21 | } 22 | ], 23 | "metadata": { 24 | "kernelspec": { 25 | "display_name": "Python 3", 26 | "language": "python", 27 | "name": "python3" 28 | }, 29 | "language_info": { 30 | "codemirror_mode": { 31 | "name": "ipython", 32 | "version": 3 33 | }, 34 | "file_extension": ".py", 35 | "mimetype": "text/x-python", 36 | "name": "python", 37 | "nbconvert_exporter": "python", 38 | "pygments_lexer": "ipython3", 39 | "version": "3.6.7" 40 | } 41 | }, 42 | "nbformat": 4, 43 | "nbformat_minor": 2 44 | } 45 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/report_mode_test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "a = 1\n", 14 | "b = 1" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "def sum(x, y):\n", 24 | " return x+y" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "print(sum(a, b))" 34 | ] 35 | } 36 | ], 37 | "metadata": { 38 | "celltoolbar": "Tags", 39 | "kernelspec": { 40 | "display_name": "Python 3", 41 | "language": "python", 42 | "name": "python3" 43 | }, 44 | "language_info": { 45 | "codemirror_mode": { 46 | "name": "ipython", 47 | "version": 3 48 | }, 49 | "file_extension": ".py", 50 | "mimetype": "text/x-python", 51 | "name": "python", 52 | "nbconvert_exporter": "python", 53 | "pygments_lexer": "ipython3", 54 | "version": "3.6.7" 55 | } 56 | }, 57 | "nbformat": 4, 58 | "nbformat_minor": 2 59 | } 60 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/result_no_exec.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "# Parameters\n", 14 | "foo = 1\n", 15 | "bar = \"hello\"\n" 16 | ] 17 | } 18 | ], 19 | "metadata": { 20 | "celltoolbar": "Tags", 21 | "hide_input": false, 22 | "kernelspec": { 23 | "display_name": "Python 3", 24 | "language": "python", 25 | "name": "python3" 26 | }, 27 | "language_info": { 28 | "codemirror_mode": { 29 | "name": "ipython", 30 | "version": 3 31 | }, 32 | "file_extension": ".py", 33 | "mimetype": "text/x-python", 34 | "name": "python", 35 | "nbconvert_exporter": "python", 36 | "pygments_lexer": "ipython3", 37 | "version": "3.6.7" 38 | }, 39 | "papermill": { 40 | "environment_variables": {}, 41 | "metrics": { 42 | "duration": 2.250469923019409 43 | }, 44 | "parameters": { 45 | "bar": "hello", 46 | "foo": 1 47 | }, 48 | "version": "0.4+2.ge10f94c.dirty" 49 | } 50 | }, 51 | "nbformat": 4, 52 | "nbformat_minor": 1 53 | } 54 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/s3/s3_in/s3-simple_notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "msg = None" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "print(msg)" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [] 31 | } 32 | ], 33 | "metadata": { 34 | "celltoolbar": "Tags", 35 | "hide_input": false, 36 | "kernelspec": { 37 | "display_name": "Python 3", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.6.1" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 1 56 | } 57 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/s3/s3_out/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/papermill/53847310586b74fdb2763fe333ab99babee367ed/papermill/tests/notebooks/s3/s3_out/.keep -------------------------------------------------------------------------------- /papermill/tests/notebooks/simple_execute.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "tags": [ 8 | "parameters" 9 | ] 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "msg = None" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "#papermill_description=DESC\n", 23 | "print(msg)" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [] 30 | } 31 | ], 32 | "metadata": { 33 | "celltoolbar": "Tags", 34 | "hide_input": false, 35 | "kernelspec": { 36 | "display_name": "Python 3", 37 | "language": "python", 38 | "name": "python3" 39 | }, 40 | "language_info": { 41 | "codemirror_mode": { 42 | "name": "ipython", 43 | "version": 3 44 | }, 45 | "file_extension": ".py", 46 | "mimetype": "text/x-python", 47 | "name": "python", 48 | "nbconvert_exporter": "python", 49 | "pygments_lexer": "ipython3", 50 | "version": "3.8.5" 51 | } 52 | }, 53 | "nbformat": 4, 54 | "nbformat_minor": 4 55 | } 56 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/sysexit.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "sys.exit()" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "print(\"Cell should not execute.\")" 28 | ] 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 3 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython3", 47 | "version": "3.6.7" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 2 52 | } 53 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/sysexit0.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "sys.exit(0)" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "print(\"Cell should not execute.\")" 28 | ] 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 3 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython3", 47 | "version": "3.6.7" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 2 52 | } 53 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/sysexit1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "sys.exit(1)" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "print(\"Cell should not execute.\")" 28 | ] 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 3 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython3", 47 | "version": "3.6.7" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 2 52 | } 53 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/systemexit.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sys" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "raise SystemExit" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "print(\"Cell should not execute.\")" 28 | ] 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 3 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython3", 47 | "version": "3.6.7" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 2 52 | } 53 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/test_autosave.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stderr", 10 | "output_type": "stream", 11 | "text": [ 12 | "INFO:test:test text\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "import time\n", 18 | "for i in range(25): # This will take 2.5 seconds\n", 19 | " time.sleep(0.1)\n", 20 | " print(i)" 21 | ] 22 | } 23 | ], 24 | "metadata": { 25 | "hide_input": false, 26 | "kernelspec": { 27 | "display_name": "Python", 28 | "language": "python", 29 | "name": "python" 30 | }, 31 | "language_info": { 32 | "codemirror_mode": { 33 | "name": "ipython", 34 | "version": 2 35 | }, 36 | "file_extension": ".py", 37 | "mimetype": "text/x-python", 38 | "name": "python", 39 | "nbconvert_exporter": "python", 40 | "pygments_lexer": "ipython2", 41 | "version": "2.7.12" 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 1 46 | } 47 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/test_logging.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stderr", 10 | "output_type": "stream", 11 | "text": [ 12 | "INFO:test:test text\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "import logging\n", 18 | "logger = logging.getLogger(\"test\")\n", 19 | "logger.setLevel(logging.INFO)\n", 20 | "logging.basicConfig(level=logging.INFO)\n", 21 | "logger.info(\"test text\")" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "hello world\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "print(\"hello world\")" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 3, 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "data": { 48 | "text/plain": [ 49 | "" 50 | ] 51 | }, 52 | "execution_count": 3, 53 | "metadata": {}, 54 | "output_type": "execute_result" 55 | }, 56 | { 57 | "data": { 58 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADzJJREFUeJzt3XGMJnV9x/H3xwMLqBWQLSWcuKhES6wiPYnG2losCS1V\nsLFWU5trSzxNNcXQpJzEVGxicjQq2qZVUdTTGhWBChWb5sBTa9KAh6KCaEA9WxC5s0rwrIEC3/7x\nzLbb83Z3VnbmYZ/f+5Vsdub3zLO/74/h9rPzm3lmUlVIktr1iGkXIEmaLoNAkhpnEEhS4wwCSWqc\nQSBJjTMIJKlxBoEkNc4gkKTGGQSS1LiDpl1AH0cddVTNz89PuwxJWlduuOGG71fV3ErbrYsgmJ+f\nZ9euXdMuQ5LWlSTf6bOdU0OS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktS4\ndfHJYs2W+a1Xj9LP7m1njNKPtN55RCBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklq\nnEEgSY0zCCSpcQaBJDXOIJCkxnnTuXVkjJu1eaM2qT0eEUhS4wwCSWqcQSBJjTMIJKlxBoEkNc4g\nkKTGGQSS1DiDQJIaZxBIUuMGD4IkG5J8Kcknu/Xjk1yX5LYkH0vyyKFrkCQtbYwjgnOAWxatXwhc\nVFVPBn4InD1CDZKkJQwaBEk2AmcA7+3WA5wKXNZtsh04a8gaJEnLG/qI4O3AXwAPduuPA+6uqvu7\n9duBYweuQZK0jMGCIMnvAHuq6oaf8f1bkuxKsmvv3r1rXJ0kacGQRwTPBV6UZDfwUSZTQu8ADk+y\ncPvrjcAdB3pzVV1cVZuqatPc3NyAZUpS2wYLgqp6fVVtrKp54GXAp6vqD4CdwEu6zTYDVw5VgyRp\nZdP4HMF5wLlJbmNyzuCSKdQgSeqM8oSyqvoM8Jlu+VvAKWP0K0lamZ8slqTGGQSS1DiDQJIaZxBI\nUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1\nziCQpMaN8oQyrR/zW6+edgmSRuYRgSQ1ziCQpMYZBJLUOINAkhrnyWLNrDFOfO/edsbgfUhD84hA\nkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSp\ncYMFQZJDklyf5MtJbk7ypq79+CTXJbktyceSPHKoGiRJKxvyiOBe4NSqegZwEnB6kmcDFwIXVdWT\ngR8CZw9YgyRpBYMFQU3s61YP7r4KOBW4rGvfDpw1VA2SpJUNeo4gyYYkNwJ7gB3AN4G7q+r+bpPb\ngWOHrEGStLxBg6CqHqiqk4CNwCnAU/u+N8mWJLuS7Nq7d+9gNUpS60a5aqiq7gZ2As8BDk+y8IjM\njcAdS7zn4qraVFWb5ubmxihTkprUKwiS/PJqf3CSuSSHd8uHAqcBtzAJhJd0m20Grlztz5YkrZ2+\nRwR/310K+qdJHtvzPccAO5N8BfgCsKOqPgmcB5yb5DbgccAlq65akrRmDlp5E6iq5yU5AfgT4IYk\n1wPvr6ody7znK8AzD9D+LSbnC2bG/Narp12CJP3Mep8jqKpbgTcw+Yv+14G/SfL1JL87VHGSpOH1\nPUfw9CQXMZnjPxV4YVX9Urd80YD1SZIG1mtqCPhb4L3A+VX1k4XGqvpukjcMUpkkaRR9g+AM4CdV\n9QBAkkcAh1TVf1XVhwarTpI0uL7nCK4BDl20fljXJkla5/oGwSGL7htEt3zYMCVJksbUNwh+nOTk\nhZUkvwL8ZJntJUnrRN9zBK8DPp7ku0CAXwR+f7CqJEmj6fuBsi8keSrwlK7pG1X138OVJUkaS98j\nAoBnAfPde05OQlV9cJCqJEmj6RUEST4EPAm4EXigay7AIJCkda7vEcEm4MSqqiGLkSSNr+9VQzcx\nOUEsSZoxfY8IjgK+1t119N6Fxqp60SBVSZJG0zcILhiyCEnS9PS9fPSzSZ4AnFBV1yQ5DNgwbGmS\npDH0vQ31K4HLgHd3TccCnxiqKEnSePpODb2GyVPFroPJQ2qS/MJgVa0hnx4mScvre9XQvVV138JK\nkoOYfI5AkrTO9Q2CzyY5Hzg0yWnAx4F/Gq4sSdJY+gbBVmAv8FXgVcCnmDy/WJK0zvW9auhB4D3d\nlyRphvS919C3OcA5gap64ppXJEka1WruNbTgEOD3gCPXvhxJ0th6nSOoqv9c9HVHVb2dyQPtJUnr\nXN+poZMXrT6CyRHCap5lIEl6mOr7y/yti5bvB3YDL13zaiRJo+t71dBvDF2IJGk6+k4Nnbvc61X1\ntrUpR5I0ttVcNfQs4Kpu/YXA9cCtQxQlSRpP3yDYCJxcVT8CSHIBcHVVvWKowiRJ4+h7i4mjgfsW\nrd/XtUmS1rm+RwQfBK5P8o/d+lnA9mFKkiSNqe9VQ29O8s/A87qmP66qLw1XliRpLH2nhgAOA+6p\nqncAtyc5fqCaJEkj6vuoyjcC5wGv75oOBv5hqKIkSePpe0TwYuBFwI8Bquq7wGOWe0OSxyfZmeRr\nSW5Ock7XfmSSHUlu7b4f8VAGIEl6aPoGwX1VVXS3ok7yqB7vuR/486o6EXg28JokJzJ5yM21VXUC\ncG23Lkmakr5BcGmSdwOHJ3klcA0rPKSmqu6sqi92yz8CbgGOBc7k/6442s7kCiRJ0pT0vWroLd2z\niu8BngL8ZVXt6NtJknngmcB1wNFVdWf30vfw8wiSNFUrBkGSDcA13Y3nev/yX/T+RwOXA6+rqnuS\n/O9rVVVJfurJZ937tgBbAI477rjVditJ6mnFqaGqegB4MMljV/vDkxzMJAQ+XFVXdM13JTmme/0Y\nYM8S/V5cVZuqatPc3Nxqu5Yk9dT3k8X7gK8m2UF35RBAVf3ZUm/I5E//S4Bb9rs76VXAZmBb9/3K\n1RYtSVo7fYPgiu5rNZ4L/CGTALmxazufSQBcmuRs4Dv4gBtJmqplgyDJcVX171W16vsKVdXngSzx\n8gtW+/MkScNY6RzBJxYWklw+cC2SpClYKQgW/0X/xCELkSRNx0pBUEssS5JmxEoni5+R5B4mRwaH\ndst061VVPz9odZKkwS0bBFW1YaxCJEnTsZrnEUiSZpBBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINA\nkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSp\ncQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNO2jaBUjr2fzWq0fpZ/e2M0bpR23yiECS\nGmcQSFLjDAJJapxBIEmNGywIkrwvyZ4kNy1qOzLJjiS3dt+PGKp/SVI/Qx4RfAA4fb+2rcC1VXUC\ncG23LkmaosGCoKo+B/xgv+Yzge3d8nbgrKH6lyT1M/Y5gqOr6s5u+XvA0SP3L0naz9ROFldVAbXU\n60m2JNmVZNfevXtHrEyS2jJ2ENyV5BiA7vuepTasqouralNVbZqbmxutQElqzdhBcBWwuVveDFw5\ncv+SpP0MefnoR4B/A56S5PYkZwPbgNOS3Ar8ZrcuSZqiwW46V1UvX+KlFwzVpyRp9fxksSQ1ziCQ\npMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklq\nnEEgSY0zCCSpcQaBJDVusCeUSVo781uvHryP3dvOGLwPPTx5RCBJjTMIJKlxBoEkNc4gkKTGGQSS\n1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ47z4qCfAOp6s1S/+9PCKQpMYZ\nBJLUOINAkhrnOQJJoxljXh1m61zEGKZyRJDk9CTfSHJbkq3TqEGSNDF6ECTZAPwd8FvAicDLk5w4\ndh2SpIlpHBGcAtxWVd+qqvuAjwJnTqEOSRLTCYJjgf9YtH571yZJmoKH7cniJFuALd3qviTfGKnr\no4Dvj9TXNLUyTmhnrK2ME1YYay4csZIB5cKHvE+f0GejaQTBHcDjF61v7Nr+n6q6GLh4rKIWJNlV\nVZvG7ndsrYwT2hlrK+OEdsY61jinMTX0BeCEJMcneSTwMuCqKdQhSWIKRwRVdX+S1wL/AmwA3ldV\nN49dhyRpYirnCKrqU8CnptF3D6NPR01JK+OEdsbayjihnbGOMs5U1Rj9SJIeprzXkCQ1rukgSPK+\nJHuS3LSo7cgkO5Lc2n0/Ypo1roUlxnlBkjuS3Nh9/fY0a1wLSR6fZGeSryW5Ock5Xfss7tOlxjpT\n+zXJIUmuT/Llbpxv6tqPT3Jdd5uaj3UXnqxry4z1A0m+vWifnrTmfbc8NZTk14B9wAer6mld218D\nP6iqbd19kI6oqvOmWedDtcQ4LwD2VdVbplnbWkpyDHBMVX0xyWOAG4CzgD9i9vbpUmN9KTO0X5ME\neFRV7UtyMPB54BzgXOCKqvpokncBX66qd06z1odqmbG+GvhkVV02VN9NHxFU1eeAH+zXfCawvVve\nzuQf17q2xDhnTlXdWVVf7JZ/BNzC5FPrs7hPlxrrTKmJfd3qwd1XAacCC78YZ2WfLjXWwTUdBEs4\nuqru7Ja/Bxw9zWIG9tokX+mmjtb9dMliSeaBZwLXMeP7dL+xwozt1yQbktwI7AF2AN8E7q6q+7tN\nZuY2NfuPtaoW9umbu316UZKfW+t+DYJl1GTebFbnzt4JPAk4CbgTeOt0y1k7SR4NXA68rqruWfza\nrO3TA4x15vZrVT1QVScxuQvBKcBTp1zSYPYfa5KnAa9nMuZnAUcCaz6taRD8tLu6+deFedg9U65n\nEFV1V/c/3YPAe5j8A1v3urnVy4EPV9UVXfNM7tMDjXVW9ytAVd0N7ASeAxyeZOFzUAe8Tc16tmis\np3fTgFVV9wLvZ4B9ahD8tKuAzd3yZuDKKdYymIVfjJ0XAzctte160Z1suwS4paretuilmdunS411\n1vZrkrkkh3fLhwKnMTkfshN4SbfZrOzTA43164v+iAmTcyFrvk9bv2roI8DzmdzJ8C7gjcAngEuB\n44DvAC+tqnV9onWJcT6fyfRBAbuBVy2aR1+Xkvwq8K/AV4EHu+bzmcydz9o+XWqsL2eG9muSpzM5\nGbyByR+ul1bVXyV5IpNnmRwJfAl4RfcX87q1zFg/DcwBAW4EXr3opPLa9N1yEEiSnBqSpOYZBJLU\nOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNe5/AEfTMGi6ZpsVAAAAAElFTkSuQmCC\n", 59 | "text/plain": [ 60 | "" 61 | ] 62 | }, 63 | "metadata": {}, 64 | "output_type": "display_data" 65 | } 66 | ], 67 | "source": [ 68 | "import warnings\n", 69 | "warnings.simplefilter(action='ignore', category=FutureWarning)\n", 70 | "from ggplot import mpg\n", 71 | "mpg['cty'].plot.hist(bins=12)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [] 80 | } 81 | ], 82 | "metadata": { 83 | "hide_input": false, 84 | "kernelspec": { 85 | "display_name": "Python 3", 86 | "language": "python", 87 | "name": "python3" 88 | }, 89 | "language_info": { 90 | "codemirror_mode": { 91 | "name": "ipython", 92 | "version": 2 93 | }, 94 | "file_extension": ".py", 95 | "mimetype": "text/x-python", 96 | "name": "python", 97 | "nbconvert_exporter": "python", 98 | "pygments_lexer": "ipython2", 99 | "version": "2.7.12" 100 | } 101 | }, 102 | "nbformat": 4, 103 | "nbformat_minor": 1 104 | } 105 | -------------------------------------------------------------------------------- /papermill/tests/notebooks/test_notebooknode_io.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": ["print('Hello World')"] 9 | } 10 | ], 11 | "metadata": { 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "language": "python", 15 | "name": "python3" 16 | } 17 | }, 18 | "nbformat": 4, 19 | "nbformat_minor": 2 20 | } 21 | -------------------------------------------------------------------------------- /papermill/tests/parameters/example.json: -------------------------------------------------------------------------------- 1 | {"foo":54321} 2 | -------------------------------------------------------------------------------- /papermill/tests/parameters/example.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | foo: 12345 3 | bar: "value" 4 | baz: 5 | k1: v1 6 | k2: v2 7 | a_date: 2019-01-01 8 | -------------------------------------------------------------------------------- /papermill/tests/test_abs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from unittest.mock import Mock, patch 4 | 5 | from azure.identity import EnvironmentCredential 6 | 7 | from ..abs import AzureBlobStore 8 | 9 | 10 | class MockBytesIO: 11 | def __init__(self): 12 | self.list = [b"hello", b"world!"] 13 | 14 | def __getitem__(self, index): 15 | return self.list[index] 16 | 17 | def seek(self, seed): 18 | pass 19 | 20 | 21 | class ABSTest(unittest.TestCase): 22 | """ 23 | Tests for `ABS` 24 | """ 25 | 26 | def setUp(self): 27 | self.list_blobs = Mock(return_value=["foo", "bar", "baz"]) 28 | self.upload_blob = Mock() 29 | self.download_blob = Mock() 30 | self._container_client = Mock(list_blobs=self.list_blobs) 31 | self._blob_client = Mock(upload_blob=self.upload_blob, download_blob=self.download_blob) 32 | self._blob_service_client = Mock( 33 | get_blob_client=Mock(return_value=self._blob_client), 34 | get_container_client=Mock(return_value=self._container_client), 35 | ) 36 | self.abs = AzureBlobStore() 37 | self.abs._blob_service_client = Mock(return_value=self._blob_service_client) 38 | os.environ["AZURE_TENANT_ID"] = "mytenantid" 39 | os.environ["AZURE_CLIENT_ID"] = "myclientid" 40 | os.environ["AZURE_CLIENT_SECRET"] = "myclientsecret" 41 | 42 | def test_split_url_raises_exception_on_invalid_url(self): 43 | with self.assertRaises(Exception) as context: 44 | AzureBlobStore._split_url("this_is_not_a_valid_url") 45 | self.assertTrue("Invalid azure blob url 'this_is_not_a_valid_url'" in str(context.exception)) 46 | 47 | def test_split_url_splits_valid_url(self): 48 | params = AzureBlobStore._split_url("abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken") 49 | self.assertEqual(params["account"], "myaccount") 50 | self.assertEqual(params["container"], "sascontainer") 51 | self.assertEqual(params["blob"], "sasblob.txt") 52 | self.assertEqual(params["sas_token"], "sastoken") 53 | 54 | def test_split_url_splits_valid_url_no_sas(self): 55 | params = AzureBlobStore._split_url("abs://myaccount.blob.core.windows.net/container/blob.txt") 56 | self.assertEqual(params["account"], "myaccount") 57 | self.assertEqual(params["container"], "container") 58 | self.assertEqual(params["blob"], "blob.txt") 59 | self.assertEqual(params["sas_token"], "") 60 | 61 | def test_split_url_splits_valid_url_with_prefix(self): 62 | params = AzureBlobStore._split_url( 63 | "abs://myaccount.blob.core.windows.net/sascontainer/A/B/sasblob.txt?sastoken" 64 | ) 65 | self.assertEqual(params["account"], "myaccount") 66 | self.assertEqual(params["container"], "sascontainer") 67 | self.assertEqual(params["blob"], "A/B/sasblob.txt") 68 | self.assertEqual(params["sas_token"], "sastoken") 69 | 70 | def test_listdir_calls(self): 71 | self.assertEqual( 72 | self.abs.listdir("abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken"), 73 | ["foo", "bar", "baz"], 74 | ) 75 | self._blob_service_client.get_container_client.assert_called_once_with("sascontainer") 76 | self.list_blobs.assert_called_once_with("sasblob.txt") 77 | 78 | @patch("papermill.abs.io.BytesIO", side_effect=MockBytesIO) 79 | def test_reads_file(self, mockBytesIO): 80 | self.assertEqual( 81 | self.abs.read("abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken"), 82 | ["hello", "world!"], 83 | ) 84 | self._blob_service_client.get_blob_client.assert_called_once_with("sascontainer", "sasblob.txt") 85 | self.download_blob.assert_called_once_with() 86 | 87 | def test_write_file(self): 88 | self.abs.write("hello world", "abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken") 89 | self._blob_service_client.get_blob_client.assert_called_once_with("sascontainer", "sasblob.txt") 90 | self.upload_blob.assert_called_once_with(data="hello world", overwrite=True) 91 | 92 | def test_blob_service_client(self): 93 | abs = AzureBlobStore() 94 | blob = abs._blob_service_client(account_name="myaccount", sas_token="sastoken") 95 | self.assertEqual(blob.account_name, "myaccount") 96 | # Credentials gets funky with v12.0.0, so I comment this out 97 | # self.assertEqual(blob.credential, "sastoken") 98 | 99 | def test_blob_service_client_environment_credentials(self): 100 | abs = AzureBlobStore() 101 | blob = abs._blob_service_client(account_name="myaccount", sas_token="") 102 | self.assertEqual(blob.account_name, "myaccount") 103 | self.assertIsInstance(blob.credential, EnvironmentCredential) 104 | self.assertEqual(blob.credential._credential._tenant_id, "mytenantid") 105 | self.assertEqual(blob.credential._credential._client_id, "myclientid") 106 | self.assertEqual(blob.credential._credential._client_credential, "myclientsecret") 107 | -------------------------------------------------------------------------------- /papermill/tests/test_adl.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import MagicMock, Mock, patch 3 | 4 | from ..adl import ADL 5 | from ..adl import core as adl_core 6 | from ..adl import lib as adl_lib 7 | 8 | 9 | class ADLTest(unittest.TestCase): 10 | """ 11 | Tests for `ADL` 12 | """ 13 | 14 | def setUp(self): 15 | self.ls = Mock(return_value=["path/to/directory/foo", "path/to/directory/bar", "path/to/directory/baz"]) 16 | self.fakeFile = MagicMock() 17 | self.fakeFile.__iter__.return_value = [b"a", b"b", b"c"] 18 | self.fakeFile.__enter__.return_value = self.fakeFile 19 | self.open = Mock(return_value=self.fakeFile) 20 | self.fakeAdapter = Mock(open=self.open, ls=self.ls) 21 | self.adl = ADL() 22 | self.adl._create_adapter = Mock(return_value=self.fakeAdapter) 23 | 24 | def test_split_url_raises_exception_on_invalid_url(self): 25 | with self.assertRaises(Exception) as context: 26 | ADL._split_url("this_is_not_a_valid_url") 27 | self.assertTrue("Invalid ADL url 'this_is_not_a_valid_url'" in str(context.exception)) 28 | 29 | def test_split_url_splits_valid_url(self): 30 | (store_name, path) = ADL._split_url("adl://foo.azuredatalakestore.net/bar/baz") 31 | self.assertEqual(store_name, "foo") 32 | self.assertEqual(path, "bar/baz") 33 | 34 | def test_listdir_calls_ls_on_adl_adapter(self): 35 | self.assertEqual( 36 | self.adl.listdir("adl://foo_store.azuredatalakestore.net/path/to/directory"), 37 | [ 38 | "adl://foo_store.azuredatalakestore.net/path/to/directory/foo", 39 | "adl://foo_store.azuredatalakestore.net/path/to/directory/bar", 40 | "adl://foo_store.azuredatalakestore.net/path/to/directory/baz", 41 | ], 42 | ) 43 | self.ls.assert_called_once_with("path/to/directory") 44 | 45 | def test_read_opens_and_reads_file(self): 46 | self.assertEqual(self.adl.read("adl://foo_store.azuredatalakestore.net/path/to/file"), ["a", "b", "c"]) 47 | self.fakeFile.__iter__.assert_called_once_with() 48 | 49 | def test_write_opens_file_and_writes_to_it(self): 50 | self.adl.write("hello world", "adl://foo_store.azuredatalakestore.net/path/to/file") 51 | self.fakeFile.write.assert_called_once_with(b"hello world") 52 | 53 | @patch.object(adl_lib, 'auth', return_value="my_token") 54 | @patch.object(adl_core, 'AzureDLFileSystem', return_value="my_adapter") 55 | def test_create_adapter(self, azure_dl_filesystem_mock, auth_mock): 56 | sut = ADL() 57 | actual = sut._create_adapter("my_store_name") 58 | assert actual == "my_adapter" 59 | auth_mock.assert_called_once_with() 60 | azure_dl_filesystem_mock.assert_called_once_with("my_token", store_name="my_store_name") 61 | -------------------------------------------------------------------------------- /papermill/tests/test_autosave.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import time 4 | import unittest 5 | from unittest.mock import patch 6 | 7 | import nbformat 8 | 9 | from .. import engines 10 | from ..engines import NotebookExecutionManager 11 | from ..execute import execute_notebook 12 | from . import get_notebook_path 13 | 14 | 15 | class TestMidCellAutosave(unittest.TestCase): 16 | def setUp(self): 17 | self.notebook_name = 'test_autosave.ipynb' 18 | self.notebook_path = get_notebook_path(self.notebook_name) 19 | self.nb = nbformat.read(self.notebook_path, as_version=4) 20 | 21 | def test_autosave_not_too_fast(self): 22 | nb_man = NotebookExecutionManager(self.nb, output_path='test.ipynb', autosave_cell_every=0.5) 23 | with patch.object(engines, 'write_ipynb') as write_mock: 24 | write_mock.reset_mock() 25 | assert write_mock.call_count == 0 # check that the mock is sane 26 | nb_man.autosave_cell() # First call to autosave shouldn't trigger save 27 | assert write_mock.call_count == 0 28 | nb_man.autosave_cell() # Call again right away. Still shouldn't save. 29 | assert write_mock.call_count == 0 30 | time.sleep(0.55) # Sleep for long enough that autosave should work 31 | nb_man.autosave_cell() 32 | assert write_mock.call_count == 1 33 | 34 | def test_autosave_disable(self): 35 | nb_man = NotebookExecutionManager(self.nb, output_path='test.ipynb', autosave_cell_every=0) 36 | with patch.object(engines, 'write_ipynb') as write_mock: 37 | write_mock.reset_mock() 38 | assert write_mock.call_count == 0 # check that the mock is sane 39 | nb_man.autosave_cell() # First call to autosave shouldn't trigger save 40 | assert write_mock.call_count == 0 41 | nb_man.autosave_cell() # Call again right away. Still shouldn't save. 42 | assert write_mock.call_count == 0 43 | time.sleep(0.55) # Sleep for long enough that autosave should work, if enabled 44 | nb_man.autosave_cell() 45 | assert write_mock.call_count == 0 # but it's disabled. 46 | 47 | def test_end2end_autosave_slow_notebook(self): 48 | test_dir = tempfile.mkdtemp() 49 | nb_test_executed_fname = os.path.join(test_dir, f'output_{self.notebook_name}') 50 | 51 | # Count how many times it writes the file w/o autosave 52 | with patch.object(engines, 'write_ipynb') as write_mock: 53 | execute_notebook(self.notebook_path, nb_test_executed_fname, autosave_cell_every=0) 54 | default_write_count = write_mock.call_count 55 | 56 | # Turn on autosave and see how many more times it gets saved. 57 | with patch.object(engines, 'write_ipynb') as write_mock: 58 | execute_notebook(self.notebook_path, nb_test_executed_fname, autosave_cell_every=1) 59 | # This notebook has a cell which takes 2.5 seconds to run. 60 | # Autosave every 1 sec should add two more saves. 61 | assert write_mock.call_count == default_write_count + 2 62 | -------------------------------------------------------------------------------- /papermill/tests/test_clientwrap.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import call, patch 3 | 4 | import nbformat 5 | 6 | from ..clientwrap import PapermillNotebookClient 7 | from ..engines import NotebookExecutionManager 8 | from ..log import logger 9 | from . import get_notebook_path 10 | 11 | 12 | class TestPapermillClientWrapper(unittest.TestCase): 13 | def setUp(self): 14 | self.nb = nbformat.read(get_notebook_path('test_logging.ipynb'), as_version=4) 15 | self.nb_man = NotebookExecutionManager(self.nb) 16 | self.client = PapermillNotebookClient(self.nb_man, log=logger, log_output=True) 17 | 18 | def test_logging_stderr_msg(self): 19 | with patch.object(logger, 'warning') as warning_mock: 20 | for output in self.nb.cells[0].get("outputs", []): 21 | self.client.log_output_message(output) 22 | warning_mock.assert_called_once_with("INFO:test:test text\n") 23 | 24 | def test_logging_stdout_msg(self): 25 | with patch.object(logger, 'info') as info_mock: 26 | for output in self.nb.cells[1].get("outputs", []): 27 | self.client.log_output_message(output) 28 | info_mock.assert_called_once_with("hello world\n") 29 | 30 | def test_logging_data_msg(self): 31 | with patch.object(logger, 'info') as info_mock: 32 | for output in self.nb.cells[2].get("outputs", []): 33 | self.client.log_output_message(output) 34 | info_mock.assert_has_calls( 35 | [ 36 | call(""), 37 | call(""), 38 | ] 39 | ) 40 | -------------------------------------------------------------------------------- /papermill/tests/test_conf.py: -------------------------------------------------------------------------------- 1 | import pytest # noqa 2 | -------------------------------------------------------------------------------- /papermill/tests/test_exceptions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import tempfile 4 | 5 | import pytest 6 | 7 | from .. import exceptions 8 | 9 | 10 | @pytest.fixture 11 | def temp_file(): 12 | """NamedTemporaryFile must be set in wb mode, closed without delete, opened with open(file, "rb"), 13 | then manually deleted. Otherwise, file fails to be read due to permission error on Windows.""" 14 | with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: 15 | yield f 16 | os.unlink(f.name) 17 | 18 | 19 | @pytest.mark.parametrize( 20 | "exc,args", 21 | [ 22 | ( 23 | exceptions.PapermillExecutionError, 24 | (1, 2, "TestSource", "Exception", Exception(), ["Traceback", "Message"]), 25 | ), 26 | (exceptions.PapermillMissingParameterException, ("PapermillMissingParameterException",)), 27 | (exceptions.AwsError, ("AwsError",)), 28 | (exceptions.FileExistsError, ("FileExistsError",)), 29 | (exceptions.PapermillException, ("PapermillException",)), 30 | (exceptions.PapermillRateLimitException, ("PapermillRateLimitException",)), 31 | ( 32 | exceptions.PapermillOptionalDependencyException, 33 | ("PapermillOptionalDependencyException",), 34 | ), 35 | ], 36 | ) 37 | def test_exceptions_are_unpickleable(temp_file, exc, args): 38 | """Ensure exceptions can be unpickled""" 39 | err = exc(*args) 40 | pickle.dump(err, temp_file) 41 | temp_file.close() # close to re-open for reading 42 | 43 | # Read the Pickled File 44 | with open(temp_file.name, "rb") as read_file: 45 | read_file.seek(0) 46 | data = read_file.read() 47 | pickled_err = pickle.loads(data) 48 | assert str(pickled_err) == str(err) 49 | -------------------------------------------------------------------------------- /papermill/tests/test_gcs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | 4 | from ..exceptions import PapermillRateLimitException 5 | from ..iorw import GCSHandler, fallback_gs_is_retriable 6 | 7 | try: 8 | try: 9 | try: 10 | from gcsfs.retry import HttpError as GCSHttpError 11 | except ImportError: 12 | from gcsfs.utils import HttpError as GCSHttpError 13 | except ImportError: 14 | from gcsfs.utils import HtmlError as GCSHttpError 15 | except ImportError: 16 | # Fall back to a sane import if gcsfs is missing 17 | GCSHttpError = Exception 18 | 19 | try: 20 | from gcsfs.utils import RateLimitException as GCSRateLimitException 21 | except ImportError: 22 | # Fall back to GCSHttpError when using older library 23 | GCSRateLimitException = GCSHttpError 24 | 25 | 26 | def mock_gcs_fs_wrapper(exception=None, max_raises=1): 27 | class MockGCSFileSystem: 28 | def __init__(self): 29 | self._file = MockGCSFile(exception, max_raises) 30 | 31 | def open(self, *args, **kwargs): 32 | return self._file 33 | 34 | def ls(self, *args, **kwargs): 35 | return [] 36 | 37 | return MockGCSFileSystem 38 | 39 | 40 | class MockGCSFile: 41 | def __init__(self, exception=None, max_raises=1): 42 | self.read_count = 0 43 | self.write_count = 0 44 | self.exception = exception 45 | self.max_raises = max_raises 46 | 47 | def __enter__(self): 48 | return self 49 | 50 | def __exit__(self, *args, **kwargs): 51 | pass 52 | 53 | def read(self): 54 | self.read_count += 1 55 | if self.exception and self.read_count <= self.max_raises: 56 | raise self.exception 57 | return self.read_count 58 | 59 | def write(self, buf): 60 | self.write_count += 1 61 | if self.exception and self.write_count <= self.max_raises: 62 | raise self.exception 63 | return self.write_count 64 | 65 | 66 | class GCSTest(unittest.TestCase): 67 | """Tests for `GCS`.""" 68 | 69 | def setUp(self): 70 | self.gcs_handler = GCSHandler() 71 | 72 | @patch('papermill.iorw.GCSFileSystem', side_effect=mock_gcs_fs_wrapper()) 73 | def test_gcs_read(self, mock_gcs_filesystem): 74 | client = self.gcs_handler._get_client() 75 | self.assertEqual(self.gcs_handler.read('gs://bucket/test.ipynb'), 1) 76 | # Check that client is only generated once 77 | self.assertIs(client, self.gcs_handler._get_client()) 78 | 79 | @patch('papermill.iorw.GCSFileSystem', side_effect=mock_gcs_fs_wrapper()) 80 | def test_gcs_write(self, mock_gcs_filesystem): 81 | client = self.gcs_handler._get_client() 82 | self.assertEqual(self.gcs_handler.write('new value', 'gs://bucket/test.ipynb'), 1) 83 | # Check that client is only generated once 84 | self.assertIs(client, self.gcs_handler._get_client()) 85 | 86 | @patch('papermill.iorw.GCSFileSystem', side_effect=mock_gcs_fs_wrapper()) 87 | def test_gcs_listdir(self, mock_gcs_filesystem): 88 | client = self.gcs_handler._get_client() 89 | self.gcs_handler.listdir('testdir') 90 | # Check that client is only generated once 91 | self.assertIs(client, self.gcs_handler._get_client()) 92 | 93 | @patch( 94 | 'papermill.iorw.GCSFileSystem', 95 | side_effect=mock_gcs_fs_wrapper(GCSRateLimitException({"message": "test", "code": 429}), 10), 96 | ) 97 | def test_gcs_handle_exception(self, mock_gcs_filesystem): 98 | with patch.object(GCSHandler, 'RETRY_DELAY', 0): 99 | with patch.object(GCSHandler, 'RETRY_MULTIPLIER', 0): 100 | with patch.object(GCSHandler, 'RETRY_MAX_DELAY', 0): 101 | with self.assertRaises(PapermillRateLimitException): 102 | self.gcs_handler.write('raise_limit_exception', 'gs://bucket/test.ipynb') 103 | 104 | @patch( 105 | 'papermill.iorw.GCSFileSystem', 106 | side_effect=mock_gcs_fs_wrapper(GCSRateLimitException({"message": "test", "code": 429}), 1), 107 | ) 108 | def test_gcs_retry(self, mock_gcs_filesystem): 109 | with patch.object(GCSHandler, 'RETRY_DELAY', 0): 110 | with patch.object(GCSHandler, 'RETRY_MULTIPLIER', 0): 111 | with patch.object(GCSHandler, 'RETRY_MAX_DELAY', 0): 112 | self.assertEqual(self.gcs_handler.write('raise_limit_exception', 'gs://bucket/test.ipynb'), 2) 113 | 114 | @patch( 115 | 'papermill.iorw.GCSFileSystem', 116 | side_effect=mock_gcs_fs_wrapper(GCSHttpError({"message": "test", "code": 429}), 1), 117 | ) 118 | def test_gcs_retry_older_exception(self, mock_gcs_filesystem): 119 | with patch.object(GCSHandler, 'RETRY_DELAY', 0): 120 | with patch.object(GCSHandler, 'RETRY_MULTIPLIER', 0): 121 | with patch.object(GCSHandler, 'RETRY_MAX_DELAY', 0): 122 | self.assertEqual(self.gcs_handler.write('raise_limit_exception', 'gs://bucket/test.ipynb'), 2) 123 | 124 | @patch('papermill.iorw.gs_is_retriable', side_effect=fallback_gs_is_retriable) 125 | @patch( 126 | 'papermill.iorw.GCSFileSystem', 127 | side_effect=mock_gcs_fs_wrapper(GCSRateLimitException({"message": "test", "code": None}), 1), 128 | ) 129 | def test_gcs_fallback_retry_unknown_failure_code(self, mock_gcs_filesystem, mock_gcs_retriable): 130 | with patch.object(GCSHandler, 'RETRY_DELAY', 0): 131 | with patch.object(GCSHandler, 'RETRY_MULTIPLIER', 0): 132 | with patch.object(GCSHandler, 'RETRY_MAX_DELAY', 0): 133 | self.assertEqual(self.gcs_handler.write('raise_limit_exception', 'gs://bucket/test.ipynb'), 2) 134 | 135 | @patch('papermill.iorw.gs_is_retriable', return_value=False) 136 | @patch( 137 | 'papermill.iorw.GCSFileSystem', 138 | side_effect=mock_gcs_fs_wrapper(GCSRateLimitException({"message": "test", "code": 500}), 1), 139 | ) 140 | def test_gcs_invalid_code(self, mock_gcs_filesystem, mock_gcs_retriable): 141 | with self.assertRaises(GCSRateLimitException): 142 | self.gcs_handler.write('fatal_exception', 'gs://bucket/test.ipynb') 143 | 144 | @patch('papermill.iorw.gs_is_retriable', side_effect=fallback_gs_is_retriable) 145 | @patch( 146 | 'papermill.iorw.GCSFileSystem', 147 | side_effect=mock_gcs_fs_wrapper(GCSRateLimitException({"message": "test", "code": 500}), 1), 148 | ) 149 | def test_fallback_gcs_invalid_code(self, mock_gcs_filesystem, mock_gcs_retriable): 150 | with self.assertRaises(GCSRateLimitException): 151 | self.gcs_handler.write('fatal_exception', 'gs://bucket/test.ipynb') 152 | 153 | @patch( 154 | 'papermill.iorw.GCSFileSystem', 155 | side_effect=mock_gcs_fs_wrapper(ValueError("not-a-retry"), 1), 156 | ) 157 | def test_gcs_unretryable(self, mock_gcs_filesystem): 158 | with self.assertRaises(ValueError): 159 | self.gcs_handler.write('no_a_rate_limit', 'gs://bucket/test.ipynb') 160 | -------------------------------------------------------------------------------- /papermill/tests/test_hdfs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import MagicMock, patch 3 | 4 | import pytest 5 | 6 | from ..iorw import HDFSHandler 7 | 8 | 9 | class MockHadoopFileSystem(MagicMock): 10 | def get_file_info(self, path): 11 | return [MockFileInfo('test1.ipynb'), MockFileInfo('test2.ipynb')] 12 | 13 | def open_input_stream(self, path): 14 | return MockHadoopFile() 15 | 16 | def open_output_stream(self, path): 17 | return MockHadoopFile() 18 | 19 | 20 | class MockHadoopFile: 21 | def __init__(self): 22 | self._content = b'Content of notebook' 23 | 24 | def __enter__(self, *args): 25 | return self 26 | 27 | def __exit__(self, *args): 28 | pass 29 | 30 | def read(self): 31 | return self._content 32 | 33 | def write(self, new_content): 34 | self._content = new_content 35 | return 1 36 | 37 | 38 | class MockFileInfo: 39 | def __init__(self, path): 40 | self.path = path 41 | 42 | 43 | @pytest.mark.skip(reason="No valid dep package for python 3.12 yet") 44 | @patch('papermill.iorw.HadoopFileSystem', side_effect=MockHadoopFileSystem()) 45 | class HDFSTest(unittest.TestCase): 46 | def setUp(self): 47 | self.hdfs_handler = HDFSHandler() 48 | 49 | def test_hdfs_listdir(self, mock_hdfs_filesystem): 50 | client = self.hdfs_handler._get_client() 51 | self.assertEqual(self.hdfs_handler.listdir("hdfs:///Projects/"), ['test1.ipynb', 'test2.ipynb']) 52 | # Check if client is the same after calling 53 | self.assertIs(client, self.hdfs_handler._get_client()) 54 | 55 | def test_hdfs_read(self, mock_hdfs_filesystem): 56 | client = self.hdfs_handler._get_client() 57 | self.assertEqual(self.hdfs_handler.read("hdfs:///Projects/test1.ipynb"), b'Content of notebook') 58 | self.assertIs(client, self.hdfs_handler._get_client()) 59 | 60 | def test_hdfs_write(self, mock_hdfs_filesystem): 61 | client = self.hdfs_handler._get_client() 62 | self.assertEqual(self.hdfs_handler.write("hdfs:///Projects/test1.ipynb", b'New content'), 1) 63 | self.assertIs(client, self.hdfs_handler._get_client()) 64 | -------------------------------------------------------------------------------- /papermill/tests/test_inspect.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from unittest.mock import MagicMock, patch 3 | 4 | import pytest 5 | from click import Context 6 | 7 | from papermill.inspection import display_notebook_help, inspect_notebook 8 | 9 | NOTEBOOKS_PATH = Path(__file__).parent / "notebooks" 10 | 11 | 12 | def _get_fullpath(name): 13 | return NOTEBOOKS_PATH / name 14 | 15 | 16 | @pytest.fixture 17 | def click_context(): 18 | mock = MagicMock(spec=Context, command=MagicMock()) 19 | mock.command.get_usage.return_value = "Dummy usage" 20 | return mock 21 | 22 | 23 | @pytest.mark.parametrize( 24 | "name, expected", 25 | [ 26 | (_get_fullpath("no_parameters.ipynb"), {}), 27 | ( 28 | _get_fullpath("simple_execute.ipynb"), 29 | {"msg": {"name": "msg", "inferred_type_name": "None", "default": "None", "help": ""}}, 30 | ), 31 | ( 32 | _get_fullpath("complex_parameters.ipynb"), 33 | { 34 | "msg": {"name": "msg", "inferred_type_name": "None", "default": "None", "help": ""}, 35 | "a": { 36 | "name": "a", 37 | "inferred_type_name": "float", 38 | "default": "2.25", 39 | "help": "Variable a", 40 | }, 41 | "b": { 42 | "name": "b", 43 | "inferred_type_name": "List[str]", 44 | "default": "['Hello','World']", 45 | "help": "Nice list", 46 | }, 47 | "c": {"name": "c", "inferred_type_name": "NoneType", "default": "None", "help": ""}, 48 | }, 49 | ), 50 | (_get_fullpath("notimplemented_translator.ipynb"), {}), 51 | ], 52 | ) 53 | def test_inspect_notebook(name, expected): 54 | assert inspect_notebook(name) == expected 55 | 56 | 57 | def test_str_path(): 58 | expected = {"msg": {"name": "msg", "inferred_type_name": "None", "default": "None", "help": ""}} 59 | assert inspect_notebook(str(_get_fullpath("simple_execute.ipynb"))) == expected 60 | 61 | 62 | @pytest.mark.parametrize( 63 | "name, expected", 64 | [ 65 | ( 66 | _get_fullpath("no_parameters.ipynb"), 67 | [ 68 | "Dummy usage", 69 | "\nParameters inferred for notebook '{name}':", 70 | "\n No cell tagged 'parameters'", 71 | ], 72 | ), 73 | ( 74 | _get_fullpath("simple_execute.ipynb"), 75 | [ 76 | "Dummy usage", 77 | "\nParameters inferred for notebook '{name}':", 78 | " msg: Unknown type (default None)", 79 | ], 80 | ), 81 | ( 82 | _get_fullpath("complex_parameters.ipynb"), 83 | [ 84 | "Dummy usage", 85 | "\nParameters inferred for notebook '{name}':", 86 | " msg: Unknown type (default None)", 87 | " a: float (default 2.25) Variable a", 88 | " b: List[str] (default ['Hello','World'])\n Nice list", 89 | " c: NoneType (default None) ", 90 | ], 91 | ), 92 | ( 93 | _get_fullpath("notimplemented_translator.ipynb"), 94 | [ 95 | "Dummy usage", 96 | "\nParameters inferred for notebook '{name}':", 97 | "\n Can't infer anything about this notebook's parameters. It may not have any parameter defined.", 98 | ], 99 | ), 100 | ], 101 | ) 102 | def test_display_notebook_help(click_context, name, expected): 103 | with patch("papermill.inspection.click.echo") as echo: 104 | display_notebook_help(click_context, str(name), None) 105 | 106 | assert echo.call_count == len(expected) 107 | for call, target in zip(echo.call_args_list, expected): 108 | assert call[0][0] == target.format(name=str(name)) 109 | -------------------------------------------------------------------------------- /papermill/tests/test_parameterize.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from datetime import datetime 3 | 4 | from ..exceptions import PapermillMissingParameterException 5 | from ..iorw import load_notebook_node 6 | from ..parameterize import add_builtin_parameters, parameterize_notebook, parameterize_path 7 | from . import get_notebook_path 8 | 9 | 10 | class TestNotebookParametrizing(unittest.TestCase): 11 | def count_nb_injected_parameter_cells(self, nb): 12 | return len([c for c in nb.cells if 'injected-parameters' in c.get('metadata', {}).get('tags', [])]) 13 | 14 | def test_no_tag_copying(self): 15 | # Test that injected cell does not copy other tags 16 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 17 | test_nb.cells[0]['metadata']['tags'].append('some tag') 18 | 19 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}) 20 | 21 | cell_zero = test_nb.cells[0] 22 | self.assertTrue('some tag' in cell_zero.get('metadata').get('tags')) 23 | self.assertTrue('parameters' in cell_zero.get('metadata').get('tags')) 24 | 25 | cell_one = test_nb.cells[1] 26 | self.assertTrue('some tag' not in cell_one.get('metadata').get('tags')) 27 | self.assertTrue('injected-parameters' in cell_one.get('metadata').get('tags')) 28 | 29 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 30 | 31 | def test_injected_parameters_tag(self): 32 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 33 | 34 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}) 35 | 36 | cell_zero = test_nb.cells[0] 37 | self.assertTrue('parameters' in cell_zero.get('metadata').get('tags')) 38 | self.assertTrue('injected-parameters' not in cell_zero.get('metadata').get('tags')) 39 | 40 | cell_one = test_nb.cells[1] 41 | self.assertTrue('injected-parameters' in cell_one.get('metadata').get('tags')) 42 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 43 | 44 | def test_repeated_run_injected_parameters_tag(self): 45 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 46 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 0) 47 | 48 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}) 49 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 50 | 51 | parameterize_notebook(test_nb, {'msg': 'Hello'}) 52 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 53 | 54 | def test_no_parameter_tag(self): 55 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 56 | test_nb.cells[0]['metadata']['tags'] = [] 57 | 58 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}) 59 | 60 | cell_zero = test_nb.cells[0] 61 | self.assertTrue('injected-parameters' in cell_zero.get('metadata').get('tags')) 62 | self.assertTrue('parameters' not in cell_zero.get('metadata').get('tags')) 63 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 64 | 65 | def test_repeated_run_no_parameters_tag(self): 66 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 67 | test_nb.cells[0]['metadata']['tags'] = [] 68 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 0) 69 | 70 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}) 71 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 72 | 73 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}) 74 | self.assertEqual(self.count_nb_injected_parameter_cells(test_nb), 1) 75 | 76 | def test_custom_comment(self): 77 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 78 | test_nb = parameterize_notebook(test_nb, {'msg': 'Hello'}, comment='This is a custom comment') 79 | 80 | cell_one = test_nb.cells[1] 81 | first_line = cell_one['source'].split('\n')[0] 82 | self.assertEqual(first_line, '# This is a custom comment') 83 | 84 | 85 | class TestBuiltinParameters(unittest.TestCase): 86 | def test_add_builtin_parameters_keeps_provided_parameters(self): 87 | with_builtin_parameters = add_builtin_parameters({"foo": "bar"}) 88 | self.assertEqual(with_builtin_parameters["foo"], "bar") 89 | 90 | def test_add_builtin_parameters_adds_dict_of_builtins(self): 91 | with_builtin_parameters = add_builtin_parameters({"foo": "bar"}) 92 | self.assertIn("pm", with_builtin_parameters) 93 | self.assertIsInstance(with_builtin_parameters["pm"], type({})) 94 | 95 | def test_add_builtin_parameters_allows_to_override_builtin(self): 96 | with_builtin_parameters = add_builtin_parameters({"pm": "foo"}) 97 | self.assertEqual(with_builtin_parameters["pm"], "foo") 98 | 99 | def test_builtin_parameters_include_run_uuid(self): 100 | with_builtin_parameters = add_builtin_parameters({"foo": "bar"}) 101 | self.assertIn("run_uuid", with_builtin_parameters["pm"]) 102 | 103 | def test_builtin_parameters_include_current_datetime_local(self): 104 | with_builtin_parameters = add_builtin_parameters({"foo": "bar"}) 105 | self.assertIn("current_datetime_local", with_builtin_parameters["pm"]) 106 | self.assertIsInstance(with_builtin_parameters["pm"]["current_datetime_local"], datetime) 107 | 108 | def test_builtin_parameters_include_current_datetime_utc(self): 109 | with_builtin_parameters = add_builtin_parameters({"foo": "bar"}) 110 | self.assertIn("current_datetime_utc", with_builtin_parameters["pm"]) 111 | self.assertIsInstance(with_builtin_parameters["pm"]["current_datetime_utc"], datetime) 112 | 113 | 114 | class TestPathParameterizing(unittest.TestCase): 115 | def test_plain_text_path_with_empty_parameters_object(self): 116 | self.assertEqual(parameterize_path("foo/bar", {}), "foo/bar") 117 | 118 | def test_plain_text_path_with_none_parameters(self): 119 | self.assertEqual(parameterize_path("foo/bar", None), "foo/bar") 120 | 121 | def test_plain_text_path_with_unused_parameters(self): 122 | self.assertEqual(parameterize_path("foo/bar", {"baz": "quux"}), "foo/bar") 123 | 124 | def test_path_with_single_parameter(self): 125 | self.assertEqual(parameterize_path("foo/bar/{baz}", {"baz": "quux"}), "foo/bar/quux") 126 | 127 | def test_path_with_boolean_parameter(self): 128 | self.assertEqual(parameterize_path("foo/bar/{baz}", {"baz": False}), "foo/bar/False") 129 | 130 | def test_path_with_dict_parameter(self): 131 | self.assertEqual(parameterize_path("foo/{bar[baz]}/", {"bar": {"baz": "quux"}}), "foo/quux/") 132 | 133 | def test_path_with_list_parameter(self): 134 | self.assertEqual(parameterize_path("foo/{bar[0]}/", {"bar": [1, 2, 3]}), "foo/1/") 135 | self.assertEqual(parameterize_path("foo/{bar[2]}/", {"bar": [1, 2, 3]}), "foo/3/") 136 | 137 | def test_path_with_none_parameter(self): 138 | self.assertEqual(parameterize_path("foo/bar/{baz}", {"baz": None}), "foo/bar/None") 139 | 140 | def test_path_with_numeric_parameter(self): 141 | self.assertEqual(parameterize_path("foo/bar/{baz}", {"baz": 42}), "foo/bar/42") 142 | 143 | def test_path_with_numeric_format_string(self): 144 | self.assertEqual(parameterize_path("foo/bar/{baz:03d}", {"baz": 42}), "foo/bar/042") 145 | 146 | def test_path_with_float_format_string(self): 147 | self.assertEqual(parameterize_path("foo/bar/{baz:.03f}", {"baz": 0.3}), "foo/bar/0.300") 148 | 149 | def test_path_with_multiple_parameter(self): 150 | self.assertEqual(parameterize_path("{foo}/{baz}", {"foo": "bar", "baz": "quux"}), "bar/quux") 151 | 152 | def test_parameterized_path_with_undefined_parameter(self): 153 | with self.assertRaises(PapermillMissingParameterException) as context: 154 | parameterize_path("{foo}", {}) 155 | self.assertEqual(str(context.exception), "Missing parameter 'foo'") 156 | 157 | def test_parameterized_path_with_none_parameters(self): 158 | with self.assertRaises(PapermillMissingParameterException) as context: 159 | parameterize_path("{foo}", None) 160 | self.assertEqual(str(context.exception), "Missing parameter 'foo'") 161 | 162 | def test_path_of_none_returns_none(self): 163 | self.assertIsNone(parameterize_path(path=None, parameters={'foo': 'bar'})) 164 | self.assertIsNone(parameterize_path(path=None, parameters=None)) 165 | 166 | def test_path_of_notebook_node_returns_input(self): 167 | test_nb = load_notebook_node(get_notebook_path("simple_execute.ipynb")) 168 | result_nb = parameterize_path(test_nb, parameters=None) 169 | self.assertIs(result_nb, test_nb) 170 | -------------------------------------------------------------------------------- /papermill/tests/test_s3.py: -------------------------------------------------------------------------------- 1 | # The following tests are purposely limited to the exposed interface by iorw.py 2 | 3 | import os.path 4 | 5 | import boto3 6 | import moto 7 | import pytest 8 | from moto import mock_aws 9 | 10 | from ..s3 import S3, Bucket, Key, Prefix 11 | 12 | 13 | @pytest.fixture 14 | def bucket_no_service(): 15 | """Returns a bucket instance with no services""" 16 | return Bucket('my_test_bucket') 17 | 18 | 19 | @pytest.fixture 20 | def bucket_with_service(): 21 | """Returns a bucket instance with a service""" 22 | return Bucket('my_sqs_bucket', ['sqs']) 23 | 24 | 25 | @pytest.fixture 26 | def bucket_sqs(): 27 | """Returns a bucket instance with a sqs service""" 28 | return Bucket('my_sqs_bucket', ['sqs']) 29 | 30 | 31 | @pytest.fixture 32 | def bucket_ec2(): 33 | """Returns a bucket instance with a ec2 service""" 34 | return Bucket('my_sqs_bucket', ['ec2']) 35 | 36 | 37 | @pytest.fixture 38 | def bucket_multiservice(): 39 | """Returns a bucket instance with a ec2 service""" 40 | return Bucket('my_sqs_bucket', ['ec2', 'sqs']) 41 | 42 | 43 | def test_bucket_init(): 44 | assert Bucket('my_test_bucket') 45 | assert Bucket('my_sqs_bucket', 'sqs') 46 | 47 | 48 | def test_bucket_defaults(): 49 | name = 'a bucket' 50 | 51 | b1 = Bucket(name) 52 | b2 = Bucket(name, None) 53 | 54 | assert b1.name == b2.name 55 | assert b1.service == b2.service 56 | 57 | 58 | def test_bucket_missing_params(): 59 | with pytest.raises(TypeError): 60 | Bucket(service=None) 61 | 62 | with pytest.raises(TypeError): 63 | Bucket() 64 | 65 | 66 | def test_bucket_list(bucket_sqs): 67 | # prefix_test = '' 68 | # assert bucket_sqs.list(prefix_test) 69 | # 70 | # prefix_test = 'abc' 71 | # assert bucket_sqs.list(prefix_test) is None 72 | # 73 | # prefix_test = 'ec2' 74 | # assert bucket_sqs.list(prefix_test) is None 75 | # 76 | # prefix_test = 'sqs' 77 | # assert bucket_sqs.list(prefix_test) 78 | pass 79 | 80 | 81 | def test_prefix_init(): 82 | with pytest.raises(TypeError): 83 | Prefix() 84 | 85 | with pytest.raises(TypeError): 86 | Prefix(service=None) 87 | 88 | with pytest.raises(TypeError): 89 | Prefix('my_test_prefix') 90 | 91 | b1 = Bucket('my_test_bucket') 92 | p1 = Prefix(b1, 'sqs_test', service='sqs') 93 | assert Prefix(b1, 'test_bucket') 94 | assert Prefix(b1, 'test_bucket', service=None) 95 | assert Prefix(b1, 'test_bucket', None) 96 | assert p1.bucket.service == p1.service 97 | 98 | 99 | def test_prefix_defaults(): 100 | bucket = Bucket('my data pool') 101 | name = 'bigdata bucket' 102 | 103 | p1 = Prefix(bucket, name) 104 | p2 = Prefix(bucket, name, None) 105 | assert p1.name == p2.name 106 | assert p1.service == p2.service 107 | 108 | 109 | def test_prefix_str(bucket_sqs): 110 | p1 = Prefix(bucket_sqs, 'sqs_prefix_test', 'sqs') 111 | assert str(p1) == f"s3://{str(bucket_sqs)}/sqs_prefix_test" 112 | 113 | 114 | def test_prefix_repr(bucket_sqs): 115 | p1 = Prefix(bucket_sqs, 'sqs_prefix_test', 'sqs') 116 | assert repr(p1) == f"s3://{str(bucket_sqs)}/sqs_prefix_test" 117 | 118 | 119 | def test_key_init(): 120 | pass 121 | 122 | 123 | def test_key_repr(): 124 | k = Key("foo", "bar") 125 | assert repr(k) == "s3://foo/bar" 126 | 127 | 128 | def test_key_defaults(): 129 | bucket = Bucket('my data pool') 130 | name = 'bigdata bucket' 131 | 132 | k1 = Key(bucket, name) 133 | k2 = Key(bucket, name, None, None, None, None, None) 134 | assert k1.size == k2.size 135 | assert k1.etag == k2.etag 136 | assert k1.storage_class == k2.storage_class 137 | assert k1.service == k2.service 138 | assert k1.is_prefix is False 139 | 140 | 141 | @mock_aws 142 | def test_s3_defaults(): 143 | s1 = S3() 144 | s2 = S3() 145 | assert s1.session == s2.session 146 | assert s1.client == s2.client 147 | assert s1.s3 == s2.s3 148 | 149 | 150 | local_dir = os.path.dirname(os.path.abspath(__file__)) 151 | test_bucket_name = 'test-pm-bucket' 152 | test_string = 'Hello' 153 | test_file_path = 'notebooks/s3/s3_in/s3-simple_notebook.ipynb' 154 | test_empty_file_path = 'notebooks/s3/s3_in/s3-empty.ipynb' 155 | 156 | with open(os.path.join(local_dir, test_file_path)) as f: 157 | test_nb_content = f.read() 158 | 159 | no_empty_lines = lambda s: "\n".join([ln for ln in s.split('\n') if ln]) 160 | test_clean_nb_content = no_empty_lines(test_nb_content) 161 | 162 | read_from_gen = lambda g: "\n".join(g) 163 | 164 | 165 | @pytest.fixture(scope="function") 166 | def s3_client(): 167 | mock_aws = moto.mock_aws() 168 | mock_aws.start() 169 | 170 | client = boto3.client('s3') 171 | client.create_bucket(Bucket=test_bucket_name, CreateBucketConfiguration={'LocationConstraint': 'us-west-2'}) 172 | client.put_object(Bucket=test_bucket_name, Key=test_file_path, Body=test_nb_content) 173 | client.put_object(Bucket=test_bucket_name, Key=test_empty_file_path, Body='') 174 | yield S3() 175 | try: 176 | client.delete_object(Bucket=test_bucket_name, Key=test_file_path) 177 | client.delete_object(Bucket=test_bucket_name, Key=f"{test_file_path}.txt") 178 | client.delete_object(Bucket=test_bucket_name, Key=test_empty_file_path) 179 | except Exception: 180 | pass 181 | mock_aws.stop() 182 | 183 | 184 | def test_s3_read(s3_client): 185 | s3_path = f"s3://{test_bucket_name}/{test_file_path}" 186 | data = read_from_gen(s3_client.read(s3_path)) 187 | assert data == test_clean_nb_content 188 | 189 | 190 | def test_s3_read_empty(s3_client): 191 | s3_path = f"s3://{test_bucket_name}/{test_empty_file_path}" 192 | data = read_from_gen(s3_client.read(s3_path)) 193 | assert data == '' 194 | 195 | 196 | def test_s3_write(s3_client): 197 | s3_path = f"s3://{test_bucket_name}/{test_file_path}.txt" 198 | s3_client.cp_string(test_string, s3_path) 199 | 200 | data = read_from_gen(s3_client.read(s3_path)) 201 | assert data == test_string 202 | 203 | 204 | def test_s3_overwrite(s3_client): 205 | s3_path = f"s3://{test_bucket_name}/{test_file_path}" 206 | s3_client.cp_string(test_string, s3_path) 207 | 208 | data = read_from_gen(s3_client.read(s3_path)) 209 | assert data == test_string 210 | 211 | 212 | def test_s3_listdir(s3_client): 213 | dir_name = os.path.dirname(test_file_path) 214 | s3_dir = f"s3://{test_bucket_name}/{dir_name}" 215 | s3_path = f"s3://{test_bucket_name}/{test_file_path}" 216 | dir_listings = s3_client.listdir(s3_dir) 217 | assert len(dir_listings) == 2 218 | assert s3_path in dir_listings 219 | -------------------------------------------------------------------------------- /papermill/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from pathlib import Path 3 | from tempfile import TemporaryDirectory 4 | from unittest.mock import Mock, call 5 | 6 | import pytest 7 | from nbformat.v4 import new_code_cell, new_notebook 8 | 9 | from ..exceptions import PapermillParameterOverwriteWarning 10 | from ..utils import ( 11 | any_tagged_cell, 12 | chdir, 13 | merge_kwargs, 14 | remove_args, 15 | retry, 16 | ) 17 | 18 | 19 | def test_no_tagged_cell(): 20 | nb = new_notebook( 21 | cells=[new_code_cell('a = 2', metadata={"tags": []})], 22 | ) 23 | assert not any_tagged_cell(nb, "parameters") 24 | 25 | 26 | def test_tagged_cell(): 27 | nb = new_notebook( 28 | cells=[new_code_cell('a = 2', metadata={"tags": ["parameters"]})], 29 | ) 30 | assert any_tagged_cell(nb, "parameters") 31 | 32 | 33 | def test_merge_kwargs(): 34 | with warnings.catch_warnings(record=True) as wrn: 35 | assert merge_kwargs({"a": 1, "b": 2}, a=3) == {"a": 3, "b": 2} 36 | assert len(wrn) == 1 37 | assert issubclass(wrn[0].category, PapermillParameterOverwriteWarning) 38 | assert wrn[0].message.__str__() == "Callee will overwrite caller's argument(s): a=3" 39 | 40 | 41 | def test_remove_args(): 42 | assert remove_args(["a"], a=1, b=2, c=3) == {"c": 3, "b": 2} 43 | 44 | 45 | def test_retry(): 46 | m = Mock(side_effect=RuntimeError(), __name__="m", __module__="test_s3", __doc__="m") 47 | wrapped_m = retry(3)(m) 48 | with pytest.raises(RuntimeError): 49 | wrapped_m("foo") 50 | m.assert_has_calls([call("foo"), call("foo"), call("foo")]) 51 | 52 | 53 | def test_chdir(): 54 | old_cwd = Path.cwd() 55 | with TemporaryDirectory() as temp_dir: 56 | with chdir(temp_dir): 57 | assert Path.cwd() != old_cwd 58 | assert Path.cwd() == Path(temp_dir) 59 | 60 | assert Path.cwd() == old_cwd 61 | -------------------------------------------------------------------------------- /papermill/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import warnings 4 | from contextlib import contextmanager 5 | from functools import wraps 6 | 7 | from .exceptions import PapermillParameterOverwriteWarning 8 | 9 | logger = logging.getLogger('papermill.utils') 10 | 11 | 12 | def any_tagged_cell(nb, tag): 13 | """Whether the notebook contains at least one cell tagged ``tag``? 14 | 15 | Parameters 16 | ---------- 17 | nb : nbformat.NotebookNode 18 | The notebook to introspect 19 | tag : str 20 | The tag to look for 21 | 22 | Returns 23 | ------- 24 | bool 25 | Whether the notebook contains a cell tagged ``tag``? 26 | """ 27 | return any([tag in cell.metadata.tags for cell in nb.cells]) 28 | 29 | 30 | def nb_kernel_name(nb, name=None): 31 | """Helper for fetching out the kernel name from a notebook object. 32 | 33 | Parameters 34 | ---------- 35 | nb : nbformat.NotebookNode 36 | The notebook to introspect 37 | name : str 38 | A provided name field 39 | 40 | Returns 41 | ------- 42 | str 43 | The name of the kernel 44 | 45 | Raises 46 | ------ 47 | ValueError 48 | If no kernel name is found or provided 49 | """ 50 | name = name or nb.metadata.get('kernelspec', {}).get('name') 51 | if not name: 52 | raise ValueError("No kernel name found in notebook and no override provided.") 53 | return name 54 | 55 | 56 | def nb_language(nb, language=None): 57 | """Helper for fetching out the programming language from a notebook object. 58 | 59 | Parameters 60 | ---------- 61 | nb : nbformat.NotebookNode 62 | The notebook to introspect 63 | language : str 64 | A provided language field 65 | 66 | Returns 67 | ------- 68 | str 69 | The programming language of the notebook 70 | 71 | Raises 72 | ------ 73 | ValueError 74 | If no notebook language is found or provided 75 | """ 76 | language = language or nb.metadata.get('language_info', {}).get('name') 77 | if not language: 78 | # v3 language path for old notebooks that didn't convert cleanly 79 | language = language or nb.metadata.get('kernelspec', {}).get('language') 80 | if not language: 81 | raise ValueError("No language found in notebook and no override provided.") 82 | return language 83 | 84 | 85 | def find_first_tagged_cell_index(nb, tag): 86 | """Find the first tagged cell ``tag`` in the notebook. 87 | 88 | Parameters 89 | ---------- 90 | nb : nbformat.NotebookNode 91 | The notebook to introspect 92 | tag : str 93 | The tag to look for 94 | 95 | Returns 96 | ------- 97 | nbformat.NotebookNode 98 | Whether the notebook contains a cell tagged ``tag``? 99 | """ 100 | parameters_indices = [] 101 | for idx, cell in enumerate(nb.cells): 102 | if tag in cell.metadata.tags: 103 | parameters_indices.append(idx) 104 | if not parameters_indices: 105 | return -1 106 | return parameters_indices[0] 107 | 108 | 109 | def merge_kwargs(caller_args, **callee_args): 110 | """Merge named argument. 111 | 112 | Function takes a dictionary of caller arguments and callee arguments as keyword arguments 113 | Returns a dictionary with merged arguments. If same argument is in both caller and callee 114 | arguments the last one will be taken and warning will be raised. 115 | 116 | Parameters 117 | ---------- 118 | caller_args : dict 119 | Caller arguments 120 | **callee_args 121 | Keyword callee arguments 122 | 123 | Returns 124 | ------- 125 | args : dict 126 | Merged arguments 127 | """ 128 | conflicts = set(caller_args) & set(callee_args) 129 | if conflicts: 130 | args = format('; '.join([f'{key}={value}' for key, value in callee_args.items()])) 131 | msg = f"Callee will overwrite caller's argument(s): {args}" 132 | warnings.warn(msg, PapermillParameterOverwriteWarning) 133 | return dict(caller_args, **callee_args) 134 | 135 | 136 | def remove_args(args=None, **kwargs): 137 | """Remove arguments from kwargs. 138 | 139 | Parameters 140 | ---------- 141 | args : list 142 | Argument names to remove from kwargs 143 | **kwargs 144 | Arbitrary keyword arguments 145 | 146 | Returns 147 | ------- 148 | kwargs : dict 149 | New dictionary of arguments 150 | """ 151 | if not args: 152 | return kwargs 153 | return {k: v for k, v in kwargs.items() if k not in args} 154 | 155 | 156 | # retry decorator 157 | def retry(num): 158 | def decorate(func): 159 | @wraps(func) 160 | def wrapper(*args, **kwargs): 161 | exception = None 162 | 163 | for i in range(num): 164 | try: 165 | return func(*args, **kwargs) 166 | except Exception as e: 167 | logger.debug(f'Retrying after: {e}') 168 | exception = e 169 | else: 170 | raise exception 171 | 172 | return wrapper 173 | 174 | return decorate 175 | 176 | 177 | @contextmanager 178 | def chdir(path): 179 | """Change working directory to `path` and restore old path on exit. 180 | 181 | `path` can be `None` in which case this is a no-op. 182 | """ 183 | if path is None: 184 | yield 185 | 186 | else: 187 | old_dir = os.getcwd() 188 | os.chdir(path) 189 | try: 190 | yield 191 | finally: 192 | os.chdir(old_dir) 193 | -------------------------------------------------------------------------------- /papermill/version.py: -------------------------------------------------------------------------------- 1 | version = '2.6.0' 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Migration to pyproject.toml is in progress 2 | 3 | # Example configuration for Black. 4 | [tool.black] 5 | line-length = 120 6 | target-version = ['py311'] 7 | skip-string-normalization = true 8 | 9 | 10 | [tool.coverage.run] 11 | branch = false 12 | 13 | [tool.coverage.report] 14 | exclude_lines = [ 15 | "if self.debug:", 16 | "pragma: no cover", 17 | "raise AssertionError", 18 | "raise NotImplementedError", 19 | "if __name__ == .__main__.:", 20 | ] 21 | omit = [ 22 | "papermill/tests/*", 23 | "papermill/version.py" 24 | ] 25 | 26 | 27 | [tool.codespell] 28 | quiet-level = 3 29 | # comma separated list of words; waiting for: 30 | # https://github.com/codespell-project/codespell/issues/2839#issuecomment-1731601603 31 | # also adding links until they ignored by its: nature 32 | # https://github.com/codespell-project/codespell/issues/2243#issuecomment-1732019960 33 | ignore-words-list = "dne, compiletime" 34 | 35 | 36 | [tool.ruff] 37 | target-version = "py38" 38 | line-length = 120 39 | # Enable Pyflakes `E` and `F` codes by default. 40 | select = [ 41 | "E", "W", # see: https://pypi.org/project/pycodestyle 42 | "F", # see: https://pypi.org/project/pyflakes 43 | "I", # isort 44 | # "D", # see: https://pypi.org/project/pydocstyle 45 | # "N", # see: https://pypi.org/project/pep8-naming 46 | "RUF100", # unnecessary noqa comment 47 | "UP", # pyupgrade 48 | ] 49 | #extend-select = [ 50 | # "C4", # see: https://pypi.org/project/flake8-comprehensions 51 | # "SIM", # see: https://pypi.org/project/flake8-simplify 52 | # "RET", # see: https://pypi.org/project/flake8-return 53 | # "PT", # see: https://pypi.org/project/flake8-pytest-style 54 | #] 55 | ignore = [ 56 | "E731", # Do not assign a lambda expression, use a def 57 | ] 58 | # Exclude a variety of commonly ignored directories. 59 | exclude = [ 60 | "docs" 61 | ] 62 | ignore-init-module-imports = true 63 | 64 | [tool.ruff.pydocstyle] 65 | # Use Google-style docstrings. 66 | convention = "google" 67 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | env = 3 | AWS_SECRET_ACCESS_KEY=foobar_secret 4 | AWS_ACCESS_KEY_ID=foobar_key 5 | filterwarnings = 6 | ignore:.*imp module is deprecated.*:DeprecationWarning 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | pyyaml 3 | nbformat >= 5.2.0 4 | nbclient >= 0.2.0 5 | tqdm >= 4.32.2 6 | requests 7 | entrypoints 8 | tenacity >= 5.0.2 9 | aiohttp >=3.9.0; python_version=="3.12" 10 | ansicolors 11 | -------------------------------------------------------------------------------- /requirements/azure.txt: -------------------------------------------------------------------------------- 1 | azure-datalake-store >= 0.0.30 2 | azure-storage-blob >= 12.1.0 3 | requests >= 2.21.0 4 | azure-identity>=1.3.1 5 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | botocore 3 | codecov 4 | coverage 5 | google_compute_engine # Need this because boto has issues with dynamic package loading during tests if other google components are there 6 | ipython>=5.0 7 | ipywidgets 8 | notebook 9 | moto >= 5.0.0,<5.1.0 10 | pytest>=4.1 11 | pytest-cov>=2.6.1 12 | pytest-mock>=1.10 13 | pytest-env>=0.6.2 14 | requests >= 2.21.0 15 | check-manifest 16 | attrs>=17.4.0 17 | pre-commit 18 | tox 19 | bumpversion 20 | recommonmark 21 | pip>=18.1 22 | wheel>=0.31.0 23 | setuptools>=38.6.0 24 | twine>=1.11.0 25 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | # Pin packages for RTD builds 2 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html#pinning-dependencies 3 | #-r ../requirements.txt 4 | Sphinx>=7.2.6 5 | furo>=2023.9.10 6 | myst-parser>=2.0.0 7 | moto>=4.2.8 8 | sphinx-copybutton>=0.5.2 9 | nbformat 10 | entrypoints 11 | -------------------------------------------------------------------------------- /requirements/gcs.txt: -------------------------------------------------------------------------------- 1 | gcsfs>=0.2.0 2 | -------------------------------------------------------------------------------- /requirements/github.txt: -------------------------------------------------------------------------------- 1 | PyGithub >= 1.55 2 | -------------------------------------------------------------------------------- /requirements/hdfs.txt: -------------------------------------------------------------------------------- 1 | pyarrow >= 2.0 2 | -------------------------------------------------------------------------------- /requirements/s3.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """" 3 | setup.py 4 | 5 | See: 6 | https://packaging.python.org/tutorials/packaging-projects/ 7 | https://packaging.python.org/en/latest/distributing.html 8 | https://github.com/pypa/sampleproject 9 | 10 | """ 11 | import os 12 | 13 | from setuptools import setup 14 | 15 | local_path = os.path.dirname(__file__) 16 | # Fix for tox which manipulates execution pathing 17 | if not local_path: 18 | local_path = '.' 19 | here = os.path.abspath(local_path) 20 | 21 | 22 | def version(): 23 | with open(f"{here}/papermill/version.py") as ver: 24 | for line in ver.readlines(): 25 | if line.startswith('version ='): 26 | return line.split(' = ')[-1].strip()[1:-1] 27 | raise ValueError('No version found in papermill/version.py') 28 | 29 | 30 | def read(fname): 31 | with open(fname) as fhandle: 32 | return fhandle.read() 33 | 34 | 35 | def read_reqs(fname, folder=None): 36 | path_dir = os.path.join(here, folder) if folder else here 37 | req_path = os.path.join(path_dir, fname) 38 | return [req.strip() for req in read(req_path).splitlines() if req.strip()] 39 | 40 | 41 | s3_reqs = read_reqs('s3.txt', folder='requirements') 42 | azure_reqs = read_reqs('azure.txt', folder='requirements') 43 | gcs_reqs = read_reqs('gcs.txt', folder='requirements') 44 | hdfs_reqs = read_reqs('hdfs.txt', folder='requirements') 45 | github_reqs = read_reqs('github.txt', folder='requirements') 46 | docs_only_reqs = read_reqs('docs.txt', folder='requirements') 47 | black_reqs = ['black >= 19.3b0'] 48 | all_reqs = s3_reqs + azure_reqs + gcs_reqs + hdfs_reqs + github_reqs + black_reqs 49 | docs_reqs = all_reqs + docs_only_reqs 50 | # Temporarily remove hdfs_reqs from dev deps until the pyarrow package is available for Python 3.12 51 | dev_reqs = read_reqs('dev.txt', folder='requirements') + s3_reqs + azure_reqs + gcs_reqs + black_reqs # all_reqs 52 | extras_require = { 53 | "test": dev_reqs, 54 | "dev": dev_reqs, 55 | "all": all_reqs, 56 | "s3": s3_reqs, 57 | "azure": azure_reqs, 58 | "gcs": gcs_reqs, 59 | "hdfs": hdfs_reqs, 60 | "github": github_reqs, 61 | "black": black_reqs, 62 | "docs": docs_reqs, 63 | } 64 | 65 | # Get the long description from the README file 66 | with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: 67 | long_description = f.read() 68 | 69 | setup( 70 | name='papermill', 71 | version=version(), 72 | description='Parameterize and run Jupyter and nteract Notebooks', 73 | author='nteract contributors', 74 | author_email='nteract@googlegroups.com', 75 | license='BSD', 76 | # Note that this is a string of words separated by whitespace, not a list. 77 | keywords='jupyter mapreduce nteract pipeline notebook', 78 | long_description=long_description, 79 | long_description_content_type='text/markdown', 80 | url='https://github.com/nteract/papermill', 81 | packages=['papermill'], 82 | python_requires='>=3.8', 83 | install_requires=read_reqs('requirements.txt'), 84 | extras_require=extras_require, 85 | entry_points={'console_scripts': ['papermill = papermill.__main__:papermill']}, 86 | project_urls={ 87 | 'Documentation': 'https://papermill.readthedocs.io', 88 | 'Funding': 'https://nteract.io', 89 | 'Source': 'https://github.com/nteract/papermill/', 90 | 'Tracker': 'https://github.com/nteract/papermill/issues', 91 | }, 92 | classifiers=[ 93 | 'Intended Audience :: Developers', 94 | 'Intended Audience :: System Administrators', 95 | 'Intended Audience :: Science/Research', 96 | 'License :: OSI Approved :: BSD License', 97 | 'Programming Language :: Python', 98 | 'Programming Language :: Python :: 3', 99 | 'Programming Language :: Python :: 3.8', 100 | 'Programming Language :: Python :: 3.9', 101 | 'Programming Language :: Python :: 3.10', 102 | 'Programming Language :: Python :: 3.11', 103 | 'Programming Language :: Python :: 3.12', 104 | ], 105 | ) 106 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = true 3 | envlist = py{38,39,310,311,312}, dist, manifest, docs, binder 4 | 5 | [gh-actions] 6 | python = 7 | 3.8: py38 8 | 3.9: py39 9 | 3.10: py310 10 | 3.11: py311, docs 11 | 3.12: py312, dist 12 | 13 | # Manifest 14 | [testenv:manifest] 15 | skip_install = true 16 | deps = check-manifest 17 | commands = check-manifest 18 | ignore = 19 | .readthedocs.yaml 20 | 21 | # Docs 22 | [testenv:docs] 23 | description = invoke sphinx-build to build the HTML docs 24 | deps = 25 | .[docs] 26 | extras = docs 27 | commands = 28 | sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs} 29 | python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' 30 | 31 | # Binder 32 | [testenv:binder] 33 | description = ensure /binder/*ipynb are runnable 34 | deps = 35 | -r binder/requirements.txt 36 | commands = python -c "import glob; import papermill as pm; [pm.execute_notebook(input, '{toxworkdir}/out.ipynb', parameters=\{'binder_dir':'binder'\}) for input in glob.glob('binder/**/*.ipynb')]" 37 | 38 | # Distro 39 | [testenv:dist] 40 | skip_install = true 41 | commands = 42 | python setup.py sdist --dist-dir={distdir} bdist_wheel --dist-dir={distdir} 43 | /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.tar.gz' 44 | /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.whl' 45 | 46 | [testenv] 47 | # disable Python's hash randomization for tests that stringify dicts, etc 48 | setenv = 49 | PYTHONHASHSEED = 0 50 | AWS_ACCESS_KEY_ID=foobar_key 51 | AWS_SECRET_ACCESS_KEY=foobar_secret 52 | passenv = * 53 | basepython = 54 | py38: python3.8 55 | py39: python3.9 56 | py310: python3.10 57 | py311: python3.11 58 | py312: python3.12 59 | manifest: python3.11 60 | dist: python3.12 61 | docs: python3.11 62 | binder: python3.11 63 | deps = .[dev] 64 | # Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line... 65 | allowlist_externals = /bin/bash 66 | # Python 3.12 breaks default pip/setuptools versions ... force an upgrade of these before anything else 67 | install_command = /bin/bash ./tox_py_installer.sh {opts} {packages} 68 | commands = pytest -v --maxfail=2 --cov=papermill -W always {posargs} 69 | -------------------------------------------------------------------------------- /tox_py_installer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python -m ensurepip --upgrade 3 | python -m pip install --upgrade setuptools 4 | # python -m pip install {opts} {packages} 5 | python -m pip install $1 $2 6 | --------------------------------------------------------------------------------