├── .bumpversion.cfg ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .readthedocs.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── RELEASING.md ├── docs ├── Makefile ├── UPDATE.md ├── changelog.md ├── conf.py ├── examples │ └── index.md ├── getting-started │ └── index.md ├── index.md ├── make.bat ├── reference │ └── index.rst └── usage │ └── index.md ├── examples ├── dataframe-example │ ├── dataframe-assertion-example.ipynb │ └── dataframe_test.py ├── requests-example │ ├── requests-example.ipynb │ └── requests_test.py └── stdout-example │ ├── stdout-assertion-example.ipynb │ └── stdout_test.py ├── pyproject.toml ├── testbook ├── __init__.py ├── client.py ├── exceptions.py ├── reference.py ├── testbook.py ├── testbooknode.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── resources │ │ ├── datamodel.ipynb │ │ ├── exception.ipynb │ │ ├── foo.ipynb │ │ ├── inject.ipynb │ │ ├── patch.ipynb │ │ └── reference.ipynb │ ├── test_client.py │ ├── test_datamodel.py │ ├── test_execute.py │ ├── test_inject.py │ ├── test_patch.py │ ├── test_reference.py │ ├── test_testbook.py │ └── test_translators.py ├── translators.py └── utils.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.4.2 3 | commit = True 4 | tag = True 5 | tag_name = {new_version} 6 | 7 | [bumpversion:file:pyproject.toml] 8 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: "*" 6 | pull_request: 7 | branches: "*" 8 | 9 | jobs: 10 | build-n-test-n-coverage: 11 | name: Build, test and code coverage 12 | runs-on: ubuntu-20.04 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v5 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install .[dev] 28 | pip install tox-gh-actions 29 | - name: Run the tests 30 | run: tox 31 | - name: Upload coverage to Codecov 32 | uses: codecov/codecov-action@v4 33 | with: 34 | file: ./coverage.xml 35 | flags: unittests 36 | name: codecov-umbrella 37 | fail_ci_if_error: false 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Editor configs 132 | .vscode/ 133 | .idea/ 134 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | formats: all 12 | 13 | build: 14 | image: latest 15 | 16 | python: 17 | version: 3.11 18 | install: 19 | - method: pip 20 | path: . 21 | extra_requirements: 22 | - docs 23 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Please read our entire [Code of Conduct](https://github.com/nteract/nteract/blob/main/CODE_OF_CONDUCT.md) 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # So You Want to Contribute to testbook! 2 | 3 | We welcome all contributions to testbook both large and small. We encourage you to join our community. 4 | 5 | ## Our Community Values 6 | 7 | We are an open and friendly community. Everybody is welcome. 8 | 9 | We encourage friendly discussions and respect for all. There are no exceptions. 10 | 11 | All contributions are equally important. Documentation, answering questions, and fixing bugs are equally as valuable as adding new features. 12 | 13 | Please read our entire code of conduct [here](https://github.com/nteract/nteract/blob/master/CODE_OF_CONDUCT.md). Also, check out the for the [Python](https://github.com/nteract/nteract/blob/master/CODE_OF_CONDUCT.md) code of conduct. 14 | 15 | ## Setting up Your Development Environment 16 | 17 | Following these instructions should give you an efficient path to opening your first pull-request. 18 | 19 | ### Cloning the testbook Repository 20 | 21 | Fork the repository to your local Github account. Clone this repository to your local development machine. 22 | 23 | ```bash 24 | git clone https://github.com//testbook 25 | cd testbook 26 | ``` 27 | 28 | ### Install an Editable Version 29 | 30 | Use you favorite editor environment, here's an example for venv: 31 | 32 | ```bash 33 | python3 -m venv dev 34 | source dev/bin/activate 35 | ``` 36 | 37 | Install testbook using: 38 | 39 | ```bash 40 | pip install -e '.[dev]' 41 | ``` 42 | 43 | _Note: When you are finished you can use `source deactivate` to go back to your base environment._ 44 | 45 | ### Running Tests Locally 46 | 47 | If you are contributing with documentation please jump to [building documentation.](#Building-Documentation) 48 | 49 | We need to install the development package before we can run the tests. If anything is confusing below, always resort to the relevant documentation. 50 | 51 | For the most basic test runs against python 3.11 use this tox subset (callable after `pip install tox`): 52 | 53 | ```bash 54 | tox -e py311 55 | ``` 56 | 57 | This will just execute the unittests against python 3.11 in a new virtual env. The first run will take longer to setup the virtualenv, but will be fast after that point. 58 | 59 | For a full test suite of all envs and linting checks simply run tox without any arguments 60 | 61 | ```bash 62 | tox 63 | ``` 64 | 65 | This will require python3.7, and python 3.8 to be installed. 66 | 67 | Alternavitely pytest can be used if you have an environment already setup which works or has custom packages not present in the tox build. 68 | 69 | ```bash 70 | pytest 71 | ``` 72 | 73 | Now there should be a working and editable installation of testbook to start making your own contributions. 74 | 75 | ### Building Documentation 76 | 77 | The documentation is built using the [Sphinx](http://www.sphinx-doc.org/en/master/) engine. To contribute, edit the [RestructuredText (`.rst`)](https://en.wikipedia.org/wiki/ReStructuredText) files in the docs directory to make changes and additions. 78 | 79 | Once you are done editing, to generate the documentation, use tox and the following command from the root directory of the repository: 80 | 81 | ```bash 82 | tox -e docs 83 | ``` 84 | 85 | This will generate `.html` files in the `/.tox/docs_out/` directory. Once you are satisfied, feel free to jump to the next section. 86 | 87 | ## So You're Ready to Pull Request 88 | 89 | The general workflow for this will be: 90 | 91 | 1. Run local tests 92 | 2. Pushed changes to your forked repository 93 | 3. Open pull request to main repository 94 | 95 | ### Run Tests Locally 96 | 97 | ```bash 98 | tox 99 | ``` 100 | 101 | Note that the manifest test reads the `MANIFEST.in` file and explicitly specify the files to include in the source distribution. You can read more about how this works [here](https://docs.python.org/3/distutils/sourcedist.html). 102 | 103 | ### Push Changes to Forked Repo 104 | 105 | Your commits should be pushed to the forked repository. To verify this type 106 | 107 | ```bash 108 | git remote -v 109 | ``` 110 | 111 | and ensure the remotes point to your GitHub. Don't work on the master branch! 112 | 113 | 1. Commit changes to local repository: 114 | ```bash 115 | git checkout -b my-feature 116 | git add 117 | git commit 118 | ``` 119 | 2. Push changes to your remote repository: 120 | ```bash 121 | git push -u origin my-feature 122 | ``` 123 | 124 | ### Create Pull Request 125 | 126 | Follow [these](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) instrucutions to create a pull request from a forked repository. If you are submitting a bug-fix for a specific issue make sure to reference the issue in the pull request. 127 | 128 | There are good references to the [Git documentation](https://git-scm.com/doc) and [Git workflows](https://docs.scipy.org/doc/numpy/dev/gitwash/development_workflow.html) for more information if any of this is unfamiliar. 129 | 130 | _Note: You might want to set a reference to the main repository to fetch/merge from there instead of your forked repository. You can do that using:_ 131 | 132 | ```bash 133 | git remote add upstream https://github.com/nteract/testbook 134 | ``` 135 | 136 | It's possible you will have conflicts between your repository and master. Here, `master` is meant to be synchronized with the `upstream` repository. GitHub has some good [documentation](https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/) on merging pull requests from the command line. 137 | 138 | Happy hacking on testbook! 139 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024, nteract 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include testbook *.py 2 | recursive-include testbook *.ipynb 3 | recursive-include testbook *.json 4 | recursive-include testbook *.yaml 5 | recursive-include testbook *.keep 6 | recursive-include testbook *.txt 7 | 8 | include tox.ini 9 | include pytest.ini 10 | include README.md 11 | include LICENSE 12 | include MANIFEST.in 13 | include *.md 14 | include *.toml 15 | include *.yml 16 | 17 | include .bumpversion.cfg 18 | 19 | # Documentation 20 | graft docs 21 | # exclude build files 22 | prune docs/_build 23 | # exclude sample notebooks for binder 24 | prune binder 25 | # Test env 26 | prune .tox 27 | # Build files 28 | prune azure-pipelines.yml 29 | # Exclude examples 30 | prune examples 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://github.com/nteract/testbook/workflows/CI/badge.svg)](https://github.com/nteract/testbook/actions) 2 | [![image](https://codecov.io/github/nteract/testbook/coverage.svg?branch=master)](https://codecov.io/github/nteract/testbook?branch=master) 3 | [![Documentation Status](https://readthedocs.org/projects/testbook/badge/?version=latest)](https://testbook.readthedocs.io/en/latest/?badge=latest) 4 | [![image](https://img.shields.io/pypi/v/testbook.svg)](https://pypi.python.org/pypi/testbook) 5 | [![image](https://img.shields.io/pypi/l/testbook.svg)](https://github.com/astral-sh/testbook/blob/main/LICENSE) 6 | [![image](https://img.shields.io/pypi/pyversions/testbook.svg)](https://pypi.python.org/pypi/testbook) 7 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) 8 | 9 | # testbook 10 | 11 | **testbook** is a unit testing framework extension for testing code in Jupyter Notebooks. 12 | 13 | Previous attempts at unit testing notebooks involved writing the tests in the notebook itself. 14 | However, testbook will allow for unit tests to be run against notebooks in separate test files, 15 | hence treating .ipynb files as .py files. 16 | 17 | testbook helps you set up **conventional unit tests for your Jupyter Notebooks**. 18 | 19 | Here is an example of a unit test written using testbook 20 | 21 | Consider the following code cell in a Jupyter Notebook `example_notebook.ipynb`: 22 | 23 | ```python 24 | def func(a, b): 25 | return a + b 26 | ``` 27 | 28 | You would write a unit test using `testbook` in a Python file `example_test.py` as follows: 29 | 30 | ```python 31 | # example_test.py 32 | from testbook import testbook 33 | 34 | 35 | @testbook('/path/to/example_notebook.ipynb', execute=True) 36 | def test_func(tb): 37 | func = tb.get("func") 38 | 39 | assert func(1, 2) == 3 40 | ``` 41 | 42 | Then [pytest](https://github.com/pytest-dev/pytest) can be used to run the test: 43 | 44 | ```{code-block} bash 45 | pytest example_test.py 46 | ``` 47 | 48 | ## Installing `testbook` 49 | 50 | ```{code-block} bash 51 | pip install testbook 52 | ``` 53 | 54 | NOTE: This does not install any kernels for running your notebooks. You'll need to install in the same way you do for running the notebooks normally. Usually this is done with `pip install ipykernel` 55 | 56 | Alternatively if you want all the same dev dependencies and the ipython kernel you can install these dependencies with: 57 | 58 | ```{code-block} bash 59 | pip install testbook[dev] 60 | ``` 61 | 62 | ## Documentation 63 | 64 | See [readthedocs](https://testbook.readthedocs.io/en/latest/) for more in-depth details. 65 | 66 | ## Development Guide 67 | 68 | Read [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines on how to setup a local development environment and make code changes back to testbook. 69 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Releasing 2 | 3 | ## Prerequisites 4 | 5 | - First check that the CHANGELOG is up to date for the next release version 6 | - Ensure dev requirements are installed `pip install ".[dev]"` 7 | 8 | ## Push to GitHub 9 | 10 | Change from patch to minor or major for appropriate version updates. 11 | 12 | ```bash 13 | bumpversion patch 14 | git push upstream && git push upstream --tags 15 | ``` 16 | 17 | ## Push to PyPI 18 | 19 | ```bash 20 | rm -rf dist/* 21 | rm -rf build/* 22 | python -m build 23 | twine upload dist/* 24 | ``` 25 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/UPDATE.md: -------------------------------------------------------------------------------- 1 | TODO: Figure out make options needed for non-api changes 2 | 3 | ``` 4 | sphinx-apidoc -f -o reference ../testbook 5 | ``` 6 | -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.4.2 4 | 5 | - Documentation and CoC updates to improve developer access (Thank you PyLadies Vancouver!) 6 | - The `text/plain` media type is now visible when calling `notebook.cell_output_text(idx)` 7 | 8 | ## 0.4.1 9 | 10 | - check for errors when `allow_errors` is true 11 | 12 | ## 0.4.0 13 | 14 | - Testbook now returns actual object for JSON serializable objects instead of reference objects. Please note that this may break tests written with prior versions. 15 | 16 | ## 0.3.0 17 | 18 | - Implemented container methods 19 | -- __len__ 20 | -- __iter__ 21 | -- __next__ 22 | -- __getitem__ 23 | -- __setitem__ 24 | -- __contains__ 25 | - Fixed testbook to work with ipykernel 5.5 26 | 27 | ## 0.2.6 28 | 29 | - Fixed Python underscore (`_`) issue 30 | 31 | ## 0.2.5 32 | 33 | - Fixed testbook decorator. 34 | 35 | ## 0.2.4 36 | 37 | - Add `cell_execute_result` to `TestbookNotebookClient` 38 | - Use testbook decorator with pytest fixture and marker 39 | 40 | ## 0.2.3 41 | 42 | - Accept notebook node as argument to testbook 43 | - Added support for specifying kernel with `kernel_name` kwarg 44 | 45 | ## 0.2.2 46 | 47 | - Added support for passing notebook as file-like object or path as str 48 | 49 | ## 0.2.1 50 | 51 | - Added support for `allow_errors` 52 | 53 | ## 0.2.0 54 | 55 | - Changed to new package name `testbook` 56 | - Supports for patch and patch_dict 57 | - Slices now supported for execute patterns 58 | - Raises TestbookRuntimeError for all exceptions that occur during cell execution 59 | 60 | ## 0.1.3 61 | 62 | - Added warning about package name change 63 | 64 | ## 0.1.2 65 | 66 | - Updated docs link in setup.py 67 | 68 | ## 0.1.1 69 | 70 | - Unpin dependencies 71 | 72 | ## 0.1.0 73 | 74 | - Initial release with basic features 75 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | from importlib.metadata import version as read_version 16 | 17 | 18 | sys.path.insert(0, os.path.abspath('..')) 19 | 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = 'testbook' 24 | copyright = '2024, nteract team' 25 | author = 'nteract team' 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.autodoc', 35 | 'sphinx.ext.intersphinx', 36 | 'sphinx.ext.mathjax', 37 | 'sphinx.ext.napoleon', 38 | 'myst_parser', 39 | ] 40 | 41 | # Add any paths that contain templates here, relative to this directory. 42 | templates_path = ['_templates'] 43 | 44 | # The suffix(es) of source filenames. 45 | # You can specify multiple suffix as a list of string: 46 | # 47 | source_suffix = ['.rst', '.md'] 48 | 49 | # The master toctree document. 50 | master_doc = 'index' 51 | 52 | # General information about the project. 53 | project = 'testbook' 54 | copyright = '2024, nteract team' 55 | author = 'nteract team' 56 | 57 | # The version info for the project you're documenting, acts as replacement for 58 | # |version| and |release|, also used in various other places throughout the 59 | # built documents. 60 | # 61 | 62 | 63 | # The short X.Y version. 64 | version = '.'.join(read_version(project).split('.')[0:2]) 65 | 66 | # The full version, including alpha/beta/rc tags. 67 | release = read_version(project) 68 | 69 | # The language for content autogenerated by Sphinx. Refer to documentation 70 | # for a list of supported languages. 71 | # 72 | # This is also used if you do content translation via gettext catalogs. 73 | # Usually you set "language" from the command line foexitr these cases. 74 | language = 'en' 75 | 76 | # List of patterns, relative to source directory, that match files and 77 | # directories to ignore when looking for source files. 78 | # This patterns also effect to html_static_path and html_extra_path 79 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'UPDATE.md'] 80 | 81 | # The name of the Pygments (syntax highlighting) style to use. 82 | pygments_style = 'sphinx' 83 | 84 | # If true, `todo` and `todoList` produce output, else they produce nothing. 85 | todo_include_todos = False 86 | 87 | 88 | # -- Options for HTML output ---------------------------------------------- 89 | 90 | # The theme to use for HTML and HTML Help pages. See the documentation for 91 | # a list of builtin themes. 92 | # 93 | html_theme = 'sphinx_book_theme' 94 | 95 | # Theme options are theme-specific and customize the look and feel of a theme 96 | # further. For a list of options available for each theme, see the 97 | # documentation. 98 | # 99 | 100 | html_theme_options = { 101 | 'path_to_docs': 'docs', 102 | 'repository_url': 'https://github.com/nteract/testbook', 103 | 'repository_branch': 'main', 104 | 'use_edit_page_button': True, 105 | } 106 | 107 | # Add any paths that contain custom static files (such as style sheets) here, 108 | # relative to this directory. They are copied after the builtin static files, 109 | # so a file named "default.css" will overwrite the builtin "default.css". 110 | # html_static_path = ['_static'] 111 | 112 | html_title = 'testbook' 113 | 114 | # -- Options for HTMLHelp output ------------------------------------------ 115 | 116 | # Output file base name for HTML help builder. 117 | htmlhelp_basename = 'testbookdoc' 118 | 119 | 120 | # -- Options for LaTeX output --------------------------------------------- 121 | 122 | latex_elements = { 123 | # The paper size ('letterpaper' or 'a4paper'). 124 | # 125 | # 'papersize': 'letterpaper', 126 | # The font size ('10pt', '11pt' or '12pt'). 127 | # 128 | # 'pointsize': '10pt', 129 | # Additional stuff for the LaTeX preamble. 130 | # 131 | # 'preamble': '', 132 | # Latex figure (float) alignment 133 | # 134 | # 'figure_align': 'htbp', 135 | } 136 | 137 | # Grouping the document tree into LaTeX files. List of tuples 138 | # (source start file, target name, title, 139 | # author, documentclass [howto, manual, or own class]). 140 | latex_documents = [ 141 | (master_doc, 'testbook.tex', 'testbook Documentation', 'nteract team', 'manual') 142 | ] 143 | 144 | 145 | # -- Options for manual page output --------------------------------------- 146 | 147 | # One entry per manual page. List of tuples 148 | # (source start file, name, description, authors, manual section). 149 | man_pages = [(master_doc, 'testbook', 'testbook Documentation', [author], 1)] 150 | 151 | 152 | # -- Options for Texinfo output ------------------------------------------- 153 | 154 | # Grouping the document tree into Texinfo files. List of tuples 155 | # (source start file, target name, title, author, 156 | # dir menu entry, description, category) 157 | texinfo_documents = [ 158 | ( 159 | master_doc, 160 | 'testbook', 161 | 'testbook Documentation', 162 | author, 163 | 'testbook', 164 | 'One line description of project.', 165 | 'Miscellaneous', 166 | ) 167 | ] 168 | 169 | # Example configuration for intersphinx: refer to the Python standard library. 170 | intersphinx_mapping = {'python': ('https://docs.python.org/', None)} 171 | 172 | # Generate heading anchors for h1, h2 and h3. 173 | myst_heading_anchors = 3 174 | -------------------------------------------------------------------------------- /docs/examples/index.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Here are some common testing patterns where testbook can help. 4 | 5 | ## Mocking requests library 6 | 7 | **Notebook:** 8 | 9 | ![mock-requests-library](https://imgur.com/GM1YExq.png) 10 | 11 | **Test:** 12 | 13 | ```python 14 | from testbook import testbook 15 | 16 | @testbook('/path/to/notebook.ipynb', execute=True) 17 | def test_get_details(tb): 18 | with tb.patch('requests.get') as mock_get: 19 | get_details = tb.get('get_details') # get reference to function 20 | get_details('https://my-api.com') 21 | 22 | mock_get.assert_called_with('https://my-api.com') 23 | ``` 24 | 25 | ## Asserting dataframe manipulations 26 | 27 | **Notebook:** 28 | 29 | ![dataframe-manip](https://imgur.com/g1DrVn2.png) 30 | 31 | **Test:** 32 | 33 | ```python 34 | from testbook import testbook 35 | 36 | @testbook('/path/to/notebook.ipynb') 37 | def test_dataframe_manipulation(tb): 38 | tb.execute_cell('imports') 39 | 40 | # Inject a dataframe with code 41 | tb.inject( 42 | """ 43 | df = pandas.DataFrame([[1, None, 3], [4, 5, 6]], columns=['a', 'b', 'c'], dtype='float') 44 | """ 45 | ) 46 | 47 | # Perform manipulation 48 | tb.execute_cell('manipulation') 49 | 50 | # Inject assertion into notebook 51 | tb.inject("assert len(df) == 1") 52 | ``` 53 | 54 | ## Asserting STDOUT of a cell 55 | 56 | **Notebook:** 57 | 58 | ![dataframe-manip](https://imgur.com/cgtvkph.png) 59 | 60 | **Test:** 61 | 62 | ```python 63 | from testbook import testbook 64 | 65 | @testbook('stdout.ipynb', execute=True) 66 | def test_stdout(tb): 67 | assert tb.cell_output_text(1) == 'hello world!' 68 | 69 | assert 'The current time is' in tb.cell_output_text(2) 70 | ``` 71 | -------------------------------------------------------------------------------- /docs/getting-started/index.md: -------------------------------------------------------------------------------- 1 | # Installation and Getting Started 2 | 3 | `testbook` is a unit testing framework for testing code in Jupyter Notebooks. 4 | 5 | ## Installing `testbook` 6 | 7 | Using a virtual environment or system Python: 8 | 9 | ```{code-block} bash 10 | pip install testbook 11 | ``` 12 | 13 | Using Anaconda: 14 | 15 | ```{code-block} bash 16 | conda install testbook 17 | ``` 18 | 19 | ## What is a Jupyter Notebook? 20 | 21 | [An introduction to Jupyter](https://jupyter-notebook-beginner-guide.readthedocs.io/en/latest/what_is_jupyter.html) 22 | 23 | ## Installing and Launching Jupyter Notebook 24 | 25 | [How to install Jupyter](https://jupyterlab.readthedocs.io/en/stable/getting_started/installation.html) 26 | 27 | ```{code-block} bash 28 | jupyter lab 29 | ``` 30 | 31 | ## Create your first test 32 | 33 | Create a new notebook: 34 | 35 | To do add image 36 | 37 | Write the following code into the first cell of a Jupyter Notebook: 38 | 39 | ```{code-block} python 40 | def foo(x): 41 | return x + 1 42 | ``` 43 | 44 | Save this Notebook as `notebook.ipynb`. 45 | 46 | Create a new `.py` file. In this new file, write the following unit test: 47 | 48 | ```{code-block} python 49 | from testbook import testbook 50 | 51 | @testbook('notebook.ipynb', execute=True) 52 | def test_foo(tb): 53 | foo = tb.get("foo") 54 | 55 | assert foo(2) == 3 56 | ``` 57 | 58 | That's it! You can now execute the test. 59 | 60 | ## General workflow when using testbook to write a unit test 61 | 62 | 1. Use `testbook.testbook` as a decorator or context manager to specify the path to the Jupyter Notebook. Passing `execute=True` will execute all the cells, and passing `execute=['cell-tag-1', 'cell-tag-2']` will only execute specific cells identified by cell tags. 63 | 64 | 2. Obtain references to objects under test using the `.get` method. 65 | 66 | 3. Write the test! 67 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to testbook 2 | 3 | [![Github-CI][github-badge]][github-link] 4 | [![Github-CI][github-ci]][github-ci-link] 5 | [![Coverage Status][codecov-badge]][codecov-link] 6 | [![Documentation Status][rtd-badge]][rtd-link] 7 | [![PyPI][pypi-badge]][pypi-link] 8 | [![image](https://img.shields.io/pypi/v/testbook.svg)](https://pypi.python.org/pypi/testbook) 9 | [![image](https://img.shields.io/pypi/l/testbook.svg)](https://github.com/astral-sh/testbook/blob/main/LICENSE) 10 | [![image](https://img.shields.io/pypi/pyversions/testbook.svg)](https://pypi.python.org/pypi/testbook) 11 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) 12 | 13 | **testbook** is a unit testing framework for testing code in Jupyter Notebooks. 14 | 15 | Previous attempts at unit testing notebooks involved writing the tests in the notebook itself. However, testbook will allow for unit tests to be run against notebooks in separate test files, hence treating `.ipynb` files as `.py` files. 16 | 17 | Here is an example of a unit test written using testbook 18 | 19 | Consider the following code cell in a Jupyter Notebook: 20 | 21 | ```{code-block} python 22 | def func(a, b): 23 | return a + b 24 | ``` 25 | 26 | You would write a unit test using `testbook` in a Python file as follows: 27 | 28 | ```python 29 | from testbook import testbook 30 | 31 | 32 | @testbook('/path/to/notebook.ipynb', execute=True) 33 | def test_func(tb): 34 | func = tb.get("func") 35 | 36 | assert func(1, 2) == 3 37 | ``` 38 | 39 | --- 40 | 41 | ## Features 42 | 43 | - Write conventional unit tests for Jupyter Notebooks 44 | - [Execute all or some specific cells before unit test](usage/index.md#using-execute-to-control-which-cells-are-executed-before-test) 45 | - [Share kernel context across multiple tests](usage/index.md#share-kernel-context-across-multiple-tests) (using pytest fixtures) 46 | - [Support for patching objects](usage/index.md#support-for-patching-objects) 47 | - Inject code into Jupyter notebooks 48 | - Works with any unit testing library - unittest, pytest or nose 49 | 50 | ## Documentation 51 | 52 | ```{toctree} 53 | :maxdepth: 3 54 | 55 | 56 | 57 | getting-started/index.md 58 | usage/index.md 59 | examples/index.md 60 | reference/index.rst 61 | changelog.md 62 | ``` 63 | 64 | [github-ci]: https://github.com/nteract/testbook/workflows/CI/badge.svg 65 | [github-ci-link]: https://github.com/nteract/testbook/actions 66 | [github-link]: https://github.com/nteract/testbook 67 | [rtd-badge]: https://readthedocs.org/projects/testbook/badge/?version=latest 68 | [rtd-link]: https://testbook.readthedocs.io/en/latest/?badge=latest 69 | [codecov-badge]: https://codecov.io/gh/nteract/testbook/branch/master/graph/badge.svg 70 | [codecov-link]: https://codecov.io/gh/nteract/testbook 71 | [github-badge]: https://img.shields.io/github/stars/nteract/testbook?label=github 72 | [pypi-badge]: https://img.shields.io/pypi/v/testbook.svg 73 | [pypi-link]: https://pypi.org/project/testbook/ 74 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | This part of the documentation lists the full API reference of all public classes and functions. 5 | 6 | testbook.client module 7 | ---------------------- 8 | 9 | .. automodule:: testbook.client 10 | :members: 11 | :undoc-members: 12 | :show-inheritance: 13 | 14 | testbook.exceptions module 15 | -------------------------- 16 | 17 | .. automodule:: testbook.exceptions 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | -------------------------------------------------------------------------------- /docs/usage/index.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | The motivation behind creating testbook was to be able to write conventional unit tests for Jupyter Notebooks. 4 | 5 | ## How it works 6 | 7 | Testbook achieves conventional unit tests to be written by setting up references to variables/functions/classes in the Jupyter Notebook. All interactions with these reference objects are internally "pushed down" into the kernel, which is where it gets executed. 8 | 9 | ## Set up Jupyter Notebook under test 10 | 11 | ### Decorator and context manager pattern 12 | 13 | These patterns are interchangeable in most cases. If there are nested decorators on your unit test function, consider using the context manager pattern instead. 14 | 15 | - Decorator pattern 16 | 17 | ```{code-block} python 18 | 19 | from testbook import testbook 20 | 21 | @testbook('/path/to/notebook.ipynb', execute=True) 22 | def test_func(tb): 23 | func = tb.get("func") 24 | 25 | assert func(1, 2) == 3 26 | ``` 27 | 28 | - Context manager pattern 29 | 30 | ```{code-block} python 31 | 32 | from testbook import testbook 33 | 34 | def test_func(): 35 | with testbook('/path/to/notebook.ipynb', execute=True) as tb: 36 | func = tb.get("func") 37 | 38 | assert func(1, 2) == 3 39 | ``` 40 | 41 | ### Using `execute` to control which cells are executed before test 42 | 43 | You may also choose to execute all or some cells: 44 | 45 | - Pass `execute=True` to execute the entire notebook before the test. In this case, it might be better to set up a [module scoped pytest fixture](#share-kernel-context-across-multiple-tests). 46 | 47 | - Pass `execute=['cell1', 'cell2']` or `execute='cell1'` to only execute the specified cell(s) before the test. 48 | 49 | - Pass `execute=slice('start-cell', 'end-cell')` or `execute=range(2, 10)` to execute all cells in the specified range. 50 | 51 | ## Obtain references to objects present in notebook 52 | 53 | ### Testing functions in Jupyter Notebook 54 | 55 | Consider the following code cell in a Jupyter Notebook: 56 | 57 | ```{code-block} python 58 | def foo(name): 59 | return f"You passed {name}!" 60 | 61 | my_list = ['list', 'from', 'notebook'] 62 | ``` 63 | 64 | Reference objects to functions can be called with, 65 | 66 | - explicit JSON serializable values (like `dict`, `list`, `int`, `float`, `str`, `bool`, etc) 67 | - other reference objects 68 | 69 | ```{code-block} python 70 | @testbook.testbook('/path/to/notebook.ipynb', execute=True) 71 | def test_foo(tb): 72 | foo = tb.get("foo") 73 | 74 | # passing in explicitly 75 | assert foo(['spam', 'eggs']) == "You passed ['spam', 'eggs']!" 76 | 77 | # passing in reference object as arg 78 | my_list = tb.get("my_list") 79 | assert foo(my_list) == "You passed ['list', 'from', 'notebook']!" 80 | ``` 81 | 82 | ### Testing function/class returning a non-serializable value 83 | 84 | Consider the following code cell in a Jupyter Notebook: 85 | 86 | ```{code-block} python 87 | class Foo: 88 | def __init__(self): 89 | self.name = name 90 | 91 | def say_hello(self): 92 | return f"Hello {self.name}!" 93 | ``` 94 | 95 | When `Foo` is instantiated from the test, the return value will be a reference object which stores a reference to the non-serializable `Foo` object. 96 | 97 | ```{code-block} python 98 | @testbook.testbook('/path/to/notebook.ipynb', execute=True) 99 | def test_say_hello(tb): 100 | Foo = tb.get("Foo") 101 | bar = Foo("bar") 102 | 103 | assert bar.say_hello() == "Hello bar!" 104 | ``` 105 | 106 | ## Share kernel context across multiple tests 107 | 108 | If your use case requires you to execute many cells (or all cells) of a Jupyter Notebook, before a test can be executed, then it would make sense to share the kernel context with multiple tests. 109 | 110 | It can be done by setting up a [module or package scoped pytest fixture][fixture]. 111 | 112 | Consider the code cells below, 113 | 114 | ```{code-block} python 115 | def foo(a, b): 116 | return a + b 117 | ``` 118 | 119 | ```{code-block} python 120 | def bar(a): 121 | return [x*2 for x in a] 122 | ``` 123 | 124 | The unit tests can be written as follows, 125 | 126 | ```{code-block} python 127 | import pytest 128 | from testbook import testbook 129 | 130 | 131 | @pytest.fixture(scope='module') 132 | def tb(): 133 | with testbook('/path/to/notebook.ipynb', execute=True) as tb: 134 | yield tb 135 | 136 | def test_foo(tb): 137 | foo = tb.get("foo") 138 | assert foo(1, 2) == 3 139 | 140 | 141 | def test_bar(tb): 142 | bar = tb.get("bar") 143 | 144 | tb.inject(""" 145 | data = [1, 2, 3] 146 | """) 147 | data = tb.get("data") 148 | 149 | assert bar(data) == [2, 4, 6] 150 | ``` 151 | 152 | ```{warning} 153 | Note that since the kernel is being shared in case of module scoped fixtures, you might run into weird state issues. Please keep in mind that changes made to an object in one test will reflect in other tests too. This will likely be fixed in future versions of testbook. 154 | ``` 155 | 156 | ## Support for patching objects 157 | 158 | Use the `patch` and `patch_dict` contextmanager to patch out objects during unit test. Learn more about how to use `patch` [here](https://docs.python.org/3/library/unittest.mock.html#unittest.mock.patch). 159 | 160 | **Example usage of `patch`:** 161 | 162 | ```{code-block} python 163 | def foo(): 164 | bar() 165 | ``` 166 | 167 | ```{code-block} python 168 | @testbook('/path/to/notebook.ipynb', execute=True) 169 | def test_method(tb): 170 | with tb.patch('__main__.bar') as mock_bar: 171 | foo = tb.get("foo") 172 | foo() 173 | 174 | mock_bar.assert_called_once() 175 | ``` 176 | 177 | **Example usage of `patch_dict`:** 178 | 179 | ```{code-block} python 180 | my_dict = {'hello': 'world'} 181 | ``` 182 | 183 | ```{code-block} python 184 | @testbook('/path/to/notebook.ipynb', execute=True) 185 | def test_my_dict(tb): 186 | with tb.patch('__main__.my_dict', {'hello' : 'new world'}) as mock_my_dict: 187 | my_dict = tb.get("my_dict") 188 | assert my_dict == {'hello' : 'new world'} 189 | 190 | ``` 191 | 192 | [fixture]: https://docs.pytest.org/en/stable/fixture.html#scope-sharing-a-fixture-instance-across-tests-in-a-class-module-or-session 193 | -------------------------------------------------------------------------------- /examples/dataframe-example/dataframe-assertion-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "6786d1ad", 7 | "metadata": { 8 | "tags": [ 9 | "imports" 10 | ] 11 | }, 12 | "outputs": [], 13 | "source": [ 14 | "import pandas as pd" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "id": "48caab90", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "df = pd.DataFrame([[1, 2, 3], [4, None, 6]], columns = ['a', 'b', 'c'], dtype='float')" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 3, 30 | "id": "03137c43", 31 | "metadata": { 32 | "tags": [ 33 | "manipulation" 34 | ] 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "df = df.dropna()" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 4, 44 | "id": "f28853da", 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/html": [ 50 | "
\n", 51 | "\n", 64 | "\n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | "
abc
01.02.03.0
\n", 82 | "
" 83 | ], 84 | "text/plain": [ 85 | " a b c\n", 86 | "0 1.0 2.0 3.0" 87 | ] 88 | }, 89 | "execution_count": 4, 90 | "metadata": {}, 91 | "output_type": "execute_result" 92 | } 93 | ], 94 | "source": [ 95 | "df" 96 | ] 97 | } 98 | ], 99 | "metadata": { 100 | "celltoolbar": "Tags", 101 | "kernelspec": { 102 | "display_name": "Python 3", 103 | "language": "python", 104 | "name": "python3" 105 | }, 106 | "language_info": { 107 | "codemirror_mode": { 108 | "name": "ipython", 109 | "version": 3 110 | }, 111 | "file_extension": ".py", 112 | "mimetype": "text/x-python", 113 | "name": "python", 114 | "nbconvert_exporter": "python", 115 | "pygments_lexer": "ipython3", 116 | "version": "3.7.6" 117 | } 118 | }, 119 | "nbformat": 4, 120 | "nbformat_minor": 5 121 | } 122 | -------------------------------------------------------------------------------- /examples/dataframe-example/dataframe_test.py: -------------------------------------------------------------------------------- 1 | from testbook import testbook 2 | 3 | 4 | @testbook('./dataframe-assertion-example.ipynb') 5 | def test_dataframe_manipulation(tb): 6 | tb.execute_cell('imports') 7 | 8 | # Inject a dataframe with code 9 | tb.inject( 10 | """ 11 | df = pd.DataFrame([[1, None, 3], [4, 5, 6]], columns=['a', 'b', 'c'], dtype='float') 12 | """ 13 | ) 14 | 15 | # Perform manipulation 16 | tb.execute_cell('manipulation') 17 | 18 | # Inject assertion into notebook 19 | tb.inject('assert len(df) == 1') 20 | -------------------------------------------------------------------------------- /examples/requests-example/requests-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import requests" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "def get_details(url):\n", 19 | " return requests.get(url).content" 20 | ] 21 | } 22 | ], 23 | "metadata": { 24 | "kernelspec": { 25 | "display_name": "Python 3", 26 | "language": "python", 27 | "name": "python3" 28 | }, 29 | "language_info": { 30 | "codemirror_mode": { 31 | "name": "ipython", 32 | "version": 3 33 | }, 34 | "file_extension": ".py", 35 | "mimetype": "text/x-python", 36 | "name": "python", 37 | "nbconvert_exporter": "python", 38 | "pygments_lexer": "ipython3", 39 | "version": "3.7.6" 40 | } 41 | }, 42 | "nbformat": 4, 43 | "nbformat_minor": 4 44 | } 45 | -------------------------------------------------------------------------------- /examples/requests-example/requests_test.py: -------------------------------------------------------------------------------- 1 | from testbook import testbook 2 | 3 | 4 | @testbook('./requests-test.ipynb', execute=True) 5 | def test_get_details(tb): 6 | with tb.patch('requests.get') as mock_get: 7 | get_details = tb.ref('get_details') # get reference to function 8 | get_details('https://my-api.com') 9 | 10 | mock_get.assert_called_with('https://my-api.com') 11 | -------------------------------------------------------------------------------- /examples/stdout-example/stdout-assertion-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "bce1bee1", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from datetime import datetime" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "b906f510", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "print(\"hello world!\")" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "c9fbf668", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "print(f\"The current time is {datetime.now().strftime('%H:%M:%S')}\")" 31 | ] 32 | } 33 | ], 34 | "metadata": { 35 | "kernelspec": { 36 | "display_name": "Python 3", 37 | "language": "python", 38 | "name": "python3" 39 | }, 40 | "language_info": { 41 | "codemirror_mode": { 42 | "name": "ipython", 43 | "version": 3 44 | }, 45 | "file_extension": ".py", 46 | "mimetype": "text/x-python", 47 | "name": "python", 48 | "nbconvert_exporter": "python", 49 | "pygments_lexer": "ipython3", 50 | "version": "3.7.6" 51 | } 52 | }, 53 | "nbformat": 4, 54 | "nbformat_minor": 5 55 | } 56 | -------------------------------------------------------------------------------- /examples/stdout-example/stdout_test.py: -------------------------------------------------------------------------------- 1 | from testbook import testbook 2 | 3 | 4 | @testbook('stdout-assertion-example.ipynb', execute=True) 5 | def test_stdout(tb): 6 | assert tb.cell_output_text(1) == 'hello world!' 7 | 8 | assert 'The current time is' in tb.cell_output_text(2) 9 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ "setuptools" ] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "testbook" 7 | version = "0.4.2" 8 | description = "A unit testing framework for Jupyter Notebooks" 9 | readme = "README.md" 10 | keywords = [ "jupyter", "notebook", "nteract", "unit-testing" ] 11 | license = { file = "LICENSE" } 12 | maintainers = [ 13 | { name = "Nteract Contributors", email = "nteract@googlegroups.com" }, 14 | { name = "Rohit Sanjay", email = "sanjay.rohit2@gmail.com" }, 15 | { name = "Matthew Seal", email = "mseal007@gmail.com" }, 16 | ] 17 | authors = [ 18 | { name = "Nteract Contributors", email = "nteract@googlegroups.com" }, 19 | { name = "Rohit Sanjay", email = "sanjay.rohit2@gmail.com" }, 20 | { name = "Matthew Seal", email = "mseal007@gmail.com" }, 21 | ] 22 | requires-python = ">=3.8" 23 | 24 | classifiers = [ 25 | "Intended Audience :: Developers", 26 | "License :: OSI Approved :: BSD License", 27 | "Programming Language :: Python", 28 | "Programming Language :: Python :: 3 :: Only", 29 | "Programming Language :: Python :: 3.8", 30 | "Programming Language :: Python :: 3.9", 31 | "Programming Language :: Python :: 3.10", 32 | "Programming Language :: Python :: 3.11", 33 | "Programming Language :: Python :: 3.12", 34 | "Topic :: Software Development :: Testing", 35 | "Topic :: Software Development :: Testing :: Mocking", 36 | "Topic :: Software Development :: Testing :: Unit", 37 | ] 38 | dependencies = [ 39 | "nbclient>=0.4", 40 | "nbformat>=5.0.4", 41 | ] 42 | optional-dependencies.dev = [ 43 | "ruff", 44 | "bumpversion", 45 | "check-manifest", 46 | "coverage", 47 | "ipykernel", 48 | "ipython", 49 | "ipywidgets", 50 | "pandas", 51 | "pip>=18.1", 52 | "pytest>=4.1", 53 | "pytest-cov>=2.6.1", 54 | "setuptools>=38.6", 55 | "tox", 56 | "build", 57 | "twine>=1.11", 58 | "xmltodict", 59 | ] 60 | optional-dependencies.docs = [ 61 | "myst-parser==3.0.1", 62 | "sphinx==7.4.7", 63 | "sphinx-book-theme==1.1.3", 64 | ] 65 | 66 | urls.Documentation = "https://testbook.readthedocs.io" 67 | urls.Funding = "https://nteract.io" 68 | urls.Issues = "https://github.com/nteract/testbook/issues" 69 | urls.Repository = "https://github.com/nteract/testbook/" 70 | 71 | [tool.pytest.ini_options] 72 | filterwarnings = "always" 73 | testpaths = [ 74 | "testbook/tests/", 75 | ] 76 | 77 | [tool.coverage.run] 78 | branch = false 79 | omit = [ 80 | "testbook/tests/*", 81 | ] 82 | 83 | [tool.coverage.report] 84 | exclude_lines = [ 85 | "if self\\.debug:", 86 | "pragma: no cover", 87 | "raise AssertionError", 88 | "raise NotImplementedError", 89 | "if __name__ == '__main__':", 90 | ] 91 | ignore_errors = true 92 | omit = [ 93 | "testbook/tests/*", 94 | ] 95 | 96 | [tool.ruff] 97 | # Exclude a variety of commonly ignored directories. 98 | exclude = [ 99 | ".bzr", 100 | ".direnv", 101 | ".eggs", 102 | ".git", 103 | ".git-rewrite", 104 | ".hg", 105 | ".ipynb_checkpoints", 106 | ".mypy_cache", 107 | ".nox", 108 | ".pants.d", 109 | ".pyenv", 110 | ".pytest_cache", 111 | ".pytype", 112 | ".ruff_cache", 113 | ".svn", 114 | ".tox", 115 | ".venv", 116 | ".vscode", 117 | "__pypackages__", 118 | "_build", 119 | "buck-out", 120 | "build", 121 | "dist", 122 | "node_modules", 123 | "site-packages", 124 | "venv", 125 | "testbook/__init__.py", 126 | ] 127 | 128 | # Same as Black. 129 | line-length = 88 130 | indent-width = 4 131 | 132 | # Assume Python 3.8 133 | target-version = "py38" 134 | 135 | [tool.ruff.lint] 136 | # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. 137 | # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or 138 | # McCabe complexity (`C901`) by default. 139 | select = ["E4", "E7", "E9", "F"] 140 | ignore = [] 141 | 142 | # Allow fix for all enabled rules (when `--fix`) is provided. 143 | fixable = ["ALL"] 144 | unfixable = [] 145 | 146 | # Allow unused variables when underscore-prefixed. 147 | dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" 148 | 149 | [tool.ruff.format] 150 | quote-style = "single" 151 | 152 | # Like Black, indent with spaces, rather than tabs. 153 | indent-style = "space" 154 | 155 | # Like Black, respect magic trailing commas. 156 | skip-magic-trailing-comma = false 157 | 158 | # Like Black, automatically detect the appropriate line ending. 159 | line-ending = "auto" 160 | 161 | # Enable auto-formatting of code examples in docstrings. Markdown, 162 | # reStructuredText code/literal blocks and doctests are all supported. 163 | # 164 | # This is currently disabled by default, but it is planned for this 165 | # to be opt-out in the future. 166 | docstring-code-format = false 167 | 168 | # Set the line length limit used when formatting code snippets in 169 | # docstrings. 170 | # 171 | # This only has an effect when the `docstring-code-format` setting is 172 | # enabled. 173 | docstring-code-line-length = "dynamic" 174 | -------------------------------------------------------------------------------- /testbook/__init__.py: -------------------------------------------------------------------------------- 1 | from .testbook import testbook 2 | -------------------------------------------------------------------------------- /testbook/client.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from inspect import getsource 3 | from textwrap import dedent 4 | from typing import Any, Dict, List, Optional, Union 5 | 6 | from nbclient import NotebookClient 7 | from nbclient.exceptions import CellExecutionError 8 | from nbformat.v4 import new_code_cell 9 | 10 | from .exceptions import ( 11 | TestbookCellTagNotFoundError, 12 | TestbookExecuteResultNotFoundError, 13 | TestbookSerializeError, 14 | TestbookRuntimeError, 15 | TestbookError, 16 | ) 17 | from .reference import TestbookObjectReference 18 | from .testbooknode import TestbookNode 19 | from .translators import PythonTranslator 20 | from .utils import random_varname, all_subclasses 21 | 22 | 23 | class TestbookNotebookClient(NotebookClient): 24 | __test__ = False 25 | 26 | def __init__(self, nb, km=None, **kw): 27 | # Fix the ipykernel 5.5 issue where execute requests after errors are aborted 28 | ea = kw.get('extra_arguments', []) 29 | if not any( 30 | arg.startswith('--Kernel.stop_on_error_timeout') 31 | for arg in self.extra_arguments 32 | ): 33 | ea.append('--Kernel.stop_on_error_timeout=0') 34 | kw['extra_arguments'] = ea 35 | super().__init__(nb, km=km, **kw) 36 | 37 | def ref(self, name: str) -> Union[TestbookObjectReference, Any]: 38 | """ 39 | Return a reference to an object in the kernel 40 | """ 41 | 42 | # Check if exists 43 | self.inject(name, pop=True) 44 | try: 45 | self.inject(f'import json; json.dumps({name})', pop=True) 46 | return self.value(name) 47 | except Exception: 48 | return TestbookObjectReference(self, name) 49 | 50 | def get(self, item): 51 | """ 52 | Return a reference to an object in the kernel 53 | 54 | Equivalent to `ref`. 55 | """ 56 | return self.ref(item) 57 | 58 | def __getitem__(self, item): 59 | return self.ref(item) 60 | 61 | @staticmethod 62 | def _construct_call_code( 63 | func_name: str, args: Optional[List] = None, kwargs: Optional[Dict] = None 64 | ) -> str: 65 | return """ 66 | {func_name}(*{args_list}, **{kwargs_dict}) 67 | """.format( 68 | func_name=func_name, 69 | args_list=PythonTranslator.translate(args) if args else [], 70 | kwargs_dict=PythonTranslator.translate(kwargs) if kwargs else {}, 71 | ) 72 | 73 | @property 74 | def cells(self): 75 | return self.nb.cells 76 | 77 | @staticmethod 78 | def _execute_result(cell) -> List: 79 | """ 80 | Return data from execute_result outputs 81 | """ 82 | 83 | return [ 84 | output['data'] 85 | for output in cell['outputs'] 86 | if output['output_type'] == 'execute_result' 87 | ] 88 | 89 | @staticmethod 90 | def _output_text(cell) -> str: 91 | if 'outputs' not in cell: 92 | raise ValueError('cell must be a code cell') 93 | 94 | text = '' 95 | for output in cell['outputs']: 96 | if 'text' in output: 97 | text += output['text'] 98 | elif 'data' in output and 'text/plain' in output['data']: 99 | text += output['data']['text/plain'] 100 | 101 | return text.strip() 102 | 103 | def _cell_index(self, tag: Union[int, str]) -> int: 104 | """ 105 | Get cell index from the cell tag 106 | """ 107 | 108 | if isinstance(tag, int): 109 | return tag 110 | elif not isinstance(tag, str): 111 | raise TypeError('expected tag as str') 112 | 113 | for idx, cell in enumerate(self.cells): 114 | metadata = cell['metadata'] 115 | if 'tags' in metadata and tag in metadata['tags']: 116 | return idx 117 | 118 | raise TestbookCellTagNotFoundError("Cell tag '{}' not found".format(tag)) 119 | 120 | def execute_cell(self, cell, **kwargs) -> Union[Dict, List[Dict]]: 121 | """ 122 | Executes a cell or list of cells 123 | """ 124 | if isinstance(cell, slice): 125 | start, stop = self._cell_index(cell.start), self._cell_index(cell.stop) 126 | if cell.step is not None: 127 | raise TestbookError('testbook does not support step argument') 128 | 129 | cell = range(start, stop + 1) 130 | elif isinstance(cell, str) or isinstance(cell, int): 131 | cell = [cell] 132 | 133 | cell_indexes = cell 134 | 135 | if all(isinstance(x, str) for x in cell): 136 | cell_indexes = [self._cell_index(tag) for tag in cell] 137 | 138 | executed_cells = [] 139 | for idx in cell_indexes: 140 | try: 141 | cell = super().execute_cell(self.nb['cells'][idx], idx, **kwargs) 142 | except CellExecutionError as ce: 143 | raise TestbookRuntimeError( 144 | ce.evalue, ce, self._get_error_class(ce.ename) 145 | ) 146 | 147 | executed_cells.append(cell) 148 | 149 | return executed_cells[0] if len(executed_cells) == 1 else executed_cells 150 | 151 | def execute(self) -> None: 152 | """ 153 | Executes all cells 154 | """ 155 | 156 | for index, cell in enumerate(self.nb.cells): 157 | super().execute_cell(cell, index) 158 | 159 | def cell_output_text(self, cell) -> str: 160 | """ 161 | Return cell text output 162 | """ 163 | cell_index = self._cell_index(cell) 164 | return self._output_text(self.nb['cells'][cell_index]) 165 | 166 | def cell_execute_result(self, cell: Union[int, str]) -> List[Dict[str, Any]]: 167 | """Return the execute results of cell at a given index or with a given tag. 168 | 169 | Each result is expressed with a dictionary for which the key is the mimetype 170 | of the data. A same result can have different representation corresponding to 171 | different mimetype. 172 | 173 | Parameters 174 | ---------- 175 | cell : int or str 176 | The index or tag to look for 177 | 178 | Returns 179 | ------- 180 | List[Dict[str, Any]] 181 | The execute results 182 | 183 | Raises 184 | ------ 185 | IndexError 186 | If index is invalid 187 | TestbookCellTagNotFoundError 188 | If tag is not found 189 | """ 190 | cell_index = self._cell_index(cell) 191 | return self._execute_result(self.nb['cells'][cell_index]) 192 | 193 | def inject( 194 | self, 195 | code: str, 196 | args: List = None, 197 | kwargs: Dict = None, 198 | run: bool = True, 199 | before: Optional[Union[str, int]] = None, 200 | after: Optional[Union[str, int]] = None, 201 | pop: bool = False, 202 | ) -> TestbookNode: 203 | """Injects and executes given code block 204 | 205 | Parameters 206 | ---------- 207 | code : str 208 | Code or function to be injected 209 | args : iterable, optional 210 | tuple of arguments to be passed to the function 211 | kwargs : dict, optional 212 | dict of keyword arguments to be passed to the function 213 | run : bool, optional 214 | Control immediate execution after injection (default is True) 215 | before, after : int, str, optional 216 | Inject code before or after cell 217 | pop : bool 218 | Pop cell after execution (default is False) 219 | 220 | Returns 221 | ------- 222 | TestbookNode 223 | Injected cell 224 | """ 225 | 226 | if isinstance(code, str): 227 | lines = dedent(code) 228 | elif callable(code): 229 | lines = getsource(code) + ( 230 | dedent(self._construct_call_code(code.__name__, args, kwargs)) 231 | if run 232 | else '' 233 | ) 234 | else: 235 | raise TypeError('can only inject function or code block as str') 236 | 237 | inject_idx = len(self.cells) 238 | 239 | if after is not None and before is not None: 240 | raise ValueError('pass either before or after as kwargs') 241 | elif before is not None: 242 | inject_idx = self._cell_index(before) 243 | elif after is not None: 244 | inject_idx = self._cell_index(after) + 1 245 | 246 | code_cell = new_code_cell(lines) 247 | self.cells.insert(inject_idx, code_cell) 248 | 249 | cell = ( 250 | TestbookNode(self.execute_cell(inject_idx)) 251 | if run 252 | else TestbookNode(code_cell) 253 | ) 254 | 255 | if self._contains_error(cell): 256 | eclass = self._get_error_class(cell.get('outputs')[0]['ename']) 257 | evalue = cell.get('outputs')[0]['evalue'] 258 | raise TestbookRuntimeError(evalue, evalue, eclass) 259 | 260 | if run and pop: 261 | self.cells.pop(inject_idx) 262 | 263 | return cell 264 | 265 | def value(self, code: str) -> Any: 266 | """ 267 | Execute given code in the kernel and return JSON serializeable result. 268 | 269 | If the result is not JSON serializeable, it raises `TestbookAttributeError`. 270 | This error object will also contain an attribute called `save_varname` which 271 | can be used to create a reference object with :meth:`ref`. 272 | 273 | Parameters 274 | ---------- 275 | code: str 276 | This can be any executable code that returns a value. 277 | It can be used the return the value of an object, or the output 278 | of a function call. 279 | 280 | Returns 281 | ------- 282 | The output of the executed code 283 | 284 | Raises 285 | ------ 286 | TestbookSerializeError 287 | 288 | """ 289 | result = self.inject(code, pop=True) 290 | 291 | if not self._execute_result(result): 292 | raise TestbookExecuteResultNotFoundError( 293 | 'code provided does not produce execute_result' 294 | ) 295 | 296 | save_varname = random_varname() 297 | 298 | inject_code = f""" 299 | import json 300 | from IPython import get_ipython 301 | from IPython.display import JSON 302 | 303 | {save_varname} = get_ipython().last_execution_result.result 304 | 305 | json.dumps({save_varname}) 306 | JSON({{"value" : {save_varname}}}) 307 | """ 308 | 309 | try: 310 | outputs = self.inject(inject_code, pop=True).outputs 311 | 312 | if outputs[0].output_type == 'error': 313 | # will receive error when `allow_errors` is set to True 314 | raise TestbookRuntimeError( 315 | outputs[0].evalue, outputs[0].traceback, outputs[0].ename 316 | ) 317 | 318 | return outputs[0].data['application/json']['value'] 319 | 320 | except TestbookRuntimeError: 321 | e = TestbookSerializeError('could not JSON serialize output') 322 | e.save_varname = save_varname 323 | raise e 324 | 325 | @contextmanager 326 | def patch(self, target, **kwargs): 327 | """Used as contextmanager to patch objects in the kernel""" 328 | mock_object = f'_mock_{random_varname()}' 329 | patcher = f'_patcher_{random_varname()}' 330 | 331 | self.inject( 332 | f""" 333 | from unittest.mock import patch 334 | {patcher} = patch( 335 | {PythonTranslator.translate(target)}, 336 | **{PythonTranslator.translate(kwargs)} 337 | ) 338 | {mock_object} = {patcher}.start() 339 | """ 340 | ) 341 | 342 | yield TestbookObjectReference(self, mock_object) 343 | 344 | self.inject(f'{patcher}.stop()') 345 | 346 | @contextmanager 347 | def patch_dict(self, in_dict, values=(), clear=False, **kwargs): 348 | """Used as contextmanager to patch dictionaries in the kernel""" 349 | mock_object = f'_mock_{random_varname()}' 350 | patcher = f'_patcher_{random_varname()}' 351 | 352 | self.inject( 353 | f""" 354 | from unittest.mock import patch 355 | {patcher} = patch.dict( 356 | {PythonTranslator.translate(in_dict)}, 357 | {PythonTranslator.translate(values)}, 358 | {PythonTranslator.translate(clear)}, 359 | **{PythonTranslator.translate(kwargs)} 360 | ) 361 | {mock_object} = {patcher}.start() 362 | """ 363 | ) 364 | 365 | yield TestbookObjectReference(self, mock_object) 366 | 367 | self.inject(f'{patcher}.stop()') 368 | 369 | @staticmethod 370 | def _get_error_class(ename): 371 | eclass = None 372 | for klass in all_subclasses(Exception): 373 | if klass.__name__ == ename: 374 | eclass = klass 375 | break 376 | return eclass 377 | 378 | @staticmethod 379 | def _contains_error(result): 380 | return result.get('outputs') and result.get('outputs')[0].output_type == 'error' 381 | -------------------------------------------------------------------------------- /testbook/exceptions.py: -------------------------------------------------------------------------------- 1 | class TestbookError(Exception): 2 | """Generic Testbook exception class""" 3 | 4 | __test__ = False 5 | 6 | 7 | class TestbookCellTagNotFoundError(TestbookError): 8 | """Raised when cell tag is not declared in notebook""" 9 | 10 | pass 11 | 12 | 13 | class TestbookSerializeError(TestbookError): 14 | """Raised when output cannot be JSON serialized""" 15 | 16 | pass 17 | 18 | 19 | class TestbookExecuteResultNotFoundError(TestbookError): 20 | """Raised when there is no execute_result""" 21 | 22 | pass 23 | 24 | 25 | class TestbookAttributeError(AttributeError): 26 | __test__ = False 27 | 28 | 29 | class TestbookRuntimeError(RuntimeError): 30 | __test__ = False 31 | 32 | def __init__(self, evalue, traceback, eclass=None): 33 | super().__init__(evalue) 34 | self.evalue = evalue 35 | self.traceback = traceback 36 | self.eclass = eclass 37 | 38 | def __str__(self): # pragma: no cover 39 | return str(self.traceback) 40 | 41 | def __repr__(self): # pragma: no cover 42 | return str(self.traceback) 43 | -------------------------------------------------------------------------------- /testbook/reference.py: -------------------------------------------------------------------------------- 1 | from .exceptions import ( 2 | TestbookExecuteResultNotFoundError, 3 | TestbookAttributeError, 4 | TestbookSerializeError, 5 | TestbookRuntimeError, 6 | ) 7 | from .utils import random_varname 8 | from .translators import PythonTranslator 9 | 10 | 11 | class TestbookObjectReference: 12 | def __init__(self, tb, name): 13 | self.tb = tb 14 | self.name: str = name 15 | 16 | @property 17 | def _type(self): 18 | return self.tb.value(f'type({self.name}).__name__') 19 | 20 | def __repr__(self): 21 | return repr(self.tb.value(f'repr({self.name})')) 22 | 23 | def __getattr__(self, name): 24 | if self.tb.value(f"hasattr({self.name}, '{name}')"): 25 | return TestbookObjectReference(self.tb, f'{self.name}.{name}') 26 | 27 | raise TestbookAttributeError(f"'{self._type}' object has no attribute {name}") 28 | 29 | def __eq__(self, rhs): 30 | return self.tb.value( 31 | '{lhs} == {rhs}'.format(lhs=self.name, rhs=PythonTranslator.translate(rhs)) 32 | ) 33 | 34 | def __len__(self): 35 | return self.tb.value(f'len({self.name})') 36 | 37 | def __iter__(self): 38 | iterobjectname = f'___iter_object_{random_varname()}' 39 | self.tb.inject(f""" 40 | {iterobjectname} = iter({self.name}) 41 | """) 42 | return TestbookObjectReference(self.tb, iterobjectname) 43 | 44 | def __next__(self): 45 | try: 46 | return self.tb.value(f'next({self.name})') 47 | except TestbookRuntimeError as e: 48 | if e.eclass is StopIteration: 49 | raise StopIteration 50 | else: 51 | raise 52 | 53 | def __getitem__(self, key): 54 | try: 55 | return self.tb.value( 56 | f'{self.name}.__getitem__({PythonTranslator.translate(key)})' 57 | ) 58 | except TestbookRuntimeError as e: 59 | if e.eclass is TypeError: 60 | raise TypeError(e.evalue) 61 | elif e.eclass is IndexError: 62 | raise IndexError(e.evalue) 63 | else: 64 | raise 65 | 66 | def __setitem__(self, key, value): 67 | try: 68 | return self.tb.inject( 69 | '{name}[{key}] = {value}'.format( 70 | name=self.name, 71 | key=PythonTranslator.translate(key), 72 | value=PythonTranslator.translate(value), 73 | ), 74 | pop=True, 75 | ) 76 | except TestbookRuntimeError as e: 77 | if e.eclass is TypeError: 78 | raise TypeError(e.evalue) 79 | elif e.eclass is IndexError: 80 | raise IndexError(e.evalue) 81 | else: 82 | raise 83 | 84 | def __contains__(self, item): 85 | return self.tb.value( 86 | f'{self.name}.__contains__({PythonTranslator.translate(item)})' 87 | ) 88 | 89 | def __call__(self, *args, **kwargs): 90 | code = self.tb._construct_call_code(self.name, args, kwargs) 91 | try: 92 | return self.tb.value(code) 93 | except TestbookExecuteResultNotFoundError: 94 | # No return value from function call 95 | pass 96 | except TestbookSerializeError as e: 97 | return TestbookObjectReference(self.tb, e.save_varname) 98 | 99 | def resolve(self): 100 | return self.tb.value(self.name) 101 | -------------------------------------------------------------------------------- /testbook/testbook.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from unittest.mock import DEFAULT 3 | 4 | import nbformat 5 | 6 | 7 | from .client import TestbookNotebookClient 8 | 9 | 10 | class testbook: 11 | """`testbook` acts as function decorator or a context manager. 12 | 13 | When the function/with statement exits the kernels started when 14 | entering the function/with statement will be terminated. 15 | 16 | If `testbook` is used as a decorator, the `TestbookNotebookClient` 17 | will be passed as first argument to the decorated function. 18 | """ 19 | 20 | # Developer notes: 21 | # 22 | # To trick pytest, we mimic the API of unittest.mock.patch in testbook. 23 | # Notably, the following elements are added: 24 | # * attribute_name, Class attribute (see below) 25 | # * new, Instance attribute (see __init__) 26 | # * patchings, wrapper attributes (see __call__) 27 | 28 | attribute_name = None 29 | 30 | def __init__( 31 | self, 32 | nb, 33 | execute=None, 34 | timeout=60, 35 | kernel_name='python3', 36 | allow_errors=False, 37 | **kwargs, 38 | ): 39 | self.execute = execute 40 | self.client = TestbookNotebookClient( 41 | nbformat.read(nb, as_version=4) 42 | if not isinstance(nb, nbformat.NotebookNode) 43 | else nb, 44 | timeout=timeout, 45 | allow_errors=allow_errors, 46 | kernel_name=kernel_name, 47 | **kwargs, 48 | ) 49 | 50 | self.new = DEFAULT 51 | 52 | def _prepare(self): 53 | if self.execute is True: 54 | self.client.execute() 55 | elif self.execute not in [None, False]: 56 | self.client.execute_cell(self.execute) 57 | 58 | def __enter__(self) -> TestbookNotebookClient: 59 | with self.client.setup_kernel(cleanup_kc=False): 60 | self._prepare() 61 | return self.client 62 | 63 | def __exit__(self, *args): 64 | self.client._cleanup_kernel() 65 | 66 | def __call__(self, func): 67 | @functools.wraps(func) 68 | def wrapper(*args, **kwargs): # pragma: no cover 69 | with self.client.setup_kernel(): 70 | self._prepare() 71 | return func(self.client, *args, **kwargs) 72 | 73 | wrapper.patchings = [self] 74 | return wrapper 75 | -------------------------------------------------------------------------------- /testbook/testbooknode.py: -------------------------------------------------------------------------------- 1 | from nbformat import NotebookNode 2 | 3 | 4 | class TestbookNode(NotebookNode): 5 | """ 6 | Extends `NotebookNode` to perform assertions 7 | """ 8 | 9 | def __init__(self, *args, **kw): 10 | super().__init__(*args, **kw) 11 | 12 | @property 13 | def output_text(self) -> str: 14 | text = '' 15 | for output in self['outputs']: 16 | if 'text' in output: 17 | text += output['text'] 18 | 19 | return text.strip() 20 | 21 | @property 22 | def execute_result(self): 23 | """Return data from execute_result outputs""" 24 | return [ 25 | output['data'] 26 | for output in self['outputs'] 27 | if output['output_type'] == 'execute_result' 28 | ] 29 | -------------------------------------------------------------------------------- /testbook/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/testbook/9d48dbb18d4b3620fcd962929e3b404af4afeea6/testbook/tests/__init__.py -------------------------------------------------------------------------------- /testbook/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | import pytest 4 | from jupyter_client import kernelspec 5 | from nbformat.notebooknode import NotebookNode 6 | from nbformat.v4 import new_notebook, new_code_cell, new_output 7 | 8 | 9 | @pytest.fixture 10 | def notebook_factory(): 11 | """Pytest fixture to generate a valid notebook.""" 12 | 13 | def notebook_generator(cells: Optional[List[NotebookNode]] = None) -> NotebookNode: 14 | """Generate an executable notebook. 15 | 16 | The notebook cells are the one passed as arguments or the hard-coded cells 17 | if no cells is provided. 18 | """ 19 | metadata = {} 20 | for name in kernelspec.find_kernel_specs(): 21 | ks = kernelspec.get_kernel_spec(name) 22 | metadata = { 23 | 'kernelspec': { 24 | 'name': name, 25 | 'language': ks.language, 26 | 'display_name': ks.display_name, 27 | } 28 | } 29 | break 30 | 31 | if cells is not None: 32 | all_cells = cells 33 | else: # Default cells 34 | all_cells = [ 35 | new_code_cell('a = 2', metadata={'tags': []}), 36 | new_code_cell('b=22\nb', metadata={'tags': ['test']}), 37 | new_code_cell( 38 | '', 39 | metadata={'tags': ['dummy-outputs']}, 40 | outputs=[new_output('execute_result', data={'text/plain': 'text'})], 41 | ), 42 | ] 43 | 44 | return new_notebook(metadata=metadata, cells=all_cells) 45 | 46 | return notebook_generator 47 | -------------------------------------------------------------------------------- /testbook/tests/resources/datamodel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "political-plaintiff", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "mylist = [1, 2, 3, 4, 5]" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "postal-contemporary", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [] 20 | } 21 | ], 22 | "metadata": { 23 | "kernelspec": { 24 | "display_name": "Python 3", 25 | "language": "python", 26 | "name": "python3" 27 | }, 28 | "language_info": { 29 | "codemirror_mode": { 30 | "name": "ipython", 31 | "version": 3 32 | }, 33 | "file_extension": ".py", 34 | "mimetype": "text/x-python", 35 | "name": "python", 36 | "nbconvert_exporter": "python", 37 | "pygments_lexer": "ipython3", 38 | "version": "3.8.6" 39 | } 40 | }, 41 | "nbformat": 4, 42 | "nbformat_minor": 5 43 | } 44 | -------------------------------------------------------------------------------- /testbook/tests/resources/exception.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "def raise_my_exception():\n", 10 | " class MyException(Exception):\n", 11 | " pass\n", 12 | " raise MyException" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [] 21 | } 22 | ], 23 | "metadata": { 24 | "kernelspec": { 25 | "display_name": "Python 3", 26 | "language": "python", 27 | "name": "python3" 28 | }, 29 | "language_info": { 30 | "codemirror_mode": { 31 | "name": "ipython", 32 | "version": 3 33 | }, 34 | "file_extension": ".py", 35 | "mimetype": "text/x-python", 36 | "name": "python", 37 | "nbconvert_exporter": "python", 38 | "pygments_lexer": "ipython3", 39 | "version": "3.7.7" 40 | } 41 | }, 42 | "nbformat": 4, 43 | "nbformat_minor": 4 44 | } 45 | -------------------------------------------------------------------------------- /testbook/tests/resources/foo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "1 + 1" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "tags": [ 17 | "test1" 18 | ] 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "print('hello world')\n", 23 | "\n", 24 | "print([1, 2, 3])" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "tags": [ 32 | "prepare_foo" 33 | ] 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "def foo():\n", 38 | " print('foo')" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "tags": [ 46 | "execute_foo" 47 | ] 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "foo()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "import pandas as pd\n", 61 | "d = {'col1': [1, 2], 'col2': [3, 4]}\n", 62 | "df = pd.DataFrame(data=d)\n", 63 | "df" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [] 72 | } 73 | ], 74 | "metadata": { 75 | "celltoolbar": "Tags", 76 | "kernelspec": { 77 | "display_name": "Python 3", 78 | "language": "python", 79 | "name": "python3" 80 | }, 81 | "language_info": { 82 | "codemirror_mode": { 83 | "name": "ipython", 84 | "version": 3 85 | }, 86 | "file_extension": ".py", 87 | "mimetype": "text/x-python", 88 | "name": "python", 89 | "nbconvert_exporter": "python", 90 | "pygments_lexer": "ipython3", 91 | "version": "3.7.7" 92 | } 93 | }, 94 | "nbformat": 4, 95 | "nbformat_minor": 2 96 | } 97 | -------------------------------------------------------------------------------- /testbook/tests/resources/inject.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "foo = 'Hello'\n", 10 | "bar = 'World'" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": { 17 | "tags": [ 18 | "hello" 19 | ] 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "def say_hello():\n", 24 | " print(\"Hello there\")" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "tags": [ 32 | "bye" 33 | ] 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "def say_bye():\n", 38 | " print(\"Bye\")" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "tags": [ 46 | "dict" 47 | ] 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "sample_dict = {'foo' : 'bar'}" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "metadata": { 58 | "tags": [ 59 | "list" 60 | ] 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "sample_list = ['foo', 'bar']" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": { 71 | "tags": [ 72 | "int" 73 | ] 74 | }, 75 | "outputs": [], 76 | "source": [ 77 | "sample_int = 42" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": { 84 | "tags": [ 85 | "str" 86 | ] 87 | }, 88 | "outputs": [], 89 | "source": [ 90 | "sample_str = 'hello world'" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "sample_list + [1]" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [] 108 | } 109 | ], 110 | "metadata": { 111 | "celltoolbar": "Tags", 112 | "kernelspec": { 113 | "display_name": "Python 3", 114 | "language": "python", 115 | "name": "python3" 116 | }, 117 | "language_info": { 118 | "codemirror_mode": { 119 | "name": "ipython", 120 | "version": 3 121 | }, 122 | "file_extension": ".py", 123 | "mimetype": "text/x-python", 124 | "name": "python", 125 | "nbconvert_exporter": "python", 126 | "pygments_lexer": "ipython3", 127 | "version": "3.8.1" 128 | } 129 | }, 130 | "nbformat": 4, 131 | "nbformat_minor": 2 132 | } 133 | -------------------------------------------------------------------------------- /testbook/tests/resources/patch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import subprocess" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "def listdir():\n", 20 | " return os.listdir()" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "def get_branch():\n", 30 | " return os.popen('git rev-parse --abbrev-ref HEAD').read().strip()" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "def env():\n", 40 | " return os.environ" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "foo = {}" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "kernelspec": { 55 | "display_name": "Python 3", 56 | "language": "python", 57 | "name": "python3" 58 | }, 59 | "language_info": { 60 | "codemirror_mode": { 61 | "name": "ipython", 62 | "version": 3 63 | }, 64 | "file_extension": ".py", 65 | "mimetype": "text/x-python", 66 | "name": "python", 67 | "nbconvert_exporter": "python", 68 | "pygments_lexer": "ipython3", 69 | "version": "3.7.7" 70 | } 71 | }, 72 | "nbformat": 4, 73 | "nbformat_minor": 4 74 | } 75 | -------------------------------------------------------------------------------- /testbook/tests/resources/reference.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "a = [1, 2, 3]" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "b = [1, 2, 3]" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 3, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "def double(ll):\n", 28 | " return [_*2 for _ in ll]" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 4, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "class Foo:\n", 38 | " def __init__(self, name):\n", 39 | " self.name = name\n", 40 | " \n", 41 | " def __repr__(self):\n", 42 | " return f\"\"\n", 43 | " \n", 44 | " def say_hello(self):\n", 45 | " return f\"Hello {self.name}!\"" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [] 54 | } 55 | ], 56 | "metadata": { 57 | "celltoolbar": "Tags", 58 | "kernelspec": { 59 | "display_name": "Python 3", 60 | "language": "python", 61 | "name": "python3" 62 | }, 63 | "language_info": { 64 | "codemirror_mode": { 65 | "name": "ipython", 66 | "version": 3 67 | }, 68 | "file_extension": ".py", 69 | "mimetype": "text/x-python", 70 | "name": "python", 71 | "nbconvert_exporter": "python", 72 | "pygments_lexer": "ipython3", 73 | "version": "3.7.7" 74 | } 75 | }, 76 | "nbformat": 4, 77 | "nbformat_minor": 2 78 | } 79 | -------------------------------------------------------------------------------- /testbook/tests/test_client.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from textwrap import dedent 3 | 4 | from ..testbook import testbook 5 | from ..client import TestbookNotebookClient 6 | from ..exceptions import ( 7 | TestbookCellTagNotFoundError, 8 | TestbookExecuteResultNotFoundError, 9 | ) 10 | 11 | 12 | @pytest.fixture(scope='module') 13 | def notebook(): 14 | with testbook('testbook/tests/resources/inject.ipynb', execute=True) as tb: 15 | yield tb 16 | 17 | 18 | @pytest.mark.parametrize('cell_index_args, expected_result', [(2, 2), ('hello', 1)]) 19 | def test_cell_index(cell_index_args, expected_result, notebook): 20 | assert notebook._cell_index(cell_index_args) == expected_result 21 | 22 | 23 | @pytest.mark.parametrize( 24 | 'cell_index_args, expected_error', 25 | [([1, 2, 3], TypeError), ('non-existent-tag', TestbookCellTagNotFoundError)], 26 | ) 27 | def test_cell_index_raises_error(cell_index_args, expected_error, notebook): 28 | with pytest.raises(expected_error): 29 | notebook._cell_index(cell_index_args) 30 | 31 | 32 | @pytest.mark.parametrize( 33 | 'var_name, expected_result', 34 | [ 35 | ('sample_dict', {'foo': 'bar'}), 36 | ('sample_list', ['foo', 'bar']), 37 | ('sample_list + ["hello world"]', ['foo', 'bar', 'hello world']), 38 | ('sample_int', 42), 39 | ('sample_int * 2', 84), 40 | ('sample_str', 'hello world'), 41 | ('sample_str + " foo"', 'hello world foo'), 42 | ], 43 | ) 44 | def test_value(var_name, expected_result, notebook): 45 | assert notebook.value(var_name) == expected_result 46 | 47 | 48 | @pytest.mark.parametrize('code', [('sample_int *= 2'), ('print(sample_int)'), ('')]) 49 | def test_value_raises_error(code, notebook): 50 | with pytest.raises(TestbookExecuteResultNotFoundError): 51 | notebook.value(code) 52 | 53 | 54 | @pytest.mark.parametrize( 55 | 'cell, expected_result', 56 | [ 57 | ( 58 | { 59 | 'cell_type': 'code', 60 | 'execution_count': 9, 61 | 'metadata': {}, 62 | 'outputs': [ 63 | { 64 | 'name': 'stdout', 65 | 'output_type': 'stream', 66 | 'text': 'hello world\n' 'foo\n' 'bar\n', 67 | } 68 | ], 69 | }, 70 | """ 71 | hello world 72 | foo 73 | bar 74 | """, 75 | ), 76 | ( 77 | {'cell_type': 'code', 'execution_count': 9, 'metadata': {}, 'outputs': []}, 78 | '', 79 | ), 80 | ], 81 | ) 82 | def test_output_text(cell, expected_result): 83 | assert TestbookNotebookClient._output_text(cell) == dedent(expected_result).strip() 84 | 85 | 86 | @pytest.mark.parametrize( 87 | 'cell', [{}, {'cell_type': 'markdown', 'metadata': {}, 'source': ['# Hello']}] 88 | ) 89 | def test_output_text_raises_error(cell): 90 | with pytest.raises(ValueError): 91 | assert TestbookNotebookClient._output_text(cell) 92 | 93 | 94 | def test_cell_execute_result_index(notebook_factory): 95 | nb = notebook_factory() 96 | with testbook(nb, execute='test') as tb: 97 | assert tb.cell_execute_result(1) == [{'text/plain': '22'}] 98 | assert tb.cell_execute_result(2) == [{'text/plain': 'text'}] 99 | 100 | 101 | def test_cell_execute_result_tag(notebook_factory): 102 | nb = notebook_factory() 103 | with testbook(nb, execute='test') as tb: 104 | assert tb.cell_execute_result('test') == [{'text/plain': '22'}] 105 | assert tb.cell_execute_result('dummy-outputs') == [{'text/plain': 'text'}] 106 | 107 | 108 | def test_cell_execute_result_indexerror(notebook_factory): 109 | nb = notebook_factory([]) 110 | with testbook(nb) as tb: 111 | with pytest.raises(IndexError): 112 | tb.cell_execute_result(1) 113 | 114 | 115 | def test_cell_execute_result_tagnotfound(notebook_factory): 116 | nb = notebook_factory([]) 117 | with testbook(nb) as tb: 118 | with pytest.raises(TestbookCellTagNotFoundError): 119 | tb.cell_execute_result('test') 120 | -------------------------------------------------------------------------------- /testbook/tests/test_datamodel.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from ..testbook import testbook 4 | 5 | 6 | @pytest.fixture(scope='module') 7 | def notebook(): 8 | with testbook('testbook/tests/resources/datamodel.ipynb', execute=True) as tb: 9 | yield tb 10 | 11 | 12 | def test_len(notebook): 13 | mylist = notebook.ref('mylist') 14 | 15 | assert len(mylist) == 5 16 | 17 | 18 | def test_iter(notebook): 19 | mylist = notebook.ref('mylist') 20 | 21 | expected = [] 22 | for x in mylist: 23 | expected.append(x) 24 | 25 | assert mylist == expected 26 | 27 | 28 | def test_getitem(notebook): 29 | mylist = notebook.ref('mylist') 30 | mylist.append(6) 31 | 32 | assert mylist[-1] == 6 33 | assert mylist.__getitem__(-1) == 6 34 | 35 | 36 | def test_getitem_raisesIndexError(notebook): 37 | mylist = notebook.ref('mylist') 38 | 39 | with pytest.raises(IndexError): 40 | mylist[100] 41 | 42 | 43 | def test_getitem_raisesTypeError(notebook): 44 | mylist = notebook.ref('mylist') 45 | 46 | with pytest.raises(TypeError): 47 | mylist['hello'] 48 | 49 | 50 | def test_setitem(notebook): 51 | notebook.inject("mydict = {'key1': 'value1', 'key2': 'value1'}") 52 | mydict = notebook.ref('mydict') 53 | 54 | mydict['key3'] = 'value3' 55 | assert mydict['key3'] == 'value3' 56 | 57 | mylist = notebook.ref('mylist') 58 | mylist[2] = 10 59 | assert mylist[2] == 10 60 | 61 | 62 | def test_setitem_raisesIndexError(notebook): 63 | mylist = notebook.ref('mylist') 64 | 65 | with pytest.raises(IndexError): 66 | mylist.__setitem__(10, 100) 67 | 68 | 69 | def test_setitem_raisesTypeError(notebook): 70 | mylist = notebook.ref('mylist') 71 | 72 | with pytest.raises(TypeError): 73 | mylist.__setitem__('key', 10) 74 | 75 | 76 | def test_contains(notebook): 77 | notebook.inject("mydict = {'key1': 'value1', 'key2': 'value1'}") 78 | mydict = notebook.ref('mydict') 79 | 80 | assert 'key1' in mydict 81 | assert 'key2' in mydict 82 | assert mydict.__contains__('key1') 83 | assert mydict.__contains__('key2') 84 | -------------------------------------------------------------------------------- /testbook/tests/test_execute.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from ..testbook import testbook 4 | from ..exceptions import TestbookRuntimeError, TestbookError 5 | 6 | 7 | @pytest.fixture(scope='module') 8 | def notebook(): 9 | with testbook('testbook/tests/resources/foo.ipynb', execute=True) as tb: 10 | yield tb 11 | 12 | 13 | def test_execute_cell(notebook): 14 | notebook.execute_cell(1) 15 | assert notebook.cell_output_text(1) == 'hello world\n[1, 2, 3]' 16 | 17 | notebook.execute_cell([2, 3]) 18 | assert notebook.cell_output_text(3) == 'foo' 19 | 20 | 21 | def test_execute_and_show_pandas_output(notebook): 22 | notebook.execute_cell(4) 23 | assert ( 24 | notebook.cell_output_text(4) 25 | == """col1 col2 26 | 0 1 3 27 | 1 2 4""" 28 | ) 29 | 30 | 31 | def test_execute_cell_tags(notebook): 32 | notebook.execute_cell('test1') 33 | assert notebook.cell_output_text('test1') == 'hello world\n[1, 2, 3]' 34 | 35 | notebook.execute_cell(['prepare_foo', 'execute_foo']) 36 | assert notebook.cell_output_text('execute_foo') == 'foo' 37 | 38 | 39 | def test_execute_cell_raises_error(notebook): 40 | with pytest.raises(TestbookRuntimeError): 41 | try: 42 | notebook.inject('1/0', pop=True) 43 | except TestbookRuntimeError as e: 44 | assert e.eclass is ZeroDivisionError 45 | raise 46 | 47 | 48 | def test_testbook_with_execute(notebook): 49 | notebook.execute_cell('execute_foo') 50 | assert notebook.cell_output_text('execute_foo') == 'foo' 51 | 52 | 53 | def test_testbook_with_execute_context_manager(notebook): 54 | notebook.execute_cell('execute_foo') 55 | assert notebook.cell_output_text('execute_foo') == 'foo' 56 | 57 | 58 | def test_testbook_range(): 59 | with testbook('testbook/tests/resources/inject.ipynb') as tb: 60 | tb.execute_cell(range(4)) 61 | assert tb.code_cells_executed == 4 62 | 63 | with testbook('testbook/tests/resources/inject.ipynb', execute=range(4)) as tb: 64 | assert tb.code_cells_executed == 4 65 | 66 | 67 | @pytest.mark.parametrize( 68 | 'slice_params, expected_result', [(('hello', 'str'), 6), ((2, 5), 4)] 69 | ) 70 | def test_testbook_slice(slice_params, expected_result): 71 | with testbook('testbook/tests/resources/inject.ipynb') as tb: 72 | tb.execute_cell(slice(*slice_params)) 73 | assert tb.code_cells_executed == expected_result 74 | 75 | with testbook( 76 | 'testbook/tests/resources/inject.ipynb', execute=slice(*slice_params) 77 | ) as tb: 78 | assert tb.code_cells_executed == expected_result 79 | 80 | 81 | def test_testbook_slice_raises_error(): 82 | with pytest.raises(TestbookError): 83 | with testbook('testbook/tests/resources/inject.ipynb', execute=slice(3, 1, -1)): 84 | pass 85 | 86 | 87 | @testbook('testbook/tests/resources/exception.ipynb', execute=True) 88 | def test_raise_exception(tb): 89 | with pytest.raises(TestbookRuntimeError): 90 | tb.ref('raise_my_exception')() 91 | 92 | 93 | @testbook('testbook/tests/resources/inject.ipynb') 94 | def test_underscore(tb): 95 | tb.inject( 96 | """ 97 | _ = 20 98 | 99 | def foo(x): 100 | return x + 1 101 | """, 102 | run=False, 103 | ) 104 | 105 | tb.execute() 106 | 107 | foo = tb.ref('foo') 108 | 109 | assert foo(2) == 3 110 | -------------------------------------------------------------------------------- /testbook/tests/test_inject.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from ..testbook import testbook 4 | 5 | 6 | @pytest.fixture(scope='module') 7 | def notebook(): 8 | with testbook('testbook/tests/resources/inject.ipynb', execute=True) as tb: 9 | yield tb 10 | 11 | 12 | def inject_helper(*args, **kwargs): 13 | pass 14 | 15 | 16 | @pytest.mark.parametrize( 17 | 'args, kwargs', 18 | [ 19 | (None, None), 20 | ([1, 2], None), 21 | ((1, 2), None), 22 | ((True, False), None), 23 | (['a', 'b'], None), 24 | ([1.1, float('nan'), float('inf'), float('-inf')], None), 25 | ([{'key1': 'value1'}, {'key2': 'value2'}], None), 26 | ((1, 2, False), {'key2': 'value2'}), 27 | ((None, None, False), {'key2': 'value2'}), 28 | ], 29 | ) 30 | def test_inject(args, kwargs, notebook): 31 | assert notebook.inject(inject_helper, args=args, kwargs=kwargs, pop=True) 32 | 33 | 34 | @pytest.mark.parametrize( 35 | 'code_block, expected_text', 36 | [ 37 | ( 38 | """ 39 | def foo(): 40 | print('I ran in the code block') 41 | foo() 42 | """, 43 | 'I ran in the code block', 44 | ), 45 | ( 46 | """ 47 | def foo(arg): 48 | print(f'You passed {arg}') 49 | foo('bar') 50 | """, 51 | 'You passed bar', 52 | ), 53 | ], 54 | ) 55 | def test_inject_code_block(code_block, expected_text, notebook): 56 | assert notebook.inject(code_block, pop=True).output_text == expected_text 57 | 58 | 59 | def test_inject_raises_exception(notebook): 60 | values = [3, {'key': 'value'}, ['a', 'b', 'c'], (1, 2, 3), {1, 2, 3}] 61 | 62 | for value in values: 63 | with pytest.raises(TypeError): 64 | notebook.inject(value) 65 | 66 | 67 | def test_inject_before_after(notebook): 68 | notebook.inject('say_hello()', run=False, after='hello') 69 | assert notebook.cells[notebook._cell_index('hello') + 1].source == 'say_hello()' 70 | 71 | notebook.inject('say_bye()', before='hello') 72 | assert notebook.cells[notebook._cell_index('hello') - 1].source == 'say_bye()' 73 | 74 | with pytest.raises(ValueError): 75 | notebook.inject('say_hello()', before='hello', after='bye') 76 | 77 | 78 | def test_inject_pop(notebook): 79 | assert notebook.inject('1+1', pop=True).execute_result == [{'text/plain': '2'}] 80 | assert notebook.cells[-1].source != '1+1' 81 | -------------------------------------------------------------------------------- /testbook/tests/test_patch.py: -------------------------------------------------------------------------------- 1 | from ..testbook import testbook 2 | from ..exceptions import TestbookRuntimeError 3 | 4 | import pytest 5 | 6 | 7 | @pytest.fixture(scope='module') 8 | def tb(): 9 | with testbook('testbook/tests/resources/patch.ipynb', execute=True) as tb: 10 | yield tb 11 | 12 | 13 | class TestPatch: 14 | @pytest.mark.parametrize( 15 | 'target, func', [('os.listdir', 'listdir'), ('os.popen', 'get_branch')] 16 | ) 17 | def test_patch_basic(self, target, func, tb): 18 | with tb.patch(target) as mock_obj: 19 | tb.ref(func)() 20 | mock_obj.assert_called_once() 21 | 22 | @pytest.mark.parametrize( 23 | 'target, func', [('os.listdir', 'listdir'), ('os.popen', 'get_branch')] 24 | ) 25 | def test_patch_raises_error(self, target, func, tb): 26 | with pytest.raises(TestbookRuntimeError), tb.patch(target) as mock_obj: 27 | mock_obj.assert_called_once() 28 | 29 | def test_patch_return_value(self, tb): 30 | with tb.patch('os.listdir', return_value=['file1', 'file2']) as mock_listdir: 31 | assert tb.ref('listdir')() == ['file1', 'file2'] 32 | mock_listdir.assert_called_once() 33 | 34 | 35 | class TestPatchDict: 36 | @pytest.mark.parametrize( 37 | 'in_dict, values', 38 | [('os.environ', {'PATH': '/usr/bin'})], 39 | ) 40 | def test_patch_dict(self, in_dict, values, tb): 41 | with tb.patch_dict(in_dict, values, clear=True): 42 | assert tb.ref(in_dict) == values 43 | -------------------------------------------------------------------------------- /testbook/tests/test_reference.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from ..testbook import testbook 4 | from ..exceptions import TestbookAttributeError, TestbookSerializeError 5 | 6 | 7 | @pytest.fixture(scope='module') 8 | def notebook(): 9 | with testbook('testbook/tests/resources/reference.ipynb', execute=True) as tb: 10 | yield tb 11 | 12 | 13 | def test_create_reference(notebook): 14 | a = notebook.ref('a') 15 | assert repr(a) == '[1, 2, 3]' 16 | 17 | 18 | def test_create_reference_getitem(notebook): 19 | a = notebook['a'] 20 | assert repr(a) == '[1, 2, 3]' 21 | 22 | 23 | def test_create_reference_get(notebook): 24 | a = notebook.get('a') 25 | assert repr(a) == '[1, 2, 3]' 26 | 27 | 28 | def test_eq_in_notebook(notebook): 29 | a = notebook.ref('a') 30 | a.append(4) 31 | assert a == [1, 2, 3, 4] 32 | 33 | 34 | def test_eq_in_notebook_ref(notebook): 35 | a, b = notebook.ref('a'), notebook.ref('b') 36 | assert a == b 37 | 38 | 39 | def test_function_call(notebook): 40 | double = notebook.ref('double') 41 | assert double([1, 2, 3]) == [2, 4, 6] 42 | 43 | 44 | def test_function_call_with_ref_object(notebook): 45 | double, a = notebook.ref('double'), notebook.ref('a') 46 | 47 | assert double(a) == [2, 4, 6] 48 | 49 | 50 | def test_reference(notebook): 51 | Foo = notebook.ref('Foo') 52 | 53 | # Check that when a non-serializeable object is returned, it returns 54 | # a reference to that object instead 55 | f = Foo('bar') 56 | 57 | assert repr(f) == '""' 58 | 59 | # Valid attribute access 60 | assert f.say_hello() 61 | 62 | # Invalid attribute access 63 | with pytest.raises(TestbookAttributeError): 64 | f.does_not_exist 65 | 66 | assert f.say_hello() == 'Hello bar!' 67 | 68 | # non JSON-serializeable output 69 | with pytest.raises(TestbookSerializeError): 70 | f.resolve() 71 | -------------------------------------------------------------------------------- /testbook/tests/test_testbook.py: -------------------------------------------------------------------------------- 1 | import nbformat 2 | 3 | import pytest 4 | 5 | from ..testbook import testbook 6 | 7 | 8 | @testbook('testbook/tests/resources/inject.ipynb', execute=True) 9 | def test_testbook_execute_all_cells(tb): 10 | for cell in tb.cells[:-1]: 11 | assert cell.execution_count 12 | 13 | 14 | @testbook('testbook/tests/resources/inject.ipynb', execute='hello') 15 | def test_testbook_class_decorator(tb): 16 | assert tb.inject('say_hello()') 17 | 18 | 19 | @testbook('testbook/tests/resources/inject.ipynb') 20 | def test_testbook_class_decorator_execute_none(tb): 21 | assert tb.code_cells_executed == 0 22 | 23 | 24 | @testbook('testbook/tests/resources/inject.ipynb', execute=True) 25 | def test_testbook_decorator_with_fixture(nb, tmp_path): 26 | assert True # Check that the decorator accept to be passed along side a fixture 27 | 28 | 29 | @testbook('testbook/tests/resources/inject.ipynb', execute=True) 30 | @pytest.mark.parametrize('cell_index_args, expected_result', [(2, 2), ('hello', 1)]) 31 | def test_testbook_decorator_with_markers(nb, cell_index_args, expected_result): 32 | assert nb._cell_index(cell_index_args) == expected_result 33 | 34 | 35 | @pytest.mark.parametrize('cell_index_args, expected_result', [(2, 2), ('hello', 1)]) 36 | @testbook('testbook/tests/resources/inject.ipynb', execute=True) 37 | def test_testbook_decorator_with_markers_order_does_not_matter( 38 | nb, cell_index_args, expected_result 39 | ): 40 | assert nb._cell_index(cell_index_args) == expected_result 41 | 42 | 43 | def test_testbook_execute_all_cells_context_manager(): 44 | with testbook('testbook/tests/resources/inject.ipynb', execute=True) as tb: 45 | for cell in tb.cells[:-1]: 46 | assert cell.execution_count 47 | 48 | 49 | def test_testbook_class_decorator_context_manager(): 50 | with testbook('testbook/tests/resources/inject.ipynb', execute='hello') as tb: 51 | assert tb.inject('say_hello()') 52 | 53 | 54 | def test_testbook_class_decorator_execute_none_context_manager(): 55 | with testbook('testbook/tests/resources/inject.ipynb') as tb: 56 | assert tb.code_cells_executed == 0 57 | 58 | 59 | def test_testbook_with_file_object(): 60 | f = open('testbook/tests/resources/inject.ipynb') 61 | 62 | with testbook(f) as tb: 63 | assert tb.code_cells_executed == 0 64 | 65 | f.close() 66 | 67 | 68 | def test_testbook_with_notebook_node(): 69 | nb = nbformat.read('testbook/tests/resources/inject.ipynb', as_version=4) 70 | 71 | with testbook(nb) as tb: 72 | assert tb.code_cells_executed == 0 73 | 74 | 75 | def test_function_with_testbook_decorator_returns_value(): 76 | @testbook('testbook/tests/resources/inject.ipynb') 77 | def test_function(tb): 78 | return 'This should be returned' 79 | 80 | assert test_function() == 'This should be returned' 81 | -------------------------------------------------------------------------------- /testbook/tests/test_translators.py: -------------------------------------------------------------------------------- 1 | """Sourced from https://github.com/nteract/papermill/blob/master/papermill/tests/test_translators.py""" 2 | 3 | import pytest 4 | 5 | from .. import translators 6 | 7 | 8 | class Foo: 9 | def __init__(self, val): 10 | self.val = val 11 | 12 | def __repr__(self): 13 | return "".format(val=self.val) 14 | 15 | 16 | @pytest.mark.parametrize( 17 | 'test_input,expected', 18 | [ 19 | ('foo', '"foo"'), 20 | ('{"foo": "bar"}', '"{\\"foo\\": \\"bar\\"}"'), 21 | ({'foo': 'bar'}, '{"foo": "bar"}'), 22 | ({'foo': '"bar"'}, '{"foo": "\\"bar\\""}'), 23 | ({'foo': ['bar']}, '{"foo": ["bar"]}'), 24 | ({'foo': {'bar': 'baz'}}, '{"foo": {"bar": "baz"}}'), 25 | ({'foo': {'bar': '"baz"'}}, '{"foo": {"bar": "\\"baz\\""}}'), 26 | (['foo'], '["foo"]'), 27 | (['foo', '"bar"'], '["foo", "\\"bar\\""]'), 28 | ([{'foo': 'bar'}], '[{"foo": "bar"}]'), 29 | ([{'foo': '"bar"'}], '[{"foo": "\\"bar\\""}]'), 30 | (12345, '12345'), 31 | (-54321, '-54321'), 32 | (1.2345, '1.2345'), 33 | (-5432.1, '-5432.1'), 34 | (float('nan'), "float('nan')"), 35 | (float('-inf'), "float('-inf')"), 36 | (float('inf'), "float('inf')"), 37 | (True, 'True'), 38 | (False, 'False'), 39 | (None, 'None'), 40 | (Foo('bar'), '''""'''), 41 | ], 42 | ) 43 | def test_translate_type_python(test_input, expected): 44 | assert translators.PythonTranslator.translate(test_input) == expected 45 | 46 | 47 | @pytest.mark.parametrize( 48 | 'test_input,expected', [(3.14, '3.14'), (False, 'false'), (True, 'true')] 49 | ) 50 | def test_translate_float(test_input, expected): 51 | assert translators.Translator.translate(test_input) == expected 52 | 53 | 54 | def test_translate_assign(): 55 | assert translators.Translator.assign('var1', [1, 2, 3]) == 'var1 = [1, 2, 3]' 56 | -------------------------------------------------------------------------------- /testbook/translators.py: -------------------------------------------------------------------------------- 1 | """Sourced from https://github.com/nteract/papermill/blob/master/papermill/translators.py""" 2 | 3 | import math 4 | import sys 5 | 6 | 7 | class Translator(object): 8 | @classmethod 9 | def translate_raw_str(cls, val): 10 | """Reusable by most interpreters""" 11 | return '{}'.format(val) 12 | 13 | @classmethod 14 | def translate_escaped_str(cls, str_val): 15 | """Reusable by most interpreters""" 16 | if isinstance(str_val, str): 17 | str_val = str_val.encode('unicode_escape') 18 | if sys.version_info >= (3, 0): 19 | str_val = str_val.decode('utf-8') 20 | str_val = str_val.replace('"', r'\"') 21 | return '"{}"'.format(str_val) 22 | 23 | @classmethod 24 | def translate_str(cls, val): 25 | """Default behavior for translation""" 26 | return cls.translate_escaped_str(val) 27 | 28 | @classmethod 29 | def translate_none(cls, val): 30 | """Default behavior for translation""" 31 | return cls.translate_raw_str(val) 32 | 33 | @classmethod 34 | def translate_int(cls, val): 35 | """Default behavior for translation""" 36 | return cls.translate_raw_str(val) 37 | 38 | @classmethod 39 | def translate_float(cls, val): 40 | """Default behavior for translation""" 41 | return cls.translate_raw_str(val) 42 | 43 | @classmethod 44 | def translate_bool(cls, val): 45 | """Default behavior for translation""" 46 | return 'true' if val else 'false' 47 | 48 | @classmethod 49 | def translate_dict(cls, val): 50 | raise NotImplementedError( 51 | 'dict type translation not implemented for {}'.format(cls) 52 | ) 53 | 54 | @classmethod 55 | def translate_list(cls, val): 56 | raise NotImplementedError( 57 | 'list type translation not implemented for {}'.format(cls) 58 | ) 59 | 60 | @classmethod 61 | def translate(cls, val): 62 | """Translate each of the standard json/yaml types to appropriate objects.""" 63 | if val is None: 64 | return cls.translate_none(val) 65 | elif isinstance(val, str): 66 | return cls.translate_str(val) 67 | # Needs to be before integer checks 68 | elif isinstance(val, bool): 69 | return cls.translate_bool(val) 70 | elif isinstance(val, int): 71 | return cls.translate_int(val) 72 | elif isinstance(val, float): 73 | return cls.translate_float(val) 74 | elif isinstance(val, dict): 75 | return cls.translate_dict(val) 76 | elif isinstance(val, list): 77 | return cls.translate_list(val) 78 | elif isinstance(val, tuple): 79 | return cls.translate_tuple(val) 80 | elif val.__class__.__name__ == 'TestbookObjectReference': 81 | return val.name 82 | 83 | # Use this generic translation as a last resort 84 | return cls.translate_escaped_str(val) 85 | 86 | @classmethod 87 | def comment(cls, cmt_str): 88 | raise NotImplementedError( 89 | 'comment translation not implemented for {}'.format(cls) 90 | ) 91 | 92 | @classmethod 93 | def assign(cls, name, str_val): 94 | return '{} = {}'.format(name, str_val) 95 | 96 | 97 | class PythonTranslator(Translator): 98 | @classmethod 99 | def translate_float(cls, val): 100 | if math.isfinite(val): 101 | return cls.translate_raw_str(val) 102 | elif math.isnan(val): 103 | return "float('nan')" 104 | elif val < 0: 105 | return "float('-inf')" 106 | else: 107 | return "float('inf')" 108 | 109 | @classmethod 110 | def translate_bool(cls, val): 111 | return cls.translate_raw_str(val) 112 | 113 | @classmethod 114 | def translate_dict(cls, val): 115 | escaped = ', '.join( 116 | [ 117 | '{}: {}'.format(cls.translate_str(k), cls.translate(v)) 118 | for k, v in val.items() 119 | ] 120 | ) 121 | return '{{{}}}'.format(escaped) 122 | 123 | @classmethod 124 | def translate_list(cls, val): 125 | escaped = ', '.join([cls.translate(v) for v in val]) 126 | return '[{}]'.format(escaped) 127 | 128 | @classmethod 129 | def translate_tuple(cls, val): 130 | escaped = ', '.join([cls.translate(v) for v in val]) + ', ' 131 | return '({})'.format(escaped) 132 | -------------------------------------------------------------------------------- /testbook/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | 4 | 5 | def random_varname(length=10): 6 | """ 7 | Creates a random variable name as string of a given length. 8 | This is used in testbook to generate temporary variables within the notebook. 9 | 10 | Parameters 11 | ---------- 12 | length (int) 13 | 14 | Returns: 15 | -------- 16 | random variable name as string of given length 17 | """ 18 | return ''.join(random.choice(string.ascii_lowercase) for _ in range(length)) 19 | 20 | 21 | def all_subclasses(klass): 22 | """ 23 | This is a function that returns a generator. 24 | 25 | Inspects subclasses associated with a given class in a recursive manner 26 | and yields them, such that subclasses of subclasses will be yielded. 27 | 28 | Parameters: 29 | ----------- 30 | klass 31 | """ 32 | for subklass in klass.__subclasses__(): 33 | yield subklass 34 | yield from all_subclasses(subklass) 35 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = true 3 | envlist = py{38,39,310,311,312}, lint, manifest, docs 4 | 5 | [gh-actions] 6 | python = 7 | 3.8: py38 8 | 3.9: py39 9 | 3.10: py310 10 | 3.11: py311, lint, manifest, docs 11 | 3.12: py312 12 | 13 | # Linters 14 | [testenv:lint] 15 | skip_install = true 16 | commands = ruff check 17 | 18 | # Manifest 19 | [testenv:manifest] 20 | skip_install = true 21 | deps = check-manifest 22 | commands = check-manifest 23 | 24 | # Docs 25 | [testenv:docs] 26 | description = invoke sphinx-build to build the HTML docs 27 | skip_install = true 28 | deps = 29 | .[docs] 30 | commands = 31 | sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -bhtml {posargs} 32 | python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' 33 | python -c "import docs.conf" 34 | 35 | # Formatter 36 | [testenv:format] 37 | description = apply ruff formatter with desired rules 38 | basepython = python3.11 39 | deps = 40 | ruff 41 | commands = ruff format 42 | 43 | [testenv] 44 | # disable Python's hash randomization for tests that stringify dicts, etc 45 | setenv = 46 | PYTHONHASHSEED = 0 47 | passenv = * 48 | basepython = 49 | py38: python3.8 50 | py39: python3.9 51 | py310: python3.10 52 | py311: python3.11 53 | py312: python3.12 54 | lint: python3.11 55 | manifest: python3.11 56 | docs: python3.11 57 | deps = .[dev] 58 | commands = pytest -vv --maxfail=2 --cov=testbook --cov-report=xml -W always {posargs} 59 | --------------------------------------------------------------------------------