├── .gitignore ├── LICENSE ├── README.rst ├── examples ├── test_series_plots.ipynb └── test_skipci.ipynb ├── img ├── pytest-ipynb_notebook.png └── pytest-ipynb_output.png ├── pytest_ipynb ├── __init__.py └── plugin.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | *.swp 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Andrea Zonca 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | * Neither the name of the {organization} nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | pytest-ipynb 2 | ============ 3 | 4 | **THE PROJECT IS NOT MAINTAINED ANYMORE** 5 | Anyone is welcome to fork and update the software using the same name without the need to ask or even notify me. 6 | 7 | Plugin for ``pytest`` to run IPython notebooks as unit tests, relies on `runipy` to interface with the Notebook. 8 | 9 | Define unit tests in IPython notebook cells (`see example on 10 | nbviewer `_): 11 | 12 | .. figure:: https://github.com/zonca/pytest-ipynb/raw/master/img/pytest-ipynb_notebook.png 13 | :alt: Example notebook 14 | 15 | Run ``py.test`` to execute them: 16 | 17 | .. figure:: https://github.com/zonca/pytest-ipynb/raw/master/img/pytest-ipynb_output.png 18 | :alt: Example output 19 | 20 | Example 21 | ------- 22 | 23 | See the ``examples/`` folder or `a preview on 24 | nbviewer `_. 25 | 26 | Features 27 | -------- 28 | 29 | - Discover files named ``test*.ipynb`` 30 | - Run each cell of the notebook as a unit test (just use ``assert``) 31 | - First line of each cell is the test name, either as docstring, 32 | comment or function name 33 | - A cell named ``fixture*`` or ``setup*`` is run before each of the 34 | following unit tests as a fixture 35 | - Add `SKIPCI` to a cell description to skip the test on Travis-CI (checks if the `CI` environment variable is defined) 36 | - IPython notebook kernel is restarted after each test 37 | - Each notebook is executed in the folder where the ``.ipynb`` file is located 38 | 39 | Requirements 40 | ------------ 41 | 42 | - Python 2.7+, Python 3.2+ 43 | - ``pytest`` 44 | - IPython Notebook 2.0+ 45 | 46 | Install 47 | ------- 48 | 49 | :: 50 | 51 | $ pip install pytest-ipynb 52 | 53 | Author 54 | ------ 55 | 56 | `Andrea Zonca `__ 57 | 58 | Credits 59 | ------- 60 | 61 | - ``__ 62 | - Thomas Kluyver for help on the IPython mailing list 63 | -------------------------------------------------------------------------------- /examples/test_series_plots.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "" 4 | }, 5 | "nbformat": 3, 6 | "nbformat_minor": 0, 7 | "worksheets": [ 8 | { 9 | "cells": [ 10 | { 11 | "cell_type": "code", 12 | "collapsed": false, 13 | "input": [ 14 | "\"\"\"fixture\"\"\"\n", 15 | "import nose\n", 16 | "import os\n", 17 | "import string\n", 18 | "from distutils.version import LooseVersion\n", 19 | "\n", 20 | "from datetime import datetime, date, timedelta\n", 21 | "\n", 22 | "from pandas import Series, DataFrame, MultiIndex, PeriodIndex, date_range\n", 23 | "from pandas.compat import range, lrange, StringIO, lmap, lzip, u, zip\n", 24 | "import pandas.util.testing as tm\n", 25 | "from pandas.util.testing import ensure_clean\n", 26 | "from pandas.core.config import set_option\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "from numpy import random\n", 30 | "from numpy.random import randn\n", 31 | "\n", 32 | "from numpy.testing import assert_array_equal\n", 33 | "from numpy.testing.decorators import slow\n", 34 | "import pandas.tools.plotting as plotting\n", 35 | "\n", 36 | "import matplotlib as mpl\n", 37 | "mpl_le_1_2_1 = str(mpl.__version__) <= LooseVersion('1.2.1')\n", 38 | "ts = tm.makeTimeSeries()\n", 39 | "ts.name = 'ts'\n", 40 | "\n", 41 | "series = tm.makeStringSeries()\n", 42 | "series.name = 'series'\n", 43 | "\n", 44 | "iseries = tm.makePeriodSeries()\n", 45 | "iseries.name = 'iseries'\n", 46 | "\n", 47 | "%matplotlib inline" 48 | ], 49 | "language": "python", 50 | "metadata": {}, 51 | "outputs": [] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "collapsed": false, 56 | "input": [ 57 | "\"\"\"test_plot_figsize_and_title\"\"\"\n", 58 | "ax = series.plot(title='Test', figsize=(16, 8))\n", 59 | "\n", 60 | "assert ax.title.get_text() == 'Test'\n", 61 | "assert_array_equal(np.round(ax.figure.get_size_inches()),\n", 62 | " np.array((16., 8.)))" 63 | ], 64 | "language": "python", 65 | "metadata": {}, 66 | "outputs": [] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "collapsed": false, 71 | "input": [ 72 | "\"\"\"test_bar_linewidth\"\"\"\n", 73 | "df = DataFrame(randn(5, 5))\n", 74 | "\n", 75 | "# regular\n", 76 | "ax = df.plot(kind='bar', linewidth=2)\n", 77 | "for r in ax.patches:\n", 78 | " assert r.get_linewidth() == 2\n", 79 | "\n", 80 | "# stacked\n", 81 | "ax = df.plot(kind='bar', stacked=True, linewidth=2)\n", 82 | "for r in ax.patches:\n", 83 | " assert r.get_linewidth() == 2\n", 84 | "\n", 85 | "# subplots\n", 86 | "axes = df.plot(kind='bar', linewidth=2, subplots=True)\n", 87 | "for ax in axes:\n", 88 | " for r in ax.patches:\n", 89 | " assert r.get_linewidth() == 2" 90 | ], 91 | "language": "python", 92 | "metadata": {}, 93 | "outputs": [] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "collapsed": false, 98 | "input": [ 99 | "\"\"\"test_irregular_datetime\"\"\"\n", 100 | "rng = date_range('1/1/2000', '3/1/2000')\n", 101 | "rng = rng[[0, 1, 2, 3, 5, 9, 10, 11, 12]]\n", 102 | "ser = Series(randn(len(rng)), rng)\n", 103 | "ax = ser.plot()\n", 104 | "xp = datetime(1999, 1, 1).toordinal()\n", 105 | "ax.set_xlim('1/1/1999', '1/1/2001')\n", 106 | "assert xp == ax.get_xlim()[0]" 107 | ], 108 | "language": "python", 109 | "metadata": {}, 110 | "outputs": [] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "collapsed": false, 115 | "input": [ 116 | "\"\"\"failing test\"\"\"\n", 117 | "1/0\n", 118 | "a = 3" 119 | ], 120 | "language": "python", 121 | "metadata": {}, 122 | "outputs": [] 123 | } 124 | ], 125 | "metadata": {} 126 | } 127 | ] 128 | } 129 | -------------------------------------------------------------------------------- /examples/test_skipci.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "" 4 | }, 5 | "nbformat": 3, 6 | "nbformat_minor": 0, 7 | "worksheets": [ 8 | { 9 | "cells": [ 10 | { 11 | "cell_type": "code", 12 | "collapsed": false, 13 | "input": [ 14 | "\"\"\"failing test unless we are on travis SKIPCI\"\"\"\n", 15 | "1/0\n", 16 | "a = 3" 17 | ], 18 | "language": "python", 19 | "metadata": {}, 20 | "outputs": [] 21 | } 22 | ], 23 | "metadata": {} 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /img/pytest-ipynb_notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zonca/pytest-ipynb/04b5fed4f280983f64254b01e3b24b7733e99224/img/pytest-ipynb_notebook.png -------------------------------------------------------------------------------- /img/pytest-ipynb_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zonca/pytest-ipynb/04b5fed4f280983f64254b01e3b24b7733e99224/img/pytest-ipynb_output.png -------------------------------------------------------------------------------- /pytest_ipynb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zonca/pytest-ipynb/04b5fed4f280983f64254b01e3b24b7733e99224/pytest_ipynb/__init__.py -------------------------------------------------------------------------------- /pytest_ipynb/plugin.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os,sys 3 | import warnings 4 | try: 5 | from exceptions import Exception, TypeError, ImportError 6 | except: 7 | pass 8 | 9 | from runipy.notebook_runner import NotebookRunner 10 | 11 | wrapped_stdin = sys.stdin 12 | sys.stdin = sys.__stdin__ 13 | sys.stdin = wrapped_stdin 14 | try: 15 | from Queue import Empty 16 | except: 17 | from queue import Empty 18 | 19 | # code copied from runipy main.py 20 | with warnings.catch_warnings(): 21 | try: 22 | from IPython.utils.shimmodule import ShimWarning 23 | warnings.filterwarnings('error', '', ShimWarning) 24 | except ImportError: 25 | class ShimWarning(Warning): 26 | """Warning issued by iPython 4.x regarding deprecated API.""" 27 | pass 28 | 29 | try: 30 | # IPython 3 31 | from IPython.nbformat import reads, NBFormatError 32 | except ShimWarning: 33 | # IPython 4 34 | from nbformat import reads, NBFormatError 35 | except ImportError: 36 | # IPython 2 37 | from IPython.nbformat.current import reads, NBFormatError 38 | finally: 39 | warnings.resetwarnings() 40 | 41 | class IPyNbException(Exception): 42 | """ custom exception for error reporting. """ 43 | 44 | def pytest_collect_file(path, parent): 45 | if path.fnmatch("test*.ipynb"): 46 | return IPyNbFile(path, parent) 47 | 48 | def get_cell_description(cell_input): 49 | """Gets cell description 50 | 51 | Cell description is the first line of a cell, 52 | in one of this formats: 53 | 54 | * single line docstring 55 | * single line comment 56 | * function definition 57 | """ 58 | try: 59 | first_line = cell_input.split("\n")[0] 60 | if first_line.startswith(('"', '#', 'def')): 61 | return first_line.replace('"','').replace("#",'').replace('def ', '').replace("_", " ").strip() 62 | except: 63 | pass 64 | return "no description" 65 | 66 | class IPyNbFile(pytest.File): 67 | def collect(self): 68 | with self.fspath.open() as f: 69 | payload = f.read() 70 | self.notebook_folder = self.fspath.dirname 71 | try: 72 | # Ipython 3 73 | self.nb = reads(payload, 3) 74 | except (TypeError, NBFormatError): 75 | # Ipython 2 76 | self.nb = reads(payload, 'json') 77 | self.runner = NotebookRunner(self.nb) 78 | 79 | cell_num = 1 80 | 81 | for cell in self.runner.iter_code_cells(): 82 | yield IPyNbCell(self.name, self, cell_num, cell) 83 | cell_num += 1 84 | 85 | def setup(self): 86 | self.fixture_cell = None 87 | 88 | def teardown(self): 89 | self.runner.shutdown_kernel() 90 | 91 | class IPyNbCell(pytest.Item): 92 | def __init__(self, name, parent, cell_num, cell): 93 | super(IPyNbCell, self).__init__(name, parent) 94 | 95 | self.cell_num = cell_num 96 | self.cell = cell 97 | self.cell_description = get_cell_description(self.cell.input) 98 | 99 | def runtest(self): 100 | self.parent.runner.km.restart_kernel() 101 | 102 | if self.parent.notebook_folder: 103 | self.parent.runner.kc.execute( 104 | """import os 105 | os.chdir("%s")""" % self.parent.notebook_folder) 106 | 107 | if ("SKIPCI" in self.cell_description) and ("CI" in os.environ): 108 | pass 109 | else: 110 | if self.parent.fixture_cell: 111 | self.parent.runner.kc.execute(self.parent.fixture_cell.input, allow_stdin=False) 112 | msg_id = self.parent.runner.kc.execute(self.cell.input, allow_stdin=False) 113 | if self.cell_description.lower().startswith("fixture") or self.cell_description.lower().startswith("setup"): 114 | self.parent.fixture_cell = self.cell 115 | timeout = 20 116 | while True: 117 | try: 118 | msg = self.parent.runner.kc.get_shell_msg(block=True, timeout=timeout) 119 | if msg.get("parent_header", None) and msg["parent_header"].get("msg_id", None) == msg_id: 120 | break 121 | except Empty: 122 | raise IPyNbException(self.cell_num, self.cell_description, 123 | self.cell.input, 124 | "Timeout of %d seconds exceeded executing cell: %s" % (timeout, self.cell.input)) 125 | 126 | reply = msg['content'] 127 | 128 | if reply['status'] == 'error': 129 | raise IPyNbException(self.cell_num, self.cell_description, self.cell.input, '\n'.join(reply['traceback'])) 130 | 131 | def repr_failure(self, excinfo): 132 | """ called when self.runtest() raises an exception. """ 133 | if isinstance(excinfo.value, IPyNbException): 134 | return "\n".join([ 135 | "Notebook execution failed", 136 | "Cell %d: %s\n\n" 137 | "Input:\n%s\n\n" 138 | "Traceback:\n%s\n" % excinfo.value.args, 139 | ]) 140 | else: 141 | return "pytest plugin exception: %s" % str(excinfo.value) 142 | 143 | def _makeid(self): 144 | description = self.parent.nodeid + "::" + self.name 145 | description += "::" + "cell %d" % self.cell_num 146 | if self.cell_description: 147 | description += ", " + self.cell_description 148 | return description 149 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | runipy 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import os.path 3 | 4 | with open("README.rst") as f: 5 | long_description = f.read() 6 | 7 | setup( 8 | name="pytest-ipynb", 9 | version="1.1.0", 10 | 11 | packages = ['pytest_ipynb'], 12 | # the following makes a plugin available to pytest 13 | entry_points = { 14 | 'pytest11': [ 15 | 'ipynb = pytest_ipynb.plugin', 16 | ] 17 | }, 18 | install_requires = ["pytest", "runipy"], 19 | 20 | # metadata for upload to PyPI 21 | author="Andrea Zonca", 22 | author_email="code@andreazonca.com", 23 | description="Use pytest's runner to discover and execute tests as cells of IPython notebooks", 24 | long_description=long_description, 25 | license="MIT", 26 | keywords="pytest test unittest ipython notebook", 27 | url="http://github.com/zonca/pytest-ipynb", 28 | classifiers=[ 29 | 'Development Status :: 3 - Alpha', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Operating System :: OS Independent', 33 | 'Programming Language :: Python :: 2', 34 | 'Programming Language :: Python :: 2.7', 35 | 'Programming Language :: Python :: 3', 36 | 'Programming Language :: Python :: 3.4', 37 | 'Programming Language :: C++', 38 | 'Topic :: Software Development :: Quality Assurance', 39 | 'Topic :: Software Development :: Testing', 40 | ], 41 | ) 42 | --------------------------------------------------------------------------------