├── .gitignore ├── LICENSE ├── README.rst ├── pytest_stepwise ├── __init__.py ├── compat.py └── plugin.py ├── setup.py ├── tests ├── conftest.py └── test_pytest_stepwise.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | eggs/ 15 | lib/ 16 | lib64/ 17 | parts/ 18 | sdist/ 19 | var/ 20 | *.egg-info/ 21 | .installed.cfg 22 | *.egg 23 | 24 | # PyInstaller 25 | # Usually these files are written by a python script from a template 26 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 27 | *.manifest 28 | *.spec 29 | 30 | # Installer logs 31 | pip-log.txt 32 | pip-delete-this-directory.txt 33 | 34 | # Unit test / coverage reports 35 | htmlcov/ 36 | .tox/ 37 | .coverage 38 | .cache 39 | nosetests.xml 40 | coverage.xml 41 | 42 | # Translations 43 | *.mo 44 | *.pot 45 | 46 | # Django stuff: 47 | *.log 48 | 49 | # Sphinx documentation 50 | docs/_build/ 51 | 52 | # PyBuilder 53 | target/ 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Niclas Olofsson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Deprecated: pytest-stepwise is now bundled with pytest 2 | ====================================================== 3 | 4 | Since pytest 3.10 this plugin has been merged into pytest itself, so you don't even need a plugin! See https://docs.pytest.org/en/latest/cache.html#stepwise 5 | 6 | 7 | Introduction 8 | ============ 9 | 10 | pytest-stepwise is a plugin for `pytest `_ that run 11 | all tests until a test fails, and then continue next test run from where 12 | the last run failed. You may think of it as a combination of the ``-x`` 13 | option (which exits the test run after a failing test) and the ``--lf`` 14 | option from pytest-cache (which only runs failing tests), except that 15 | it does not restart the test run from the beginning as soon as a test 16 | passes. 17 | 18 | How to use it? 19 | ============== 20 | 21 | 0. Install the plugin - ``pip install pytest-stepwise``. 22 | 1. Run ``py.test --stepwise`` (you can also use the alias ``--sw``). 23 | 2. Watch the test fail and fix it. 24 | 3. Run ``py.test --stepwise`` again. The test suite will continue to run 25 | right from where it was. 26 | 27 | Use the ``--skip`` option to ignore one failing test and stop the 28 | test execution on the second failing test instead. This is useful if you 29 | get stuck on a failing test and just want to ignore it until later. 30 | 31 | 32 | When is this useful? 33 | ==================== 34 | 35 | pytest-stepwise was written for use when a large part of the test suite 36 | is failing. In this case, pytest-stepwise allows you to focus on fixing 37 | one test at the time instead of being overwhelmed by all failing 38 | tests. It should however be noted that all tests need to be re-run after 39 | to make sure that any changes made when fixing one test has not broken 40 | some other test. 41 | 42 | Please submit an issue if you have any suggestions regarding use cases 43 | of pytest-stepwise. 44 | 45 | 46 | Compatibility 47 | ============= 48 | 49 | pytest-stepwise is compatible with pytest 2.2 -> 2.8. 50 | For pytest 2.7 and earlier, ``pytest-cache`` is required as a dependency. 51 | 52 | 53 | Changelog 54 | ========= 55 | 56 | * 0.1 - Initial version. 57 | * 0.2 - Clear cache after test run when the plugin is not active. 58 | Added ``--skip`` option. 59 | * 0.3 - Fixed issue when failing tests are removed. 60 | Fixed compatibility with ``--pdb`` option. 61 | Stop on errors as well as on test failures. 62 | * 0.4 - Refactoring, pytest 2.8 compatiblity. Stop test execution on 63 | collection errors. 64 | -------------------------------------------------------------------------------- /pytest_stepwise/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.4' 2 | -------------------------------------------------------------------------------- /pytest_stepwise/compat.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | try: 4 | from _pytest.cacheprovider import Cache 5 | except ImportError: 6 | from pytest_cache import Cache 7 | 8 | try: 9 | # pytest 3.7+ 10 | Cache = Cache.for_config 11 | except AttributeError: 12 | pass 13 | 14 | 15 | if hasattr(pytest, 'hookimpl'): 16 | tryfirst = pytest.hookimpl(tryfirst=True) 17 | else: 18 | tryfirst = pytest.mark.tryfirst 19 | -------------------------------------------------------------------------------- /pytest_stepwise/plugin.py: -------------------------------------------------------------------------------- 1 | from .compat import Cache, tryfirst 2 | 3 | 4 | def pytest_addoption(parser): 5 | group = parser.getgroup('general') 6 | group.addoption('--sw', action='store_true', dest='stepwise', 7 | help='alias for --stepwise') 8 | group.addoption('--stepwise', action='store_true', dest='stepwise', 9 | help='exit on test fail and continue from last failing test next time') 10 | group.addoption('--skip', action='store_true', dest='skip', 11 | help='ignore the first failing test but stop on the next failing test') 12 | 13 | 14 | @tryfirst 15 | def pytest_configure(config): 16 | config.cache = Cache(config) 17 | config.pluginmanager.register(StepwisePlugin(config), 'stepwiseplugin') 18 | 19 | 20 | class StepwisePlugin: 21 | def __init__(self, config): 22 | self.config = config 23 | self.active = config.getvalue('stepwise') 24 | self.session = None 25 | 26 | if self.active: 27 | self.lastfailed = config.cache.get('cache/stepwise', None) 28 | self.skip = config.getvalue('skip') 29 | 30 | def pytest_sessionstart(self, session): 31 | self.session = session 32 | 33 | def pytest_collection_modifyitems(self, session, config, items): 34 | if not self.active or not self.lastfailed: 35 | return 36 | 37 | already_passed = [] 38 | found = False 39 | 40 | # Make a list of all tests that has been runned before the last failing one. 41 | for item in items: 42 | if item.nodeid == self.lastfailed: 43 | found = True 44 | break 45 | else: 46 | already_passed.append(item) 47 | 48 | # If the previously failed test was not found among the test items, 49 | # do not skip any tests. 50 | if not found: 51 | already_passed = [] 52 | 53 | for item in already_passed: 54 | items.remove(item) 55 | 56 | config.hook.pytest_deselected(items=already_passed) 57 | 58 | def pytest_collectreport(self, report): 59 | if self.active and report.failed: 60 | self.session.shouldstop = 'Error when collecting test, stopping test execution.' 61 | 62 | def pytest_runtest_logreport(self, report): 63 | # Skip this hook if plugin is not active or the test is xfailed. 64 | if not self.active or 'xfail' in report.keywords: 65 | return 66 | 67 | if report.failed: 68 | if self.skip: 69 | # Remove test from the failed ones (if it exists) and unset the skip option 70 | # to make sure the following tests will not be skipped. 71 | if report.nodeid == self.lastfailed: 72 | self.lastfailed = None 73 | 74 | self.skip = False 75 | else: 76 | # Mark test as the last failing and interrupt the test session. 77 | self.lastfailed = report.nodeid 78 | self.session.shouldstop = 'Test failed, continuing from this test next run.' 79 | 80 | else: 81 | # If the test was actually run and did pass. 82 | if report.when == 'call': 83 | # Remove test from the failed ones, if exists. 84 | if report.nodeid == self.lastfailed: 85 | self.lastfailed = None 86 | 87 | def pytest_sessionfinish(self, session): 88 | if self.active: 89 | self.config.cache.set('cache/stepwise', self.lastfailed) 90 | else: 91 | # Clear the list of failing tests if the plugin is not active. 92 | self.config.cache.set('cache/stepwise', []) 93 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import codecs 5 | from setuptools import setup 6 | 7 | 8 | def read(fname): 9 | file_path = os.path.join(os.path.dirname(__file__), fname) 10 | return codecs.open(file_path, encoding='utf-8').read() 11 | 12 | setup( 13 | name='pytest-stepwise', 14 | version=__import__('pytest_stepwise').__version__, 15 | author='Niclas Olofsson', 16 | author_email='n@niclasolofsson.se', 17 | maintainer='Niclas Olofsson', 18 | maintainer_email='n@niclasolofsson.se', 19 | license='MIT', 20 | url='https://github.com/nip3o/pytest-stepwise', 21 | description='Run a test suite one failing test at a time.', 22 | long_description=read('README.rst'), 23 | packages=['pytest_stepwise'], 24 | install_requires=['pytest >= 2.2'], 25 | classifiers=['Development Status :: 4 - Beta', 26 | 'Intended Audience :: Developers', 27 | 'Topic :: Software Development :: Testing', 28 | 'Programming Language :: Python', 29 | 'Operating System :: OS Independent', 30 | 'License :: OSI Approved :: MIT License'], 31 | entry_points={'pytest11': ['stepwise = pytest_stepwise.plugin']}, 32 | ) 33 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | pytest_plugins = 'pytester' 2 | -------------------------------------------------------------------------------- /tests/test_pytest_stepwise.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def stepwise_testdir(testdir): 6 | # Rather than having to modify our testfile between tests, we introduce 7 | # a flag for wether or not the second test should fail. 8 | testdir.makeconftest(''' 9 | def pytest_addoption(parser): 10 | group = parser.getgroup('general') 11 | group.addoption('--fail', action='store_true', dest='fail') 12 | group.addoption('--fail-last', action='store_true', dest='fail_last') 13 | ''') 14 | 15 | # Create a simple test suite. 16 | testdir.makepyfile(test_stepwise=''' 17 | def test_success_before_fail(): 18 | assert 1 19 | 20 | def test_fail_on_flag(request): 21 | assert not request.config.getvalue('fail') 22 | 23 | def test_success_after_fail(): 24 | assert 1 25 | 26 | def test_fail_last_on_flag(request): 27 | assert not request.config.getvalue('fail_last') 28 | 29 | def test_success_after_last_fail(): 30 | assert 1 31 | ''') 32 | 33 | testdir.makepyfile(testfile_b=''' 34 | def test_success(): 35 | assert 1 36 | ''') 37 | 38 | return testdir 39 | 40 | 41 | @pytest.fixture 42 | def error_testdir(testdir): 43 | testdir.makepyfile(test_stepwise=''' 44 | def test_error(nonexisting_fixture): 45 | assert 1 46 | 47 | def test_success_after_fail(): 48 | assert 1 49 | ''') 50 | 51 | return testdir 52 | 53 | 54 | @pytest.fixture 55 | def broken_testdir(testdir): 56 | testdir.makepyfile(working_testfile='def test_proper(): assert 1', broken_testfile='foobar') 57 | return testdir 58 | 59 | 60 | def test_run_without_stepwise(stepwise_testdir): 61 | result = stepwise_testdir.runpytest('-v', '--strict', '--fail') 62 | 63 | result.stdout.fnmatch_lines(['*test_success_before_fail PASSED*']) 64 | result.stdout.fnmatch_lines(['*test_fail_on_flag FAILED*']) 65 | result.stdout.fnmatch_lines(['*test_success_after_fail PASSED*']) 66 | 67 | 68 | def test_fail_and_continue_with_stepwise(stepwise_testdir): 69 | # Run the tests with a failing second test. 70 | result = stepwise_testdir.runpytest('-v', '--strict', '--stepwise', '--fail') 71 | assert not result.stderr.str() 72 | 73 | stdout = result.stdout.str() 74 | # Make sure we stop after first failing test. 75 | assert 'test_success_before_fail PASSED' in stdout 76 | assert 'test_fail_on_flag FAILED' in stdout 77 | assert 'test_success_after_fail' not in stdout 78 | 79 | # "Fix" the test that failed in the last run and run it again. 80 | result = stepwise_testdir.runpytest('-v', '--strict', '--stepwise') 81 | assert not result.stderr.str() 82 | 83 | stdout = result.stdout.str() 84 | # Make sure the latest failing test runs and then continues. 85 | assert 'test_success_before_fail' not in stdout 86 | assert 'test_fail_on_flag PASSED' in stdout 87 | assert 'test_success_after_fail PASSED' in stdout 88 | 89 | 90 | def test_run_with_skip_option(stepwise_testdir): 91 | result = stepwise_testdir.runpytest('-v', '--strict', '--stepwise', '--skip', 92 | '--fail', '--fail-last') 93 | assert not result.stderr.str() 94 | 95 | stdout = result.stdout.str() 96 | # Make sure first fail is ignore and second fail stops the test run. 97 | assert 'test_fail_on_flag FAILED' in stdout 98 | assert 'test_success_after_fail PASSED' in stdout 99 | assert 'test_fail_last_on_flag FAILED' in stdout 100 | assert 'test_success_after_last_fail' not in stdout 101 | 102 | 103 | def test_fail_on_errors(error_testdir): 104 | result = error_testdir.runpytest('-v', '--strict', '--stepwise') 105 | 106 | assert not result.stderr.str() 107 | stdout = result.stdout.str() 108 | 109 | assert 'test_error ERROR' in stdout 110 | assert 'test_success_after_fail' not in stdout 111 | 112 | 113 | def test_change_testfile(stepwise_testdir): 114 | result = stepwise_testdir.runpytest('-v', '--strict', '--stepwise', '--fail', 115 | 'test_stepwise.py') 116 | assert not result.stderr.str() 117 | 118 | stdout = result.stdout.str() 119 | assert 'test_fail_on_flag FAILED' in stdout 120 | 121 | # Make sure the second test run starts from the beginning, since the 122 | # test to continue from does not exist in testfile_b. 123 | result = stepwise_testdir.runpytest('-v', '--strict', '--stepwise', 124 | 'testfile_b.py') 125 | assert not result.stderr.str() 126 | 127 | stdout = result.stdout.str() 128 | assert 'test_success PASSED' in stdout 129 | 130 | 131 | def test_stop_on_collection_errors(broken_testdir): 132 | result = broken_testdir.runpytest('-v', '--strict', '--stepwise', 'working_testfile.py', 'broken_testfile.py') 133 | 134 | stdout = result.stdout.str() 135 | if pytest.__version__ < '3.0.0': 136 | assert 'Error when collecting test' in stdout 137 | else: 138 | assert 'errors during collection' in stdout 139 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27-{pytest23,pytest27}, {py27,py36}-{pytest28,pytest29,pytest30,pytest37} 3 | 4 | [testenv] 5 | deps = 6 | pytest23: pytest==2.3 7 | pytest23: pytest-cache 8 | pytest27: pytest==2.7 9 | pytest27: pytest-cache 10 | pytest28: pytest==2.8 11 | pytest29: pytest==2.9 12 | pytest30: pytest==3.0 13 | pytest37: pytest==3.7 14 | commands = 15 | py.test tests {posargs} 16 | --------------------------------------------------------------------------------