├── .gitignore ├── pyproject.toml ├── RELEASING.rst ├── testing ├── conftest.py ├── test_boxed.py └── test_xfail_behavior.py ├── .github └── workflows │ ├── deploy.yml │ └── test.yml ├── .pre-commit-config.yaml ├── LICENSE ├── CHANGELOG.rst ├── tox.ini ├── setup.py ├── README.rst ├── example └── boxed.txt └── src └── pytest_forked └── __init__.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo 3 | .cache/ 4 | .eggs/ 5 | .tox/ 6 | build/ 7 | dist/ 8 | src/pytest_forked.egg-info/ 9 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools >= 41.4', 'setuptools_scm >= 3.3'] 3 | build-backend = 'setuptools.build_meta' 4 | -------------------------------------------------------------------------------- /RELEASING.rst: -------------------------------------------------------------------------------- 1 | Here are the steps on how to make a new release. 2 | 3 | 1. Create a ``release-VERSION`` branch from ``upstream/master``. 4 | 2. Update ``CHANGELOG.rst``. 5 | 3. Push a branch with the changes. 6 | 4. Once all builds pass, push a tag to ``upstream``. 7 | 5. Merge the PR. 8 | -------------------------------------------------------------------------------- /testing/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | pytest_plugins = "pytester" 5 | 6 | 7 | @pytest.fixture(autouse=True) 8 | def _divert_atexit(request, monkeypatch): 9 | import atexit 10 | 11 | atexit_fns = [] 12 | 13 | def atexit_register(func, *args, **kwargs): 14 | atexit_fns.append(lambda: func(*args, **kwargs)) 15 | 16 | def finish(): 17 | while atexit_fns: 18 | atexit_fns.pop()() 19 | 20 | monkeypatch.setattr(atexit, "register", atexit_register) 21 | request.addfinalizer(finish) 22 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | 10 | deploy: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Build and Check Package 17 | uses: hynek/build-and-inspect-python-package@v1.5 18 | - name: Download Package 19 | uses: actions/download-artifact@v3 20 | with: 21 | name: Packages 22 | path: dist 23 | - name: Publish package to PyPI 24 | uses: pypa/gh-action-pypi-publish@release/v1 25 | with: 26 | user: __token__ 27 | password: ${{ secrets.pypi_token }} 28 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/asottile/pyupgrade 3 | rev: v2.32.1 4 | hooks: 5 | - id: pyupgrade 6 | args: [--py36-plus] 7 | - repo: https://github.com/psf/black 8 | rev: 22.3.0 9 | hooks: 10 | - id: black 11 | args: [--safe, --quiet] 12 | - repo: https://github.com/pre-commit/pre-commit-hooks 13 | rev: v4.2.0 14 | hooks: 15 | - id: trailing-whitespace 16 | - id: end-of-file-fixer 17 | - repo: local 18 | hooks: 19 | - id: rst 20 | name: rst 21 | entry: rst-lint --encoding utf-8 22 | files: ^(CHANGELOG|README.rst)$ 23 | language: python 24 | additional_dependencies: [pygments, restructuredtext_lint] 25 | - repo: https://github.com/asottile/reorder_python_imports 26 | rev: v3.1.0 27 | hooks: 28 | - id: reorder-python-imports 29 | args: ['--application-directories=.:src'] 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Permission is hereby granted, free of charge, to any person obtaining a copy 3 | of this software and associated documentation files (the "Software"), to deal 4 | in the Software without restriction, including without limitation the rights 5 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 6 | copies of the Software, and to permit persons to whom the Software is 7 | furnished to do so, subject to the following conditions: 8 | 9 | The above copyright notice and this permission notice shall be included in all 10 | copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 13 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 15 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 17 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 18 | SOFTWARE. 19 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | python: ["3.7", "3.8", "3.9", "3.10", "3.11"] 14 | include: 15 | - python: "3.7" 16 | tox_env: "py37" 17 | - python: "3.8" 18 | tox_env: "py38" 19 | - python: "3.9" 20 | tox_env: "py39" 21 | - python: "3.10" 22 | tox_env: "py310" 23 | - python: "3.11" 24 | tox_env: "py311" 25 | 26 | steps: 27 | - uses: actions/checkout@v3 28 | - name: Set up Python 29 | uses: actions/setup-python@v2 30 | with: 31 | python-version: ${{ matrix.python }} 32 | - name: Install tox 33 | run: | 34 | python -m pip install --upgrade pip 35 | pip install tox 36 | - name: Test 37 | run: | 38 | tox -e ${{ matrix.tox_env }} 39 | 40 | check-package: 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v3 44 | - name: Build and Check Package 45 | uses: hynek/build-and-inspect-python-package@v1.5 46 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | v1.6.0 2 | ====== 3 | 4 | * Relaxed dependency requirements (`#77 `__). 5 | 6 | v1.5.0 7 | ====== 8 | 9 | * Dropped support for Python 3.6. 10 | * Added official support for Python 3.11. 11 | 12 | v1.4.0 13 | ====== 14 | 15 | * Dropped support for Python 2.7 and 3.5. 16 | * Added official support for Python 3.10. 17 | 18 | v1.3.0 19 | ====== 20 | 21 | * Add support for pytest 6 (issue #45 / PR #46) 22 | * Replace `@pytest.mark.tryfirst` with newer `@pytest.hookimpl` (PR #46) 23 | * Invoke `pytest_runtest_logstart` and `pytest_runtest_logfinish` hooks in `runtest_protocol` (issue #31 / PR #46) 24 | 25 | v1.2.0 26 | ====== 27 | 28 | * Add limited support for xfail marker (issue #33 / PR #34). 29 | * Fix support for pytest 5.4.0+ (issue #30 / PR #32). 30 | * Drop support for Python 3.4 as it is EOL (PR #39). 31 | 32 | v1.1.3 33 | ====== 34 | 35 | * Another dummy release to sort out missing wheels (hopefully). 36 | 37 | v1.1.2 38 | ====== 39 | 40 | * Another dummy release to sort out missing wheels (hopefully). 41 | 42 | v1.1.1 43 | ====== 44 | 45 | * Dummy release to sort out CI issues. 46 | 47 | v1.1.0 48 | ====== 49 | 50 | * New marker `pytest.mark.forked` to fork before individual tests. 51 | 52 | v1.0.2 53 | ====== 54 | 55 | * Fix support for pytest 4.2. 56 | 57 | v1.0.1 58 | ====== 59 | 60 | * Fix support for pytest 4.1. 61 | 62 | v1.0 63 | ===== 64 | 65 | * just a takeout of pytest-xdist 66 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | # if you change the envlist, please update .travis.yml file as well 3 | minversion = 3.7.0 4 | isolated_build = true 5 | envlist= 6 | py{37,38,39,310,311}-pytest{310,46,54,62,latest} 7 | flakes 8 | build-dists 9 | metadata-validation 10 | 11 | [testenv] 12 | deps = 13 | pycmd 14 | # to avoid .eggs 15 | setuptools_scm 16 | pytest310: pytest~=3.10 17 | pytest46: pytest~=4.6 18 | pytest54: pytest~=5.4 19 | pytest62: pytest~=6.2 20 | pytestlatest: pytest 21 | pytestmain: git+https://github.com/pytest-dev/pytest.git@main 22 | platform=linux|darwin 23 | commands= 24 | pytest {posargs} 25 | 26 | [testenv:flakes] 27 | changedir= 28 | deps = flake8 29 | commands = flake8 setup.py testing src/pytest_forked/ 30 | 31 | [testenv:build-dists] 32 | basepython = python3 33 | isolated_build = true 34 | # `usedevelop = true` overrides `skip_install` instruction, it's unwanted 35 | usedevelop = false 36 | # don't install pytest-forked itself in this env 37 | skip_install = true 38 | deps = 39 | pep517 >= 0.7.0 40 | commands = 41 | rm -rfv {toxinidir}/dist/ 42 | {envpython} -m pep517.build \ 43 | --source \ 44 | --binary \ 45 | --out-dir {toxinidir}/dist/ \ 46 | {toxinidir} 47 | whitelist_externals = 48 | rm 49 | 50 | [testenv:metadata-validation] 51 | description = 52 | Verify that dists under the dist/ dir have valid metadata 53 | depends = 54 | build-dists 55 | deps = 56 | twine 57 | usedevelop = false 58 | skip_install = true 59 | commands = 60 | twine check {toxinidir}/dist/* 61 | 62 | [pytest] 63 | addopts = -rsfxX 64 | 65 | [flake8] 66 | max-line-length = 88 67 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="pytest-forked", 5 | use_scm_version=True, 6 | description="run tests in isolated forked subprocesses", 7 | long_description=open("README.rst").read(), 8 | long_description_content_type="text/x-rst", 9 | license="MIT", 10 | author="pytest-dev", 11 | author_email="pytest-dev@python.org", 12 | url="https://github.com/pytest-dev/pytest-forked", 13 | platforms=["linux", "osx"], 14 | packages=["pytest_forked"], 15 | package_dir={"": "src"}, 16 | entry_points={ 17 | "pytest11": [ 18 | "pytest_forked = pytest_forked", 19 | ], 20 | }, 21 | zip_safe=False, 22 | install_requires=["py", "pytest>=3.10"], 23 | setup_requires=["setuptools_scm"], 24 | python_requires=">=3.7", 25 | classifiers=[ 26 | "Development Status :: 7 - Inactive", 27 | "Framework :: Pytest", 28 | "Intended Audience :: Developers", 29 | "License :: OSI Approved :: MIT License", 30 | "Operating System :: POSIX", 31 | "Operating System :: MacOS :: MacOS X", 32 | "Topic :: Software Development :: Testing", 33 | "Topic :: Software Development :: Quality Assurance", 34 | "Topic :: Utilities", 35 | "Programming Language :: Python", 36 | "Programming Language :: Python :: 3", 37 | "Programming Language :: Python :: 3.7", 38 | "Programming Language :: Python :: 3.8", 39 | "Programming Language :: Python :: 3.9", 40 | "Programming Language :: Python :: 3.10", 41 | "Programming Language :: Python :: 3.11", 42 | "Programming Language :: Python :: 3 :: Only", 43 | ], 44 | ) 45 | -------------------------------------------------------------------------------- /testing/test_boxed.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | needsfork = pytest.mark.skipif(not hasattr(os, "fork"), reason="os.fork required") 6 | 7 | 8 | @needsfork 9 | def test_functional_boxed(testdir): 10 | p1 = testdir.makepyfile( 11 | """ 12 | import os 13 | def test_function(): 14 | os.kill(os.getpid(), 15) 15 | """ 16 | ) 17 | result = testdir.runpytest(p1, "--forked") 18 | result.stdout.fnmatch_lines(["*CRASHED*", "*1 failed*"]) 19 | 20 | 21 | @needsfork 22 | def test_functional_boxed_per_test(testdir): 23 | p1 = testdir.makepyfile( 24 | """ 25 | import os 26 | import pytest 27 | 28 | @pytest.mark.forked 29 | def test_function(): 30 | os.kill(os.getpid(), 15) 31 | """ 32 | ) 33 | result = testdir.runpytest(p1) 34 | result.stdout.fnmatch_lines(["*CRASHED*", "*1 failed*"]) 35 | 36 | 37 | @needsfork 38 | @pytest.mark.parametrize( 39 | "capmode", 40 | [ 41 | "no", 42 | pytest.param("sys", marks=pytest.mark.xfail(reason="capture cleanup needed")), 43 | pytest.param("fd", marks=pytest.mark.xfail(reason="capture cleanup needed")), 44 | ], 45 | ) 46 | def test_functional_boxed_capturing(testdir, capmode): 47 | p1 = testdir.makepyfile( 48 | """ 49 | import os 50 | import sys 51 | def test_function(): 52 | sys.stdout.write("hello\\n") 53 | sys.stderr.write("world\\n") 54 | os.kill(os.getpid(), 15) 55 | """ 56 | ) 57 | result = testdir.runpytest(p1, "--forked", "--capture=%s" % capmode) 58 | result.stdout.fnmatch_lines( 59 | """ 60 | *CRASHED* 61 | *stdout* 62 | hello 63 | *stderr* 64 | world 65 | *1 failed* 66 | """ 67 | ) 68 | 69 | 70 | def test_is_not_boxed_by_default(testdir): 71 | config = testdir.parseconfig(testdir.tmpdir) 72 | assert not config.option.forked 73 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | pytest-forked: run each test in a forked subprocess 2 | ==================================================== 3 | 4 | 5 | .. warning:: 6 | 7 | this is a extraction of the xdist --forked module, 8 | future maintenance beyond the bare minimum is not planned until a new maintainer is found. 9 | 10 | 11 | This plugin **does not work on Windows** because there's no ``fork`` support. 12 | 13 | 14 | * ``--forked``: run each test in a forked 15 | subprocess to survive ``SEGFAULTS`` or otherwise dying processes. 16 | 17 | |python| |version| |ci| |pre-commit| |black| 18 | 19 | .. |version| image:: http://img.shields.io/pypi/v/pytest-forked.svg 20 | :target: https://pypi.python.org/pypi/pytest-forked 21 | 22 | .. |ci| image:: https://github.com/pytest-dev/pytest-forked/workflows/build/badge.svg 23 | :target: https://github.com/pytest-dev/pytest-forked/actions 24 | 25 | .. |python| image:: https://img.shields.io/pypi/pyversions/pytest-forked.svg 26 | :target: https://pypi.python.org/pypi/pytest-forked/ 27 | 28 | .. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg 29 | :target: https://github.com/ambv/black 30 | 31 | .. |pre-commit| image:: https://results.pre-commit.ci/badge/github/pytest-dev/pytest-forked/master.svg 32 | :target: https://results.pre-commit.ci/latest/github/pytest-dev/pytest-forked/master 33 | 34 | Installation 35 | ----------------------- 36 | 37 | Install the plugin with:: 38 | 39 | pip install pytest-forked 40 | 41 | or use the package in develope/in-place mode with 42 | a checkout of the `pytest-forked repository`_ :: 43 | 44 | pip install -e . 45 | 46 | 47 | Usage examples 48 | --------------------- 49 | 50 | If you have tests involving C or C++ libraries you might have to deal 51 | with tests crashing the process. For this case you may use the boxing 52 | options:: 53 | 54 | pytest --forked 55 | 56 | which will run each test in a subprocess and will report if a test 57 | crashed the process. You can also combine this option with 58 | running multiple processes via pytest-xdist to speed up the test run 59 | and use your CPU cores:: 60 | 61 | pytest -n3 --forked 62 | 63 | this would run 3 testing subprocesses in parallel which each 64 | create new forked subprocesses for each test. 65 | 66 | 67 | You can also fork for individual tests:: 68 | 69 | @pytest.mark.forked 70 | def test_with_leaky_state(): 71 | run_some_monkey_patches() 72 | 73 | 74 | This test will be unconditionally boxed, regardless of CLI flag. 75 | 76 | 77 | .. _`pytest-forked repository`: https://github.com/pytest-dev/pytest-forked 78 | -------------------------------------------------------------------------------- /example/boxed.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | If your testing involves C or C++ libraries you might have to deal 4 | with crashing processes. The xdist-plugin provides the ``--boxed`` option 5 | to run each test in a controlled subprocess. Here is a basic example:: 6 | 7 | # content of test_module.py 8 | 9 | import pytest 10 | import os 11 | import time 12 | 13 | # run test function 50 times with different argument 14 | @pytest.mark.parametrize("arg", range(50)) 15 | def test_func(arg): 16 | time.sleep(0.05) # each tests takes a while 17 | if arg % 19 == 0: 18 | os.kill(os.getpid(), 15) 19 | 20 | If you run this with:: 21 | 22 | $ pytest --forked 23 | =========================== test session starts ============================ 24 | platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8 25 | plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov 26 | collecting ... collected 50 items 27 | 28 | test_module.py f..................f..................f........... 29 | 30 | ================================= FAILURES ================================= 31 | _______________________________ test_func[0] _______________________________ 32 | /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15 33 | ______________________________ test_func[19] _______________________________ 34 | /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15 35 | ______________________________ test_func[38] _______________________________ 36 | /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15 37 | =================== 3 failed, 47 passed in 3.41 seconds ==================== 38 | 39 | You'll see that a couple of tests are reported as crashing, indicated 40 | by lower-case ``f`` and the respective failure summary. You can also use 41 | the xdist-provided parallelization feature to speed up your testing:: 42 | 43 | $ pytest --forked -n3 44 | =========================== test session starts ============================ 45 | platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8 46 | plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov 47 | gw0 I / gw1 I / gw2 I 48 | gw0 [50] / gw1 [50] / gw2 [50] 49 | 50 | scheduling tests via LoadScheduling 51 | ..f...............f..................f............ 52 | ================================= FAILURES ================================= 53 | _______________________________ test_func[0] _______________________________ 54 | [gw0] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python 55 | /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15 56 | ______________________________ test_func[19] _______________________________ 57 | [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python 58 | /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15 59 | ______________________________ test_func[38] _______________________________ 60 | [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python 61 | /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15 62 | =================== 3 failed, 47 passed in 2.03 seconds ==================== 63 | -------------------------------------------------------------------------------- /src/pytest_forked/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import warnings 3 | 4 | import py 5 | import pytest 6 | from _pytest import runner 7 | 8 | # we know this bit is bad, but we cant help it with the current pytest setup 9 | 10 | 11 | # copied from xdist remote 12 | def serialize_report(rep): 13 | import py 14 | 15 | d = rep.__dict__.copy() 16 | if hasattr(rep.longrepr, "toterminal"): 17 | d["longrepr"] = str(rep.longrepr) 18 | else: 19 | d["longrepr"] = rep.longrepr 20 | for name in d: 21 | if isinstance(d[name], py.path.local): 22 | d[name] = str(d[name]) 23 | elif name == "result": 24 | d[name] = None # for now 25 | return d 26 | 27 | 28 | def pytest_addoption(parser): 29 | group = parser.getgroup("forked", "forked subprocess test execution") 30 | group.addoption( 31 | "--forked", 32 | action="store_true", 33 | dest="forked", 34 | default=False, 35 | help="box each test run in a separate process (unix)", 36 | ) 37 | 38 | 39 | def pytest_load_initial_conftests(early_config, parser, args): 40 | early_config.addinivalue_line( 41 | "markers", 42 | "forked: Always fork for this test.", 43 | ) 44 | 45 | 46 | @pytest.hookimpl(tryfirst=True) 47 | def pytest_runtest_protocol(item): 48 | if item.config.getvalue("forked") or item.get_closest_marker("forked"): 49 | ihook = item.ihook 50 | ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) 51 | reports = forked_run_report(item) 52 | for rep in reports: 53 | ihook.pytest_runtest_logreport(report=rep) 54 | ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location) 55 | return True 56 | 57 | 58 | def forked_run_report(item): 59 | # for now, we run setup/teardown in the subprocess 60 | # XXX optionally allow sharing of setup/teardown 61 | from _pytest.runner import runtestprotocol 62 | 63 | EXITSTATUS_TESTEXIT = 4 64 | import marshal 65 | 66 | def runforked(): 67 | try: 68 | reports = runtestprotocol(item, log=False) 69 | except KeyboardInterrupt: 70 | os._exit(EXITSTATUS_TESTEXIT) 71 | return marshal.dumps([serialize_report(x) for x in reports]) 72 | 73 | ff = py.process.ForkedFunc(runforked) 74 | result = ff.waitfinish() 75 | if result.retval is not None: 76 | report_dumps = marshal.loads(result.retval) 77 | return [runner.TestReport(**x) for x in report_dumps] 78 | else: 79 | if result.exitstatus == EXITSTATUS_TESTEXIT: 80 | pytest.exit(f"forked test item {item} raised Exit") 81 | return [report_process_crash(item, result)] 82 | 83 | 84 | def report_process_crash(item, result): 85 | from _pytest._code import getfslineno 86 | 87 | path, lineno = getfslineno(item) 88 | info = "%s:%s: running the test CRASHED with signal %d" % ( 89 | path, 90 | lineno, 91 | result.signal, 92 | ) 93 | from _pytest import runner 94 | 95 | # pytest >= 4.1 96 | has_from_call = getattr(runner.CallInfo, "from_call", None) is not None 97 | if has_from_call: 98 | call = runner.CallInfo.from_call(lambda: 0 / 0, "???") 99 | else: 100 | call = runner.CallInfo(lambda: 0 / 0, "???") 101 | call.excinfo = info 102 | rep = runner.pytest_runtest_makereport(item, call) 103 | if result.out: 104 | rep.sections.append(("captured stdout", result.out)) 105 | if result.err: 106 | rep.sections.append(("captured stderr", result.err)) 107 | 108 | xfail_marker = item.get_closest_marker("xfail") 109 | if not xfail_marker: 110 | return rep 111 | 112 | rep.outcome = "skipped" 113 | rep.wasxfail = ( 114 | "reason: {xfail_reason}; " 115 | "pytest-forked reason: {crash_info}".format( 116 | xfail_reason=xfail_marker.kwargs["reason"], 117 | crash_info=info, 118 | ) 119 | ) 120 | warnings.warn( 121 | "pytest-forked xfail support is incomplete at the moment and may " 122 | "output a misleading reason message", 123 | RuntimeWarning, 124 | ) 125 | 126 | return rep 127 | -------------------------------------------------------------------------------- /testing/test_xfail_behavior.py: -------------------------------------------------------------------------------- 1 | """Tests for xfail support.""" 2 | import os 3 | import signal 4 | 5 | import pytest 6 | 7 | IS_PYTEST4_PLUS = int(pytest.__version__[0]) >= 4 # noqa: WPS609 8 | FAILED_WORD = "FAILED" if IS_PYTEST4_PLUS else "FAIL" 9 | PYTEST_GTE_7_2 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 2) # type: ignore[attr-defined] 10 | PYTEST_GTE_8_0 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (8, 0) # type: ignore[attr-defined] 11 | 12 | pytestmark = pytest.mark.skipif( # pylint: disable=invalid-name 13 | not hasattr(os, "fork"), # noqa: WPS421 14 | reason="os.fork required", 15 | ) 16 | 17 | 18 | @pytest.mark.parametrize( 19 | ("is_crashing", "is_strict"), 20 | ( 21 | pytest.param(True, True, id="strict xfail"), 22 | pytest.param(False, True, id="strict xpass"), 23 | pytest.param(True, False, id="non-strict xfail"), 24 | pytest.param(False, False, id="non-strict xpass"), 25 | ), 26 | ) 27 | def test_xfail(is_crashing, is_strict, testdir): 28 | """Test xfail/xpass/strict permutations.""" 29 | # pylint: disable=possibly-unused-variable 30 | sig_num = signal.SIGTERM.numerator 31 | 32 | test_func_body = ( 33 | "os.kill(os.getpid(), signal.SIGTERM)" if is_crashing else "assert True" 34 | ) 35 | 36 | if is_crashing: 37 | # marked xfailed and crashing, no matter strict or not 38 | expected_letter = "x" # XFAILED 39 | expected_lowercase = "xfailed" 40 | expected_word = "XFAIL" 41 | elif is_strict: 42 | # strict and not failing as expected should cause failure 43 | expected_letter = "F" # FAILED 44 | expected_lowercase = "failed" 45 | expected_word = FAILED_WORD 46 | elif not is_strict: 47 | # non-strict and not failing as expected should cause xpass 48 | expected_letter = "X" # XPASS 49 | expected_lowercase = "xpassed" 50 | expected_word = "XPASS" 51 | 52 | session_start_title = "*==== test session starts ====*" 53 | loaded_pytest_plugins = "plugins:* forked*" 54 | collected_tests_num = "collected 1 item" 55 | expected_progress = f"test_xfail.py {expected_letter!s}*" 56 | failures_title = "*==== FAILURES ====*" 57 | failures_test_name = "*____ test_function ____*" 58 | failures_test_reason = "[XPASS(strict)] The process gets terminated" 59 | short_test_summary_title = "*==== short test summary info ====*" 60 | short_test_summary = f"{expected_word!s} test_xfail.py::test_function" 61 | if expected_lowercase == "xpassed": 62 | # XPASS wouldn't have the crash message from 63 | # pytest-forked because the crash doesn't happen 64 | if PYTEST_GTE_8_0: 65 | short_test_summary += " -" 66 | short_test_summary += " The process gets terminated" 67 | 68 | reason_string = ( 69 | f"reason: The process gets terminated; " 70 | f"pytest-forked reason: " 71 | f"*:*: running the test CRASHED with signal {sig_num:d}" 72 | ) 73 | if expected_lowercase == "xfailed" and PYTEST_GTE_7_2: 74 | short_test_summary += " - " + reason_string 75 | total_summary_line = f"*==== 1 {expected_lowercase!s} in 0.*s* ====*" 76 | 77 | expected_lines = ( 78 | session_start_title, 79 | loaded_pytest_plugins, 80 | collected_tests_num, 81 | expected_progress, 82 | ) 83 | if expected_word == FAILED_WORD: 84 | # XPASS(strict) 85 | expected_lines += ( 86 | failures_title, 87 | failures_test_name, 88 | failures_test_reason, 89 | ) 90 | expected_lines += ( 91 | short_test_summary_title, 92 | short_test_summary, 93 | ) 94 | if expected_lowercase == "xpassed" and expected_word == FAILED_WORD: 95 | # XPASS(strict) 96 | expected_lines += (" " + reason_string,) 97 | expected_lines += (total_summary_line,) 98 | 99 | test_module = testdir.makepyfile( 100 | f""" 101 | import os 102 | import signal 103 | 104 | import pytest 105 | 106 | # The current implementation emits RuntimeWarning. 107 | pytestmark = pytest.mark.filterwarnings('ignore:pytest-forked xfail') 108 | 109 | @pytest.mark.xfail( 110 | reason='The process gets terminated', 111 | strict={is_strict!s}, 112 | ) 113 | @pytest.mark.forked 114 | def test_function(): 115 | {test_func_body!s} 116 | """ 117 | ) 118 | 119 | pytest_run_result = testdir.runpytest(test_module, "-ra") 120 | pytest_run_result.stdout.fnmatch_lines(expected_lines) 121 | --------------------------------------------------------------------------------