├── tests ├── __init__.py ├── conftest.py └── detect_test_pollution_test.py ├── .gitignore ├── requirements-dev.txt ├── setup.py ├── .github └── workflows │ └── main.yml ├── tox.ini ├── LICENSE ├── setup.cfg ├── .pre-commit-config.yaml ├── README.md └── detect_test_pollution.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | /.coverage 4 | /.tox 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | covdefaults>=2.1 2 | coverage 3 | pytest 4 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | pytest_plugins = 'pytester' 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from setuptools import setup 4 | setup() 5 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: main 2 | 3 | on: 4 | push: 5 | branches: [main, test-me-*] 6 | tags: '*' 7 | pull_request: 8 | 9 | jobs: 10 | main: 11 | uses: asottile/workflows/.github/workflows/tox.yml@v1.8.1 12 | with: 13 | env: '["py310", "py311", "py312"]' 14 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py,pre-commit 3 | 4 | [testenv] 5 | deps = -rrequirements-dev.txt 6 | commands = 7 | coverage erase 8 | coverage run -m pytest {posargs:tests} 9 | coverage report 10 | 11 | [testenv:pre-commit] 12 | skip_install = true 13 | deps = pre-commit 14 | commands = pre-commit run --all-files --show-diff-on-failure 15 | 16 | [pep8] 17 | ignore = E265,E501,W504 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Anthony Sottile 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = detect_test_pollution 3 | version = 1.2.0 4 | description = a tool to detect test pollution 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | url = https://github.com/asottile/detect-test-pollution 8 | author = Anthony Sottile 9 | author_email = asottile@umich.edu 10 | license = MIT 11 | license_files = LICENSE 12 | classifiers = 13 | Programming Language :: Python :: 3 14 | Programming Language :: Python :: 3 :: Only 15 | Programming Language :: Python :: Implementation :: CPython 16 | Programming Language :: Python :: Implementation :: PyPy 17 | 18 | [options] 19 | py_modules = detect_test_pollution 20 | python_requires = >=3.10 21 | 22 | [options.entry_points] 23 | console_scripts = 24 | detect-test-pollution = detect_test_pollution:main 25 | 26 | [bdist_wheel] 27 | universal = True 28 | 29 | [coverage:run] 30 | plugins = covdefaults 31 | 32 | [mypy] 33 | check_untyped_defs = true 34 | disallow_any_generics = true 35 | disallow_incomplete_defs = true 36 | disallow_untyped_defs = true 37 | warn_redundant_casts = true 38 | warn_unused_ignores = true 39 | 40 | [mypy-testing.*] 41 | disallow_untyped_defs = false 42 | 43 | [mypy-tests.*] 44 | disallow_untyped_defs = false 45 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v6.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: debug-statements 9 | - id: double-quote-string-fixer 10 | - id: name-tests-test 11 | - id: requirements-txt-fixer 12 | - repo: https://github.com/asottile/setup-cfg-fmt 13 | rev: v3.1.0 14 | hooks: 15 | - id: setup-cfg-fmt 16 | - repo: https://github.com/asottile/reorder-python-imports 17 | rev: v3.16.0 18 | hooks: 19 | - id: reorder-python-imports 20 | args: [--py310-plus, --add-import, 'from __future__ import annotations'] 21 | - repo: https://github.com/asottile/add-trailing-comma 22 | rev: v4.0.0 23 | hooks: 24 | - id: add-trailing-comma 25 | - repo: https://github.com/asottile/pyupgrade 26 | rev: v3.21.2 27 | hooks: 28 | - id: pyupgrade 29 | args: [--py310-plus] 30 | - repo: https://github.com/hhatto/autopep8 31 | rev: v2.3.2 32 | hooks: 33 | - id: autopep8 34 | - repo: https://github.com/PyCQA/flake8 35 | rev: 7.3.0 36 | hooks: 37 | - id: flake8 38 | - repo: https://github.com/pre-commit/mirrors-mypy 39 | rev: v1.19.1 40 | hooks: 41 | - id: mypy 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![build status](https://github.com/asottile/detect-test-pollution/actions/workflows/main.yml/badge.svg)](https://github.com/asottile/detect-test-pollution/actions/workflows/main.yml) 2 | [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/asottile/detect-test-pollution/main.svg)](https://results.pre-commit.ci/latest/github/asottile/detect-test-pollution/main) 3 | 4 | detect-test-pollution 5 | ===================== 6 | 7 | a tool to detect test pollution 8 | 9 | ## installation 10 | 11 | ```bash 12 | pip install detect-test-pollution 13 | ``` 14 | 15 | ## what is test pollution? 16 | 17 | [![video about test pollution](https://i.ytimg.com/vi/FRteianaPMo/mqdefault.jpg)](https://youtu.be/FRteianaPMo) 18 | 19 | test pollution is where a test fails due to the side-effects of some other test 20 | in the test suite. 21 | 22 | it usually appears as a "test flake" something where the test fails 23 | mysteriously but passes when run by itself. 24 | 25 | a simple example of this is the following python code: 26 | 27 | ```python 28 | k = 1 29 | 30 | def test_k(): 31 | assert k == 1 32 | 33 | def test_k2(): 34 | global k 35 | 36 | k = 2 37 | assert k == 2 38 | ``` 39 | 40 | now this example is a little bit silly, you probably wouldn't write code this 41 | poorly but helps us demonstrate the problem here. 42 | 43 | when run normally -- these tests pass: 44 | 45 | ```console 46 | $ pytest -q t.py 47 | .. [100%] 48 | 2 passed in 0.00s 49 | ``` 50 | 51 | but, if the tests were run in some other order (due to something like 52 | [pytest-randomly] or [pytest-xdist]) then the pollution would be apparent: 53 | 54 | ```console 55 | $ pytest -q t.py::test_k2 t.py::test_k 56 | .F [100%] 57 | =================================== FAILURES =================================== 58 | ____________________________________ test_k ____________________________________ 59 | 60 | def test_k(): 61 | > assert k == 1 62 | E assert 2 == 1 63 | 64 | t.py:4: AssertionError 65 | =========================== short test summary info ============================ 66 | FAILED t.py::test_k - assert 2 == 1 67 | 1 failed, 1 passed in 0.03s 68 | ``` 69 | 70 | often this flake happens in a codebase with hundreds or thousands of tests 71 | and it's difficult to track down which test is causing the global side-effects. 72 | 73 | that's where this tool comes in handy! it helps you find the pair of tests 74 | which error when run in order. 75 | 76 | [pytest-randomly]: https://github.com/pytest-dev/pytest-randomly 77 | [pytest-xdist]: https://github.com/pytest-dev/pytest-xdist 78 | 79 | ## usage 80 | 81 | [![video about using detect-test-pollution](https://i.ytimg.com/vi/w5O4zTusyJ0/mqdefault.jpg)](https://www.youtube.com/watch?v=w5O4zTusyJ0) 82 | 83 | once you have identified a failing test, you'll be able to feed it into 84 | `detect-test-pollution` to find the causal test. 85 | 86 | the basic mode is to run: 87 | 88 | ```bash 89 | detect-test-pollution \ 90 | --failing-test test.py::test_id_here \ 91 | --tests ./tests 92 | ``` 93 | 94 | where `test.py::test_id_here` is the identifier of the failing test and 95 | `./tests` is the directory where your testsuite lives. 96 | 97 | if you've already narrowed down the list of testids further than that, you 98 | can specify a `--testids-file` instead of `--tests` to speed up discovery: 99 | 100 | ```bash 101 | detect-test-pollution \ 102 | --failing-test test.py::test_id_here \ 103 | --testids-file ./testids 104 | ``` 105 | 106 | you can usually get a list of testids via `pytest --collect-only -q` (though 107 | you'll need to strip some unrelated lines at the end, such as timing and 108 | warning info). 109 | 110 | then `detect-test-pollution` will bisect the list of tests to find the failing 111 | one. here's an example bisection from a [bug in pytest] 112 | 113 | ```console 114 | $ detect-test-pollution --tests ./testing --failing-test testing/io/test_terminalwriter.py::test_should_do_markup_FORCE_COLOR 115 | discovering all tests... 116 | -> discovered 3140 tests! 117 | ensuring test passes by itself... 118 | -> OK! 119 | ensuring test fails with test group... 120 | -> OK! 121 | running step 1: 122 | - 3139 tests remaining (about 12 steps) 123 | running step 2: 124 | - 1570 tests remaining (about 11 steps) 125 | running step 3: 126 | - 785 tests remaining (about 10 steps) 127 | running step 4: 128 | - 393 tests remaining (about 9 steps) 129 | running step 5: 130 | - 197 tests remaining (about 8 steps) 131 | running step 6: 132 | - 99 tests remaining (about 7 steps) 133 | running step 7: 134 | - 50 tests remaining (about 6 steps) 135 | running step 8: 136 | - 25 tests remaining (about 5 steps) 137 | running step 9: 138 | - 12 tests remaining (about 4 steps) 139 | running step 10: 140 | - 6 tests remaining (about 3 steps) 141 | running step 11: 142 | - 3 tests remaining (about 2 steps) 143 | double checking we found it... 144 | -> the polluting test is: testing/test_terminal.py::TestTerminal::test_report_teststatus_explicit_markup 145 | ``` 146 | 147 | [bug in pytest]: https://github.com/pytest-dev/pytest/issues/9708 148 | 149 | ## fuzzing 150 | 151 | `detect-test-pollution` can also be used to "fuzz" out failing tests. 152 | 153 | it does this by shuffling the test ids and running the testsuite until it 154 | fails. 155 | 156 | here's an example execution on a silly testsuite: 157 | 158 | ```console 159 | $ detect-test-pollution --fuzz --tests t.py 160 | discovering all tests... 161 | -> discovered 1002 tests! 162 | run 1... 163 | -> OK! 164 | run 2... 165 | -> found failing test! 166 | try `detect-test-pollution --failing-test t.py::test_k --tests t.py`! 167 | ``` 168 | 169 | afterwards you can use the normal mode of `detect-test-pollution` to find the 170 | failing pair. 171 | 172 | ## supported test runners 173 | 174 | at the moment only `pytest` is supported -- though in theory the tool could 175 | be adapted to support other python test runners, or even other languages. 176 | -------------------------------------------------------------------------------- /tests/detect_test_pollution_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | 5 | import pytest 6 | 7 | import detect_test_pollution 8 | from detect_test_pollution import _common_testpath 9 | from detect_test_pollution import _discover_tests 10 | from detect_test_pollution import _format_cmd 11 | from detect_test_pollution import _parse_testids_file 12 | from detect_test_pollution import _passed_with_testlist 13 | from detect_test_pollution import main 14 | 15 | 16 | def test_pytest_plugin_does_not_crash_when_not_enabled(pytester): 17 | res = pytester.inline_runsource( 18 | 'def test(): pass', 19 | '-p', detect_test_pollution.__name__, 20 | ) 21 | assert res.ret == 0 22 | 23 | 24 | def test_pytest_plugin_outputs_testids(tmp_path, pytester): 25 | src = '''\ 26 | import pytest 27 | 28 | @pytest.mark.parametrize('s', (1, 2, 3)) 29 | def test(s): pass 30 | ''' 31 | out = tmp_path.joinpath('testids') 32 | res = pytester.inline_runsource( 33 | src, 34 | '--collect-only', '-q', 35 | '-p', detect_test_pollution.__name__, 36 | # use `=` to avoid pytest's basedir detection 37 | f'--dtp-testids-output-file={out}', 38 | ) 39 | assert res.ret == 0 40 | 41 | assert out.read_text() == '''\ 42 | test_pytest_plugin_outputs_testids.py::test[1] 43 | test_pytest_plugin_outputs_testids.py::test[2] 44 | test_pytest_plugin_outputs_testids.py::test[3] 45 | ''' 46 | 47 | 48 | def test_pytest_plugin_inputs_testids(tmp_path, pytester): 49 | src = '''\ 50 | import pytest 51 | 52 | @pytest.mark.parametrize('s', (1, 2, 3)) 53 | def test(s): pass 54 | ''' 55 | inp = tmp_path.joinpath('testids') 56 | inp.write_text('test_pytest_plugin_inputs_testids.py::test[1]') 57 | res = pytester.inline_runsource( 58 | src, 59 | '-p', detect_test_pollution.__name__, 60 | # use `=` to avoid pytest's basedir detection 61 | f'--dtp-testids-input-file={inp}', 62 | ) 63 | assert res.ret == 0 64 | passed, failed, skipped = res.listoutcomes() 65 | assert len(passed) == 1 66 | assert len(failed) == 0 67 | assert len(skipped) == 0 68 | 69 | 70 | def test_pytest_plugin_results_output(tmp_path, pytester): 71 | src = ''' 72 | def test1(): assert False 73 | def test2(): pass 74 | ''' 75 | out = tmp_path.joinpath('out.json') 76 | res = pytester.inline_runsource( 77 | src, 78 | '-p', detect_test_pollution.__name__, 79 | # use `=` to avoid pytest's basedir detection 80 | f'--dtp-results-output-file={out}', 81 | ) 82 | assert res.ret == 1 83 | 84 | with open(out) as f: 85 | contents = json.load(f) 86 | 87 | assert contents == { 88 | 'test_pytest_plugin_results_output.py::test1': False, 89 | 'test_pytest_plugin_results_output.py::test2': True, 90 | } 91 | 92 | 93 | def test_pytest_plugin_results_output_error(tmp_path, pytester): 94 | src = '''\ 95 | import pytest 96 | 97 | def test1(): pass 98 | 99 | @pytest.fixture 100 | def e(): assert False 101 | def test2(e): pass 102 | ''' 103 | 104 | out = tmp_path.joinpath('out.json') 105 | res = pytester.inline_runsource( 106 | src, 107 | '-p', detect_test_pollution.__name__, 108 | # use `=` to avoid pytest's basedir detection 109 | f'--dtp-results-output-file={out}', 110 | ) 111 | assert res.ret == 1 112 | 113 | with open(out) as f: 114 | contents = json.load(f) 115 | 116 | assert contents == { 117 | 'test_pytest_plugin_results_output_error.py::test1': True, 118 | 'test_pytest_plugin_results_output_error.py::test2': False, 119 | } 120 | 121 | 122 | def test_parse_testids_file(tmp_path): 123 | f = tmp_path.joinpath('t.json') 124 | f.write_text('test.py::test1\ntest.py::test2') 125 | 126 | assert _parse_testids_file(f) == ['test.py::test1', 'test.py::test2'] 127 | 128 | 129 | def test_parse_testids_file_blank_line(tmp_path): 130 | f = tmp_path.joinpath('t.json') 131 | f.write_text('test.py::test1\n\ntest.py::test2') 132 | 133 | assert _parse_testids_file(f) == ['test.py::test1', 'test.py::test2'] 134 | 135 | 136 | def test_discover_tests(tmp_path): 137 | f = tmp_path.joinpath('t.py') 138 | f.write_text('def test_one(): pass\ndef test_two(): pass\n') 139 | 140 | assert _discover_tests(f) == ['t.py::test_one', 't.py::test_two'] 141 | 142 | 143 | @pytest.mark.parametrize( 144 | ('inputs', 'expected'), 145 | ( 146 | ([], '.'), 147 | (['a', 'a/b'], 'a'), 148 | (['a', 'b'], '.'), 149 | (['a/b/c', 'a/b/d', 'a/b/e'], 'a/b'), 150 | (['a/b/c', 'a/b/c'], 'a/b/c'), 151 | ), 152 | ) 153 | def test_common_testpath(inputs, expected): 154 | assert _common_testpath(inputs) == expected 155 | 156 | 157 | def test_passed_with_testlist_failing(tmp_path): 158 | f = tmp_path.joinpath('t.py') 159 | f.write_text('def test1(): pass\ndef test2(): assert False\n') 160 | assert _passed_with_testlist(f, 't.py::test2', ['t.py::test1']) is False 161 | 162 | 163 | def test_passed_with_testlist_passing(tmp_path): 164 | f = tmp_path.joinpath('t.py') 165 | f.write_text('def test1(): pass\ndef test2(): pass\n') 166 | assert _passed_with_testlist(f, 't.py::test2', ['t.py::test1']) is True 167 | 168 | 169 | def test_format_cmd_with_tests(): 170 | ret = _format_cmd('t.py::test1', 'this t.py', None) 171 | assert ret == ( 172 | 'detect-test-pollution --failing-test t.py::test1 ' 173 | "--tests 'this t.py'" 174 | ) 175 | 176 | 177 | def test_format_cmd_with_testids_filename(): 178 | ret = _format_cmd('t.py::test1', None, 't.txt') 179 | assert ret == ( 180 | 'detect-test-pollution --failing-test t.py::test1 ' 181 | '--testids-filename t.txt' 182 | ) 183 | 184 | 185 | def test_integration_missing_failing_test(tmpdir, capsys): 186 | f = tmpdir.join('t.py') 187 | f.write('def test1(): pass') 188 | 189 | with tmpdir.as_cwd(): 190 | ret = main(('--tests', str(f), '--failing-test', 't.py::test2')) 191 | assert ret == 1 192 | 193 | out, _ = capsys.readouterr() 194 | assert out == '''\ 195 | discovering all tests... 196 | -> discovered 1 tests! 197 | -> failing test was not part of discovered tests! 198 | ''' 199 | 200 | 201 | def test_integration_test_does_not_pass_by_itself(tmpdir, capsys): 202 | f = tmpdir.join('t.py') 203 | f.write('def test1(): pass\ndef test2(): assert False') 204 | 205 | with tmpdir.as_cwd(): 206 | ret = main(('--tests', str(f), '--failing-test', 't.py::test2')) 207 | assert ret == 1 208 | 209 | out, _ = capsys.readouterr() 210 | assert out == '''\ 211 | discovering all tests... 212 | -> discovered 2 tests! 213 | ensuring test passes by itself... 214 | -> test failed! (output printed above) 215 | ''' 216 | 217 | 218 | def test_integration_does_not_fail_with_all_tests(tmpdir, capsys): 219 | f = tmpdir.join('t.py') 220 | f.write('def test1(): pass\ndef test2(): pass') 221 | 222 | with tmpdir.as_cwd(): 223 | ret = main(('--tests', str(f), '--failing-test', 't.py::test2')) 224 | assert ret == 1 225 | 226 | out, _ = capsys.readouterr() 227 | assert out == '''\ 228 | discovering all tests... 229 | -> discovered 2 tests! 230 | ensuring test passes by itself... 231 | -> OK! 232 | ensuring test fails with test group... 233 | -> expected failure -- but it passed? 234 | ''' 235 | 236 | 237 | def test_integration_finds_pollution(tmpdir, capsys): 238 | src = '''\ 239 | k = 1 240 | 241 | def test_other(): 242 | pass 243 | 244 | def test_other2(): 245 | pass 246 | 247 | def test_k(): 248 | assert k == 1 249 | 250 | def test_k2(): 251 | global k 252 | k = 2 253 | assert k == 2 254 | ''' 255 | f = tmpdir.join('t.py') 256 | f.write(src) 257 | 258 | with tmpdir.as_cwd(): 259 | ret = main(('--tests', str(f), '--failing-test', 't.py::test_k')) 260 | assert ret == 0 261 | 262 | out, _ = capsys.readouterr() 263 | assert out == '''\ 264 | discovering all tests... 265 | -> discovered 4 tests! 266 | ensuring test passes by itself... 267 | -> OK! 268 | ensuring test fails with test group... 269 | -> OK! 270 | running step 1: 271 | - 3 tests remaining (about 2 steps) 272 | running step 2: 273 | - 2 tests remaining (about 1 steps) 274 | double checking we found it... 275 | -> the polluting test is: t.py::test_k2 276 | ''' 277 | 278 | 279 | def test_integration_pre_supplied_test_list(tmpdir, capsys): 280 | src = '''\ 281 | k = 1 282 | 283 | def test_other(): 284 | pass 285 | 286 | def test_other2(): 287 | pass 288 | 289 | def test_k(): 290 | assert k == 1 291 | 292 | def test_k2(): 293 | global k 294 | k = 2 295 | assert k == 2 296 | ''' 297 | testlist = tmpdir.join('testlist') 298 | testlist.write( 299 | 't.py::test_k\n' 300 | 't.py::test_k2\n' 301 | 't.py::test_other\n', 302 | ) 303 | f = tmpdir.join('t.py') 304 | f.write(src) 305 | 306 | with tmpdir.as_cwd(): 307 | ret = main(( 308 | '--testids-file', str(testlist), 309 | '--failing-test', 't.py::test_k', 310 | )) 311 | assert ret == 0 312 | 313 | out, _ = capsys.readouterr() 314 | assert out == '''\ 315 | discovering all tests... 316 | -> pre-discovered 3 tests! 317 | ensuring test passes by itself... 318 | -> OK! 319 | ensuring test fails with test group... 320 | -> OK! 321 | running step 1: 322 | - 2 tests remaining (about 1 steps) 323 | double checking we found it... 324 | -> the polluting test is: t.py::test_k2 325 | ''' 326 | 327 | 328 | def test_integration_fuzz(tmpdir, capsys): 329 | src = '''\ 330 | k = 1 331 | 332 | def test_other(): 333 | pass 334 | 335 | def test_other2(): 336 | pass 337 | 338 | def test_k(): 339 | assert k == 1 340 | 341 | def test_k2(): 342 | global k 343 | k = 2 344 | assert k == 2 345 | ''' 346 | 347 | f = tmpdir.join('t.py') 348 | f.write(src) 349 | 350 | with tmpdir.as_cwd(): 351 | ret = main(('--fuzz', '--tests', str(f))) 352 | assert ret == 1 353 | 354 | out, err = capsys.readouterr() 355 | assert out == f'''\ 356 | discovering all tests... 357 | -> discovered 4 tests! 358 | run 1... 359 | -> OK! 360 | run 2... 361 | -> found failing test! 362 | try `detect-test-pollution --failing-test t.py::test_k --tests {f}`! 363 | ''' 364 | -------------------------------------------------------------------------------- /detect_test_pollution.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import argparse 4 | import contextlib 5 | import json 6 | import math 7 | import os.path 8 | import random 9 | import shlex 10 | import subprocess 11 | import sys 12 | import tempfile 13 | from collections.abc import Sequence 14 | 15 | import pytest 16 | 17 | TESTIDS_INPUT_OPTION = '--dtp-testids-input-file' 18 | TESTIDS_OUTPUT_OPTION = '--dtp-testids-output-file' 19 | RESULTS_OUTPUT_OPTION = '--dtp-results-output-file' 20 | PYTEST_OPTIONS = ( 21 | '-p', __name__, 22 | # disable known test-randomization plugins 23 | '-p', 'no:randomly', 24 | # we don't read the output at all 25 | '--quiet', '--quiet', 26 | ) 27 | 28 | 29 | def pytest_addoption(parser: pytest.Parser) -> None: 30 | parser.addoption(TESTIDS_INPUT_OPTION) 31 | parser.addoption(TESTIDS_OUTPUT_OPTION) 32 | parser.addoption(RESULTS_OUTPUT_OPTION) 33 | 34 | 35 | def pytest_collection_modifyitems( 36 | config: pytest.Config, 37 | items: list[pytest.Item], 38 | ) -> None: 39 | read_option = config.getoption(TESTIDS_INPUT_OPTION) 40 | write_option = config.getoption(TESTIDS_OUTPUT_OPTION) 41 | if read_option is not None: 42 | by_id = {item.nodeid: item for item in items} 43 | testids = _parse_testids_file(read_option) 44 | items[:] = [by_id[testid] for testid in testids] 45 | elif write_option is not None: 46 | with open(write_option, 'w', encoding='UTF-8') as f: 47 | for item in items: 48 | f.write(f'{item.nodeid}\n') 49 | 50 | 51 | class CollectResults: 52 | def __init__(self, filename: str) -> None: 53 | self.filename = filename 54 | self.results: dict[str, bool] = {} 55 | 56 | def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: 57 | if report.when == 'call': 58 | self.results[report.nodeid] = report.outcome == 'passed' 59 | elif report.outcome == 'failed': 60 | self.results[report.nodeid] = False 61 | 62 | def pytest_terminal_summary(self, config: pytest.Config) -> None: 63 | with open(self.filename, 'w') as f: 64 | f.write(json.dumps(self.results, indent=2)) 65 | 66 | def pytest_unconfigure(self, config: pytest.Config) -> None: 67 | config.pluginmanager.unregister(self) 68 | 69 | 70 | def pytest_configure(config: pytest.Config) -> None: 71 | results_filename = config.getoption(RESULTS_OUTPUT_OPTION) 72 | if results_filename is not None: 73 | config.pluginmanager.register(CollectResults(results_filename)) 74 | 75 | 76 | def _run_pytest(*args: str) -> None: 77 | # XXX: this is potentially difficult to debug? maybe --verbose? 78 | subprocess.check_call( 79 | (sys.executable, '-mpytest', *PYTEST_OPTIONS, *args), 80 | stdout=subprocess.DEVNULL, 81 | ) 82 | 83 | 84 | def _parse_testids_file(filename: str) -> list[str]: 85 | with open(filename) as f: 86 | return [line for line in f.read().splitlines() if line] 87 | 88 | 89 | def _discover_tests(path: str) -> list[str]: 90 | with tempfile.TemporaryDirectory() as tmpdir: 91 | testids_filename = os.path.join(tmpdir, 'testids.txt') 92 | _run_pytest( 93 | path, 94 | # use `=` to avoid pytest's basedir detection 95 | f'{TESTIDS_OUTPUT_OPTION}={testids_filename}', 96 | '--collect-only', 97 | ) 98 | 99 | return _parse_testids_file(testids_filename) 100 | 101 | 102 | def _common_testpath(testids: list[str]) -> str: 103 | paths = [testid.split('::')[0] for testid in testids] 104 | if not paths: 105 | return '.' 106 | else: 107 | return os.path.commonpath(paths) or '.' 108 | 109 | 110 | def _passed_with_testlist(path: str, test: str, testids: list[str]) -> bool: 111 | with tempfile.TemporaryDirectory() as tmpdir: 112 | testids_filename = os.path.join(tmpdir, 'testids.txt') 113 | with open(testids_filename, 'w') as f: 114 | for testid in testids: 115 | f.write(f'{testid}\n') 116 | f.write(f'{test}\n') 117 | 118 | results_json = os.path.join(tmpdir, 'results.json') 119 | 120 | with contextlib.suppress(subprocess.CalledProcessError): 121 | _run_pytest( 122 | path, 123 | # use `=` to avoid pytest's basedir detection 124 | f'{TESTIDS_INPUT_OPTION}={testids_filename}', 125 | f'{RESULTS_OUTPUT_OPTION}={results_json}', 126 | ) 127 | 128 | with open(results_json) as f: 129 | contents = json.load(f) 130 | 131 | return contents[test] 132 | 133 | 134 | def _format_cmd( 135 | victim: str, 136 | cmd_tests: str | None, 137 | cmd_testids_filename: str | None, 138 | ) -> str: 139 | args = ['detect-test-pollution', '--failing-test', victim] 140 | if cmd_tests is not None: 141 | args.extend(('--tests', cmd_tests)) 142 | elif cmd_testids_filename is not None: 143 | args.extend(('--testids-filename', cmd_testids_filename)) 144 | else: 145 | raise AssertionError('unreachable?') 146 | return shlex.join(args) 147 | 148 | 149 | def _fuzz( 150 | testpath: str, 151 | testids: list[str], 152 | cmd_tests: str | None, 153 | cmd_testids_filename: str | None, 154 | ) -> int: 155 | # make shuffling "deterministic" 156 | r = random.Random() 157 | r.seed(1542676187, version=2) 158 | 159 | with tempfile.TemporaryDirectory() as tmpdir: 160 | testids_filename = os.path.join(tmpdir, 'testids.txt') 161 | results_json = os.path.join(tmpdir, 'results.json') 162 | 163 | i = 0 164 | while True: 165 | i += 1 166 | print(f'run {i}...') 167 | 168 | r.shuffle(testids) 169 | with open(testids_filename, 'w') as f: 170 | for testid in testids: 171 | f.write(f'{testid}\n') 172 | 173 | try: 174 | _run_pytest( 175 | testpath, 176 | '--maxfail=1', 177 | # use `=` to avoid pytest's basedir detection 178 | f'{TESTIDS_INPUT_OPTION}={testids_filename}', 179 | f'{RESULTS_OUTPUT_OPTION}={results_json}', 180 | ) 181 | except subprocess.CalledProcessError: 182 | print('-> found failing test!') 183 | else: 184 | print('-> OK!') 185 | continue 186 | 187 | with open(results_json) as f: 188 | contents = json.load(f) 189 | 190 | testids = list(contents) 191 | victim = testids[-1] 192 | 193 | cmd = _format_cmd(victim, cmd_tests, cmd_testids_filename) 194 | print(f'try `{cmd}`!') 195 | return 1 196 | 197 | 198 | def _bisect(testpath: str, failing_test: str, testids: list[str]) -> int: 199 | if failing_test not in testids: 200 | print('-> failing test was not part of discovered tests!') 201 | return 1 202 | 203 | # step 2: make sure the failing test passes on its own 204 | print('ensuring test passes by itself...') 205 | if _passed_with_testlist(testpath, failing_test, []): 206 | print('-> OK!') 207 | else: 208 | print('-> test failed! (output printed above)') 209 | return 1 210 | 211 | # we'll be bisecting testids 212 | testids.remove(failing_test) 213 | 214 | # step 3: ensure test fails 215 | print('ensuring test fails with test group...') 216 | if _passed_with_testlist(testpath, failing_test, testids): 217 | print('-> expected failure -- but it passed?') 218 | return 1 219 | else: 220 | print('-> OK!') 221 | 222 | # step 4: bisect time! 223 | n = 0 224 | while len(testids) != 1: 225 | n += 1 226 | print(f'running step {n}:') 227 | n_left = len(testids) 228 | steps_s = f'(about {math.ceil(math.log(n_left, 2))} steps)' 229 | print(f'- {n_left} tests remaining {steps_s}') 230 | 231 | pivot = len(testids) // 2 232 | part1 = testids[:pivot] 233 | part2 = testids[pivot:] 234 | 235 | if _passed_with_testlist(testpath, failing_test, part1): 236 | testids = part2 237 | else: 238 | testids = part1 239 | 240 | # step 5: make sure it still fails 241 | print('double checking we found it...') 242 | if _passed_with_testlist(testpath, failing_test, testids): 243 | raise AssertionError('unreachable? unexpected pass? report a bug?') 244 | else: 245 | print(f'-> the polluting test is: {testids[0]}') 246 | return 0 247 | 248 | 249 | def main(argv: Sequence[str] | None = None) -> int: 250 | parser = argparse.ArgumentParser() 251 | 252 | mutex1 = parser.add_mutually_exclusive_group(required=True) 253 | mutex1.add_argument( 254 | '--fuzz', 255 | action='store_true', 256 | help='repeatedly shuffle the test suite searching for failures', 257 | ) 258 | mutex1.add_argument( 259 | '--failing-test', 260 | help=( 261 | 'the identifier of the failing test, ' 262 | 'for example `tests/my_test.py::test_name_here`' 263 | ), 264 | ) 265 | 266 | mutex2 = parser.add_mutually_exclusive_group(required=True) 267 | mutex2.add_argument( 268 | '--tests', 269 | help='where tests will be discovered from, often `--tests=tests/', 270 | ) 271 | mutex2.add_argument( 272 | '--testids-file', 273 | help='optional pre-discovered test ids (one per line)', 274 | ) 275 | args = parser.parse_args(argv) 276 | 277 | # step 1: discover all the tests 278 | print('discovering all tests...') 279 | if args.testids_file: 280 | testids = _parse_testids_file(args.testids_file) 281 | print(f'-> pre-discovered {len(testids)} tests!') 282 | else: 283 | testids = _discover_tests(args.tests) 284 | print(f'-> discovered {len(testids)} tests!') 285 | 286 | testpath = _common_testpath(testids) 287 | 288 | if args.fuzz: 289 | return _fuzz(testpath, testids, args.tests, args.testids_file) 290 | else: 291 | return _bisect(testpath, args.failing_test, testids) 292 | 293 | 294 | if __name__ == '__main__': 295 | raise SystemExit(main()) 296 | --------------------------------------------------------------------------------