├── tests ├── unittests │ ├── data │ │ └── a │ │ │ ├── b.py │ │ │ ├── py.py │ │ │ ├── c │ │ │ ├── d.py │ │ │ └── __init__.py │ │ │ └── __init__.py │ ├── test_util.py │ ├── test_work_item.py │ ├── test_ast.py │ ├── conftest.py │ ├── test_mutate_and_test.py │ ├── test_config.py │ ├── operators │ │ ├── test_binary_operator_replacement.py │ │ └── test_operator_samples.py │ ├── test_find_modules.py │ └── test_command_line_processing.py ├── resources │ ├── example_project │ │ ├── eve │ │ │ ├── __init__.py │ │ │ └── eve.py │ │ ├── empty │ │ │ └── __init__.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_init_order.py │ │ │ ├── test_eve.py │ │ │ └── test_adam.py │ │ ├── .gitignore │ │ ├── init_order │ │ │ ├── __init__.py │ │ │ ├── first.py │ │ │ └── second.py │ │ ├── cosmic-ray.pytest.local.conf │ │ ├── cosmic-ray.unittest.local.conf │ │ ├── cosmic-ray.empty.conf │ │ ├── cosmic-ray.with-pytest-filter.conf │ │ ├── cosmic-ray.inexisting.conf │ │ ├── cosmic-ray.init_order.conf │ │ ├── adam │ │ │ ├── __init__.py │ │ │ ├── adam_2.py │ │ │ └── adam_1.py │ │ ├── cosmic-ray.pytest.http.conf │ │ ├── cosmic-ray.unittest.http.conf │ │ └── cosmic-ray.import.conf │ └── fast_tests │ │ ├── calculator.py │ │ ├── cr.conf │ │ ├── test_calculator.py │ │ └── README.md ├── tools │ ├── test_http_workers.py │ ├── test_xml.py │ ├── test_filter_pragma.py │ ├── test_rate.py │ ├── test_filter_git.py │ ├── test_html_report.py │ ├── test_report.py │ ├── conftest.py │ └── test_filter_operators.py ├── e2e │ ├── test_fast.py │ └── test_e2e.py └── conftest.py ├── src └── cosmic_ray │ ├── tools │ ├── __init__.py │ ├── filters │ │ ├── __init__.py │ │ ├── operators_filter.py │ │ ├── filter_app.py │ │ ├── pragma_no_mutate.py │ │ └── git.py │ ├── badge.py │ ├── report.py │ ├── survival_rate.py │ ├── xml.py │ └── http_workers.py │ ├── operators │ ├── __init__.py │ ├── util.py │ ├── break_continue.py │ ├── keyword_replacer.py │ ├── no_op.py │ ├── zero_iteration_for_loop.py │ ├── remove_decorator.py │ ├── number_replacer.py │ ├── provider.py │ ├── exception_replacer.py │ ├── binary_operator_replacement.py │ ├── boolean_replacer.py │ ├── operator.py │ ├── variable_inserter.py │ ├── comparison_operator_replacement.py │ ├── unary_operator_replacement.py │ └── variable_replacer.py │ ├── distribution │ ├── __init__.py │ ├── distributor.py │ └── local.py │ ├── __init__.py │ ├── version.py │ ├── exceptions.py │ ├── commands │ ├── __init__.py │ ├── new_config.py │ ├── execute.py │ └── init.py │ ├── util.py │ ├── timing.py │ ├── modules.py │ ├── plugins.py │ ├── testing.py │ ├── work_item.py │ ├── ast │ ├── __init__.py │ └── ast_query.py │ ├── config.py │ └── progress.py ├── MANIFEST.in ├── docs ├── source │ ├── tutorials │ │ ├── intro │ │ │ ├── mod.1.py │ │ │ ├── test_mod.1.py │ │ │ └── tutorial.toml.1 │ │ └── distributed │ │ │ ├── mod.1.py │ │ │ ├── test_mod.1.py │ │ │ ├── config.1.toml │ │ │ └── config.2.toml │ ├── cr-in-action.gif │ ├── how-tos │ │ ├── distributor.rst │ │ ├── index.rst │ │ ├── implementation.rst │ │ └── filters.rst │ ├── reference │ │ ├── api │ │ │ ├── modules.rst │ │ │ ├── cosmic_ray.ast.rst │ │ │ ├── cosmic_ray.commands.rst │ │ │ ├── cosmic_ray.distribution.rst │ │ │ ├── cosmic_ray.tools.filters.rst │ │ │ ├── cosmic_ray.tools.rst │ │ │ ├── cosmic_ray.rst │ │ │ └── cosmic_ray.operators.rst │ │ ├── index.rst │ │ ├── badge.rst │ │ ├── continuous_integration.rst │ │ ├── cli.rst │ │ └── tests.rst │ ├── legal │ │ ├── cosmic-ray-entity-cla.pdf │ │ └── cosmic-ray-individual-cla.pdf │ ├── theory.rst │ ├── index.rst │ └── conf.py ├── rtd-requirements.txt ├── requirements.txt ├── Makefile └── make.bat ├── deploy_key.enc ├── .landscape.yaml ├── .github ├── dependabot.yaml └── workflows │ └── python-package.yml ├── .coveragerc ├── tools └── inspector.py ├── .readthedocs.yml ├── LICENCE.txt ├── README.rst ├── .gitignore ├── AGENTS.md ├── pyproject.toml └── CONTRIBUTING.rst /tests/unittests/data/a/b.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unittests/data/a/py.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unittests/data/a/c/d.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unittests/data/a/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unittests/data/a/c/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/filters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/example_project/eve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/example_project/empty/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/example_project/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | include README.md 3 | -------------------------------------------------------------------------------- /tests/resources/example_project/tests/test_init_order.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/source/tutorials/intro/mod.1.py: -------------------------------------------------------------------------------- 1 | def func(): 2 | return 1234 3 | -------------------------------------------------------------------------------- /tests/resources/example_project/.gitignore: -------------------------------------------------------------------------------- 1 | *.session.json 2 | *.sqlite -------------------------------------------------------------------------------- /docs/source/tutorials/distributed/mod.1.py: -------------------------------------------------------------------------------- 1 | def func(): 2 | return 1234 3 | -------------------------------------------------------------------------------- /src/cosmic_ray/distribution/__init__.py: -------------------------------------------------------------------------------- 1 | # cosmic_ray/execution/__init__.py 2 | -------------------------------------------------------------------------------- /tests/resources/example_project/init_order/__init__.py: -------------------------------------------------------------------------------- 1 | initialized = False 2 | -------------------------------------------------------------------------------- /docs/rtd-requirements.txt: -------------------------------------------------------------------------------- 1 | # Requirements for building on readthedocs 2 | sphinx 3 | -------------------------------------------------------------------------------- /tests/resources/fast_tests/calculator.py: -------------------------------------------------------------------------------- 1 | def mul(x, y): 2 | return x * y 3 | -------------------------------------------------------------------------------- /deploy_key.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sixty-north/cosmic-ray/HEAD/deploy_key.enc -------------------------------------------------------------------------------- /src/cosmic_ray/__init__.py: -------------------------------------------------------------------------------- 1 | """Cosmic Ray is a mutation testing tool for Python.""" 2 | -------------------------------------------------------------------------------- /docs/source/cr-in-action.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sixty-north/cosmic-ray/HEAD/docs/source/cr-in-action.gif -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Requirements for building the docs locally (i.e. not on readthedocs) 2 | sphinx 3 | sphinx_rtd_theme 4 | -------------------------------------------------------------------------------- /docs/source/how-tos/distributor.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Distributors 3 | ============ 4 | 5 | **TODO**: Explain how to create a distributor. -------------------------------------------------------------------------------- /tests/resources/example_project/init_order/first.py: -------------------------------------------------------------------------------- 1 | import init_order 2 | 3 | init_order.initialized = True 4 | 5 | print("first") 6 | -------------------------------------------------------------------------------- /tests/resources/example_project/init_order/second.py: -------------------------------------------------------------------------------- 1 | import init_order 2 | 3 | assert init_order.initialized 4 | 5 | print("second") 6 | -------------------------------------------------------------------------------- /tests/tools/test_http_workers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.skip(reason="TODO") 5 | def test_smoke_test(): 6 | pass 7 | -------------------------------------------------------------------------------- /docs/source/reference/api/modules.rst: -------------------------------------------------------------------------------- 1 | Cosmic Ray API 2 | ============== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | cosmic_ray 8 | -------------------------------------------------------------------------------- /docs/source/legal/cosmic-ray-entity-cla.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sixty-north/cosmic-ray/HEAD/docs/source/legal/cosmic-ray-entity-cla.pdf -------------------------------------------------------------------------------- /src/cosmic_ray/version.py: -------------------------------------------------------------------------------- 1 | """Cosmic Ray version info.""" 2 | 3 | __version__ = "8.4.3" 4 | __version_info__ = tuple(__version__.split(".")) 5 | -------------------------------------------------------------------------------- /docs/source/legal/cosmic-ray-individual-cla.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sixty-north/cosmic-ray/HEAD/docs/source/legal/cosmic-ray-individual-cla.pdf -------------------------------------------------------------------------------- /src/cosmic_ray/exceptions.py: -------------------------------------------------------------------------------- 1 | class CosmicRayTestingException(Exception): 2 | """Exception that we use for exception replacement.""" 3 | 4 | pass 5 | -------------------------------------------------------------------------------- /.landscape.yaml: -------------------------------------------------------------------------------- 1 | doc-warnings: yes 2 | test-warnings: no 3 | strictness: veryhigh 4 | max-line-length: 120 5 | ignore-paths: 6 | - experiments 7 | - tools 8 | -------------------------------------------------------------------------------- /docs/source/how-tos/index.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | How-tos 3 | ======= 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | filters 9 | distributor 10 | implementation 11 | operators 12 | -------------------------------------------------------------------------------- /docs/source/tutorials/intro/test_mod.1.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import mod 4 | 5 | 6 | class Tests(unittest.TestCase): 7 | def test_func(self): 8 | self.assertEqual(mod.func(), 1234) 9 | -------------------------------------------------------------------------------- /docs/source/reference/index.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Reference 3 | ========= 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | api/modules 9 | cli 10 | tests 11 | continuous_integration 12 | badge -------------------------------------------------------------------------------- /docs/source/tutorials/distributed/test_mod.1.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import mod 4 | 5 | 6 | class Tests(unittest.TestCase): 7 | def test_func(self): 8 | self.assertEqual(mod.func(), 1234) 9 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.pytest.local.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "adam" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m pytest -x tests" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/fast_tests/cr.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "calculator.py" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover test_calculator" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.unittest.local.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "adam" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover tests" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/example_project/tests/test_eve.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from eve import eve 4 | 5 | 6 | class Tests(unittest.TestCase): 7 | def test_constant_42(self): 8 | self.assertEqual(eve.constant_42(), 42) 9 | -------------------------------------------------------------------------------- /docs/source/tutorials/intro/tutorial.toml.1: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "mod.py" 3 | timeout = 10.0 4 | excluded-modules = [] 5 | test-command = "python -m unittest test_mod.py" 6 | 7 | [cosmic-ray.distributor] 8 | name = "local" 9 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.empty.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "empty/__init__.py" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover tests" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.with-pytest-filter.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "adam" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m pytest -x tests -k 'not foo'" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/fast_tests/test_calculator.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from calculator import mul 4 | 5 | 6 | class CalculatorTest(TestCase): 7 | def test_mul(self): 8 | self.assertEqual(mul(2, 2), 4) 9 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.inexisting.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "example/unknown_file.py" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover tests" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.init_order.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "init_order/second.py" 3 | timeout = 100 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover tests" 6 | distributor.name = "local" 7 | -------------------------------------------------------------------------------- /tests/resources/example_project/eve/eve.py: -------------------------------------------------------------------------------- 1 | # this module is used to test the import mechanisms 2 | # of Cosmic Ray. For more info see: 3 | # https://github.com/sixty-north/cosmic-ray/issues/157 4 | 5 | 6 | def constant_42(): 7 | return 42 8 | -------------------------------------------------------------------------------- /tests/unittests/test_util.py: -------------------------------------------------------------------------------- 1 | import parso 2 | 3 | from cosmic_ray.ast import dump_node 4 | 5 | 6 | def test_dump_node(): 7 | s = open(__file__).read() 8 | node = parso.parse(s) 9 | d = dump_node(node) 10 | assert isinstance(d, str) 11 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: github-actions 5 | directory: / 6 | schedule: 7 | interval: weekly 8 | groups: 9 | all-gh-actions-dependencies: 10 | patterns: 11 | - "*" 12 | -------------------------------------------------------------------------------- /tests/resources/example_project/adam/__init__.py: -------------------------------------------------------------------------------- 1 | # adam 2 | """ 3 | A set of function which exercise specific mutation operators. This 4 | is paired up with a test suite. The idea is that cosmic-ray should 5 | kill every mutant when that suite is run; if it doesn't, then we've 6 | got a problem. 7 | """ 8 | -------------------------------------------------------------------------------- /docs/source/tutorials/distributed/config.1.toml: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "mod.py" 3 | timeout = 10.0 4 | excluded-modules = [] 5 | test-command = "python -m unittest test_mod.py" 6 | 7 | [cosmic-ray.distributor] 8 | name = "http" 9 | 10 | [cosmic-ray.distributor.http] 11 | worker-urls = ["http://localhost:9876"] 12 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.pytest.http.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "adam" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m pytest -x tests" 6 | distributor.name = "http" 7 | 8 | [cosmic-ray.distributor.http] 9 | worker-urls = ["http://localhost:9876", "http://localhost:9877"] 10 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.unittest.http.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "adam" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover tests" 6 | distributor.name = "http" 7 | 8 | [cosmic-ray.distributor.http] 9 | worker-urls = ["http://localhost:9876", "http://localhost:9877"] 10 | -------------------------------------------------------------------------------- /docs/source/tutorials/distributed/config.2.toml: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "mod.py" 3 | timeout = 10.0 4 | excluded-modules = [] 5 | test-command = "python -m unittest test_mod.py" 6 | 7 | [cosmic-ray.distributor] 8 | name = "http" 9 | 10 | [cosmic-ray.distributor.http] 11 | worker-urls = ["http://localhost:9876", "http://localhost:9877"] 12 | -------------------------------------------------------------------------------- /tests/unittests/test_work_item.py: -------------------------------------------------------------------------------- 1 | "Tests covering types in work_item." 2 | 3 | from pathlib import Path 4 | 5 | from cosmic_ray.work_item import MutationSpec 6 | 7 | 8 | def test_mutation_spec_accepts_str_module_path(): 9 | mutation = MutationSpec("foo/bar.py", "operator", 0, (0, 0), (0, 1)) 10 | 11 | assert mutation.module_path == Path("foo/bar.py") 12 | -------------------------------------------------------------------------------- /src/cosmic_ray/commands/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | These are the core implementations of the various commands in cosmic ray. 3 | 4 | Not all commands are represented here, just the ones which seem big enough to 5 | justify a separate module. 6 | """ 7 | 8 | from .execute import execute # NOQA 9 | from .init import init # NOQA 10 | from .new_config import new_config # NOQA 11 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.ast.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray.ast package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | cosmic\_ray.ast.ast\_query module 8 | --------------------------------- 9 | 10 | .. automodule:: cosmic_ray.ast.ast_query 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: cosmic_ray.ast 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /tests/resources/fast_tests/README.md: -------------------------------------------------------------------------------- 1 | This is intended to be a very fast test suite. On some platform (e.g. Windows) we've found that test suites like this 2 | can run faster than the resolution of the filesystem timestamps. This leads to problems where Python doesn't regenerate 3 | pycs files when necessary, leading to incorrect mutation testing results. 4 | 5 | We've modified CR to work around these problems, and this test (as driven by the pytest suite) will hopefully detect 6 | regressions in our workaround. -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = . 3 | branch = True 4 | 5 | # Wrong warning caused by include used in [report] (Coveragepy 4.4.2) 6 | # Ref: https://bitbucket.org/ned/coveragepy/issues/621 7 | disable_warnings = include-ignored 8 | 9 | [report] 10 | include = cosmic_ray/*,test/* 11 | exclude_lines = 12 | raise NotImplementedError 13 | pragma: no cover 14 | # Branch coverage: ignore default and assert statements. 15 | partial_branches = 16 | \s+#\s*(pragma|PRAGMA)[:\s]?\s*(no|NO)\s*(branch|BRANCH) 17 | ^\s*assert\s 18 | -------------------------------------------------------------------------------- /src/cosmic_ray/distribution/distributor.py: -------------------------------------------------------------------------------- 1 | "Base distributor implementation details." 2 | 3 | import abc 4 | 5 | 6 | class Distributor(metaclass=abc.ABCMeta): 7 | "Base class for work distribution strategies." 8 | 9 | @abc.abstractmethod 10 | def __call__(self, pending_work, test_command, timeout, distributor_config, on_task_complete): 11 | """Execute jobs in `pending_work_items`. 12 | 13 | Spend no more than `timeout` seconds for a single job, using `distributor_config` to 14 | distribute the work. 15 | """ 16 | -------------------------------------------------------------------------------- /tests/tools/test_xml.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | 5 | def test_smoke_test_initialized_session(initialized_session): 6 | command = [sys.executable, "-m", "cosmic_ray.tools.xml", str(initialized_session.session)] 7 | 8 | subprocess.check_call(command, cwd=str(initialized_session.session.parent)) 9 | 10 | 11 | def test_smoke_test_execd_session(execd_session): 12 | command = [sys.executable, "-m", "cosmic_ray.tools.xml", str(execd_session.session)] 13 | 14 | subprocess.check_call(command, cwd=str(execd_session.session.parent)) 15 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/util.py: -------------------------------------------------------------------------------- 1 | "Utilities for implementing operators." 2 | 3 | 4 | def extend_name(suffix): 5 | """A factory for class decorators that modify the class name by appending some text to it. 6 | 7 | Example: 8 | 9 | .. code-block:: python 10 | 11 | @extend_name('_Foo') 12 | class Class: 13 | pass 14 | 15 | assert Class.__name__ == 'Class_Foo' 16 | """ 17 | 18 | def dec(cls): 19 | name = f"{cls.__name__}{suffix}" 20 | setattr(cls, "__name__", name) 21 | return cls 22 | 23 | return dec 24 | -------------------------------------------------------------------------------- /tools/inspector.py: -------------------------------------------------------------------------------- 1 | # This is just a simple example of how to inspect ASTs visually. 2 | # 3 | # This can be useful for developing new operators, etc. 4 | 5 | import ast 6 | 7 | from cosmic_ray.mutating import MutatingCore 8 | from cosmic_ray.operators.comparison_operator_replacement import MutateComparisonOperator 9 | 10 | code = "((x is not y) ^ (x is y))" 11 | node = ast.parse(code) 12 | print() 13 | print(ast.dump(node)) 14 | 15 | core = MutatingCore(0) 16 | operator = MutateComparisonOperator(core) 17 | new_node = operator.visit(node) 18 | print() 19 | print(ast.dump(new_node)) 20 | -------------------------------------------------------------------------------- /tests/resources/example_project/cosmic-ray.import.conf: -------------------------------------------------------------------------------- 1 | [cosmic-ray] 2 | module-path = "eve/eve.py" 3 | timeout = 10 4 | excluded-modules = [] 5 | test-command = "python -m unittest discover tests" 6 | distributor.name = "local" 7 | 8 | # [[cosmic-ray.operators."core/VariableReplacer"]] 9 | # cause_variable = "foo" 10 | # effect_variable = "bar" 11 | 12 | # [[cosmic-ray.operators."core/VariableReplacer"]] 13 | # cause_variable = "llama" 14 | # effect_variable = "yak" 15 | 16 | # [[cosmic-ray.operators."core/VariableInserter"]] 17 | # cause_variable = "foo" 18 | # effect_variable = "bar" 19 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/break_continue.py: -------------------------------------------------------------------------------- 1 | """Implementation of the replace-break-with-continue and 2 | replace-continue-with-break operators. 3 | """ 4 | 5 | from .keyword_replacer import KeywordReplacementOperator 6 | 7 | 8 | class ReplaceBreakWithContinue(KeywordReplacementOperator): 9 | "Operator which replaces 'break' with 'continue'." 10 | 11 | from_keyword = "break" 12 | to_keyword = "continue" 13 | 14 | 15 | class ReplaceContinueWithBreak(KeywordReplacementOperator): 16 | "Operator which replaces 'continue' with 'break'." 17 | 18 | from_keyword = "continue" 19 | to_keyword = "break" 20 | -------------------------------------------------------------------------------- /tests/tools/test_filter_pragma.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | 5 | def test_smoke_test_on_initialized_session(initialized_session, fast_tests_root): 6 | command = [sys.executable, "-m", "cosmic_ray.tools.filters.pragma_no_mutate", str(initialized_session.session)] 7 | 8 | subprocess.check_call(command, cwd=str(fast_tests_root)) 9 | 10 | 11 | def test_smoke_test_on_execd_session(execd_session, fast_tests_root): 12 | command = [sys.executable, "-m", "cosmic_ray.tools.filters.pragma_no_mutate", str(execd_session.session)] 13 | 14 | subprocess.check_call(command, cwd=str(fast_tests_root)) 15 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/source/reference/badge.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Badge 3 | ===== 4 | 5 | Utility to generate badge useful to decorate your preferred 6 | Continuous Integration system (github, gitlab, ...). 7 | The badge indicate the percentage of failing migrations. 8 | 9 | This utility is based on `anybadge `__. 10 | 11 | Command 12 | ======= 13 | 14 | :: 15 | 16 | cr-badge [--config ] 17 | 18 | Configuration 19 | ============= 20 | 21 | :: 22 | 23 | [cosmic-ray.badge] 24 | label = "mutation" 25 | format = "%.2f %%" 26 | 27 | [cosmic-ray.badge.thresholds] 28 | 50 = 'red' 29 | 70 = 'orange' 30 | 100 = 'yellow' 31 | 101 = 'green' 32 | -------------------------------------------------------------------------------- /tests/tools/test_rate.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | 5 | def test_smoke_test_for_initialized_session(initialized_session): 6 | command = [sys.executable, "-m", "cosmic_ray.tools.survival_rate", str(initialized_session.session)] 7 | 8 | proc = subprocess.run(command, cwd=str(initialized_session.session.parent), capture_output=True) 9 | assert proc.returncode == 0 10 | assert float(proc.stdout) == 0 11 | 12 | 13 | def test_smoke_test_for_execd_session(execd_session): 14 | command = [sys.executable, "-m", "cosmic_ray.tools.survival_rate", str(execd_session.session)] 15 | 16 | proc = subprocess.run(command, cwd=str(execd_session.session.parent), capture_output=True) 17 | assert proc.returncode == 0 18 | assert float(proc.stdout) == 18.18 19 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 2 | 3 | # Required 4 | version: 2 5 | 6 | build: 7 | os: ubuntu-24.04 8 | tools: 9 | python: "3.13" 10 | jobs: 11 | pre_create_environment: 12 | - asdf plugin add uv 13 | - asdf install uv latest 14 | - asdf global uv latest 15 | create_environment: 16 | - uv venv "${READTHEDOCS_VIRTUALENV_PATH}" 17 | install: 18 | - UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --frozen 19 | 20 | # Build documentation in the docs/source/ directory with Sphinx 21 | sphinx: 22 | configuration: docs/source/conf.py 23 | 24 | # Optionally build your docs in additional formats such as PDF and ePub 25 | formats: all 26 | -------------------------------------------------------------------------------- /tests/unittests/test_ast.py: -------------------------------------------------------------------------------- 1 | from cosmic_ray.ast import get_ast_from_path 2 | 3 | 4 | def test_call_get_ast_from_path(tmp_path): 5 | module_filepath = tmp_path / "module.py" 6 | input_code = "def foo():\n pass\n" 7 | module_filepath.write_text(input_code) 8 | ast = get_ast_from_path(module_filepath) 9 | assert ast.get_code() == input_code 10 | 11 | 12 | def test_call_get_ast_from_path_with_non_utf8_encoding(tmp_path): 13 | module_filepath = tmp_path / "module.py" 14 | input_code = """# -*- coding: latin-1 -*- 15 | def foo(): 16 | pass 17 | """.encode("latin-1") 18 | with module_filepath.open("wb") as f: 19 | f.write(input_code) 20 | ast = get_ast_from_path(module_filepath) 21 | assert ast.get_code().encode("latin-1") == input_code 22 | -------------------------------------------------------------------------------- /tests/unittests/conftest.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import os 3 | from pathlib import Path 4 | 5 | import pytest 6 | 7 | _THIS_DIR = Path(os.path.dirname(os.path.realpath(__file__))) 8 | 9 | 10 | @pytest.fixture 11 | def data_dir(): 12 | "Directory containing test data" 13 | return _THIS_DIR / "data" 14 | 15 | 16 | class PathUtils: 17 | "Path utilities for testing." 18 | 19 | @staticmethod 20 | @contextlib.contextmanager 21 | def excursion(directory): 22 | """Context manager for temporarily setting `directory` as the current working 23 | directory. 24 | """ 25 | old_dir = os.getcwd() 26 | os.chdir(str(directory)) 27 | try: 28 | yield 29 | finally: 30 | os.chdir(old_dir) 31 | 32 | 33 | @pytest.fixture 34 | def path_utils(): 35 | "Path utilities for testing." 36 | return PathUtils 37 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/keyword_replacer.py: -------------------------------------------------------------------------------- 1 | "Common implementation for operators that replace keywords." 2 | 3 | from parso.python.tree import Keyword 4 | 5 | from .operator import Example, Operator 6 | 7 | # pylint: disable=E1101 8 | 9 | 10 | class KeywordReplacementOperator(Operator): 11 | """A base class for operators that replace one keyword with another""" 12 | 13 | def mutation_positions(self, node): 14 | if isinstance(node, Keyword): 15 | if node.value.strip() == self.from_keyword: 16 | yield (node.start_pos, node.end_pos) 17 | 18 | def mutate(self, node, index): 19 | assert isinstance(node, Keyword) 20 | assert node.value == self.from_keyword 21 | 22 | node.value = self.to_keyword 23 | return node 24 | 25 | @classmethod 26 | def examples(cls): 27 | return (Example(cls.from_keyword, cls.to_keyword),) 28 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/no_op.py: -------------------------------------------------------------------------------- 1 | "Implementation of the no-op operator." 2 | 3 | from .operator import Example, Operator 4 | 5 | 6 | class NoOp(Operator): 7 | """An operator that makes no changes. 8 | 9 | This is primarily for baselining and debugging. It behaves like any other operator, but it makes no changes. 10 | Obviously this means that, if your test suite passes on unmutated code, it will still pass after applying this 11 | operator. Use with care. 12 | """ 13 | 14 | def mutation_positions(self, node): 15 | yield (node.start_pos, node.end_pos) 16 | 17 | def mutate(self, node, index): 18 | return node 19 | 20 | @classmethod 21 | def examples(cls): 22 | return ( 23 | Example("@foo\ndef bar(): pass", "@foo\ndef bar(): pass"), 24 | Example("def bar(): pass", "def bar(): pass"), 25 | Example("1 + 1", "1 + 1"), 26 | ) 27 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.commands.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray.commands package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | cosmic\_ray.commands.execute module 8 | ----------------------------------- 9 | 10 | .. automodule:: cosmic_ray.commands.execute 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | cosmic\_ray.commands.init module 16 | -------------------------------- 17 | 18 | .. automodule:: cosmic_ray.commands.init 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | cosmic\_ray.commands.new\_config module 24 | --------------------------------------- 25 | 26 | .. automodule:: cosmic_ray.commands.new_config 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: cosmic_ray.commands 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/zero_iteration_for_loop.py: -------------------------------------------------------------------------------- 1 | "Implementation of the zero-iteration-loop operator." 2 | 3 | import parso 4 | from parso.python.tree import ForStmt 5 | 6 | from .operator import Example, Operator 7 | 8 | 9 | class ZeroIterationForLoop(Operator): 10 | """An operator that modified for-loops to have zero iterations.""" 11 | 12 | def mutation_positions(self, node): 13 | if isinstance(node, ForStmt): 14 | expr = node.children[3] 15 | yield (expr.start_pos, expr.end_pos) 16 | 17 | def mutate(self, node, index): 18 | "Modify the For loop to evaluate to None" 19 | assert index == 0 20 | assert isinstance(node, ForStmt) 21 | 22 | empty_list = parso.parse(" []") 23 | node.children[3] = empty_list 24 | return node 25 | 26 | @classmethod 27 | def examples(cls): 28 | return (Example("for i in rang(1,2): pass", "for i in []: pass"),) 29 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.distribution.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray.distribution package 2 | ================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | cosmic\_ray.distribution.distributor module 8 | ------------------------------------------- 9 | 10 | .. automodule:: cosmic_ray.distribution.distributor 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | cosmic\_ray.distribution.http module 16 | ------------------------------------ 17 | 18 | .. automodule:: cosmic_ray.distribution.http 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | cosmic\_ray.distribution.local module 24 | ------------------------------------- 25 | 26 | .. automodule:: cosmic_ray.distribution.local 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: cosmic_ray.distribution 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /tests/tools/test_filter_git.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | 5 | import pytest 6 | 7 | # Skip these tests when running in GitHub Actions where a proper git 8 | # workspace isn't available in the test environment. 9 | skip_in_ci = pytest.mark.xfail( 10 | os.environ.get("GITHUB_ACTIONS") == "true", 11 | reason="This test failes on non-master branches in CI, so we ignore this problem for now.", 12 | ) 13 | 14 | 15 | @skip_in_ci 16 | def test_smoke_test_on_initialized_session(initialized_session, fast_tests_root): 17 | command = [sys.executable, "-m", "cosmic_ray.tools.filters.git", str(initialized_session.session)] 18 | 19 | subprocess.check_call(command, cwd=str(fast_tests_root)) 20 | 21 | 22 | @skip_in_ci 23 | def test_smoke_test_on_execd_session(execd_session, fast_tests_root): 24 | command = [sys.executable, "-m", "cosmic_ray.tools.filters.git", str(execd_session.session)] 25 | 26 | subprocess.check_call(command, cwd=str(fast_tests_root)) 27 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/remove_decorator.py: -------------------------------------------------------------------------------- 1 | "Implementation of the remove-decorator operator." 2 | 3 | from parso.python.tree import Decorator 4 | 5 | from .operator import Example, Operator 6 | 7 | 8 | class RemoveDecorator(Operator): 9 | """An operator that removes decorators.""" 10 | 11 | def mutation_positions(self, node): 12 | if isinstance(node, Decorator): 13 | yield (node.start_pos, node.end_pos) 14 | 15 | def mutate(self, node, index): 16 | assert isinstance(node, Decorator) 17 | assert index == 0 18 | 19 | @classmethod 20 | def examples(cls): 21 | return ( 22 | Example("@foo\ndef bar(): pass", "def bar(): pass"), 23 | Example("@first\n@second\ndef bar(): pass", "@second\ndef bar(): pass"), 24 | Example("@first\n@second\ndef bar(): pass", "@first\ndef bar(): pass", occurrence=1), 25 | Example("@first\n@second\n@third\ndef bar(): pass", "@first\n@third\ndef bar(): pass", occurrence=1), 26 | ) 27 | -------------------------------------------------------------------------------- /src/cosmic_ray/distribution/local.py: -------------------------------------------------------------------------------- 1 | """Cosmic Ray distributor that runs tests sequentially and locally. 2 | 3 | Enabling the distributor 4 | ======================== 5 | 6 | To use the local distributor, set ``cosmic-ray.distributor.name = "local"`` in your Cosmic Ray configuration: 7 | 8 | .. code-block:: toml 9 | 10 | [cosmic-ray.distributor] 11 | name = "local" 12 | """ 13 | 14 | import logging 15 | 16 | from cosmic_ray.distribution.distributor import Distributor 17 | from cosmic_ray.mutating import mutate_and_test 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | 22 | class LocalDistributor(Distributor): 23 | "The local distributor." 24 | 25 | def __call__(self, pending_work, test_command, timeout, _distributor_config, on_task_complete): 26 | for work_item in pending_work: 27 | result = mutate_and_test( 28 | mutations=work_item.mutations, 29 | test_command=test_command, 30 | timeout=timeout, 31 | ) 32 | on_task_complete(work_item.job_id, result) 33 | -------------------------------------------------------------------------------- /src/cosmic_ray/util.py: -------------------------------------------------------------------------------- 1 | import tokenize 2 | from contextlib import contextmanager 3 | from pathlib import Path 4 | 5 | 6 | def read_python_source(module_filepath): 7 | """Load the code in a Python source file. 8 | 9 | Use this whenever reading source code from a Python source file! 10 | This takes care of handling the encoding of the file. 11 | 12 | Args: 13 | module_path: The path to the Python source file. 14 | 15 | Returns: A string contining the decoded source code from the file. 16 | """ 17 | with tokenize.open(module_filepath) as handle: 18 | source = handle.read() 19 | return source 20 | 21 | 22 | @contextmanager 23 | def restore_contents(filepath: Path): 24 | """Restore the original contents of a file after a context-manager. 25 | 26 | Args: 27 | filepath (Path): Path to the file. 28 | 29 | Yields: 30 | bytes: The original contents of the file. 31 | """ 32 | contents = filepath.read_bytes() 33 | try: 34 | yield contents 35 | finally: 36 | filepath.write_bytes(contents) 37 | -------------------------------------------------------------------------------- /tests/unittests/test_mutate_and_test.py: -------------------------------------------------------------------------------- 1 | "Tests for worker." 2 | 3 | from pathlib import Path 4 | 5 | from cosmic_ray.mutating import mutate_and_test 6 | from cosmic_ray.work_item import MutationSpec, WorkResult, WorkerOutcome 7 | 8 | 9 | def test_no_test_return_value(path_utils, data_dir): 10 | with path_utils.excursion(data_dir): 11 | result = mutate_and_test( 12 | [ 13 | MutationSpec( 14 | Path("a/b.py"), 15 | "core/ReplaceTrueWithFalse", 16 | 100, 17 | # TODO: As in other places, these are placeholder position values. How can we not have to provide them? 18 | (0, 0), 19 | (0, 1), 20 | ) 21 | ], 22 | "python -m unittest tests", 23 | 1000, 24 | ) 25 | 26 | expected = WorkResult( 27 | output=None, 28 | test_outcome=None, 29 | diff=None, 30 | worker_outcome=WorkerOutcome.NO_TEST, 31 | ) 32 | assert result == expected 33 | -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2017 Sixty North AS 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/cosmic_ray/timing.py: -------------------------------------------------------------------------------- 1 | """Support for timing the execution of functions. 2 | 3 | This is primarily intended to support baselining, but it's got some reasonable 4 | generic functionality. 5 | """ 6 | 7 | import datetime 8 | 9 | 10 | class Timer: 11 | """A simple context manager for timing events. 12 | 13 | Generally use it like this: 14 | 15 | .. code-block:: 16 | 17 | with Timer() as t: 18 | do_something() 19 | print(t.elapsed()) 20 | """ 21 | 22 | def __init__(self): 23 | self._start = None 24 | self.reset() 25 | 26 | def reset(self): 27 | """Set the elapsed time back to 0.""" 28 | self._start = datetime.datetime.now() 29 | 30 | @property 31 | def elapsed(self): 32 | """Get the elapsed time between the last call to `reset` and now. 33 | 34 | Returns a `datetime.timedelta` object. 35 | """ 36 | return datetime.datetime.now() - self._start 37 | 38 | def __enter__(self): 39 | self.reset() 40 | return self 41 | 42 | def __exit__(self, ex_type, ex_value, ex_traceback): 43 | pass 44 | -------------------------------------------------------------------------------- /tests/tools/test_html_report.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import subprocess 3 | import sys 4 | 5 | import pytest 6 | 7 | ONLY_COMPLETED_OPTIONS = (None, "--only-completed", "--not-only-completed") 8 | SKIP_SUCCESS_OPTIONS = (None, "--skip-success", "--include-success") 9 | OPTION_COMBINATIONS = ( 10 | list(filter(None, combo)) for combo in itertools.product(ONLY_COMPLETED_OPTIONS, SKIP_SUCCESS_OPTIONS) 11 | ) 12 | 13 | 14 | @pytest.fixture(params=OPTION_COMBINATIONS) 15 | def options(request): 16 | "All valid combinations of command line options for cr-report." 17 | return request.param 18 | 19 | 20 | def test_smoke_test_on_initialized_session(initialized_session, options): 21 | command = [sys.executable, "-m", "cosmic_ray.tools.html"] + options + [str(initialized_session.session)] 22 | 23 | subprocess.check_call(command, cwd=str(initialized_session.session.parent)) 24 | 25 | 26 | def test_smoke_test_on_execd_session(execd_session, options): 27 | command = [sys.executable, "-m", "cosmic_ray.tools.html"] + options + [str(execd_session.session)] 28 | 29 | subprocess.check_call(command, cwd=str(execd_session.session.parent)) 30 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.tools.filters.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray.tools.filters package 2 | ================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | cosmic\_ray.tools.filters.filter\_app module 8 | -------------------------------------------- 9 | 10 | .. automodule:: cosmic_ray.tools.filters.filter_app 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | cosmic\_ray.tools.filters.git module 16 | ------------------------------------ 17 | 18 | .. automodule:: cosmic_ray.tools.filters.git 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | cosmic\_ray.tools.filters.operators\_filter module 24 | -------------------------------------------------- 25 | 26 | .. automodule:: cosmic_ray.tools.filters.operators_filter 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | cosmic\_ray.tools.filters.pragma\_no\_mutate module 32 | --------------------------------------------------- 33 | 34 | .. automodule:: cosmic_ray.tools.filters.pragma_no_mutate 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: cosmic_ray.tools.filters 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /tests/resources/example_project/adam/adam_2.py: -------------------------------------------------------------------------------- 1 | """adam.adam_2""" 2 | 3 | 4 | def trigger_infinite_loop(): 5 | result = None 6 | # When `break` becomes `continue`, this should enter an infinite loop. This 7 | # helps us test timeouts. 8 | # Any object which isn't None passes the truth value testing so here 9 | # we use `while object()` instead of `while True` b/c the later becomes 10 | # `while False` when ReplaceTrueFalse is applied and we don't trigger an 11 | # infinite loop. 12 | while object(): 13 | result = object() 14 | break 15 | 16 | # when `while object()` becomes `while not object()` 17 | # the code below will be triggered 18 | return result 19 | 20 | 21 | def single_iteration(): 22 | result = None 23 | iterable = [object()] 24 | 25 | for i in iterable: # pylint: disable=W0612 26 | result = True 27 | 28 | return result 29 | 30 | 31 | def handle_exception(): 32 | result = None 33 | try: 34 | raise OSError 35 | except OSError: 36 | result = True 37 | 38 | return result 39 | 40 | 41 | def decorator(func): 42 | func.cosmic_ray = True 43 | return func 44 | 45 | 46 | @decorator 47 | def decorated_func(): 48 | result = None 49 | if decorated_func.cosmic_ray: 50 | result = True 51 | 52 | return result 53 | -------------------------------------------------------------------------------- /tests/tools/test_report.py: -------------------------------------------------------------------------------- 1 | "Tests for cr-report." 2 | 3 | import itertools 4 | import subprocess 5 | import sys 6 | 7 | import pytest 8 | 9 | SHOW_OUTPUT_OPTIONS = (None, "--show-output", "--no-show-output") 10 | SHOW_DIFF_OPTIONS = (None, "--show-diff", "--no-show-diff") 11 | SHOW_PENDING_OPTIONS = (None, "--show-pending", "--no-show-pending") 12 | SURVIVING_ONLY_OPTIONS = (None, "--surviving-only", "--all-mutations") 13 | OPTION_COMBINATIONS = ( 14 | list(filter(None, combo)) 15 | for combo in itertools.product(SHOW_OUTPUT_OPTIONS, SHOW_DIFF_OPTIONS, SHOW_PENDING_OPTIONS, SURVIVING_ONLY_OPTIONS) 16 | ) 17 | 18 | 19 | @pytest.fixture(params=OPTION_COMBINATIONS) 20 | def options(request): 21 | "All valid combinations of command line options for cr-report." 22 | return request.param 23 | 24 | 25 | def test_smoke_test_for_report_on_initialized_session(initialized_session, options): 26 | command = [sys.executable, "-m", "cosmic_ray.tools.report"] + options + [str(initialized_session.session)] 27 | 28 | subprocess.check_call(command, cwd=str(initialized_session.session.parent)) 29 | 30 | 31 | def test_smoke_test_for_report_on_executed_session(execd_session): 32 | subprocess.check_call( 33 | [sys.executable, "-m", "cosmic_ray.tools.report", str(execd_session.session)], 34 | cwd=str(execd_session.session.parent), 35 | ) 36 | -------------------------------------------------------------------------------- /tests/e2e/test_fast.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import subprocess 3 | import sys 4 | 5 | import pytest 6 | 7 | from cosmic_ray.tools.survival_rate import survival_rate 8 | from cosmic_ray.work_db import WorkDB, use_db 9 | 10 | 11 | @pytest.fixture(scope="session") 12 | def project_root(pytestconfig): 13 | root = pathlib.Path(str(pytestconfig.rootdir)) 14 | return root / "tests" / "resources" / "fast_tests" 15 | 16 | 17 | def test_fast_tests(project_root, session): 18 | """This tests that CR works correctly on suites that execute very rapidly. 19 | 20 | A single mutation-test round can be faster than the resolution of file timestamps for some filesystems. When this 21 | happens, we found that Python would not correctly create new pyc files - because it had no way to know do do so! We 22 | modified CR to work around this problem, and this test tries to ensure that we don't regress. 23 | """ 24 | subprocess.check_call( 25 | [sys.executable, "-m", "cosmic_ray.cli", "init", "cr.conf", str(session)], cwd=str(project_root) 26 | ) 27 | 28 | subprocess.check_call( 29 | [sys.executable, "-m", "cosmic_ray.cli", "exec", "cr.conf", str(session)], cwd=str(project_root) 30 | ) 31 | 32 | session_path = project_root / session 33 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 34 | rate = survival_rate(work_db) 35 | assert round(rate, 2) == 18.18 36 | -------------------------------------------------------------------------------- /docs/source/how-tos/implementation.rst: -------------------------------------------------------------------------------- 1 | Implementation 2 | ============== 3 | 4 | Cosmic Ray works by parsing the module under test (MUT) and its submodules into 5 | abstract syntax trees using `parso `_. It 6 | walks the parse trees produced by parso, allowing mutation operators to modify 7 | or delete them. These modified parse trees are then turned back into code which 8 | is written to disk for use in a test run. 9 | 10 | For each individual mutation, Cosmic Ray applies a mutation to the code on disk. 11 | It then uses user-supplied test commands to run tests against mutated code. 12 | 13 | In effect, the mutation testing algorithm is something like this: 14 | 15 | .. code:: python 16 | 17 | for mod in modules_under_test: 18 | for op in mutation_operators: 19 | for site in mutation_sites(op, mod): 20 | mutant_ast = mutate_ast(op, mod, site) 21 | write_to_disk(mutant_ast) 22 | 23 | try: 24 | if discover_and_run_tests(): 25 | print('Oh no! The mutant survived!') 26 | else: 27 | print('The mutant was killed.') 28 | except Exception: 29 | print('The mutant was incompetent.') 30 | 31 | Obviously this can result in a lot of tests, and it can take some time 32 | if your test suite is large and/or slow. 33 | -------------------------------------------------------------------------------- /src/cosmic_ray/modules.py: -------------------------------------------------------------------------------- 1 | """Functions related to finding modules for testing.""" 2 | 3 | import glob 4 | from pathlib import Path 5 | 6 | 7 | def find_modules(module_paths): 8 | """Find all modules in the module (possibly package) represented by ``module_path``. 9 | 10 | Args: 11 | module_paths: A list of pathlib.Path to Python packages or modules. 12 | 13 | Returns: 14 | An iterable of paths Python modules (i.e. \\*py files). 15 | """ 16 | for module_path in module_paths: 17 | if not module_path.exists(): 18 | raise FileNotFoundError(f"Could not find module path {module_path}") 19 | if module_path.is_file(): 20 | if module_path.suffix == ".py": 21 | yield module_path 22 | elif module_path.is_dir(): 23 | pyfiles = glob.glob(f"{module_path}/**/*.py", recursive=True) 24 | yield from (Path(pyfile) for pyfile in pyfiles) 25 | 26 | 27 | def filter_paths(paths, excluded_paths): 28 | """Filter out path matching one of excluded_paths glob 29 | 30 | Args: 31 | paths: path to filter. 32 | excluded_paths: List for glob of modules to exclude. 33 | 34 | Returns: 35 | An iterable of paths Python modules (i.e. \\*py files). 36 | """ 37 | excluded = set(Path(f) for excluded_path in excluded_paths for f in glob.glob(excluded_path, recursive=True)) 38 | return set(paths) - excluded 39 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/badge.py: -------------------------------------------------------------------------------- 1 | """Tool for creating badge.""" 2 | 3 | import os 4 | from logging import getLogger 5 | 6 | import click 7 | from anybadge import Badge 8 | 9 | from cosmic_ray.config import load_config 10 | from cosmic_ray.tools.survival_rate import survival_rate 11 | from cosmic_ray.work_db import WorkDB, use_db 12 | 13 | log = getLogger() 14 | 15 | 16 | @click.command() 17 | @click.argument("config_file", type=click.Path(exists=True, dir_okay=False, readable=True)) 18 | @click.argument("badge_file", type=click.Path(dir_okay=False, writable=True)) 19 | @click.argument("session_file", type=click.Path(exists=True, dir_okay=False, readable=True)) 20 | def generate_badge(config_file, badge_file, session_file): 21 | """Generate badge file.""" 22 | 23 | with use_db(session_file, WorkDB.Mode.open) as db: 24 | config = load_config(config_file) 25 | 26 | percent = 100 - survival_rate(db) 27 | 28 | config = config["badge"] 29 | 30 | badge = Badge( 31 | label=config["label"], 32 | value=percent, 33 | value_format=config["format"], 34 | thresholds=config["thresholds"], 35 | ) 36 | 37 | log.info("Generating badge: " + config["format"], percent) # pylint: disable=logging-not-lazy 38 | 39 | try: 40 | os.unlink(badge_file) 41 | except OSError: 42 | pass 43 | 44 | badge.write_badge(badge_file) 45 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/number_replacer.py: -------------------------------------------------------------------------------- 1 | """Implementation of the NumberReplacer operator.""" 2 | 3 | import parso 4 | 5 | from ..ast import is_number 6 | from .operator import Example, Operator 7 | 8 | # List of offsets that we apply to numbers in the AST. Each index into the list 9 | # corresponds to single mutation. 10 | OFFSETS = [ 11 | +1, 12 | -1, 13 | ] 14 | 15 | 16 | class NumberReplacer(Operator): 17 | """An operator that modifies numeric constants.""" 18 | 19 | def mutation_positions(self, node): 20 | if is_number(node): 21 | for _ in OFFSETS: 22 | yield (node.start_pos, node.end_pos) 23 | 24 | def mutate(self, node, index): 25 | """Modify the numeric value on `node`.""" 26 | 27 | assert index < len(OFFSETS), "received count with no associated offset" 28 | assert isinstance(node, parso.python.tree.Number) 29 | 30 | val = eval(node.value) + OFFSETS[index] # pylint: disable=W0123 31 | return parso.python.tree.Number(" " + str(val), node.start_pos) 32 | 33 | @classmethod 34 | def examples(cls): 35 | return ( 36 | Example("x = 1", "x = 2"), 37 | Example("x = 1", "x = 0", occurrence=1), 38 | Example("x = 4.2", "x = 5.2"), 39 | Example("x = 4.2", "x = 3.2", occurrence=1), 40 | Example("x = 1j", "x = (1+1j)"), 41 | Example("x = 1j", "x = (-1+1j)", occurrence=1), 42 | ) 43 | -------------------------------------------------------------------------------- /docs/source/theory.rst: -------------------------------------------------------------------------------- 1 | Theory 2 | ====== 3 | 4 | Mutation testing is conceptually simple and elegant. You make certain kinds of controlled changes (mutations) to your 5 | *code under test* [1]_, and then you run your test suite over this mutated code. If your test suite fails, then we say that 6 | your tests "killed" (i.e. detected) the mutant. If the changes cause your code to simply crash, then we say the mutant 7 | is "incompetent". If your test suite passes, however, we say that the mutant has "survived". 8 | 9 | Needless to say, we want to kill all of the mutants. 10 | 11 | The goal of mutation testing is to verify that your test suite is 12 | actually testing all of the parts of your code that it needs to, and 13 | that it is doing so in a meaningful way. If a mutant survives your test 14 | suite, this is an indication that your test suite is not adequately 15 | checking the code that was changed. This means that either a) you need 16 | more or better tests or b) you've got code which you don't need. 17 | 18 | You can read more about mutation testing at `the repository of all human 19 | knowledge `__. Lionel 20 | Brian has a `nice set of 21 | slides `__ 22 | introducing mutation testing as well. 23 | 24 | .. [1] By "code under test", we mean the code that your test suite is testing. Mutation testing is trying 25 | to ensure that your unaltered test suite can detect explicitly incorrect behavior in your code. 26 | 27 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.tools.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray.tools package 2 | ========================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | cosmic_ray.tools.filters 10 | 11 | Submodules 12 | ---------- 13 | 14 | cosmic\_ray.tools.badge module 15 | ------------------------------ 16 | 17 | .. automodule:: cosmic_ray.tools.badge 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | cosmic\_ray.tools.html module 23 | ----------------------------- 24 | 25 | .. automodule:: cosmic_ray.tools.html 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | cosmic\_ray.tools.http\_workers module 31 | -------------------------------------- 32 | 33 | .. automodule:: cosmic_ray.tools.http_workers 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | cosmic\_ray.tools.report module 39 | ------------------------------- 40 | 41 | .. automodule:: cosmic_ray.tools.report 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | cosmic\_ray.tools.survival\_rate module 47 | --------------------------------------- 48 | 49 | .. automodule:: cosmic_ray.tools.survival_rate 50 | :members: 51 | :undoc-members: 52 | :show-inheritance: 53 | 54 | cosmic\_ray.tools.xml module 55 | ---------------------------- 56 | 57 | .. automodule:: cosmic_ray.tools.xml 58 | :members: 59 | :undoc-members: 60 | :show-inheritance: 61 | 62 | 63 | Module contents 64 | --------------- 65 | 66 | .. automodule:: cosmic_ray.tools 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | -------------------------------------------------------------------------------- /tests/resources/example_project/adam/adam_1.py: -------------------------------------------------------------------------------- 1 | """adam.adam_1""" 2 | 3 | # pylint: disable=C0111 4 | import operator 5 | from math import * # noqa: F401,F403 6 | 7 | # Add mutation points for comparison operators. 8 | 9 | 10 | def constant_number(): 11 | return 42 12 | 13 | 14 | def constant_true(): 15 | return True 16 | 17 | 18 | def constant_false(): 19 | return False 20 | 21 | 22 | def bool_and(): 23 | return object() and None 24 | 25 | 26 | def bool_or(): 27 | return object() or None 28 | 29 | 30 | def bool_expr_with_not(): 31 | return not object() 32 | 33 | 34 | def bool_if(): 35 | if object(): 36 | return True 37 | 38 | raise Exception("bool_if() failed") 39 | 40 | 41 | def if_expression(): 42 | return True if object() else None 43 | 44 | 45 | def assert_in_func(): 46 | assert object() 47 | return True 48 | 49 | 50 | def unary_sub(): 51 | return -1 52 | 53 | 54 | def unary_add(): 55 | return +1 56 | 57 | 58 | def binary_add(): 59 | return 5 + 6 60 | 61 | 62 | def equals(vals): 63 | def constraint(x, y): 64 | return operator.xor(x == y, x != y) 65 | 66 | return all([constraint(x, y) for x in vals for y in vals]) 67 | 68 | 69 | def use_break(limit): 70 | for x in range(limit): 71 | break 72 | return x 73 | 74 | 75 | def use_continue(limit): 76 | for x in range(limit): 77 | continue 78 | return x 79 | 80 | 81 | def use_star_args(*args): 82 | pass 83 | 84 | 85 | def use_extended_call_syntax(x): 86 | use_star_args(*x) 87 | 88 | 89 | def use_star_expr(x): 90 | a, *b = x 91 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Cosmic Ray documentation documentation master file, created by 2 | sphinx-quickstart on Fri Oct 27 12:29:41 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Cosmic Ray: mutation testing for Python 7 | ======================================= 8 | 9 | "Four human beings -- changed by space-born cosmic rays into something more than merely human." 10 | 11 | -- The Fantastic Four 12 | 13 | Cosmic Ray is a mutation testing tool for Python 3. It makes small changes to your production source code, running your 14 | test suite for each change. If a test suite passes on mutated code, then you have a mismatch between your tests and your 15 | functionality. 16 | 17 | Like coverage analysis, mutation testing helps ensure that you're testing all of your code. But while coverage only 18 | tells you if a line of code is executed, mutation testing will determine if your tests actually check the behavior of your 19 | code. This adds tremendous value to your test suite by helping it fulfill its primary role: making sure your code 20 | does what you expect it to do! 21 | 22 | Cosmic Ray has been successfully used on a wide variety of projects ranging from 23 | assemblers to oil exploration software. 24 | 25 | Contents 26 | ======== 27 | 28 | .. toctree:: 29 | :maxdepth: 1 30 | 31 | theory 32 | tutorials/intro/index 33 | tutorials/distributed/index 34 | concepts 35 | how-tos/index 36 | reference/index 37 | 38 | 39 | Indices and tables 40 | ================== 41 | 42 | * :ref:`genindex` 43 | * :ref:`modindex` 44 | * :ref:`search` 45 | -------------------------------------------------------------------------------- /docs/source/reference/continuous_integration.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Continuous Integration 3 | ======================== 4 | 5 | Cosmic Ray has a continuous integration system based on `Travis 6 | `__. Whenever we push new changes to our github 7 | repository, travis runs a set of tests. These :doc:`tests ` include 8 | low-level unit tests, end-to-end integration tests, static analysis (e.g. 9 | linting), and testing documentation builds. Generally speaking, these tests are 10 | run on all versions of Python which we support. 11 | 12 | Automated release deployment 13 | ============================ 14 | 15 | Cosmic Ray also has an automated release deployment scheme. Whenever you push 16 | changes to `the release 17 | branch `__, travis attempts 18 | to make a new release. This process involves determining the release version by 19 | reading ``cosmic_ray/version.py``, creating and uploading PyPI distributions, and 20 | creating new release tags in git. 21 | 22 | Releasing a new version 23 | ----------------------- 24 | 25 | As described above, the release process for Cosmic Ray is largely automatic. In 26 | order to do a new release, you simply need to: 27 | 28 | 1. Bump the version with `bumpversion`. 29 | 2. Push it to ``master`` on github. 30 | 3. Push the changes to the ``release`` branch on github. 31 | 32 | Once the push is made to ``release``, the automated release system will take over. 33 | 34 | Note that only the Python 3.6 travis build will attempt to make a release 35 | deployment. So to see the progress of your release, check the output for that 36 | build. 37 | -------------------------------------------------------------------------------- /src/cosmic_ray/commands/new_config.py: -------------------------------------------------------------------------------- 1 | """Implementation of the 'new-config' command.""" 2 | 3 | import os.path 4 | 5 | import qprompt 6 | 7 | from cosmic_ray.config import ConfigDict 8 | from cosmic_ray.plugins import distributor_names 9 | 10 | MODULE_PATH_HELP = """The path to the module that will be mutated. 11 | 12 | If this is a package (as opposed to a single file module), 13 | then all modules in the package and its subpackages will be 14 | mutated. 15 | 16 | This path can be absolute or relative to the location of the 17 | config file. 18 | """ 19 | 20 | 21 | TEST_COMMAND_HELP = """The command to execute to run the tests on mutated code. 22 | """ 23 | 24 | 25 | def new_config(): 26 | """Prompt user for config variables and generate new config. 27 | 28 | Returns: A new ConfigDict. 29 | """ 30 | config = ConfigDict() 31 | config["module-path"] = qprompt.ask_str( 32 | "Top-level module path", blk=False, vld=os.path.exists, hlp=MODULE_PATH_HELP 33 | ) 34 | 35 | timeout = qprompt.ask_str( 36 | "Test execution timeout (seconds)", 37 | vld=float, 38 | blk=False, 39 | hlp="The number of seconds to let a test run before terminating it.", 40 | ) 41 | config["timeout"] = float(timeout) 42 | config["excluded-modules"] = [] 43 | 44 | config["test-command"] = qprompt.ask_str("Test command", blk=False, hlp=TEST_COMMAND_HELP) 45 | 46 | menu = qprompt.Menu() 47 | for at_pos, distributor_name in enumerate(distributor_names()): 48 | menu.add(str(at_pos), distributor_name) 49 | config["distributor"] = ConfigDict() 50 | config["distributor"]["name"] = menu.show(header="Distributor", returns="desc") 51 | 52 | return config 53 | -------------------------------------------------------------------------------- /tests/tools/conftest.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import subprocess 3 | import sys 4 | import tempfile 5 | from pathlib import Path 6 | 7 | import pytest 8 | from attrs import define 9 | 10 | 11 | @pytest.fixture(scope="session") 12 | def fast_tests_root(resources_dirpath): 13 | "Root directory for 'fast_tests'." 14 | return resources_dirpath / "fast_tests" 15 | 16 | 17 | @define 18 | class SessionData: 19 | config: Path 20 | session: Path 21 | 22 | 23 | @pytest.fixture(scope="module") 24 | def initialized_session(fast_tests_root): 25 | "Initialize a session in 'fast_tests_root' and return its path." 26 | config = fast_tests_root / "cr.conf" 27 | with tempfile.TemporaryDirectory() as tmp_path: 28 | tmp_path = pathlib.Path(tmp_path) 29 | session = tmp_path / "cr.db" 30 | subprocess.check_call( 31 | [sys.executable, "-m", "cosmic_ray.cli", "init", str(config), str(session)], cwd=str(fast_tests_root) 32 | ) 33 | yield SessionData(config, session) 34 | 35 | 36 | @pytest.fixture(scope="module") 37 | def execd_session(fast_tests_root): 38 | "Initialize and exec a session in 'fast_test_root' and return its path." 39 | config = fast_tests_root / "cr.conf" 40 | with tempfile.TemporaryDirectory() as tmp_path: 41 | tmp_path = pathlib.Path(tmp_path) 42 | session = tmp_path / "cr.db" 43 | 44 | subprocess.check_call( 45 | [sys.executable, "-m", "cosmic_ray.cli", "init", str(config), str(session)], cwd=str(fast_tests_root) 46 | ) 47 | 48 | subprocess.check_call( 49 | [sys.executable, "-m", "cosmic_ray.cli", "exec", str(config), str(session)], cwd=str(fast_tests_root) 50 | ) 51 | 52 | yield SessionData(config, session) 53 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | THIS_DIR = Path(__file__).parent 6 | 7 | 8 | @pytest.fixture 9 | def tmpdir_path(tmpdir): 10 | """A temporary directory as a pathlib.Path.""" 11 | return Path(str(tmpdir)) 12 | 13 | 14 | @pytest.fixture(scope="session") 15 | def resources_dirpath(): 16 | return THIS_DIR / "resources" 17 | 18 | 19 | @pytest.fixture 20 | def session(tmpdir_path): 21 | """A temp session file (pathlib.Path)""" 22 | return tmpdir_path / "cr-session.sqlite" 23 | 24 | 25 | def pytest_addoption(parser): 26 | "Add our custom command line options" 27 | parser.addoption("--e2e-distributor", action="append", default=[], help="List of distributors to test with.") 28 | 29 | parser.addoption("--e2e-tester", action="append", default=[], help="List of test systems to use in e2e tests.") 30 | 31 | parser.addoption("--run-slow", action="store_true", default=False, help="run slow tests") 32 | 33 | 34 | def pytest_configure(config): 35 | config.addinivalue_line("markers", "slow: mark test as slow to run") 36 | 37 | 38 | def pytest_collection_modifyitems(config, items): 39 | if not config.getoption("--run-slow"): 40 | skip_slow = pytest.mark.skip(reason="need --run-slow option to run") 41 | for item in items: 42 | if "slow" in item.keywords: 43 | item.add_marker(skip_slow) 44 | 45 | 46 | def pytest_generate_tests(metafunc): 47 | "Resolve the 'distributor' and 'tester' fixtures." 48 | if "distributor" in metafunc.fixturenames: 49 | metafunc.parametrize("distributor", set(metafunc.config.getoption("--e2e-distributor"))) 50 | 51 | if "tester" in metafunc.fixturenames: 52 | metafunc.parametrize("tester", set(metafunc.config.getoption("--e2e-tester"))) 53 | -------------------------------------------------------------------------------- /src/cosmic_ray/commands/execute.py: -------------------------------------------------------------------------------- 1 | "Implementation of the 'execute' command." 2 | 3 | import logging 4 | import os 5 | 6 | from cosmic_ray.config import ConfigDict 7 | from cosmic_ray.plugins import get_distributor 8 | from cosmic_ray.progress import reports_progress 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | _progress_messages = {} # pylint: disable=invalid-name 13 | 14 | 15 | def _update_progress(work_db): 16 | num_work_items = work_db.num_work_items 17 | pending = num_work_items - work_db.num_results 18 | total = num_work_items 19 | remaining = total - pending 20 | message = f"{remaining} out of {total} completed" 21 | _progress_messages[work_db.name] = message 22 | 23 | 24 | def _report_progress(stream): 25 | for db_name, progress_message in _progress_messages.items(): 26 | session = os.path.splitext(db_name)[0] 27 | print(f"{session} : {progress_message}", file=stream) 28 | 29 | 30 | @reports_progress(_report_progress) 31 | def execute(work_db, config: ConfigDict): 32 | """Execute any pending work in the database `work_db`, 33 | recording the results. 34 | 35 | This looks for any work in `work_db` which has no results, schedules it to 36 | be executed, and records any results that arrive. 37 | """ 38 | _update_progress(work_db) 39 | distributor = get_distributor(config.distributor_name) 40 | 41 | def on_task_complete(job_id, work_result): 42 | work_db.set_result(job_id, work_result) 43 | _update_progress(work_db) 44 | log.info("Job %s complete", job_id) 45 | 46 | log.info("Beginning execution") 47 | distributor( 48 | work_db.pending_work_items, 49 | config.test_command, 50 | config.timeout, 51 | config.distributor_config, 52 | on_task_complete=on_task_complete, 53 | ) 54 | log.info("Execution finished") 55 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |Python version| |Python version windows| |Build Status| |Documentation| 2 | 3 | Cosmic Ray: mutation testing for Python 4 | ======================================= 5 | 6 | 7 | "Four human beings -- changed by space-born cosmic rays into something more than merely human." 8 | 9 | -- The Fantastic Four 10 | 11 | Cosmic Ray is a mutation testing tool for Python 3. 12 | 13 | It makes small changes to your source code, running your test suite for each 14 | one. Here's how the mutations look: 15 | 16 | .. image:: docs/source/cr-in-action.gif 17 | 18 | |full_documentation|_ 19 | 20 | Contributing 21 | ------------ 22 | 23 | The easiest way to contribute is to use Cosmic Ray and submit reports for defects or any other issues you come across. 24 | Please see CONTRIBUTING.rst for more details. 25 | 26 | .. |Python version| image:: https://img.shields.io/badge/Python_version-3.9+-blue.svg 27 | :target: https://www.python.org/ 28 | .. |Python version windows| image:: https://img.shields.io/badge/Python_version_(windows)-3.9+-blue.svg 29 | :target: https://www.python.org/ 30 | .. |Build Status| image:: https://github.com/sixty-north/cosmic-ray/actions/workflows/python-package.yml/badge.svg 31 | :target: https://github.com/sixty-north/cosmic-ray/actions/workflows/python-package.yml 32 | .. |Code Health| image:: https://landscape.io/github/sixty-north/cosmic-ray/master/landscape.svg?style=flat 33 | :target: https://landscape.io/github/sixty-north/cosmic-ray/master 34 | .. |Code Coverage| image:: https://codecov.io/gh/sixty-north/cosmic-ray/branch/master/graph/badge.svg 35 | :target: https://codecov.io/gh/Vimjas/covimerage/branch/master 36 | .. |Documentation| image:: https://readthedocs.org/projects/cosmic-ray/badge/?version=latest 37 | :target: http://cosmic-ray.readthedocs.org/en/latest/ 38 | .. |full_documentation| replace:: **Read the full documentation at readthedocs.** 39 | .. _full_documentation: http://cosmic-ray.readthedocs.org/en/latest/ 40 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: ["*"] 9 | tags: ["release/*"] 10 | pull_request: 11 | branches: [master] 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: [3.9, "3.10", "3.11", "3.12", "3.13"] 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Install the latest version of uv 22 | uses: astral-sh/setup-uv@v3 23 | with: 24 | version: "latest" 25 | enable-cache: true 26 | - name: Install Python 27 | run: uv python install ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: uv sync 30 | - name: Lint with ruff 31 | run: | 32 | uv run ruff check . 33 | uv run ruff format --check . 34 | - name: Test with pytest 35 | run: uv run pytest tests --run-slow 36 | 37 | pypi-publish: 38 | name: Upload release to PyPI 39 | needs: test 40 | if: startsWith(github.ref, 'refs/tags/release') 41 | runs-on: ubuntu-latest 42 | environment: 43 | name: pypi 44 | url: https://pypi.org/p/cosmic-ray 45 | permissions: 46 | id-token: write 47 | steps: 48 | - uses: actions/checkout@v4 49 | - name: Install the latest version of uv 50 | uses: astral-sh/setup-uv@v3 51 | with: 52 | version: "latest" 53 | enable-cache: true 54 | - name: Install Python 55 | run: uv python install 3.12 56 | - name: "Build distribution" 57 | run: | 58 | uv venv 59 | uv pip install build 60 | uv run python -m build 61 | - name: Publish package distributions to PyPI 62 | uses: pypa/gh-action-pypi-publish@release/v1 63 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/provider.py: -------------------------------------------------------------------------------- 1 | """Operator-provider plugin for all core cosmic ray operators.""" 2 | 3 | import itertools 4 | 5 | from . import ( 6 | binary_operator_replacement, 7 | boolean_replacer, 8 | break_continue, 9 | comparison_operator_replacement, 10 | exception_replacer, 11 | no_op, 12 | number_replacer, 13 | remove_decorator, 14 | unary_operator_replacement, 15 | variable_inserter, 16 | variable_replacer, 17 | zero_iteration_for_loop, 18 | ) 19 | 20 | # NB: The no_op operator gets special handling. We don't include it in iteration of the 21 | # available operators. However, you can request it from the provider by name. This lets us 22 | # use it in a special way: to request that a worker perform a no-op test run while preventing 23 | # it from being used in normal mutations testing runs. 24 | 25 | _OPERATORS = { 26 | op.__name__: op 27 | for op in itertools.chain( 28 | binary_operator_replacement.operators(), 29 | comparison_operator_replacement.operators(), 30 | unary_operator_replacement.operators(), 31 | ( 32 | boolean_replacer.AddNot, 33 | boolean_replacer.ReplaceTrueWithFalse, 34 | boolean_replacer.ReplaceFalseWithTrue, 35 | boolean_replacer.ReplaceAndWithOr, 36 | boolean_replacer.ReplaceOrWithAnd, 37 | break_continue.ReplaceBreakWithContinue, 38 | break_continue.ReplaceContinueWithBreak, 39 | exception_replacer.ExceptionReplacer, 40 | number_replacer.NumberReplacer, 41 | remove_decorator.RemoveDecorator, 42 | zero_iteration_for_loop.ZeroIterationForLoop, 43 | variable_replacer.VariableReplacer, 44 | variable_inserter.VariableInserter, 45 | ), 46 | ) 47 | } 48 | 49 | 50 | class OperatorProvider: 51 | """Provider for all of the core Cosmic Ray operators.""" 52 | 53 | def __iter__(self): 54 | return iter(_OPERATORS) 55 | 56 | def __getitem__(self, name): 57 | if name == "NoOp": 58 | return no_op.NoOp 59 | 60 | return _OPERATORS[name] 61 | -------------------------------------------------------------------------------- /tests/unittests/test_config.py: -------------------------------------------------------------------------------- 1 | """Tests for config loading functions.""" 2 | 3 | import io 4 | from unittest import mock 5 | 6 | import pytest 7 | 8 | from cosmic_ray.config import ConfigDict, ConfigError, load_config, serialize_config 9 | 10 | 11 | def test_load_valid_stdin(): 12 | temp_stdin = io.StringIO() 13 | temp_stdin.name = "stringio" 14 | config = ConfigDict() 15 | config["key"] = "value" 16 | temp_stdin.write(serialize_config(config)) 17 | temp_stdin.seek(0) 18 | with mock.patch("cosmic_ray.config.sys.stdin", temp_stdin): 19 | assert load_config()["key"] == "value" 20 | 21 | 22 | def test_load_invalid_stdin_raises_ConfigError(): 23 | temp_stdin = io.StringIO() 24 | temp_stdin.name = "stringio" 25 | temp_stdin.write("{invalid") 26 | temp_stdin.seek(0) 27 | 28 | with mock.patch("cosmic_ray.config.sys.stdin", temp_stdin): 29 | with pytest.raises(ConfigError): 30 | load_config() 31 | 32 | 33 | def test_load_from_valid_config_file(tmpdir): 34 | config_path = tmpdir / "config.conf" 35 | config = ConfigDict() 36 | config["key"] = "value" 37 | with config_path.open(mode="wt", encoding="utf-8") as handle: 38 | handle.write(serialize_config(config)) 39 | assert load_config(str(config_path))["key"] == "value" 40 | 41 | 42 | def test_load_non_existent_file_raises_ConfigError(): 43 | with pytest.raises(ConfigError): 44 | load_config("/foo/bar/this/does/no-exist/I/hope") 45 | 46 | 47 | def test_load_from_invalid_config_file_raises_ConfigError(tmpdir): 48 | config_path = tmpdir / "config.yml" 49 | with config_path.open(mode="wt", encoding="utf-8") as handle: 50 | handle.write("{asdf") 51 | with pytest.raises(ConfigError): 52 | load_config(str(config_path)) 53 | 54 | 55 | def test_load_from_non_utf8_file_raises_ConfigError(tmpdir): 56 | config_path = tmpdir / "config.conf" 57 | config = {"key": "value"} 58 | with config_path.open(mode="wb") as handle: 59 | handle.write(serialize_config(config).encode("utf-16")) 60 | with pytest.raises(ConfigError): 61 | load_config(str(config_path)) 62 | -------------------------------------------------------------------------------- /docs/source/reference/cli.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Command Line Interface 3 | ====================== 4 | 5 | This documents each of the command line programs provided with Cosmic Ray. You can also get help on the command line by passed `--help` to any command. 6 | 7 | ``cosmic-ray`` 8 | ============== 9 | 10 | This primary program provided by Cosmic Ray is, unsurprisingly, ``cosmic-ray``. This program initializes sessions, performs mutations, and executes tests. 11 | 12 | .. click:: cosmic_ray.cli:cli 13 | :prog: cosmic-ray 14 | :nested: full 15 | 16 | Concurrency 17 | ----------- 18 | 19 | Note that most Cosmic Ray commands can be safely executed while ``exec`` is 20 | running. One exception is ``init`` since that will rewrite the work manifest. 21 | 22 | For example, you can run ``cr-report`` on a session while that session is being 23 | executed. This will tell you what progress has been made. 24 | 25 | ``cr-html`` 26 | =========== 27 | 28 | .. click:: cosmic_ray.tools.html:report_html 29 | :prog: cr-html 30 | :nested: full 31 | 32 | ``cr-report`` 33 | ============= 34 | 35 | .. click:: cosmic_ray.tools.report:report 36 | :prog: cr-report 37 | :nested: full 38 | 39 | Use ``--surviving-only`` alongside ``--show-diff`` (and/or ``--show-output``) to focus the detailed listings on 40 | mutants whose tests survived, e.g. ``cr-report session.sqlite --show-diff --surviving-only``. 41 | 42 | ``cr-badge`` 43 | ============ 44 | 45 | .. click:: cosmic_ray.tools.badge:generate_badge 46 | :prog: cr-badge 47 | :nested: full 48 | 49 | ``cr-rate`` 50 | =========== 51 | 52 | .. click:: cosmic_ray.tools.survival_rate:format_survival_rate 53 | :prog: cr-rate 54 | :nested: full 55 | 56 | ``cr-xml`` 57 | ========== 58 | 59 | .. click:: cosmic_ray.tools.xml:report_xml 60 | :prog: cr-xml 61 | :nested: full 62 | 63 | ``cr-http-workers`` 64 | =================== 65 | 66 | .. click:: cosmic_ray.tools.http_workers:main 67 | :prog: cr-http-workers 68 | :nested: full 69 | 70 | ``cr-filter-operators`` 71 | ======================= 72 | 73 | **TODO** 74 | 75 | ``cr-filter-pragma`` 76 | ==================== 77 | 78 | **TODO** 79 | 80 | ``cr-filter-git`` 81 | ================= 82 | 83 | **TODO** 84 | -------------------------------------------------------------------------------- /docs/source/reference/tests.rst: -------------------------------------------------------------------------------- 1 | Tests 2 | ===== 3 | 4 | Cosmic Ray has a number of test suites to help ensure that it works. To 5 | install the necessary dependencies for testing, run: 6 | 7 | :: 8 | 9 | pip install -e .[dev,test] 10 | 11 | ``pytest`` suite 12 | ---------------- 13 | 14 | The first suite is a `pytest `__ test suite that 15 | validates some if its internals. You can run that like this: 16 | 17 | :: 18 | 19 | pytest tests/test_suite 20 | 21 | The "adam" tests 22 | ---------------- 23 | 24 | There is also a set of tests which verify the various mutation 25 | operators. These tests comprise a specially prepared body of code, 26 | ``adam.py``, and a full-coverage test-suite. The idea here is that 27 | Cosmic Ray should be 100% lethal against the mutants of ``adam.py`` or 28 | there's a problem. 29 | 30 | We have "adam" configurations for each of the 31 | test-runner/execution-engine combinations. For example, the 32 | configuration which uses ``unittest`` and the ``local`` execution 33 | engine is in ``test_project/cosmic-ray.unittest.local.conf``. 34 | 35 | To run an "adam" test, first switch to the ``test_project`` directory: 36 | 37 | :: 38 | 39 | cd tests/example_project 40 | 41 | Then initialize a new session using one of the configurations. Here's an 42 | example using the ``pytest``/``local`` configuration: 43 | 44 | :: 45 | 46 | cosmic-ray init cosmic-ray.pytest.local.conf pytest-local.sqlite 47 | 48 | (Note that if you were going to use the ``celery4`` engine instead, you 49 | need to make sure that celery workers were running.) 50 | 51 | Execute the session like this: 52 | 53 | :: 54 | 55 | cosmic-ray exec pytest-local.sqlite 56 | 57 | Finally, view the results of this test with ``dump`` and ``cr-report``: 58 | 59 | :: 60 | 61 | cr-report pytest-local.sqlite 62 | 63 | You should see a 0% survival rate at the end of the report. 64 | 65 | The full test suite 66 | ------------------- 67 | 68 | While the "adam" tests verify the various mutation operators in Cosmic 69 | Ray, the full test suite comprises a few more tests for other behaviors 70 | and functionality. To run all of these tests, it's often simplest to use tox. Just run:: 71 | 72 | $ tox 73 | 74 | at the root of the project. 75 | -------------------------------------------------------------------------------- /tests/resources/example_project/tests/test_adam.py: -------------------------------------------------------------------------------- 1 | "Tests for the adam packages." 2 | 3 | # pylint: disable=C0111 4 | 5 | import copy 6 | import unittest 7 | import uuid 8 | 9 | import adam.adam_1 10 | import adam.adam_2 11 | 12 | 13 | class Tests(unittest.TestCase): 14 | def test_constant_number(self): 15 | self.assertEqual(adam.adam_1.constant_number(), 42) 16 | 17 | def test_constant_true(self): 18 | self.assertEqual(adam.adam_1.constant_true(), True) 19 | 20 | def test_constant_false(self): 21 | self.assertEqual(adam.adam_1.constant_false(), False) 22 | 23 | def test_bool_and(self): 24 | self.assertFalse(adam.adam_1.bool_and()) 25 | 26 | def test_bool_or(self): 27 | self.assertTrue(adam.adam_1.bool_or()) 28 | 29 | def test_bool_expr_with_not(self): 30 | self.assertFalse(adam.adam_1.bool_expr_with_not()) 31 | 32 | def test_bool_if(self): 33 | self.assertTrue(adam.adam_1.bool_if()) 34 | 35 | def test_if_expression(self): 36 | self.assertTrue(adam.adam_1.if_expression()) 37 | 38 | def test_assert_in_func(self): 39 | self.assertTrue(adam.adam_1.assert_in_func()) 40 | 41 | def test_unary_sub(self): 42 | self.assertEqual(adam.adam_1.unary_sub(), -1) 43 | 44 | def test_unary_add(self): 45 | self.assertEqual(adam.adam_1.unary_add(), +1) 46 | 47 | def test_binary_add(self): 48 | self.assertEqual(adam.adam_1.binary_add(), 11) 49 | 50 | def test_equals(self): 51 | vals = [uuid.uuid4(), uuid.uuid4()] 52 | vals.append(copy.copy(vals[0])) 53 | self.assertTrue(adam.adam_1.equals(vals)) 54 | 55 | def test_break_to_continue(self): 56 | self.assertEqual(adam.adam_1.use_break(10), 0) 57 | 58 | def test_continue_to_break(self): 59 | self.assertEqual(adam.adam_1.use_continue(10), 9) 60 | 61 | def test_trigger_infinite_loop(self): 62 | self.assertTrue(adam.adam_2.trigger_infinite_loop()) 63 | 64 | def test_single_iteration(self): 65 | self.assertTrue(adam.adam_2.single_iteration()) 66 | 67 | def test_handle_exception(self): 68 | self.assertTrue(adam.adam_2.handle_exception()) 69 | 70 | def test_decorator(self): 71 | self.assertTrue(adam.adam_2.decorated_func()) 72 | -------------------------------------------------------------------------------- /src/cosmic_ray/plugins.py: -------------------------------------------------------------------------------- 1 | """Query and retrieve the various plugins in Cosmic Ray.""" 2 | 3 | import logging 4 | 5 | from stevedore import ExtensionManager, driver 6 | 7 | log = logging.getLogger() # pylint: disable=invalid-name 8 | 9 | 10 | def _log_extension_loading_failure(_mgr, extension_point, err): 11 | # We have to log at the `error` level here as opposed to, say, `info` 12 | # because logging isn't configure when we reach here. We need this infor to 13 | # print with the default logging settings. 14 | log.error('Operator provider load failure: extension-point="%s", err="%s"', extension_point, err) 15 | 16 | 17 | OPERATOR_PROVIDERS = { 18 | extension.name: extension.plugin() 19 | for extension in ExtensionManager( 20 | "cosmic_ray.operator_providers", on_load_failure_callback=_log_extension_loading_failure 21 | ) 22 | } 23 | 24 | 25 | def get_operator(name): 26 | """Get an operator class from a provider plugin. 27 | 28 | Args: 29 | name: The name of the operator class. 30 | 31 | Returns: 32 | The operator *class object* (i.e. not an instance). 33 | """ 34 | sep = name.index("/") 35 | provider_name = name[:sep] 36 | operator_name = name[sep + 1 :] 37 | 38 | provider = OPERATOR_PROVIDERS[provider_name] 39 | return provider[operator_name] 40 | 41 | 42 | def operator_names(): 43 | """Get all operator names. 44 | 45 | Returns: 46 | A sequence of operator names. 47 | """ 48 | return tuple( 49 | f"{provider_name}/{operator_name}" 50 | for provider_name, provider in OPERATOR_PROVIDERS.items() 51 | for operator_name in provider 52 | ) 53 | 54 | 55 | def get_distributor(name): 56 | """Get the distributor by name.""" 57 | manager = driver.DriverManager( 58 | namespace="cosmic_ray.distributors", 59 | name=name, 60 | invoke_on_load=True, 61 | on_load_failure_callback=_log_extension_loading_failure, 62 | ) 63 | 64 | return manager.driver 65 | 66 | 67 | def distributor_names(): 68 | """Get all distributor plugin names. 69 | 70 | Returns: 71 | A sequence of distributor names. 72 | """ 73 | return ExtensionManager( 74 | "cosmic_ray.distributors", 75 | on_load_failure_callback=_log_extension_loading_failure, 76 | ).names() 77 | -------------------------------------------------------------------------------- /src/cosmic_ray/testing.py: -------------------------------------------------------------------------------- 1 | "Support for running tests in a subprocess." 2 | 3 | import logging 4 | import os 5 | import shlex 6 | import subprocess 7 | import traceback 8 | 9 | from cosmic_ray.work_item import TestOutcome 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | # We use an asyncio-subprocess-based approach here instead of a simple 14 | # subprocess.run()-based approach because there are problems with timeouts and 15 | # reading from stderr in subprocess.run. Since we have to be prepared for test 16 | # processes that run longer than timeout (and, indeed, which run forever), the 17 | # broken subprocess stuff simply doesn't work. So we do this, which seems to 18 | # work on all platforms. 19 | 20 | 21 | def run_tests(command, timeout): 22 | """Run test command in a subprocess. 23 | 24 | If the command exits with status 0, then we assume that all tests passed. If 25 | it exits with any other code, we assume a test failed. If the call to launch 26 | the subprocess throws an exception, we consider the test 'incompetent'. 27 | 28 | Tests which time out are considered 'killed' as well. 29 | 30 | Args: 31 | command (str): The command to execute. 32 | timeout (number): The maximum number of seconds to allow the tests to run. 33 | 34 | Return: A tuple `(TestOutcome, output)` where the `output` is a string 35 | containing the output of the command. 36 | """ 37 | log.info("Running test (timeout=%s): %s", timeout, command) 38 | 39 | # We want to avoid writing pyc files in case our changes happen too fast for Python to 40 | # notice them. If the timestamps between two changes are too small, Python won't recompile 41 | # the source. 42 | env = dict(os.environ) 43 | env["PYTHONDONTWRITEBYTECODE"] = "1" 44 | 45 | try: 46 | proc = subprocess.run(shlex.split(command), check=True, env=env, timeout=timeout, capture_output=True) 47 | assert proc.returncode == 0 48 | return (TestOutcome.SURVIVED, proc.stdout.decode("utf-8")) 49 | 50 | except subprocess.CalledProcessError as err: 51 | return (TestOutcome.KILLED, err.output.decode("utf-8")) 52 | 53 | except subprocess.TimeoutExpired: 54 | return (TestOutcome.KILLED, "timeout") 55 | 56 | except Exception: # pylint: disable=W0703 57 | return (TestOutcome.INCOMPETENT, traceback.format_exc()) 58 | -------------------------------------------------------------------------------- /tests/unittests/operators/test_binary_operator_replacement.py: -------------------------------------------------------------------------------- 1 | from cosmic_ray.mutating import mutate_code 2 | from cosmic_ray.operators.binary_operator_replacement import ( 3 | ReplaceBinaryOperator_Add_Mul, 4 | ReplaceBinaryOperator_BitOr_Add, 5 | ReplaceBinaryOperator_Mul_Add, 6 | ReplaceBinaryOperator_Sub_Add, 7 | ) 8 | 9 | 10 | def test_pipe_operator_in_assignment_annotation_not_mutated_as_binary_operator(): 11 | code = "my_var: str | int = 10" 12 | mutated = mutate_code(code, ReplaceBinaryOperator_BitOr_Add(), 0) 13 | assert mutated is None 14 | 15 | 16 | def test_pipe_operator_in_local_assignment_annotation_not_mutated_as_binary_operator(): 17 | code = r""" 18 | def my_function(): 19 | local_var: str | int = 3 20 | """ 21 | mutated = mutate_code(code, ReplaceBinaryOperator_BitOr_Add(), 0) 22 | assert mutated is None 23 | 24 | 25 | def test_pipe_in_function_argument_type_annotation_mutated_as_binary_operator(): 26 | code = r""" 27 | def my_function(arg: str | int = 10): 28 | local_var = arg 29 | """ 30 | mutated = mutate_code(code, ReplaceBinaryOperator_BitOr_Add(), 0) 31 | assert mutated is not None 32 | 33 | 34 | def test_bitwise_or_mutated_as_binary_operator(): 35 | code = "my_var = 10 | 2" 36 | mutated = mutate_code(code, ReplaceBinaryOperator_BitOr_Add(), 0) 37 | assert mutated is not None 38 | 39 | 40 | def test_import_star_not_mutated_as_binary_operator(): 41 | code = "from math import *" 42 | mutated = mutate_code(code, ReplaceBinaryOperator_Mul_Add(), 0) 43 | assert mutated is None 44 | 45 | 46 | def test_star_expr_not_mutated_as_binary_operator(): 47 | code = "a, *b = x" 48 | mutated = mutate_code(code, ReplaceBinaryOperator_Mul_Add(), 0) 49 | assert mutated is None 50 | 51 | 52 | def test_star_args_not_mutated_as_binary_operator(): 53 | code = "def foo(*args): pass" 54 | mutated = mutate_code(code, ReplaceBinaryOperator_Mul_Add(), 0) 55 | assert mutated is None 56 | 57 | 58 | def test_unary_minus_not_mutated_as_binary_operator(): 59 | code = "-1" 60 | mutated = mutate_code(code, ReplaceBinaryOperator_Sub_Add(), 0) 61 | assert mutated is None 62 | 63 | 64 | def test_unary_plus_not_mutated_as_binary_operator(): 65 | code = "+1" 66 | mutated = mutate_code(code, ReplaceBinaryOperator_Add_Mul(), 0) 67 | assert mutated is None 68 | -------------------------------------------------------------------------------- /tests/unittests/operators/test_operator_samples.py: -------------------------------------------------------------------------------- 1 | """Tests for the various mutation operators.""" 2 | 3 | import parso 4 | import pytest 5 | 6 | from cosmic_ray.mutating import MutationVisitor 7 | from cosmic_ray.operators.binary_operator_replacement import ReplaceBinaryOperator_Add_Mul 8 | from cosmic_ray.operators.operator import Example 9 | from cosmic_ray.operators.unary_operator_replacement import ReplaceUnaryOperator_USub_UAdd 10 | from cosmic_ray.operators.variable_inserter import VariableInserter 11 | from cosmic_ray.operators.variable_replacer import VariableReplacer 12 | from cosmic_ray.plugins import get_operator, operator_names 13 | 14 | 15 | class Sample: 16 | def __init__(self, operator, example): 17 | self.operator = operator 18 | self.example = example 19 | 20 | 21 | OPERATOR_PROVIDED_SAMPLES = tuple( 22 | Sample(operator_class, example) 23 | for operator_class in map(get_operator, operator_names()) 24 | for example in operator_class.examples() 25 | ) 26 | 27 | EXTRA_SAMPLES = tuple( 28 | Sample(*args) 29 | for args in ( 30 | # Make sure unary and binary op mutators don't pick up the wrong kinds of operators 31 | (ReplaceUnaryOperator_USub_UAdd, Example("x + 1", "x + 1")), 32 | (ReplaceBinaryOperator_Add_Mul, Example("+1", "+1")), 33 | ) 34 | ) 35 | 36 | OPERATOR_SAMPLES = OPERATOR_PROVIDED_SAMPLES + EXTRA_SAMPLES 37 | 38 | 39 | @pytest.mark.parametrize("sample", OPERATOR_SAMPLES, ids=lambda s: str(s.operator.__name__)) 40 | def test_mutation_changes_ast(sample: Sample): 41 | if sample.operator in {VariableReplacer, VariableInserter}: 42 | pytest.xfail(f"{sample.operator} tests fail because they produce random output.") 43 | 44 | node = parso.parse(sample.example.pre_mutation_code) 45 | visitor = MutationVisitor(sample.example.occurrence, sample.operator(**sample.example.operator_args)) 46 | mutant = visitor.walk(node) 47 | 48 | assert mutant.get_code() == sample.example.post_mutation_code 49 | 50 | 51 | @pytest.mark.parametrize("sample", OPERATOR_SAMPLES) 52 | def test_no_mutation_leaves_ast_unchanged(sample): 53 | print(sample.operator, sample.example) 54 | node = parso.parse(sample.example.pre_mutation_code) 55 | visitor = MutationVisitor(-1, sample.operator(**sample.example.operator_args)) 56 | mutant = visitor.walk(node) 57 | 58 | assert mutant.get_code() == sample.example.pre_mutation_code 59 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/exception_replacer.py: -------------------------------------------------------------------------------- 1 | "Implementation of the exception-replacement operator." 2 | 3 | from parso.python.tree import Name, PythonNode 4 | 5 | from cosmic_ray.exceptions import CosmicRayTestingException 6 | 7 | from .operator import Example, Operator 8 | 9 | 10 | class ExceptionReplacer(Operator): 11 | """An operator that modifies exception handlers.""" 12 | 13 | def mutation_positions(self, node): 14 | if isinstance(node, PythonNode): 15 | if node.type == "except_clause": 16 | for name in self._name_nodes(node): 17 | yield (name.start_pos, name.end_pos) 18 | 19 | def mutate(self, node, index): 20 | assert isinstance(node, PythonNode) 21 | assert node.type == "except_clause" 22 | 23 | name_nodes = self._name_nodes(node) 24 | assert index < len(name_nodes) 25 | name_nodes[index].value = CosmicRayTestingException.__name__ 26 | return node 27 | 28 | @staticmethod 29 | def _name_nodes(node): 30 | if isinstance(node.children[1], Name): 31 | return (node.children[1],) 32 | 33 | atom = node.children[1] 34 | test_list = atom.children[1] 35 | if isinstance(test_list, Name): 36 | return (test_list,) 37 | return test_list.children[::2] 38 | 39 | @classmethod 40 | def examples(cls): 41 | return ( 42 | Example( 43 | "try: raise OSError\nexcept OSError: pass", 44 | f"try: raise OSError\nexcept {CosmicRayTestingException.__name__}: pass", 45 | ), 46 | Example( 47 | "try: raise OSError\nexcept (OSError): pass", 48 | f"try: raise OSError\nexcept ({CosmicRayTestingException.__name__}): pass", 49 | ), 50 | Example( 51 | "try: raise OSError\nexcept (OSError, ValueError): pass", 52 | f"try: raise OSError\nexcept (OSError, {CosmicRayTestingException.__name__}): pass", 53 | occurrence=1, 54 | ), 55 | Example( 56 | "try: raise OSError\nexcept (OSError, ValueError, KeyError): pass", 57 | f"try: raise OSError\nexcept (OSError, {CosmicRayTestingException.__name__}, KeyError): pass", 58 | occurrence=1, 59 | ), 60 | Example("try: pass\nexcept: pass", "try: pass\nexcept: pass"), 61 | ) 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | deploy_key 2 | docs/source/_build 3 | .DS_Store 4 | .ruff_cache 5 | 6 | # Created by https://www.gitignore.io/api/python,visualstudiocode 7 | # Edit at https://www.gitignore.io/?templates=python,visualstudiocode 8 | 9 | ### Python ### 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | share/python-wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .nox/ 52 | .coverage 53 | .coverage.* 54 | .cache 55 | nosetests.xml 56 | coverage.xml 57 | *.cover 58 | .hypothesis/ 59 | .pytest_cache/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | db.sqlite3 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # celery beat schedule file 94 | celerybeat-schedule 95 | 96 | # SageMath parsed files 97 | *.sage.py 98 | 99 | # Environments 100 | .env 101 | .venv 102 | env/ 103 | venv/ 104 | ENV/ 105 | env.bak/ 106 | venv.bak/ 107 | 108 | # Spyder project settings 109 | .spyderproject 110 | .spyproject 111 | 112 | # Rope project settings 113 | .ropeproject 114 | 115 | # mkdocs documentation 116 | /site 117 | 118 | # mypy 119 | .mypy_cache/ 120 | .dmypy.json 121 | dmypy.json 122 | 123 | # Pyre type checker 124 | .pyre/ 125 | 126 | ### Python Patch ### 127 | .venv/ 128 | 129 | ### VisualStudioCode ### 130 | .vscode/ 131 | 132 | ### VisualStudioCode Patch ### 133 | # Ignore all local history of files 134 | .history 135 | 136 | # End of https://www.gitignore.io/api/python,visualstudiocode 137 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/filters/operators_filter.py: -------------------------------------------------------------------------------- 1 | """An filter that removes operators based on regular expressions.""" 2 | 3 | import logging 4 | import re 5 | import sys 6 | from argparse import ArgumentParser, Namespace 7 | 8 | from cosmic_ray.config import load_config 9 | from cosmic_ray.tools.filters.filter_app import FilterApp 10 | from cosmic_ray.work_db import WorkDB 11 | from cosmic_ray.work_item import WorkResult, WorkerOutcome 12 | 13 | log = logging.getLogger() 14 | 15 | 16 | class OperatorsFilter(FilterApp): 17 | "Implemenents the operators-filter." 18 | 19 | def description(self): 20 | return __doc__ 21 | 22 | def _skip_filtered(self, work_db, exclude_operators): 23 | if not exclude_operators: 24 | return 25 | 26 | re_exclude_operators = re.compile("|".join(f"(:?{e})" for e in exclude_operators)) 27 | 28 | for item in work_db.pending_work_items: 29 | for mutation in item.mutations: 30 | if re_exclude_operators.match(mutation.operator_name): 31 | log.info( 32 | "operator skipping %s %s %s %s %s %s", 33 | item.job_id, 34 | mutation.operator_name, 35 | mutation.occurrence, 36 | mutation.module_path, 37 | mutation.start_pos, 38 | mutation.end_pos, 39 | ) 40 | 41 | work_db.set_result( 42 | item.job_id, 43 | WorkResult( 44 | output="Filtered operator", 45 | worker_outcome=WorkerOutcome.SKIPPED, 46 | ), 47 | ) 48 | 49 | break 50 | 51 | def filter(self, work_db: WorkDB, args: Namespace): 52 | """Mark as skipped all work item with filtered operator""" 53 | 54 | config = load_config(args.config) 55 | 56 | exclude_operators = config.sub("filters", "operators-filter").get("exclude-operators", ()) 57 | self._skip_filtered(work_db, exclude_operators) 58 | 59 | def add_args(self, parser: ArgumentParser): 60 | parser.add_argument("config", help="Config file to use") 61 | 62 | 63 | def main(argv=None): 64 | """Run the operators-filter with the specified command line arguments.""" 65 | return OperatorsFilter().main(argv) 66 | 67 | 68 | if __name__ == "__main__": 69 | sys.exit(main()) 70 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/filters/filter_app.py: -------------------------------------------------------------------------------- 1 | """A simple base for creating common types of work-db filters.""" 2 | 3 | import argparse 4 | import logging 5 | import sys 6 | 7 | from exit_codes import ExitCode 8 | 9 | from cosmic_ray.work_db import use_db 10 | 11 | 12 | class FilterApp: 13 | """Base class for simple WorkDB filters. 14 | 15 | This provides command-line handling for common filter options like 16 | the session and verbosity level. Subclasses can add their own arguments 17 | as well. This provides a `main()` function that open the session's WorkDB 18 | and passes it to the subclass's `filter()` function. 19 | 20 | **It is by no means required that you inherit from or otherwise use 21 | this class in order to build a filter.** You can build a filter using 22 | any technique you want. Typically all a filter does is modify a session 23 | database in some way, so you can use the Cosmic Ray API for that directly 24 | if you want. 25 | """ 26 | 27 | def add_args(self, parser: argparse.ArgumentParser): 28 | """Add any arguments that the subclass needs to the parser. 29 | 30 | Args: 31 | parser: The ArgumentParser for command-line processing. 32 | """ 33 | 34 | def description(self): 35 | """The description of the filter. 36 | 37 | This is used for the command-line help message. 38 | """ 39 | return None 40 | 41 | def main(self, argv=None): 42 | """The main function for the app. 43 | 44 | Args: 45 | argv: Command line argument list of parse. 46 | """ 47 | if argv is None: 48 | argv = sys.argv[1:] 49 | 50 | parser = argparse.ArgumentParser( 51 | description=self.description(), 52 | ) 53 | parser.add_argument("session", help="Path to the session on which to operate") 54 | parser.add_argument("--verbosity", help="Verbosity level for logging", default="WARNING") 55 | self.add_args(parser) 56 | args = parser.parse_args(argv) 57 | 58 | logging.basicConfig(level=getattr(logging, args.verbosity)) 59 | 60 | with use_db(args.session) as db: 61 | self.filter(db, args) 62 | 63 | return ExitCode.OK 64 | 65 | def filter(self, work_db, args): 66 | """Apply this filter to a WorkDB. 67 | 68 | This should modify the WorkDB in place. 69 | 70 | Args: 71 | work_db: An open WorkDB instance. 72 | args: The argparse Namespace for the command line. 73 | """ 74 | raise NotImplementedError() 75 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/filters/pragma_no_mutate.py: -------------------------------------------------------------------------------- 1 | """A filter that uses "# pragma: no mutate" to determine when specific mutations 2 | should be skipped. 3 | """ 4 | 5 | import logging 6 | import re 7 | import sys 8 | from functools import lru_cache 9 | 10 | from cosmic_ray.tools.filters.filter_app import FilterApp 11 | from cosmic_ray.work_item import WorkResult, WorkerOutcome 12 | 13 | log = logging.getLogger() 14 | 15 | 16 | class PragmaNoMutateFilter(FilterApp): 17 | """Implements the pragma-no-mutate filter.""" 18 | 19 | def description(self): 20 | return __doc__ 21 | 22 | def filter(self, work_db, _args): 23 | """Mark lines with "# pragma: no mutate" as SKIPPED 24 | 25 | For all work_item in db, if the LAST line of the working zone is marked 26 | with "# pragma: no mutate", This work_item will be skipped. 27 | """ 28 | 29 | @lru_cache 30 | def file_contents(file_path): 31 | "A simple cache of file contents." 32 | with file_path.open(mode="rt") as handle: 33 | return handle.readlines() 34 | 35 | re_is_mutate = re.compile(r".*#.*pragma:.*no mutate.*") 36 | 37 | for item in work_db.work_items: 38 | for mutation in item.mutations: 39 | print(mutation.module_path) 40 | lines = file_contents(mutation.module_path) 41 | try: 42 | # item.{start,end}_pos[0] seems to be 1-based. 43 | line_number = mutation.end_pos[0] - 1 44 | if mutation.end_pos[1] == 0: 45 | # The working zone ends at begin of line, 46 | # consider the previous line. 47 | line_number -= 1 48 | line = lines[line_number] 49 | if re_is_mutate.match(line): 50 | work_db.set_result( 51 | item.job_id, 52 | WorkResult(output=None, test_outcome=None, diff=None, worker_outcome=WorkerOutcome.SKIPPED), 53 | ) 54 | except Exception as ex: 55 | raise Exception( 56 | f"module_path: {mutation.module_path}, start_pos: {mutation.start_pos}, end_pos: {mutation.end_pos}, len(lines): {len(lines)}" 57 | ) from ex 58 | 59 | 60 | def main(argv=None): 61 | """Run pragma-no-mutate filter with specified command line arguments.""" 62 | return PragmaNoMutateFilter().main(argv) 63 | 64 | 65 | if __name__ == "__main__": 66 | sys.exit(main()) 67 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/report.py: -------------------------------------------------------------------------------- 1 | "Tool for printing reports on mutation testing sessions." 2 | 3 | import click 4 | 5 | from cosmic_ray.tools.survival_rate import kills_count, survival_rate 6 | from cosmic_ray.work_db import WorkDB, use_db 7 | 8 | 9 | @click.command() 10 | @click.option("--show-output/--no-show-output", default=False, help="Display output of test executions") 11 | @click.option("--show-diff/--no-show-diff", default=False, help="Display diff of mutants") 12 | @click.option("--show-pending/--no-show-pending", default=False, help="Display results for incomplete tasks") 13 | @click.option( 14 | "--surviving-only/--all-mutations", 15 | default=False, 16 | help="Only display completed work items whose tests survived", 17 | ) 18 | @click.argument("session-file", type=click.Path(dir_okay=False, readable=True, exists=True)) 19 | def report(show_output, show_diff, show_pending, surviving_only, session_file): 20 | """Print a nicely formatted report of test results and some basic statistics.""" 21 | 22 | with use_db(session_file, WorkDB.Mode.open) as db: 23 | for work_item, result in db.completed_work_items: 24 | if surviving_only and result.is_killed: 25 | continue 26 | 27 | display_work_item(work_item) 28 | 29 | print(f"worker outcome: {result.worker_outcome}, test outcome: {result.test_outcome}") 30 | 31 | if show_output: 32 | print("=== OUTPUT ===") 33 | print(result.output) 34 | print("==============") 35 | 36 | if show_diff: 37 | print("=== DIFF ===") 38 | print(result.diff) 39 | print("============") 40 | 41 | if show_pending: 42 | for work_item in db.pending_work_items: 43 | display_work_item(work_item) 44 | 45 | num_items = db.num_work_items 46 | num_complete = db.num_results 47 | 48 | print(f"total jobs: {num_items}") 49 | 50 | if num_complete > 0: 51 | print(f"complete: {num_complete} ({num_complete / num_items * 100:.2f}%)") 52 | num_killed = kills_count(db) 53 | print(f"surviving mutants: {num_complete - num_killed} ({survival_rate(db):.2f}%)") 54 | else: 55 | print("no jobs completed") 56 | 57 | 58 | def display_work_item(work_item): 59 | print(f"[job-id] {work_item.job_id}") 60 | for mutation in work_item.mutations: 61 | print(f"{mutation.module_path} {mutation.operator_name} {mutation.occurrence}") 62 | 63 | 64 | if __name__ == "__main__": 65 | report() # no-qa: no-value-for-parameter 66 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/survival_rate.py: -------------------------------------------------------------------------------- 1 | "Tool for printing the survival rate in a session." 2 | 3 | import math 4 | import sys 5 | 6 | import click 7 | 8 | from cosmic_ray.work_db import WorkDB, use_db 9 | 10 | SUPPORTED_Z_SCORES = {800: 1.282, 900: 1.645, 950: 1.960, 980: 2.326, 990: 2.576, 995: 2.807, 998: 3.080, 999: 3.291} 11 | 12 | 13 | @click.command() 14 | @click.option( 15 | "--estimate/--no-estimate", default=False, help="Print the lower bound, estimate and upper bound of survival rate" 16 | ) 17 | @click.option( 18 | "--confidence", 19 | type=click.Choice(sorted([str(z / 10) for z in SUPPORTED_Z_SCORES])), 20 | default="95.0", 21 | help="Specify the confidence levels for estimates", 22 | ) 23 | @click.option( 24 | "--fail-over", 25 | type=click.FloatRange(0, 100), 26 | default=None, 27 | help="Exit with a non-zero code if the survival rate is larger than or the calculated confidence interval " 28 | "is above the (if --estimate is used). Specified as percentage.", 29 | ) 30 | @click.argument("session-file", type=click.Path(dir_okay=False, readable=True, exists=True)) 31 | def format_survival_rate(estimate, confidence, fail_over, session_file): 32 | """Calculate the survival rate of a session.""" 33 | confidence = float(confidence) 34 | try: 35 | z_score = SUPPORTED_Z_SCORES[int(float(confidence) * 10)] 36 | except KeyError: 37 | raise ValueError(f"Unsupported confidence interval: {confidence}") 38 | 39 | with use_db(session_file, WorkDB.Mode.open) as db: 40 | rate = survival_rate(db) 41 | num_items = db.num_work_items 42 | num_complete = db.num_results 43 | 44 | if estimate: 45 | if not num_complete: 46 | conf_int = 0 47 | else: 48 | conf_int = ( 49 | math.sqrt(rate * (100 - rate) / num_complete) * z_score * (1 - math.sqrt(num_complete / num_items)) 50 | ) 51 | min_rate = rate - conf_int 52 | print(f"{min_rate:.2f} {rate:.2f} {rate + conf_int:.2f}") 53 | 54 | else: 55 | print(f"{rate:.2f}") 56 | min_rate = rate 57 | 58 | if fail_over and min_rate > float(fail_over): 59 | sys.exit(1) 60 | 61 | 62 | def kills_count(work_db): 63 | """Return the number of killed mutants.""" 64 | return sum(r.is_killed for _, r in work_db.results) 65 | 66 | 67 | def survival_rate(work_db): 68 | """Calculate the survival rate for the results in a WorkDB.""" 69 | kills = kills_count(work_db) 70 | num_results = work_db.num_results 71 | 72 | if not num_results: 73 | return 0 74 | 75 | return (1 - kills / num_results) * 100 76 | 77 | 78 | if __name__ == "__main__": 79 | format_survival_rate() 80 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray package 2 | =================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | cosmic_ray.ast 10 | cosmic_ray.commands 11 | cosmic_ray.distribution 12 | cosmic_ray.operators 13 | cosmic_ray.tools 14 | 15 | Submodules 16 | ---------- 17 | 18 | cosmic\_ray.cli module 19 | ---------------------- 20 | 21 | .. automodule:: cosmic_ray.cli 22 | :members: 23 | :undoc-members: 24 | :show-inheritance: 25 | 26 | cosmic\_ray.config module 27 | ------------------------- 28 | 29 | .. automodule:: cosmic_ray.config 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | 34 | cosmic\_ray.exceptions module 35 | ----------------------------- 36 | 37 | .. automodule:: cosmic_ray.exceptions 38 | :members: 39 | :undoc-members: 40 | :show-inheritance: 41 | 42 | cosmic\_ray.modules module 43 | -------------------------- 44 | 45 | .. automodule:: cosmic_ray.modules 46 | :members: 47 | :undoc-members: 48 | :show-inheritance: 49 | 50 | cosmic\_ray.mutating module 51 | --------------------------- 52 | 53 | .. automodule:: cosmic_ray.mutating 54 | :members: 55 | :undoc-members: 56 | :show-inheritance: 57 | 58 | cosmic\_ray.plugins module 59 | -------------------------- 60 | 61 | .. automodule:: cosmic_ray.plugins 62 | :members: 63 | :undoc-members: 64 | :show-inheritance: 65 | 66 | cosmic\_ray.progress module 67 | --------------------------- 68 | 69 | .. automodule:: cosmic_ray.progress 70 | :members: 71 | :undoc-members: 72 | :show-inheritance: 73 | 74 | cosmic\_ray.testing module 75 | -------------------------- 76 | 77 | .. automodule:: cosmic_ray.testing 78 | :members: 79 | :undoc-members: 80 | :show-inheritance: 81 | 82 | cosmic\_ray.timing module 83 | ------------------------- 84 | 85 | .. automodule:: cosmic_ray.timing 86 | :members: 87 | :undoc-members: 88 | :show-inheritance: 89 | 90 | cosmic\_ray.version module 91 | -------------------------- 92 | 93 | .. automodule:: cosmic_ray.version 94 | :members: 95 | :undoc-members: 96 | :show-inheritance: 97 | 98 | cosmic\_ray.work\_db module 99 | --------------------------- 100 | 101 | .. automodule:: cosmic_ray.work_db 102 | :members: 103 | :undoc-members: 104 | :show-inheritance: 105 | 106 | cosmic\_ray.work\_item module 107 | ----------------------------- 108 | 109 | .. automodule:: cosmic_ray.work_item 110 | :members: 111 | :undoc-members: 112 | :show-inheritance: 113 | 114 | 115 | Module contents 116 | --------------- 117 | 118 | .. automodule:: cosmic_ray 119 | :members: 120 | :undoc-members: 121 | :show-inheritance: 122 | -------------------------------------------------------------------------------- /AGENTS.md: -------------------------------------------------------------------------------- 1 | # Repository Guidelines 2 | 3 | ## Project Structure & Module Organization 4 | Cosmic Ray follows a `src/` layout: packages live in `src/cosmic_ray`, metadata in `src/cosmic_ray/version.py`, and CLI surfaces in `src/cosmic_ray/cli`. Tests live in `tests/unittests`, `tests/tools`, and `tests/e2e`, with fixtures/configs parked under `tests/resources`. Documentation resides in `docs/source`, and helper utilities live in `tools/`. The canonical upstream is `https://github.com/sixty-north/cosmic-ray`; match this layout when mirroring across forks. 5 | 6 | ## Build, Test, and Development Commands 7 | - `python -m venv .venv && source .venv/bin/activate` isolates dependencies before syncing/installing. 8 | - `pip install -e .[dev]` (or `uv sync --dev`) installs the CLI plus dev tooling. 9 | - `pytest` runs the default suite; `pytest --run-slow tests/e2e` adds the long mutation workflows. 10 | - `ruff check src tests` (optionally `--fix`) keeps linting/ordering consistent. 11 | 12 | ## Coding Style & Naming Conventions 13 | Target idiomatic Python 3.9+ and PEP 8, but favor clarity when a rule conflicts with readability. `ruff` enforces a 120-character limit and `isort`-style grouping; let it rewrite imports instead of hand-tuning. CI expects `uv ruff .` and `uv ruff format .` before every commit. Modules stay snake_case, classes use CapWords, and CLI flags remain kebab-case (`cosmic-ray exec`). Keep shared configuration (`*.conf`, `pyproject.toml`) in the repo root or `tests/resources` so tooling can load them without extra paths. 14 | 15 | ## Testing Guidelines 16 | Pytest backs every layer: unit coverage in `tests/unittests`, tool coverage in `tests/tools`, and slow integration runs in `tests/e2e`. Name files `test_.py`, keep test functions descriptive, and centralize fixtures in `tests/conftest.py`. Tag long-lived scenarios with `@pytest.mark.slow` so contributors can opt in via `--run-slow`. Add regression tests whenever you alter a mutation operator, CLI flag, or persistence layer. 17 | 18 | ## Commit & Pull Request Guidelines 19 | Follow the format from `CONTRIBUTING.rst`: subjects are imperative and reference an issue (`Issue #1234 - Make operator ordering deterministic`) or start with `Doc -` for prose-only commits. Commit bodies should state problem, impact, and fix. Before opening a PR, run `git diff --check`, `ruff check`, and the relevant `pytest` targets; summarize those results plus reproduction steps (and screenshots/logs for reporting tools such as `cr-html`) in the PR description. Keep PRs focused and call out dependency or configuration changes in `pyproject.toml`. 20 | 21 | ## Security & Configuration Tips 22 | Do not commit plaintext secrets—`deploy_key.enc` shows the expected encrypted form. Store experimental configs under `tests/resources` or ignore them locally. When developing HTTP distributors or remote workers, prefer the mocked services in `tests/tools` and document any new ports, certificates, or environment variables directly in the pull request. 23 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/binary_operator_replacement.py: -------------------------------------------------------------------------------- 1 | """Implementation of the binary-operator-replacement operator.""" 2 | 3 | import itertools 4 | from enum import Enum 5 | 6 | import parso 7 | 8 | from .operator import Example, Operator 9 | from .util import extend_name 10 | 11 | 12 | class BinaryOperators(Enum): 13 | "All binary operators that we mutate." 14 | 15 | Add = "+" 16 | Sub = "-" 17 | Mul = "*" 18 | Div = "/" 19 | FloorDiv = "//" 20 | Mod = "%" 21 | Pow = "**" 22 | RShift = ">>" 23 | LShift = "<<" 24 | BitOr = "|" 25 | BitAnd = "&" 26 | BitXor = "^" 27 | 28 | 29 | def _create_replace_binary_operator(from_op, to_op): 30 | @extend_name(f"_{from_op.name}_{to_op.name}") 31 | class ReplaceBinaryOperator(Operator): 32 | f"An operator that replaces binary {from_op.name} with binary {to_op.name}." 33 | 34 | def mutation_positions(self, node): 35 | if _is_binary_operator(node): 36 | if node.value == from_op.value: 37 | yield (node.start_pos, node.end_pos) 38 | 39 | def mutate(self, node, index): 40 | assert _is_binary_operator(node) 41 | assert index == 0 42 | 43 | node.value = to_op.value 44 | return node 45 | 46 | @classmethod 47 | def examples(cls): 48 | return (Example(f"x {from_op.value} y", f"x {to_op.value} y"),) 49 | 50 | return ReplaceBinaryOperator 51 | 52 | 53 | # Parent types of operators which indicate that the operator isn't binary. 54 | _NON_BINARY_PARENTS = { 55 | "factor", # unary operators, e.g. -1 56 | "argument", # extended function definitions, e.g. def foo(*args) 57 | "star_expr", # destructuring, e.g. a, *b = x 58 | "import_from", # star import, e.g. from module import * 59 | } 60 | 61 | 62 | def _is_binary_operator(node): 63 | if isinstance(node, parso.python.tree.Operator): 64 | # This catches extended call syntax, e.g. call(*x) 65 | if isinstance(node.parent, parso.python.tree.Param): 66 | return False 67 | 68 | operator_is_pipe_in_assignment_annotation = ( 69 | (node.value == "|") 70 | and (node.parent and node.parent.type == "expr") 71 | and (node.parent.parent and node.parent.parent.type == "annassign") 72 | ) 73 | 74 | if node.parent.type in _NON_BINARY_PARENTS or operator_is_pipe_in_assignment_annotation: 75 | return False 76 | 77 | return True 78 | 79 | return False 80 | 81 | 82 | # Build all of the binary replacement operators 83 | _MUTATION_OPERATORS = tuple( 84 | _create_replace_binary_operator(from_op, to_op) for from_op, to_op in itertools.permutations(BinaryOperators, 2) 85 | ) 86 | 87 | # Inject operators into module namespace 88 | for op_cls in _MUTATION_OPERATORS: 89 | globals()[op_cls.__name__] = op_cls 90 | 91 | 92 | def operators(): 93 | "Iterable of all binary operator replacement mutation operators." 94 | return _MUTATION_OPERATORS 95 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/boolean_replacer.py: -------------------------------------------------------------------------------- 1 | """Implementation of the boolean replacement operators.""" 2 | 3 | import parso.python.tree 4 | 5 | from .keyword_replacer import KeywordReplacementOperator 6 | from .operator import Example, Operator 7 | 8 | 9 | class ReplaceTrueWithFalse(KeywordReplacementOperator): 10 | """An that replaces True with False.""" 11 | 12 | from_keyword = "True" 13 | to_keyword = "False" 14 | 15 | 16 | class ReplaceFalseWithTrue(KeywordReplacementOperator): 17 | """An that replaces False with True.""" 18 | 19 | from_keyword = "False" 20 | to_keyword = "True" 21 | 22 | 23 | class ReplaceAndWithOr(KeywordReplacementOperator): 24 | """An operator that swaps 'and' with 'or'.""" 25 | 26 | from_keyword = "and" 27 | to_keyword = "or" 28 | 29 | @classmethod 30 | def examples(cls): 31 | return (Example("x and y", "x or y"),) 32 | 33 | 34 | class ReplaceOrWithAnd(KeywordReplacementOperator): 35 | """An operator that swaps 'or' with 'and'.""" 36 | 37 | from_keyword = "or" 38 | to_keyword = "and" 39 | 40 | @classmethod 41 | def examples(cls): 42 | return (Example("x or y", "x and y"),) 43 | 44 | 45 | class AddNot(Operator): 46 | """ 47 | An operator that adds the 'not' keyword to boolean expressions. 48 | 49 | NOTE: 'not' as unary operator is mutated in 50 | ``unary_operator_replacement.py``, including deletion of the same 51 | operator. 52 | """ 53 | 54 | NODE_TYPES = (parso.python.tree.IfStmt, parso.python.tree.WhileStmt, parso.python.tree.AssertStmt) 55 | 56 | def mutation_positions(self, node): 57 | if isinstance(node, self.NODE_TYPES): 58 | expr = node.children[1] 59 | yield (expr.start_pos, expr.end_pos) 60 | elif isinstance(node, parso.python.tree.PythonNode) and node.type == "test": 61 | # ternary conditional 62 | expr = node.children[2] 63 | yield (expr.start_pos, expr.end_pos) 64 | 65 | def mutate(self, node, index): 66 | assert index == 0 67 | 68 | if isinstance(node, self.NODE_TYPES): 69 | expr_node = node.children[1] 70 | mutated_code = f" not{expr_node.get_code()}" 71 | mutated_node = parso.parse(mutated_code) 72 | node.children[1] = mutated_node 73 | 74 | else: 75 | assert node.type == "test" 76 | expr_node = node.children[2] 77 | mutated_code = f" not{expr_node.get_code()}" 78 | mutated_node = parso.parse(mutated_code) 79 | node.children[2] = mutated_node 80 | 81 | return node 82 | 83 | @classmethod 84 | def examples(cls): 85 | return ( 86 | Example("if True or False: pass", "if not True or False: pass"), 87 | Example("A if B else C", "A if not B else C"), 88 | Example("assert isinstance(node, ast.Break)", "assert not isinstance(node, ast.Break)"), 89 | Example("while True: pass", "while not True: pass"), 90 | ) 91 | -------------------------------------------------------------------------------- /tests/unittests/test_find_modules.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | from cosmic_ray.modules import filter_paths, find_modules 6 | 7 | 8 | def test_small_directory_tree(data_dir): 9 | paths = (("a", "__init__.py"), ("a", "b.py"), ("a", "py.py"), ("a", "c", "__init__.py"), ("a", "c", "d.py")) 10 | expected = sorted(data_dir / Path(*path) for path in paths) 11 | results = sorted(find_modules([data_dir / "a"])) 12 | assert expected == results 13 | 14 | 15 | def test_finding_modules_via_dir_name(data_dir): 16 | paths = (("a", "c", "__init__.py"), ("a", "c", "d.py")) 17 | expected = sorted(data_dir / Path(*path) for path in paths) 18 | results = sorted(find_modules([data_dir / "a" / "c"])) 19 | assert expected == results 20 | 21 | 22 | def test_finding_modules_via_dir_name_and_filename_ending_in_py(data_dir): 23 | paths = (("a", "c", "d.py"),) 24 | expected = sorted(data_dir / Path(*path) for path in paths) 25 | results = sorted(find_modules([data_dir / "a" / "c" / "d.py"])) 26 | assert expected == results 27 | 28 | 29 | def test_finding_module_py_dot_py_using_dots(data_dir): 30 | paths = (("a", "py.py"),) 31 | expected = sorted(data_dir / Path(*path) for path in paths) 32 | results = sorted(find_modules([data_dir / "a" / "py.py"])) 33 | assert expected == results 34 | 35 | 36 | def test_finding_modules_with_missing_file(data_dir): 37 | path = data_dir / "a" / "inexisting_file.py" 38 | with pytest.raises(FileNotFoundError) as exc_info: 39 | tuple(find_modules((data_dir / "a", path, data_dir / "a" / "c"))) 40 | assert str(exc_info.value) == f"Could not find module path {path}" 41 | 42 | 43 | def test_finding_modules_py_dot_py_using_slashes_with_full_filename(data_dir): 44 | paths = (("a", "py.py"),) 45 | expected = sorted(data_dir / Path(*path) for path in paths) 46 | results = sorted(find_modules([data_dir / "a" / "py.py"])) 47 | assert expected == results 48 | 49 | 50 | def test_small_directory_tree_with_excluding_files(data_dir, path_utils): 51 | paths = (("a", "b.py"), ("a", "py.py"), ("a", "c", "d.py")) 52 | excluded_modules = ["**/__init__.py"] 53 | expected = set(Path(*path) for path in paths) 54 | 55 | with path_utils.excursion(data_dir): 56 | results = find_modules([Path("a")]) 57 | results = filter_paths(results, excluded_modules) 58 | assert expected == results 59 | 60 | 61 | def test_small_directory_tree_with_excluding_dir(data_dir, path_utils): 62 | paths = (("a", "__init__.py"), ("a", "b.py"), ("a", "py.py")) 63 | excluded_modules = ["*/c/*"] 64 | expected = set(Path(*path) for path in paths) 65 | 66 | with path_utils.excursion(data_dir): 67 | results = find_modules([Path("a")]) 68 | results = filter_paths(results, excluded_modules) 69 | assert expected == results 70 | 71 | 72 | def test_multiple_module_paths(data_dir): 73 | paths = (("a", "b.py"), ("a", "c", "__init__.py"), ("a", "c", "d.py")) 74 | expected = sorted(data_dir / Path(*path) for path in paths) 75 | results = sorted(find_modules([data_dir / "a" / "b.py", data_dir / "a" / "c"])) 76 | assert expected == results 77 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/xml.py: -------------------------------------------------------------------------------- 1 | "A tool for creating XML reports." 2 | 3 | import sys 4 | import xml.etree.ElementTree 5 | 6 | import click 7 | 8 | from cosmic_ray.work_db import WorkDB, use_db 9 | from cosmic_ray.work_item import TestOutcome, WorkerOutcome 10 | 11 | 12 | @click.command() 13 | @click.argument("session-file", type=click.Path(dir_okay=False, readable=True, exists=True)) 14 | def report_xml(session_file): 15 | """Print an XML formatted report of test results for continuous integration systems""" 16 | with use_db(session_file, WorkDB.Mode.open) as db: 17 | xml_elem = _create_xml_report(db) 18 | xml_elem.write(sys.stdout.buffer, encoding="utf-8", xml_declaration=True) 19 | 20 | 21 | def _create_xml_report(db): 22 | errors = 0 23 | failed = 0 24 | skipped = 0 25 | root_elem = xml.etree.ElementTree.Element("testsuite") 26 | 27 | for work_item, result in db.completed_work_items: 28 | if result.worker_outcome in {WorkerOutcome.EXCEPTION, WorkerOutcome.ABNORMAL}: 29 | errors += 1 30 | if result.is_killed: 31 | failed += 1 32 | if result.worker_outcome == WorkerOutcome.SKIPPED: 33 | skipped += 1 34 | 35 | subelement = _create_element_from_work_item(work_item) 36 | subelement = _update_element_with_result(subelement, result) 37 | root_elem.append(subelement) 38 | 39 | for work_item in db.pending_work_items: 40 | subelement = _create_element_from_work_item(work_item) 41 | root_elem.append(subelement) 42 | 43 | root_elem.set("errors", str(errors)) 44 | root_elem.set("failures", str(failed)) 45 | root_elem.set("skips", str(skipped)) 46 | root_elem.set("tests", str(db.num_work_items)) 47 | return xml.etree.ElementTree.ElementTree(root_elem) 48 | 49 | 50 | def _create_element_from_work_item(work_item): 51 | sub_elem = xml.etree.ElementTree.Element("testcase") 52 | 53 | for mutation in work_item.mutations: 54 | mutation_elem = xml.etree.ElementTree.Element("mutation") 55 | mutation_elem.set("classname", work_item.job_id) 56 | mutation_elem.set("line", str(mutation.start_pos[0])) 57 | mutation_elem.set("file", str(mutation.module_path)) 58 | sub_elem.append(mutation_elem) 59 | 60 | return sub_elem 61 | 62 | 63 | def _update_element_with_result(sub_elem, result): 64 | data = result.output 65 | outcome = result.worker_outcome 66 | 67 | if outcome == WorkerOutcome.EXCEPTION: 68 | error_elem = xml.etree.ElementTree.SubElement(sub_elem, "error") 69 | error_elem.set("message", "Worker has encountered exception") 70 | error_elem.text = str(data) + "\n".join(result.diff) 71 | elif _evaluation_success(result): 72 | failure_elem = xml.etree.ElementTree.SubElement(sub_elem, "failure") 73 | failure_elem.set("message", "Mutant has survived your unit tests") 74 | failure_elem.text = str(data) + result.diff 75 | 76 | return sub_elem 77 | 78 | 79 | def _evaluation_success(result): 80 | return result.worker_outcome == WorkerOutcome.NORMAL and result.test_outcome in { 81 | TestOutcome.SURVIVED, 82 | TestOutcome.INCOMPETENT, 83 | } 84 | 85 | 86 | if __name__ == "__main__": 87 | report_xml() # pylint: disable=no-value-for-parameter 88 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/operator.py: -------------------------------------------------------------------------------- 1 | "Implementation of operator base class." 2 | 3 | from abc import ABC, abstractmethod 4 | from collections.abc import Sequence 5 | from typing import Optional 6 | 7 | from attrs import define, field 8 | 9 | 10 | class Operator(ABC): 11 | """The mutation operator base class.""" 12 | 13 | @abstractmethod 14 | def mutation_positions(self, node): 15 | """All positions where this operator can mutate ``node``. 16 | 17 | An operator might be able to mutate a node in multiple ways, and this 18 | function should produce a position description for each of these 19 | mutations. Critically, if an operator can make multiple mutations to the 20 | same position, this should produce a position for each of these 21 | mutations (i.e. multiple identical positions). 22 | 23 | Args: 24 | node: The AST node being mutated. 25 | 26 | Returns: 27 | An iterable of ``((start-line, start-col), (stop-line, stop-col))`` 28 | tuples describing the locations where this operator will mutate ``node``. 29 | """ 30 | 31 | @abstractmethod 32 | def mutate(self, node, index): 33 | """Mutate a node in an operator-specific manner. 34 | 35 | Return the new, mutated node. Return ``None`` if the node has 36 | been deleted. Return ``node`` if there is no mutation at all for 37 | some reason. 38 | """ 39 | 40 | @classmethod 41 | def arguments(cls) -> Sequence["Argument"]: 42 | """Sequence of Arguments that the operator accepts. 43 | 44 | Returns: A Sequence of Argument instances 45 | """ 46 | return () 47 | 48 | @classmethod 49 | @abstractmethod 50 | def examples(cls): 51 | """Examples of the mutations that this operator can make. 52 | 53 | This is primarily for testing purposes, but it could also be used for 54 | documentation. 55 | 56 | Each example takes the following arguments: 57 | pre_mutation_code: code prior to applying the mutation. 58 | post_mutation_code: code after (successfully) applying the mutation. 59 | occurrence: the index of the occurrence to which the mutation is 60 | applied (optional, default=0). 61 | operator_args: a dictionary of arguments to be **-unpacked to the 62 | operator (optional, default={}). 63 | 64 | Returns: An iterable of Examples. 65 | """ 66 | 67 | 68 | @define(frozen=True) 69 | class Argument: 70 | name: str = field() 71 | description: str = field() 72 | 73 | 74 | @define(frozen=True) 75 | class Example: 76 | """A structure to store pre and post mutation operator code snippets, 77 | including optional specification of occurrence and operator args. 78 | 79 | This is used for testing whether the pre-mutation code is correctly 80 | mutated to the post-mutation code at the given occurrence (if specified) 81 | and for the given operator args (if specified). 82 | """ 83 | 84 | pre_mutation_code: str = field() 85 | post_mutation_code: str = field() 86 | occurrence: Optional[int] = field(default=0) 87 | operator_args: Optional[dict] = field(default=None) 88 | 89 | def __attrs_post_init__(self): 90 | if not self.operator_args: 91 | object.__setattr__(self, "operator_args", {}) 92 | -------------------------------------------------------------------------------- /src/cosmic_ray/work_item.py: -------------------------------------------------------------------------------- 1 | """Classes for describing work and results.""" 2 | 3 | import enum 4 | from pathlib import Path 5 | from typing import Any, Optional 6 | 7 | from attrs import define, field 8 | 9 | 10 | class StrEnum(str, enum.Enum): 11 | "An Enum subclass with str values." 12 | 13 | 14 | class WorkerOutcome(StrEnum): 15 | """Possible outcomes for a worker.""" 16 | 17 | NORMAL = "normal" # The worker exited normally, producing valid output 18 | EXCEPTION = "exception" # The worker exited with an exception 19 | ABNORMAL = "abnormal" # The worker did not exit normally or with an exception (e.g. a segfault) 20 | NO_TEST = "no-test" # The worker had no test to run 21 | SKIPPED = "skipped" # The job was skipped (worker was not executed) 22 | 23 | 24 | class TestOutcome(StrEnum): 25 | """A enum of the possible outcomes for any mutant test run.""" 26 | 27 | SURVIVED = "survived" 28 | KILLED = "killed" 29 | INCOMPETENT = "incompetent" 30 | 31 | 32 | @define(frozen=True) 33 | class WorkResult: 34 | """The result of a single mutation and test run.""" 35 | 36 | worker_outcome: WorkerOutcome = field() 37 | output: Optional[str] = field(default=None) 38 | test_outcome: Optional[TestOutcome] = field(default=None) 39 | diff: Optional[str] = field(default=None) 40 | 41 | def __attrs_post_init__(self): 42 | if self.worker_outcome is None: 43 | raise ValueError("Worker outcome must always have a value.") 44 | 45 | if self.test_outcome is not None: 46 | object.__setattr__(self, "test_outcome", TestOutcome(self.test_outcome)) 47 | 48 | object.__setattr__(self, "worker_outcome", WorkerOutcome(self.worker_outcome)) 49 | 50 | @property 51 | def is_killed(self): 52 | "Whether the mutation should be considered 'killed'" 53 | return self.test_outcome != TestOutcome.SURVIVED 54 | 55 | 56 | @define(frozen=True) 57 | class MutationSpec: 58 | "Description of a single mutation." 59 | 60 | module_path: Path = field(converter=Path) 61 | operator_name: str = field() 62 | occurrence: int = field(converter=int) 63 | start_pos: tuple[int, int] = field() 64 | end_pos: tuple[int, int] = field() 65 | operator_args: dict[str, Any] = field(factory=dict) 66 | 67 | @end_pos.validator 68 | def _validate_positions(self, attribute, value): 69 | start_line, start_col = self.start_pos 70 | end_line, end_col = value 71 | 72 | if start_line > end_line or (start_line == end_line and start_col >= end_col): 73 | raise ValueError("End position must come after start position.") 74 | 75 | 76 | @define(frozen=True) 77 | class WorkItem: 78 | """A collection (possibly empty) of mutations to perform for a single test. 79 | 80 | This ability to perform more than one mutation for a single test run is how we support 81 | higher-order mutations. 82 | """ 83 | 84 | job_id: str = field() 85 | mutations: tuple[MutationSpec, ...] = field(converter=tuple) 86 | 87 | @classmethod 88 | def single(cls, job_id, mutation: MutationSpec): 89 | """Construct a WorkItem with a single mutation. 90 | 91 | Args: 92 | job_id: The ID of the job. 93 | mutation: The single mutation for the WorkItem. 94 | 95 | Returns: 96 | A new `WorkItem` instance. 97 | """ 98 | return cls(job_id, (mutation,)) 99 | -------------------------------------------------------------------------------- /src/cosmic_ray/ast/__init__.py: -------------------------------------------------------------------------------- 1 | "Tools for working with parso ASTs." 2 | 3 | import io 4 | from abc import ABC, abstractmethod 5 | from pathlib import Path 6 | 7 | import parso.python.tree 8 | import parso.tree 9 | 10 | from cosmic_ray.util import read_python_source 11 | 12 | 13 | class Visitor(ABC): 14 | """AST visitor for parso trees. 15 | 16 | This supports both simple traversal as well as editing of the tree. 17 | """ 18 | 19 | def walk(self, node): 20 | "Walk a parse tree, calling visit for each node." 21 | node = self.visit(node) 22 | 23 | if node is None: 24 | return None 25 | 26 | if isinstance(node, parso.tree.BaseNode): 27 | walked = map(self.walk, node.children) 28 | node.children = [child for child in walked if child is not None] 29 | 30 | return node 31 | 32 | @abstractmethod 33 | def visit(self, node): 34 | """Called for each node in the walk. 35 | 36 | This should return a node that will replace the node argument in the AST. This can be 37 | the node argument itself, a new node, or None. If None is returned, then the node is 38 | removed from the tree. 39 | 40 | Args: 41 | node: The node currently being visited. 42 | 43 | Returns: 44 | A node or `None`. 45 | """ 46 | 47 | 48 | def ast_nodes(node): 49 | """Iterable of all nodes in a tree. 50 | 51 | Args: 52 | node: The top node in a parso tree to iterate. 53 | 54 | Yields: 55 | All of the nodes in the tree. 56 | """ 57 | yield node 58 | 59 | if isinstance(node, parso.tree.BaseNode): 60 | for child in node.children: 61 | yield from ast_nodes(child) 62 | 63 | 64 | def get_ast_from_path(module_path: Path): 65 | """Get the AST for the code in a file. 66 | 67 | Args: 68 | module_path: pathlib.Path to the file containing the code. 69 | 70 | Returns: 71 | The parso parse tree for the code in `module_path`. 72 | """ 73 | source = read_python_source(module_path) 74 | return get_ast(source) 75 | 76 | 77 | def get_ast(source: str): 78 | """Parse the AST for a code string. 79 | 80 | Args: 81 | code (str): _description_ 82 | """ 83 | return parso.parse(source) 84 | 85 | 86 | def is_none(node): 87 | "Determine if a node is the `None` keyword." 88 | return isinstance(node, parso.python.tree.Keyword) and node.value == "None" 89 | 90 | 91 | def is_number(node): 92 | "Determine if a node is a number." 93 | return isinstance(node, parso.python.tree.Number) 94 | 95 | 96 | def dump_node(node): 97 | "Generate string version of node." 98 | buffer = io.StringIO() 99 | write = buffer.write 100 | 101 | def do_dump(node, indent=""): 102 | write(f"{indent}{type(node).__name__}({node.type}") 103 | value = getattr(node, "value", None) 104 | if value: 105 | value = value.replace("\n", "\\n") 106 | write(f", '{value}'") 107 | children = getattr(node, "children", None) 108 | if children: 109 | write(", [\n") 110 | for child in children: 111 | do_dump(child, indent + " " * 4) 112 | write(",\n") 113 | write(f"{indent}]") 114 | write(")") 115 | if not indent: 116 | write("\n") 117 | 118 | do_dump(node) 119 | return buffer.getvalue() 120 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/variable_inserter.py: -------------------------------------------------------------------------------- 1 | """Implementation of the variable-inserter operator.""" 2 | 3 | import random 4 | 5 | import parso.python.tree 6 | from parso.python.tree import Name, PythonNode 7 | 8 | from .operator import Argument, Example, Operator 9 | 10 | 11 | class VariableInserter(Operator): 12 | """An operator that replaces usages of named variables to particular statements.""" 13 | 14 | def __init__(self, cause_variable, effect_variable): 15 | self.cause_variable = cause_variable 16 | self.effect_variable = effect_variable 17 | 18 | @classmethod 19 | def arguments(cls): 20 | return ( 21 | Argument("cause_variable", "The cause variable"), 22 | Argument("effect_variable", "The effect variable"), 23 | ) 24 | 25 | def mutation_positions(self, node): 26 | """Find expressions or terms that define the effect variable. These nodes can be 27 | mutated to introduce an effect of the cause variable. 28 | """ 29 | if isinstance(node, PythonNode) and (node.type == "arith_expr" or node.type == "term"): 30 | expr_node = node.search_ancestor("expr_stmt") 31 | if expr_node: 32 | effect_variable_names = [v.value for v in expr_node.get_defined_names()] 33 | if self.effect_variable in effect_variable_names: 34 | cause_variables = list(self._get_causes_from_expr_node(expr_node)) 35 | if node not in cause_variables: 36 | yield (node.start_pos, node.end_pos) 37 | 38 | def mutate(self, node, index): 39 | """Join the node with cause variable using a randomly sampled arithmetic operator.""" 40 | assert isinstance(node, PythonNode) 41 | assert node.type == "arith_expr" or node.type == "term" 42 | 43 | arith_operator = random.choice(["+", "*", "-"]) 44 | arith_operator_node_start_pos = self._iterate_col(node.end_pos) 45 | cause_node_start_pos = self._iterate_col(arith_operator_node_start_pos) 46 | arith_operator_node = parso.python.tree.Operator(arith_operator, start_pos=arith_operator_node_start_pos) 47 | cause_node = Name(self.cause_variable, start_pos=cause_node_start_pos) 48 | replacement_node = parso.python.tree.PythonNode("arith_expr", [node, arith_operator_node, cause_node]) 49 | return replacement_node 50 | 51 | def _get_causes_from_expr_node(self, expr_node): 52 | rhs = expr_node.get_rhs().children 53 | return self._flatten_expr(rhs) 54 | 55 | def _flatten_expr(self, expr): 56 | for item in expr: 57 | # Convert PythonNode to list of its children 58 | try: 59 | item_to_flatten = item.children 60 | except AttributeError: 61 | item_to_flatten = item 62 | # 63 | try: 64 | yield from self._flatten_expr(item_to_flatten) 65 | except TypeError: 66 | yield item_to_flatten 67 | 68 | @staticmethod 69 | def _iterate_col(position_tuple): 70 | return tuple(sum(x) for x in zip(position_tuple, (0, 1))) 71 | 72 | @classmethod 73 | def examples(cls): 74 | return ( 75 | Example("y = x + z", "y = x + z * j", operator_args={"cause_variable": "j", "effect_variable": "y"}), 76 | Example( 77 | "j = x + z\ny = x + z", 78 | "j = x + z + x\ny = x + z", 79 | operator_args={"cause_variable": "x", "effect_variable": "j"}, 80 | ), 81 | ) 82 | -------------------------------------------------------------------------------- /src/cosmic_ray/ast/ast_query.py: -------------------------------------------------------------------------------- 1 | "Tools for querying ASTs." 2 | 3 | 4 | class ASTQuery: 5 | """ 6 | Allowing to navigate into any object and test attribute of any object: 7 | 8 | Examples: 9 | >>> ASTQuery(node).parent.match(Node, type='node').ok 10 | 11 | Test if node.parent isinstance of Node and node.parent.type == 'node' 12 | At each step (each '.' (dot)) you receive an ObjTest object, then 13 | 14 | Navigation: 15 | You can call any properties or functions of the base object 16 | >>> ASTQuery(node).parent.children[2].get_next_sibling() 17 | 18 | Test: 19 | >>> ASTQuery(node).match(attr='value').match(Class) 20 | All in once: 21 | >>> ASTQuery(node).match(Class, attr='value') 22 | 23 | Conditional navigation: 24 | >>> ASTQuery(node).IF.match(attr='intermediate').parent.FI 25 | 26 | Final result: 27 | >>> ASTQuery(node).ok 28 | >>> bool(ASTQuery(node)) 29 | 30 | """ 31 | 32 | def __init__(self, obj): 33 | self.obj = obj 34 | 35 | def _clone(self, obj) -> "ASTQuery": 36 | "Clone this query." 37 | return type(self)(obj) 38 | 39 | def match(self, cls=None, **kwargs) -> "ASTQuery": 40 | "Check if node matches a class." 41 | obj = self.obj 42 | if obj is None: 43 | return self 44 | 45 | if cls is None or isinstance(obj, cls): 46 | for k, v in kwargs.items(): 47 | op = None 48 | k__op = k.split("__") 49 | if len(k__op) == 2: 50 | k, op = k__op 51 | node_value = getattr(obj, k) 52 | if op is None: 53 | if node_value != v: 54 | break 55 | elif op == "in": 56 | if node_value not in v: 57 | break 58 | else: 59 | raise ValueError(f"Can't handle operator {op}") 60 | else: 61 | # All is true, continue recursion 62 | return self 63 | 64 | # A test fails 65 | return self._clone(None) 66 | 67 | @property 68 | def ok(self): 69 | "Is the query ok." 70 | return bool(self.obj) 71 | 72 | def __bool__(self): 73 | return self.ok 74 | 75 | def __getattr__(self, item) -> "ASTQuery": 76 | obj = self.obj 77 | if obj is None: 78 | return self 79 | return self._clone(getattr(obj, item)) 80 | 81 | @property 82 | def IF(self): 83 | "Conditional navigation." 84 | return ASTQueryOptional(self.obj, obj_test=self) 85 | 86 | def __call__(self, *args, **kwargs) -> "ASTQuery": 87 | if self.obj is None: 88 | return self 89 | return self._clone(self.obj(*args, **kwargs)) 90 | 91 | def __getitem__(self, item) -> "ASTQuery": 92 | if self.obj is None: 93 | return self 94 | return self._clone(self.obj[item]) 95 | 96 | 97 | class ASTQueryOptional(ASTQuery): 98 | "Manages conditional navigation." 99 | 100 | def __init__(self, obj, obj_test=None): 101 | super().__init__(obj) 102 | self._initial = obj_test 103 | 104 | def _clone(self, obj): 105 | o = super()._clone(obj) 106 | o._initial = self._initial # pylint: disable=protected-access 107 | 108 | return o 109 | 110 | @property 111 | def FI(self): 112 | "End of conditional navigation." 113 | if self: 114 | return self._initial._clone(self.obj) # pylint: disable=protected-access 115 | return self._initial 116 | -------------------------------------------------------------------------------- /tests/unittests/test_command_line_processing.py: -------------------------------------------------------------------------------- 1 | "Tests for the command line interface and return codes." 2 | 3 | # pylint: disable=C0111,W0621,W0613 4 | 5 | import stat 6 | 7 | import pytest 8 | from exit_codes import ExitCode 9 | 10 | import cosmic_ray.cli 11 | import cosmic_ray.config 12 | import cosmic_ray.modules 13 | import cosmic_ray.mutating 14 | import cosmic_ray.plugins 15 | 16 | 17 | @pytest.fixture 18 | def config_file(tmpdir): 19 | return str(tmpdir.join("config.toml")) 20 | 21 | 22 | def _make_config(test_command="python -m unittest discover tests", timeout=100, distributor="local"): 23 | return { 24 | "module-path": "foo.py", 25 | "timeout": timeout, 26 | "test-command": test_command, 27 | "distributor": {"name": distributor}, 28 | "excluded-modules": [], 29 | } 30 | 31 | 32 | @pytest.fixture 33 | def local_unittest_config(config_file): 34 | """Creates a valid config file for local, unittest-based execution, returning 35 | the path to the config. 36 | """ 37 | with open(config_file, mode="w") as handle: 38 | config = _make_config() 39 | config_str = cosmic_ray.config.serialize_config(config) 40 | handle.write(config_str) 41 | return config_file 42 | 43 | 44 | @pytest.fixture 45 | def lobotomize(monkeypatch): 46 | "Short-circuit some of CR's core functionality to make testing simpler." 47 | # This effectively prevent init from actually trying to scan the module in the config. 48 | monkeypatch.setattr(cosmic_ray.modules, "find_modules", lambda *args: []) 49 | 50 | # Make cosmic_ray.mutating.mutate_and_test just return a simple empty dict. 51 | monkeypatch.setattr(cosmic_ray.mutating, "mutate_and_test", lambda *args: {}) 52 | 53 | 54 | def test_invalid_command_line_returns_EX_USAGE(): 55 | assert cosmic_ray.cli.main(["init", "foo"]) == 2 56 | 57 | 58 | def test_non_existent_session_file_returns_EX_NOINPUT(local_unittest_config): 59 | assert cosmic_ray.cli.main(["exec", str(local_unittest_config), "foo.session"]) == ExitCode.NO_INPUT 60 | 61 | 62 | def test_non_existent_config_file_returns_EX_NOINPUT(session, local_unittest_config): 63 | cosmic_ray.cli.main(["init", local_unittest_config, str(session)]) 64 | assert cosmic_ray.cli.main(["exec", "no-such-file", str(session)]) == ExitCode.CONFIG 65 | 66 | 67 | @pytest.mark.skip("need to sort this API out") 68 | def test_unreadable_file_returns_EX_PERM(tmpdir, local_unittest_config): 69 | p = tmpdir.ensure("bogus.session.sqlite") 70 | p.chmod(stat.S_IRUSR) 71 | assert cosmic_ray.cli.main(["exec", local_unittest_config, str(p.realpath())]) == ExitCode.NO_PERM 72 | 73 | 74 | def test_new_config_success_returns_EX_OK(monkeypatch, config_file): 75 | monkeypatch.setattr(cosmic_ray.commands, "new_config", lambda *args: "") 76 | errcode = cosmic_ray.cli.main(["new-config", config_file]) 77 | assert errcode == ExitCode.OK 78 | 79 | 80 | # NOTE: We have integration tests for the happy-path for many commands, so we don't cover them explicitly here. 81 | 82 | 83 | def test_dump_success_returns_EX_OK(lobotomize, local_unittest_config, session): 84 | errcode = cosmic_ray.cli.main(["init", local_unittest_config, str(session)]) 85 | assert errcode == ExitCode.OK 86 | 87 | errcode = cosmic_ray.cli.main(["dump", str(session)]) 88 | assert errcode == ExitCode.OK 89 | 90 | 91 | def test_operators_success_returns_EX_OK(): 92 | assert cosmic_ray.cli.main(["operators"]) == ExitCode.OK 93 | 94 | 95 | # def test_mutate_and_test_success_returns_EX_OK(lobotomize, local_unittest_config): 96 | # cmd = ["worker", "some_module", "core/ReplaceTrueWithFalse", "0", local_unittest_config] 97 | # assert cosmic_ray.cli.main(cmd) == ExitCode.OK 98 | -------------------------------------------------------------------------------- /docs/source/reference/api/cosmic_ray.operators.rst: -------------------------------------------------------------------------------- 1 | cosmic\_ray.operators package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | cosmic\_ray.operators.binary\_operator\_replacement module 8 | ---------------------------------------------------------- 9 | 10 | .. automodule:: cosmic_ray.operators.binary_operator_replacement 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | cosmic\_ray.operators.boolean\_replacer module 16 | ---------------------------------------------- 17 | 18 | .. automodule:: cosmic_ray.operators.boolean_replacer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | cosmic\_ray.operators.break\_continue module 24 | -------------------------------------------- 25 | 26 | .. automodule:: cosmic_ray.operators.break_continue 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | cosmic\_ray.operators.comparison\_operator\_replacement module 32 | -------------------------------------------------------------- 33 | 34 | .. automodule:: cosmic_ray.operators.comparison_operator_replacement 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | cosmic\_ray.operators.exception\_replacer module 40 | ------------------------------------------------ 41 | 42 | .. automodule:: cosmic_ray.operators.exception_replacer 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | cosmic\_ray.operators.keyword\_replacer module 48 | ---------------------------------------------- 49 | 50 | .. automodule:: cosmic_ray.operators.keyword_replacer 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | cosmic\_ray.operators.no\_op module 56 | ----------------------------------- 57 | 58 | .. automodule:: cosmic_ray.operators.no_op 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | cosmic\_ray.operators.number\_replacer module 64 | --------------------------------------------- 65 | 66 | .. automodule:: cosmic_ray.operators.number_replacer 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | cosmic\_ray.operators.operator module 72 | ------------------------------------- 73 | 74 | .. automodule:: cosmic_ray.operators.operator 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | cosmic\_ray.operators.provider module 80 | ------------------------------------- 81 | 82 | .. automodule:: cosmic_ray.operators.provider 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | cosmic\_ray.operators.remove\_decorator module 88 | ---------------------------------------------- 89 | 90 | .. automodule:: cosmic_ray.operators.remove_decorator 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | cosmic\_ray.operators.unary\_operator\_replacement module 96 | --------------------------------------------------------- 97 | 98 | .. automodule:: cosmic_ray.operators.unary_operator_replacement 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | cosmic\_ray.operators.util module 104 | --------------------------------- 105 | 106 | .. automodule:: cosmic_ray.operators.util 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | cosmic\_ray.operators.zero\_iteration\_for\_loop module 112 | ------------------------------------------------------- 113 | 114 | .. automodule:: cosmic_ray.operators.zero_iteration_for_loop 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: cosmic_ray.operators 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools'] 3 | build-backend = 'setuptools.build_meta' 4 | 5 | [project] 6 | name = "cosmic_ray" 7 | requires-python = ">= 3.9" 8 | dynamic = ["version"] 9 | authors = [{ name = "Sixty North AS", email = "austin@sixty-north.com" }] 10 | description = "Mutation testing" 11 | readme = { file = "README.rst", content-type = "text/x-rst" } 12 | license = { file = "LICENCE.txt" } 13 | classifiers = [ 14 | "Development Status :: 4 - Beta", 15 | "Environment :: Console", 16 | "Intended Audience :: Developers", 17 | "License :: OSI Approved :: MIT License", 18 | "Operating System :: OS Independent", 19 | "Programming Language :: Python", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | "Programming Language :: Python :: 3.13", 25 | "Topic :: Software Development :: Libraries :: Python Modules", 26 | "Topic :: Software Development :: Testing", 27 | 28 | ] 29 | dependencies = [ 30 | "attrs", 31 | "aiohttp", 32 | "anybadge", 33 | "click", 34 | "decorator", 35 | "exit_codes", 36 | "gitpython", 37 | "parso", 38 | "qprompt", 39 | "rich", 40 | "sqlalchemy", 41 | "stevedore", 42 | "toml", 43 | "yattag", 44 | ] 45 | 46 | [project.scripts] 47 | cosmic-ray = "cosmic_ray.cli:main" 48 | cr-html = "cosmic_ray.tools.html:report_html" 49 | cr-report = "cosmic_ray.tools.report:report" 50 | cr-badge = "cosmic_ray.tools.badge:generate_badge" 51 | cr-rate = "cosmic_ray.tools.survival_rate:format_survival_rate" 52 | cr-xml = "cosmic_ray.tools.xml:report_xml" 53 | cr-filter-operators = "cosmic_ray.tools.filters.operators_filter:main" 54 | cr-filter-pragma = "cosmic_ray.tools.filters.pragma_no_mutate:main" 55 | cr-filter-git = "cosmic_ray.tools.filters.git:main" 56 | cr-http-workers = "cosmic_ray.tools.http_workers:main" 57 | 58 | [project.entry-points."cosmic_ray.operator_providers"] 59 | core = "cosmic_ray.operators.provider:OperatorProvider" 60 | 61 | [project.entry-points."cosmic_ray.distributors"] 62 | http = "cosmic_ray.distribution.http:HttpDistributor" 63 | local = "cosmic_ray.distribution.local:LocalDistributor" 64 | 65 | [project.urls] 66 | repository = "https://github.com/sixty-north/cosmic-ray" 67 | 68 | [dependency-groups] 69 | dev = [ 70 | "bump-my-version", 71 | "hypothesis", 72 | "nox", 73 | "pytest", 74 | "ruff", 75 | "sphinx", 76 | "sphinx-click>=6.0.0", 77 | "sphinx-rtd-theme", 78 | ] 79 | 80 | [[tool.uv.index]] 81 | url = "https://pypi.org/simple" 82 | 83 | [tool.setuptools.packages.find] 84 | where = ["src"] 85 | 86 | [tool.setuptools.dynamic] 87 | version = { attr = "cosmic_ray.version.__version__" } 88 | 89 | 90 | [tool.bumpversion] 91 | current_version = "8.4.3" 92 | parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" 93 | serialize = ["{major}.{minor}.{patch}"] 94 | tag = true 95 | commit = true 96 | message = "Bump version: {current_version} → {new_version}" 97 | tag_name = "release/v{new_version}" 98 | tag_message = "Bump version: {current_version} → {new_version}" 99 | 100 | [[tool.bumpversion.files]] 101 | filename = "src/cosmic_ray/version.py" 102 | 103 | [tool.ruff] 104 | line-length = 120 105 | 106 | [tool.ruff.lint] 107 | select = [ 108 | "UP", # PYUPGRADE 109 | "I", # ISORT 110 | ] 111 | 112 | [tool.ruff.lint.isort] 113 | case-sensitive = true 114 | known-first-party = ["cosmic_ray"] 115 | 116 | [tool.pytest.ini_options] 117 | minversion = "6.0" 118 | testpaths = ["tests"] 119 | addopts = ["--e2e-distributor=local", "--e2e-tester=unittest", "--e2e-tester=pytest", "--ignore=tests/resources"] 120 | norecursedirs = "tests/resources" 121 | -------------------------------------------------------------------------------- /src/cosmic_ray/commands/init.py: -------------------------------------------------------------------------------- 1 | "Implementation of the 'init' command." 2 | 3 | import logging 4 | import uuid 5 | from collections.abc import Iterable 6 | 7 | import cosmic_ray.modules 8 | import cosmic_ray.plugins 9 | from cosmic_ray.ast import ast_nodes, get_ast_from_path 10 | from cosmic_ray.work_db import WorkDB 11 | from cosmic_ray.work_item import MutationSpec, WorkItem 12 | 13 | log = logging.getLogger() 14 | 15 | 16 | def _operators(operator_cfgs): 17 | """Find all Operator instances that should be used for the session. 18 | 19 | Args: 20 | operator_cfgs (Mapping[str, Iterable[Mapping[str, Any]]]): Mapping 21 | of operator names to arguments sets for that operator. Each 22 | argument set for an operator will result in an instance of the 23 | operator. 24 | 25 | Yields: 26 | operators: tuples of (operator name, argument dict, operator instance). 27 | 28 | Raises: 29 | TypeError: The arguments supplied to an operator are invalid. 30 | """ 31 | # TODO: Can/should we check for operator arguments which don't correspond to 32 | # *any* known operator? 33 | 34 | for operator_name in cosmic_ray.plugins.operator_names(): 35 | operator_class = cosmic_ray.plugins.get_operator(operator_name) 36 | if not operator_class.arguments(): 37 | if operator_name in operator_cfgs: 38 | raise TypeError(f"Arguments provided for operator {operator_name} which accepts no arguments") 39 | 40 | yield operator_name, {}, operator_class() 41 | else: 42 | for operator_args in operator_cfgs.get(operator_name, ()): 43 | yield operator_name, operator_args, operator_class(**operator_args) 44 | 45 | 46 | def _all_work_items(module_paths, operator_cfgs) -> Iterable[WorkItem]: 47 | """Iterable of all WorkItems for the given inputs. 48 | 49 | Raises: 50 | TypeError: If an operator is provided with a parameterization it can't use. 51 | """ 52 | 53 | for module_path in module_paths: 54 | module_ast = get_ast_from_path(module_path) 55 | 56 | for operator_name, operator_args, operator in _operators(operator_cfgs): 57 | positions = ( 58 | (start_pos, end_pos) 59 | for node in ast_nodes(module_ast) 60 | for start_pos, end_pos in operator.mutation_positions(node) 61 | ) 62 | 63 | for occurrence, (start_pos, end_pos) in enumerate(positions): 64 | mutation = MutationSpec( 65 | module_path=str(module_path), 66 | operator_name=operator_name, 67 | operator_args=operator_args, 68 | occurrence=occurrence, 69 | start_pos=start_pos, 70 | end_pos=end_pos, 71 | ) 72 | yield WorkItem.single(job_id=uuid.uuid4().hex, mutation=mutation) 73 | 74 | 75 | def init(module_paths, work_db: WorkDB, operator_cfgs): 76 | """Clear and initialize a work-db with work items. 77 | 78 | Any existing data in the work-db will be cleared and replaced with entirely 79 | new work orders. In particular, this means that any results in the db are 80 | removed. 81 | 82 | Args: 83 | module_paths: iterable of pathlib.Paths of modules to mutate. 84 | work_db: A `WorkDB` instance into which the work orders will be saved. 85 | operator_cfgs: A dict mapping operator names to parameterization dicts. 86 | 87 | Raises: 88 | TypeError: Arguments provided for an operator are invalid. 89 | """ 90 | # By default each operator will be parameterized with an empty dict. 91 | work_db.clear() 92 | work_db.add_work_items(_all_work_items(module_paths, operator_cfgs)) 93 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/comparison_operator_replacement.py: -------------------------------------------------------------------------------- 1 | """This module contains mutation operators which replace one 2 | comparison operator with another. 3 | """ 4 | 5 | import itertools 6 | from enum import Enum 7 | 8 | import parso.python.tree 9 | 10 | from ..ast import is_none, is_number 11 | from .operator import Example, Operator 12 | from .util import extend_name 13 | 14 | 15 | class ComparisonOperators(Enum): 16 | "All comparison operators that we mutate." 17 | 18 | Eq = "==" 19 | NotEq = "!=" 20 | Lt = "<" 21 | LtE = "<=" 22 | Gt = ">" 23 | GtE = ">=" 24 | Is = "is" 25 | IsNot = "is not" 26 | 27 | 28 | def _create_operator(from_op, to_op): 29 | @extend_name(f"_{from_op.name}_{to_op.name}") 30 | class ReplaceComparisonOperator(Operator): 31 | f"An operator that replaces {from_op.name} with {to_op.name}" 32 | 33 | def mutation_positions(self, node): 34 | if node.type == "comparison": 35 | # Every other child starting at 1 is a comparison operator of some sort 36 | for _, comparison_op in self._mutation_points(node): 37 | yield (comparison_op.start_pos, comparison_op.end_pos) 38 | 39 | def mutate(self, node, index): 40 | points = list(itertools.islice(self._mutation_points(node), index, index + 1)) 41 | assert len(points) == 1 42 | op_idx, _ = points[0] 43 | mutated_comparison_op = parso.parse(" " + to_op.value) 44 | node.children[op_idx * 2 + 1] = mutated_comparison_op 45 | return node 46 | 47 | @staticmethod 48 | def _mutation_points(node): 49 | for op_idx, comparison_op in enumerate(node.children[1::2]): 50 | if comparison_op.get_code().strip() == from_op.value: 51 | rhs = node.children[(op_idx + 1) * 2] 52 | if _allowed(to_op, from_op, rhs): 53 | yield op_idx, comparison_op 54 | 55 | @classmethod 56 | def examples(cls): 57 | return (Example(f"x {from_op.value} y", f"x {to_op.value} y"),) 58 | 59 | return ReplaceComparisonOperator 60 | 61 | 62 | # Build all of the binary replacement operators 63 | _OPERATORS = tuple( 64 | _create_operator(from_op, to_op) for from_op, to_op in itertools.permutations(ComparisonOperators, 2) 65 | ) 66 | 67 | # Inject the operators into the module namespace 68 | for op_cls in _OPERATORS: 69 | globals()[op_cls.__name__] = op_cls 70 | 71 | 72 | def operators(): 73 | "Iterable of all binary operator replacement mutation operators." 74 | return iter(_OPERATORS) 75 | 76 | 77 | # This determines the allowed from-to mutations when the RHS is None. 78 | _RHS_IS_NONE_OPS = { 79 | ComparisonOperators.Eq: [ComparisonOperators.IsNot], 80 | ComparisonOperators.NotEq: [ComparisonOperators.Is], 81 | ComparisonOperators.Is: [ComparisonOperators.IsNot], 82 | ComparisonOperators.IsNot: [ComparisonOperators.Is], 83 | } 84 | 85 | # This determines the allowed to mutations when the RHS is a number 86 | _RHS_IS_INTEGER_OPS = set( 87 | [ 88 | ComparisonOperators.Eq, 89 | ComparisonOperators.NotEq, 90 | ComparisonOperators.Lt, 91 | ComparisonOperators.LtE, 92 | ComparisonOperators.Gt, 93 | ComparisonOperators.GtE, 94 | ] 95 | ) 96 | 97 | 98 | def _allowed(to_op, from_op, rhs): 99 | "Determine if a mutation from `from_op` to `to_op` is allowed given a particular `rhs` node." 100 | if is_none(rhs): 101 | return to_op in _RHS_IS_NONE_OPS.get(from_op, ()) 102 | 103 | if is_number(rhs): 104 | return to_op in _RHS_IS_INTEGER_OPS 105 | 106 | return True 107 | -------------------------------------------------------------------------------- /src/cosmic_ray/config.py: -------------------------------------------------------------------------------- 1 | """Configuration module.""" 2 | 3 | import logging 4 | import sys 5 | from contextlib import contextmanager 6 | 7 | import toml 8 | 9 | log = logging.getLogger() 10 | 11 | 12 | def load_config(filename=None): 13 | """Load a configuration from a file or stdin. 14 | 15 | If `filename` is `None` or "-", then configuration gets read from stdin. 16 | 17 | Returns: A `ConfigDict`. 18 | 19 | Raises: ConfigError: If there is an error loading the config. 20 | """ 21 | try: 22 | with _config_stream(filename) as handle: 23 | filename = handle.name 24 | return deserialize_config(handle.read()) 25 | except (OSError, toml.TomlDecodeError, UnicodeDecodeError) as exc: 26 | raise ConfigError(f"Error loading configuration from {filename}") from exc 27 | 28 | 29 | def deserialize_config(sz) -> "ConfigDict": 30 | "Parse a serialized config into a ConfigDict." 31 | return toml.loads(sz, _dict=ConfigDict)["cosmic-ray"] 32 | 33 | 34 | def serialize_config(config): 35 | "Return the serialized form of `config`." 36 | return toml.dumps({"cosmic-ray": config}) 37 | 38 | 39 | class ConfigError(Exception): 40 | "Base class for exceptions raised by ConfigDict." 41 | 42 | 43 | class ConfigKeyError(ConfigError, KeyError): 44 | "KeyError subclass raised by ConfigDict." 45 | 46 | 47 | class ConfigValueError(ConfigError, ValueError): 48 | "ValueError subclass raised by ConfigDict." 49 | 50 | 51 | class ConfigDict(dict): 52 | """A dictionary subclass that contains the application configuration.""" 53 | 54 | def __getitem__(self, key): 55 | try: 56 | return super().__getitem__(key) 57 | except KeyError as exc: 58 | raise ConfigKeyError(*exc.args) 59 | 60 | def sub(self, *segments): 61 | "Get a sub-configuration." 62 | d = self 63 | for segment in segments: 64 | try: 65 | d = d[segment] 66 | except KeyError: 67 | return ConfigDict({}) 68 | return d 69 | 70 | @property 71 | def test_command(self): 72 | """The command to run to execute tests.""" 73 | return self["test-command"] 74 | 75 | @property 76 | def timeout(self): 77 | "The timeout (seconds) for tests." 78 | return float(self["timeout"]) 79 | 80 | @property 81 | def distributor_name(self): 82 | "The name of the distributor to use." 83 | return self["distributor"]["name"] 84 | 85 | @property 86 | def distributor_config(self): 87 | "The configuration for the named distributor." 88 | name = self.distributor_name 89 | return self["distributor"].get(name, ConfigDict()) 90 | 91 | @property 92 | def operators_config(self): 93 | """The configuration for specified operators. 94 | 95 | This is a dict mapping operator names to dicts which represent keyword-arguments 96 | for parameterizing an operator. Each keyword arg dict is a single parameterization 97 | of the operator, and each parameterized operator will be executed once for each 98 | parameterization. 99 | """ 100 | return self.get("operators", {}) 101 | 102 | 103 | @contextmanager 104 | def _config_stream(filename): 105 | """Given a configuration's filename, this returns a stream from which a configuration can be read. 106 | 107 | If `filename` is `None` or '-' then stream will be `sys.stdin`. Otherwise, 108 | it's the open file handle for the filename. 109 | """ 110 | if filename is None or filename == "-": 111 | log.info("Reading config from stdin") 112 | yield sys.stdin 113 | else: 114 | with open(filename) as handle: 115 | log.info("Reading config from %r", filename) 116 | yield handle 117 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/unary_operator_replacement.py: -------------------------------------------------------------------------------- 1 | """Implementation of the unary-operator-replacement operator.""" 2 | 3 | from enum import Enum 4 | from itertools import permutations 5 | 6 | from parso.python.tree import Keyword, Operator, PythonNode 7 | 8 | from . import operator 9 | from .util import extend_name 10 | 11 | 12 | class UnaryOperators(Enum): 13 | "All unary operators that we mutate." 14 | 15 | UAdd = "+" 16 | USub = "-" 17 | Invert = "~" 18 | Not = "not " 19 | Nothing = None 20 | 21 | 22 | def _create_replace_unary_operators(from_op, to_op): 23 | if to_op.value is None: 24 | suffix = f"_Delete_{from_op.name}" 25 | else: 26 | suffix = f"_{from_op.name}_{to_op.name}" 27 | 28 | @extend_name(suffix) 29 | class ReplaceUnaryOperator(operator.Operator): 30 | f"An operator that replaces unary {from_op.name} with unary {to_op.name}." 31 | 32 | def mutation_positions(self, node): 33 | if _is_unary_operator(node): 34 | op = node.children[0] 35 | if op.value.strip() == from_op.value.strip(): 36 | yield (op.start_pos, op.end_pos) 37 | 38 | def mutate(self, node, index): 39 | assert index == 0 40 | assert _is_unary_operator(node) 41 | 42 | if to_op.value is None: 43 | # This is a bit goofy since it can result in "return not x" 44 | # becoming "return x" (i.e. with two spaces). But it's correct 45 | # enough. 46 | node.children[0].value = "" 47 | else: 48 | node.children[0].value = to_op.value 49 | return node 50 | 51 | @classmethod 52 | def examples(cls): 53 | from_code = f"{from_op.value}1" 54 | to_code = from_code[len(from_op.value) :] 55 | 56 | if to_op is not UnaryOperators.Nothing: 57 | to_code = to_op.value + to_code 58 | elif from_op is UnaryOperators.Not: 59 | to_code = " " + to_code 60 | 61 | return (operator.Example(from_code, to_code),) 62 | 63 | return ReplaceUnaryOperator 64 | 65 | 66 | def _is_factor(node): 67 | return ( 68 | isinstance(node, PythonNode) 69 | and node.type in {"factor", "not_test"} 70 | and len(node.children) > 0 71 | and isinstance(node.children[0], Operator) 72 | ) 73 | 74 | 75 | def _is_not_test(node): 76 | return ( 77 | isinstance(node, PythonNode) 78 | and node.type == "not_test" 79 | and len(node.children) > 0 80 | and isinstance(node.children[0], Keyword) 81 | and node.children[0].value == "not" 82 | ) 83 | 84 | 85 | def _is_unary_operator(node): 86 | return _is_factor(node) or _is_not_test(node) 87 | 88 | 89 | def _prohibited(from_op, to_op): 90 | "Determines if from_op is allowed to be mutated to to_op." 91 | # 'not' can only be removed but not replaced with 92 | # '+', '-' or '~' b/c that may lead to strange results 93 | if from_op is UnaryOperators.Not: 94 | if to_op is not UnaryOperators.Nothing: 95 | return True 96 | 97 | # '+1' => '1' yields equivalent mutations 98 | if from_op is UnaryOperators.UAdd: 99 | if to_op is UnaryOperators.Nothing: 100 | return True 101 | 102 | return False 103 | 104 | 105 | _MUTATION_OPERATORS = tuple( 106 | _create_replace_unary_operators(from_op, to_op) 107 | for (from_op, to_op) in permutations(UnaryOperators, 2) 108 | if from_op.value is not None 109 | if not _prohibited(from_op, to_op) 110 | ) 111 | 112 | for op_cls in _MUTATION_OPERATORS: 113 | globals()[op_cls.__name__] = op_cls 114 | 115 | 116 | def operators(): 117 | "Iterable of unary operator mutation operators." 118 | return _MUTATION_OPERATORS 119 | -------------------------------------------------------------------------------- /tests/tools/test_filter_operators.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | from cosmic_ray.tools.filters import operators_filter 5 | from cosmic_ray.work_item import MutationSpec, WorkItem, WorkResult, WorkerOutcome 6 | 7 | 8 | def test_smoke_test_on_initialized_session(initialized_session): 9 | command = [ 10 | sys.executable, 11 | "-m", 12 | "cosmic_ray.tools.filters.operators_filter", 13 | str(initialized_session.session), 14 | str(initialized_session.config), 15 | ] 16 | 17 | subprocess.check_call(command, cwd=str(initialized_session.session.parent)) 18 | 19 | 20 | def test_smoke_test_on_execd_session(execd_session): 21 | command = [ 22 | sys.executable, 23 | "-m", 24 | "cosmic_ray.tools.filters.operators_filter", 25 | str(execd_session.session), 26 | str(execd_session.config), 27 | ] 28 | 29 | subprocess.check_call(command, cwd=str(execd_session.session.parent)) 30 | 31 | 32 | class FakeWorkDB: 33 | def __init__(self): 34 | self.count = 0 35 | self.results = [] 36 | 37 | def new_work_item(self, operator_name, job_id): 38 | self.count += 1 39 | return WorkItem.single( 40 | job_id, 41 | MutationSpec( 42 | module_path=f"{self.count}.py", 43 | operator_name=operator_name, 44 | occurrence=self.count, 45 | start_pos=(self.count, self.count), 46 | end_pos=(self.count + 1, self.count + 1), 47 | ), 48 | ) 49 | 50 | @property 51 | def pending_work_items(self): 52 | return [ 53 | self.new_work_item("Op1", "id1"), 54 | self.new_work_item("Op2", "id2"), 55 | self.new_work_item("Op3", "id3"), 56 | self.new_work_item("Op2", "id4"), 57 | self.new_work_item("Opregex1", "regex1"), 58 | self.new_work_item("Opregex2", "regex2"), 59 | self.new_work_item("Opregex3", "regex3"), 60 | self.new_work_item("Complex1", "regex4"), 61 | self.new_work_item("CompLex2", "regex5"), 62 | ] 63 | 64 | def set_result(self, job_id, work_result: WorkResult): 65 | self.results.append((job_id, work_result.worker_outcome)) 66 | 67 | @property 68 | def expected_after_filter(self): 69 | return [ 70 | ("id1", WorkerOutcome.SKIPPED), 71 | ("id2", WorkerOutcome.SKIPPED), 72 | ("id4", WorkerOutcome.SKIPPED), 73 | ("regex1", WorkerOutcome.SKIPPED), 74 | ("regex2", WorkerOutcome.SKIPPED), 75 | ("regex4", WorkerOutcome.SKIPPED), 76 | ] 77 | 78 | @property 79 | def expected_all_filtered(self): 80 | return [ 81 | ("id1", WorkerOutcome.SKIPPED), 82 | ("id2", WorkerOutcome.SKIPPED), 83 | ("id3", WorkerOutcome.SKIPPED), 84 | ("id4", WorkerOutcome.SKIPPED), 85 | ("regex1", WorkerOutcome.SKIPPED), 86 | ("regex2", WorkerOutcome.SKIPPED), 87 | ("regex3", WorkerOutcome.SKIPPED), 88 | ("regex4", WorkerOutcome.SKIPPED), 89 | ("regex5", WorkerOutcome.SKIPPED), 90 | ] 91 | 92 | 93 | def test_operators_filter(): 94 | data = FakeWorkDB() 95 | exclude = ["Op1", "Op2", "Opregex[12]", r"(?:.[oO]m(?:p|P)lex).*"] 96 | operators_filter.OperatorsFilter()._skip_filtered(data, exclude) 97 | assert data.results == data.expected_after_filter 98 | 99 | 100 | def test_operators_filter_empty_excludes(): 101 | data = FakeWorkDB() 102 | exclude = [] 103 | operators_filter.OperatorsFilter()._skip_filtered(data, exclude) 104 | assert data.results == [] 105 | 106 | 107 | def test_operators_filter_all_excluded(): 108 | data = FakeWorkDB() 109 | exclude = [r"."] 110 | operators_filter.OperatorsFilter()._skip_filtered(data, exclude) 111 | assert data.results == data.expected_all_filtered 112 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/filters/git.py: -------------------------------------------------------------------------------- 1 | """A filter that uses git to determine when specific files/lines 2 | should be skipped. 3 | """ 4 | 5 | import logging 6 | import re 7 | import subprocess 8 | import sys 9 | from argparse import Namespace 10 | from collections import defaultdict 11 | from pathlib import Path 12 | 13 | from cosmic_ray.config import ConfigDict, load_config 14 | from cosmic_ray.tools.filters.filter_app import FilterApp 15 | from cosmic_ray.work_db import WorkDB 16 | from cosmic_ray.work_item import WorkResult, WorkerOutcome 17 | 18 | log = logging.getLogger() 19 | 20 | 21 | class GitFilter(FilterApp): 22 | """Implements the git filter.""" 23 | 24 | def description(self): 25 | return __doc__ 26 | 27 | def _git_news(self, branch): 28 | """Get the set of new lines by file""" 29 | # we could use interlap, but do not want to 30 | # add new dependency at the moment 31 | git_command = ["git", "diff", "--relative", "-U0", branch, "."] 32 | log.info(f"Executing {' '.join(git_command)}") 33 | try: 34 | output = subprocess.check_output(git_command, stderr=subprocess.PIPE) 35 | except subprocess.CalledProcessError as exc: 36 | log.error( 37 | f"'git diff' call failed: {exc}\n[stdout]\n{exc.stdout.decode()}\n[stderr]\n{exc.stderr.decode()}" 38 | ) 39 | raise 40 | 41 | regex = re.compile(r"@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*") 42 | current_file = None 43 | res = defaultdict(set) 44 | for diff_line in output.decode("utf-8").split("\n"): 45 | if diff_line.startswith("@@"): 46 | m = regex.match(diff_line) 47 | if m is None: 48 | continue 49 | start = int(m.group(1)) 50 | length = int(m.group(2)) if m.group(2) is not None else 1 51 | for line in range(start, start + length): 52 | res[current_file].add(line) 53 | if diff_line.startswith("+++ b/"): 54 | current_file = Path(diff_line[6:]) 55 | return res 56 | 57 | def _skip_filtered(self, work_db, branch): 58 | git_news = self._git_news(branch) 59 | 60 | job_ids = [] 61 | 62 | for item in work_db.pending_work_items: 63 | for mutation in item.mutations: 64 | if mutation.module_path not in git_news or not ( 65 | git_news[mutation.module_path] & set(range(mutation.start_pos[0], mutation.end_pos[0] + 1)) 66 | ): 67 | log.info( 68 | "git skipping %s %s %s %s %s %s", 69 | item.job_id, 70 | mutation.operator_name, 71 | mutation.occurrence, 72 | mutation.module_path, 73 | mutation.start_pos, 74 | mutation.end_pos, 75 | ) 76 | 77 | job_ids.append(item.job_id) 78 | 79 | if job_ids: 80 | work_db.set_multiple_results( 81 | job_ids, 82 | WorkResult( 83 | output="Filtered git", 84 | worker_outcome=WorkerOutcome.SKIPPED, 85 | ), 86 | ) 87 | 88 | def filter(self, work_db: WorkDB, args: Namespace): 89 | """Mark as skipped all work item that is not new""" 90 | 91 | config = ConfigDict() 92 | if args.config is not None: 93 | config = load_config(args.config) 94 | 95 | branch = config.sub("filters", "git-filter").get("branch", "master") 96 | log.info(f"Base git branch: {branch}") 97 | self._skip_filtered(work_db, branch) 98 | 99 | def add_args(self, parser): 100 | parser.add_argument("--config", help="Config file to use") 101 | 102 | 103 | def main(argv=None): 104 | """Run the operators-filter with the specified command line arguments.""" 105 | return GitFilter().main(argv) 106 | 107 | 108 | if __name__ == "__main__": 109 | sys.exit(main()) 110 | -------------------------------------------------------------------------------- /src/cosmic_ray/progress.py: -------------------------------------------------------------------------------- 1 | """Management of progress reporting. 2 | 3 | The design of this subsystem is such that progress reporting is 4 | decoupled from updating the current progress. This allows for 5 | progress reporting functions which can be invoked from another 6 | context, such as a SIGINFO signal handler. 7 | 8 | As such, reporter callables are responsible only for displaying 9 | current progress, and must be capable of retrieving the latest 10 | progress state from elsewhere when invoked. This may be global state 11 | or reached through references that the reporter callable was 12 | constructed with. Typically the latest progress state will be 13 | updated by the routine whose progress is being monitored. 14 | 15 | To report the current progress invoke report_progress(). 16 | 17 | To manage installation and deinstallation of progress reporting 18 | functions use the reports_progress() decorator for whole-function 19 | contexts, or the progress_reporter() context manager for narrower 20 | contexts. 21 | 22 | It is the responsibility of the client to manage any thread-safety 23 | issues. You should assume that progress reporting functions can be 24 | called asynchronously, at any time, from the main thread. 25 | 26 | Example:: 27 | 28 | _progress_message = "" 29 | 30 | def _update_foo_progress(i, n): 31 | global _progress_message 32 | _progress_message = "{i} of {n} complete".format(i=i, n=n) 33 | 34 | def _report_foo_progress(stream): 35 | print(_progress_message, file=stream) 36 | 37 | @reports_progress(_report_foo_progress) 38 | def foo(n): 39 | for i in range(n): 40 | _update_foo_progress(i, n) 41 | 42 | # ... 43 | 44 | signal.signal(signal.SIGINFO, 45 | lambda *args: report_progress()) 46 | 47 | """ 48 | 49 | # Currently installed zero-argument callables used to report progress. 50 | import sys 51 | from contextlib import contextmanager 52 | from functools import wraps 53 | 54 | _reporters = [] # pylint: disable=invalid-name 55 | 56 | 57 | def report_progress(stream=None): 58 | """Report progress from any currently installed reporters. 59 | 60 | Args: 61 | stream: The text stream (default: sys.stderr) to which 62 | progress will be reported. 63 | """ 64 | if stream is None: 65 | stream = sys.stderr 66 | for reporter in _reporters: 67 | reporter(stream) 68 | 69 | 70 | @contextmanager 71 | def progress_reporter(reporter): 72 | """A context manager to install and remove a progress reporting function. 73 | 74 | Args: 75 | reporter: A zero-argument callable to report progress. 76 | The callable provided should have the means to both 77 | retrieve and display current progress information. 78 | """ 79 | install_progress_reporter(reporter) 80 | yield 81 | uninstall_progress_reporter(reporter) 82 | 83 | 84 | def reports_progress(reporter): 85 | """A decorator factory to mark functions which report progress. 86 | 87 | Args: 88 | reporter: A zero-argument callable to report progress. 89 | The callable provided should have the means to both 90 | retrieve and display current progress information. 91 | """ 92 | 93 | def decorator(func): # pylint: disable=missing-docstring 94 | @wraps(func) 95 | def wrapper(*args, **kwargs): # pylint: disable=missing-docstring 96 | with progress_reporter(reporter): 97 | return func(*args, **kwargs) 98 | 99 | return wrapper 100 | 101 | return decorator 102 | 103 | 104 | def install_progress_reporter(reporter): 105 | """Install a progress reporter. 106 | 107 | Where possible prefer to use the progress_reporter() context 108 | manager or reports_progress() decorator factory. 109 | 110 | Args: 111 | reporter: A zero-argument callable to report progress. 112 | The callable provided should have the means to both 113 | retrieve and display current progress information. 114 | """ 115 | _reporters.append(reporter) 116 | 117 | 118 | def uninstall_progress_reporter(reporter): 119 | """Uninstall a progress reporter. 120 | 121 | Where possible prefer to use the progress_reporter() context 122 | manager or reports_progress() decorator factory. 123 | 124 | Args: 125 | reporter: A callable previously installed by 126 | install_progress_reporter(). 127 | """ 128 | _reporters.remove(reporter) 129 | -------------------------------------------------------------------------------- /src/cosmic_ray/operators/variable_replacer.py: -------------------------------------------------------------------------------- 1 | """Implementation of the variable-replacement operator.""" 2 | 3 | from random import randint 4 | 5 | from parso.python.tree import ExprStmt, Leaf, Number 6 | 7 | from .operator import Argument, Example, Operator 8 | 9 | 10 | class VariableReplacer(Operator): 11 | """An operator that replaces usages of named variables.""" 12 | 13 | def __init__(self, cause_variable, effect_variable=None): 14 | self.cause_variable = cause_variable 15 | self.effect_variable = effect_variable 16 | 17 | @classmethod 18 | def arguments(cls): 19 | return ( 20 | Argument("cause_variable", "The cause variable"), 21 | Argument("effect_variable", "The effect variable"), 22 | ) 23 | 24 | def mutation_positions(self, node): 25 | """Mutate usages of the specified cause variable. If an effect variable is also 26 | specified, then only mutate usages of the cause variable in definitions of the 27 | effect variable.""" 28 | 29 | if isinstance(node, ExprStmt): 30 | # Confirm that name node is used on right hand side of the expression 31 | cause_variables = list(self._get_causes_from_expr_node(node)) 32 | cause_variable_names = [cause_variable.value for cause_variable in cause_variables] 33 | if self.cause_variable in cause_variable_names: 34 | mutation_position = (node.start_pos, node.end_pos) 35 | 36 | # If an effect variable is specified, confirm that it appears on left hand 37 | # side of the expression 38 | if self.effect_variable: 39 | effect_variable_names = [v.value for v in node.get_defined_names()] 40 | if self.effect_variable in effect_variable_names: 41 | yield mutation_position 42 | 43 | # If no effect variable is specified, any occurrence of the cause variable 44 | # on the right hand side of an expression can be mutated 45 | else: 46 | yield mutation_position 47 | 48 | def mutate(self, node, index): 49 | """Replace cause variable with random constant.""" 50 | assert isinstance(node, ExprStmt) 51 | # Find all occurrences of the cause node in the ExprStatement and replace with a random number 52 | rhs = node.get_rhs() 53 | new_rhs = self._replace_named_variable_in_expr(rhs, self.cause_variable) 54 | node.children[2] = new_rhs 55 | return node 56 | 57 | def _get_causes_from_expr_node(self, expr_node): 58 | rhs = expr_node.get_rhs().children 59 | return self._flatten_expr(rhs) 60 | 61 | def _flatten_expr(self, expr): 62 | for item in expr: 63 | # Convert PythonNode to list of its children 64 | try: 65 | item_to_flatten = item.children 66 | except AttributeError: 67 | item_to_flatten = item 68 | # 69 | try: 70 | yield from self._flatten_expr(item_to_flatten) 71 | except TypeError: 72 | yield item_to_flatten 73 | 74 | def _replace_named_variable_in_expr(self, node, variable_name): 75 | if isinstance(node, Leaf): 76 | if node.value == variable_name: 77 | return Number(start_pos=node.start_pos, value=str(randint(-100, 100))) 78 | else: 79 | return node 80 | 81 | updated_child_nodes = [] 82 | for child_node in node.children: 83 | updated_child_nodes.append(self._replace_named_variable_in_expr(child_node, variable_name)) 84 | node.children = updated_child_nodes 85 | return node 86 | 87 | @classmethod 88 | def examples(cls): 89 | return ( 90 | Example("y = x + z", "y = 10 + z", operator_args={"cause_variable": "x"}), 91 | Example( 92 | "j = x + z\ny = x + z", 93 | "j = x + z\ny = -2 + z", 94 | operator_args={"cause_variable": "x", "effect_variable": "y"}, 95 | ), 96 | Example( 97 | "j = x + z\ny = x + z", 98 | "j = 1 + z\ny = x + z", 99 | operator_args={"cause_variable": "x", "effect_variable": "j"}, 100 | ), 101 | Example("y = 2*x + 10 + j + x**2", "y=2*10 + 10 + j + -4**2", operator_args={"cause_variable": "x"}), 102 | ) 103 | -------------------------------------------------------------------------------- /src/cosmic_ray/tools/http_workers.py: -------------------------------------------------------------------------------- 1 | """A tool for launching HTTP workers and executing a session using them. 2 | 3 | This reads the 'distributor.http.worker-urls' field of a config to see where workers are expected to be 4 | running. For each worker, it makes a clone of the git repository that's going to be tested, optionally changing to a 5 | directory under the root of the clone before starting the worker. It then starts the workers with the correct options to 6 | provide the configured URLs. 7 | """ 8 | 9 | import asyncio 10 | import contextlib 11 | import logging 12 | import shutil 13 | import tempfile 14 | from pathlib import Path 15 | 16 | import click 17 | import git 18 | import yarl 19 | 20 | import cosmic_ray.config 21 | 22 | log = logging.getLogger() 23 | 24 | 25 | async def run(config_file, repo_url, location): 26 | """Start the configured workers in their own git clones. 27 | 28 | Args: 29 | config_file: The Cosmic Ray configuration file describing the distributor URLs. 30 | repo_url: The git repository to clone for each worker. 31 | location: The relative path into the cloned repository to use as the cwd for 32 | each worker. 33 | """ 34 | config = cosmic_ray.config.load_config(config_file) 35 | worker_urls = config.sub("distributor", "http").get("worker-urls", ()) 36 | 37 | worker_args = tuple(_urls_to_args(worker_urls, Path(config_file).resolve())) 38 | if not worker_args: 39 | log.warning("No valid worker URLs found in config %s", config_file) 40 | 41 | with contextlib.ExitStack() as stack: 42 | procs = [ 43 | await asyncio.create_subprocess_shell( 44 | f"cosmic-ray --verbosity INFO http-worker {option} {value}", 45 | cwd=stack.enter_context(_create_clone(repo_url)) / location, 46 | ) 47 | for option, value in worker_args 48 | ] 49 | 50 | await asyncio.gather(*[proc.communicate() for proc in procs]) 51 | 52 | 53 | @click.command(help=__doc__) 54 | @click.argument("config_file", type=click.Path(exists=True, dir_okay=False, readable=True)) 55 | @click.argument("repo_url") 56 | @click.option("--location", default="", help="The relative path under a repo clone at which to run the worker") 57 | def main(config_file, repo_url, location): 58 | logging.basicConfig(level=logging.INFO) 59 | asyncio.get_event_loop().run_until_complete(run(config_file, repo_url, location)) 60 | 61 | 62 | @contextlib.contextmanager 63 | def _create_clone(source_repo_url): 64 | """Clone a git repository into a temporary directory. 65 | 66 | This is a context-manager that yields the directory used for the clone:: 67 | 68 | with _create_clone('http://github.com/sixty-north/cosmic-ray') as clone_dir: 69 | . . . 70 | 71 | This attempts to clean up the clone directory after the context ends. NB: that there are 72 | known problems with this on Windows, so it's possible that the directory will not be 73 | removed. 74 | """ 75 | # Normally I'd use the context manager tempfile.TemporaryDirectory, but that has problems 76 | # on windows: https://github.com/sixty-north/cosmic-ray/issues/521 77 | 78 | root = tempfile.mkdtemp() 79 | 80 | try: 81 | destination_dir = Path(root) 82 | url = yarl.URL(source_repo_url) 83 | if url.scheme == "": 84 | url = yarl.URL.build(scheme="file", path=str(Path(url.path).resolve())) 85 | 86 | log.info("Cloning %s to %s", url, destination_dir) 87 | git.Repo.clone_from(str(url), destination_dir, depth=1) 88 | 89 | yield destination_dir 90 | finally: 91 | try: 92 | shutil.rmtree(root) 93 | except (RecursionError, PermissionError): 94 | log.warning(f"Unable to remove directory: {root}") 95 | 96 | 97 | LOCALHOST_ADDRESSES = ( 98 | "localhost", 99 | "0.0.0.0", 100 | "127.0.0.1", 101 | ) 102 | 103 | 104 | def _urls_to_args(urls, config_filepath: Path): 105 | for url in urls: 106 | url = yarl.URL(url) 107 | if url.scheme == "": 108 | socket_path = config_filepath.parent / url.path 109 | yield ("--path", socket_path) 110 | elif url.scheme in ("http", "https"): 111 | if url.port is None: 112 | log.warning("HTTP(S) URL %s has no port", url) 113 | 114 | elif url.host.lower() not in LOCALHOST_ADDRESSES: 115 | log.warning("%s does not appear to be on localhost", url) 116 | 117 | else: 118 | yield ("--port", url.port) 119 | else: 120 | log.warning("The scheme of URL %s is not supported", url) 121 | 122 | 123 | if __name__ == "__main__": 124 | main() 125 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | How to contribute 3 | ================= 4 | 5 | Third-party patches are welcomed for improving Cosmic Ray. There is plenty of work to be done on bug fixes, 6 | documentation, new features, and improved tooling. 7 | 8 | Although we want to keep it as easy as possible to contribute changes that 9 | get things working in your environment, there are a few guidelines that we 10 | need contributors to follow so that we can have a chance of keeping on 11 | top of things. 12 | 13 | 14 | Getting Started 15 | =============== 16 | 17 | The easiest way to help is by submitting issues reporting defects or 18 | requesting additional features. 19 | 20 | * Make sure you have a `GitHub account `_ 21 | 22 | * Submit an issue, assuming one does not already exist. 23 | 24 | * Clearly describe the issue including steps to reproduce when it is a bug. 25 | 26 | * If appropriate, include a Cosmic Ray config file and, if possible, some way for us to get access to 27 | the code you're working with. 28 | 29 | * Make sure you mention the earliest version that you know has the issue. 30 | 31 | * Fork the repository on GitHub 32 | 33 | 34 | Making Changes 35 | ============== 36 | 37 | * You must own the copyright to the patch you're submitting, and be in a 38 | position to transfer the copyright to Sixty North by agreeing to the either 39 | the |ICLA| 40 | (for private individuals) or the |ECLA| 41 | (for corporations or other organisations). 42 | * Make small commits in logical units. 43 | * Ensure your code is in the spirit of `PEP 8 `_, 44 | although we accept that much of what is in PEP 8 are guidelines 45 | rather than rules, so we value readability over strict compliance. 46 | * Check for unnecessary whitespace with ``git diff --check`` before committing. 47 | * Make sure your commit messages are in the proper format:: 48 | 49 | 50 | Issue #1234 - Make the example in CONTRIBUTING imperative and concrete 51 | 52 | Without this patch applied the example commit message in the CONTRIBUTING 53 | document is not a concrete example. This is a problem because the 54 | contributor is left to imagine what the commit message should look like 55 | based on a description rather than an example. This patch fixes the 56 | problem by making the example concrete and imperative. 57 | 58 | The first line is a real life imperative statement with an issue number 59 | from our issue tracker. The body describes the behavior without the patch, 60 | why this is a problem, and how the patch fixes the problem when applied. 61 | 62 | 63 | * Make sure you have added the necessary tests for your changes. 64 | * Run **all** the tests to assure nothing else was accidentally broken. 65 | 66 | Making Trivial Changes 67 | ====================== 68 | 69 | Documentation 70 | ------------- 71 | 72 | For changes of a trivial nature to comments and documentation, it is not 73 | always necessary to create a new issue. In this case, it is appropriate 74 | to start the first line of a commit with 'Doc -' instead of an issue 75 | number:: 76 | 77 | Doc - Add documentation commit example to CONTRIBUTING 78 | 79 | There is no example for contributing a documentation commit 80 | to the Cosmic Ray repository. This is a problem because the contributor 81 | is left to assume how a commit of this nature may appear. 82 | 83 | The first line is a real life imperative statement with 'Doc -' in 84 | place of what would have been the ticket number in a 85 | non-documentation related commit. The body describes the nature of 86 | the new documentation or comments added. 87 | 88 | Submitting Changes 89 | ================== 90 | 91 | * Agree to the |ICLA| or the |ECLA| 92 | by attaching a copy of the current CLA to an email (so we know which 93 | version you're agreeing to). The body of the message should contain 94 | the text "I, , [representing ] have read the 95 | attached CLA and agree to its terms." Send the email to austin@sixty-north.com 96 | * Push your changes to a topic branch in your fork of the repository. 97 | * Submit a pull request to the repository in the sixty-north organization. 98 | 99 | 100 | Additional Resources 101 | ==================== 102 | 103 | * |ICLA| 104 | * |ECLA| 105 | * `PEP 8 `_ 106 | * `General GitHub documentation `_ 107 | * `GitHub pull request documentation `_ 108 | 109 | .. |ICLA| replace:: `Individual Contributors License Agreement `__ 110 | .. |ECLA| replace:: `Entity Contributor License Agreement `__ -------------------------------------------------------------------------------- /docs/source/how-tos/filters.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Filters 3 | ======= 4 | 5 | The ``cosmic-ray init`` commands scans a module for all possible mutations, but we don't always want to execute all of 6 | these. For example, we may know that some of these mutations will result in *equivalent mutants*, so we need a way to 7 | prevent these mutations from actually being run. 8 | 9 | To account for this, Cosmic Ray includes a number of *filters*. Filters are nothing more than programs - generally small 10 | ones - that modify a session in some way, often by marking certains mutations as "skipped", thereby preventing them from 11 | running. The name "filter" is actually a bit misleading since these programs could modify a session in ways other than 12 | simply skipping some mutations. In practice, though, the need to skip certain tests is by far the most common use of 13 | these programs. 14 | 15 | Using filters 16 | ============= 17 | 18 | Generally speaking, filters will be run immediately after running ``cosmic-ray init``. It's up to you to decide which to 19 | run, and often they will be run along with ``init`` in a batch script or CI configuration. 20 | 21 | For example, if you wanted to apply the ``cr-filter-pragma`` filter to your session, you could do something like this: 22 | 23 | .. code-block:: bash 24 | 25 | cosmic-ray init cr.conf session.sqlite 26 | cr-filter-pragma session.sqlite 27 | 28 | The ``init`` would first create a session where *all* mutation would be run, and then the ``cr-filter-pragma`` call 29 | would mark as skipped all mutations which are on a line with the pragma comment. 30 | 31 | Filters included with Cosmic Ray 32 | ================================ 33 | 34 | Cosmic Ray comes with a number of filters. Remember, though, that they are nothing more than simple programs that modify 35 | a session in some way; it should be straightforward to write your own filters should the need arise. 36 | 37 | cr-filter-operators 38 | ------------------- 39 | 40 | ``cr-filter-operators`` allows you to filter out operators according to their names. You provide the filter with a set 41 | of regular expressions, and any Cosmic Ray operator who's name matches a one of these expressions will be skipped 42 | entirely. 43 | 44 | The configuration is provided through a TOML file such as a standard Cosmic Ray configuration. The expressions must be 45 | in a list at the key "cosmic-ray.filters.operators-filter.exclude-operators". Here's an example: 46 | 47 | .. code-block:: toml 48 | 49 | [cosmic-ray.filters.operators-filter] 50 | exclude-operators = [ 51 | "core/ReplaceComparisonOperator_Is(Not)?_(Not)?(Eq|[LG]tE?)", 52 | "core/ReplaceComparisonOperator_(Not)?(Eq|[LG]tE?)_Is(Not)?", 53 | "core/ReplaceComparisonOperator_LtE_Eq", 54 | "core/ReplaceComparisonOperator_Lt_NotEq", 55 | ] 56 | 57 | The first regular expression here is skipping the following operators: 58 | 59 | - core/ReplaceComparisonOperator_Is_Eq 60 | - core/ReplaceComparisonOperator_Is_Lt 61 | - core/ReplaceComparisonOperator_Is_LtE 62 | - core/ReplaceComparisonOperator_Is_Gt 63 | - core/ReplaceComparisonOperator_Is_GtE 64 | - core/ReplaceComparisonOperator_Is_NotEq 65 | - core/ReplaceComparisonOperator_Is_NotLt 66 | - core/ReplaceComparisonOperator_Is_NotLtE 67 | - core/ReplaceComparisonOperator_Is_NotGt 68 | - core/ReplaceComparisonOperator_Is_NotGtE 69 | - core/ReplaceComparisonOperator_IsNot_Eq 70 | - core/ReplaceComparisonOperator_IsNot_Lt 71 | - core/ReplaceComparisonOperator_IsNot_LtE 72 | - core/ReplaceComparisonOperator_IsNot_Gt 73 | - core/ReplaceComparisonOperator_IsNot_GtE 74 | - core/ReplaceComparisonOperator_IsNot_NotEq 75 | - core/ReplaceComparisonOperator_IsNot_NotLt 76 | - core/ReplaceComparisonOperator_IsNot_NotLtE 77 | - core/ReplaceComparisonOperator_IsNot_NotGt 78 | - core/ReplaceComparisonOperator_IsNot_NotGtE 79 | 80 | While all of the entries in `operators-filter.exclude-operators` are treated as regular expressions, you don't need to 81 | us "fancy" regular expression features in them. As in the last two entries in the example above, you can do matching 82 | against an exact string; these are still regular expressions, albeit simple ones. 83 | 84 | For a list of all operators in your Cosmic Ray installation, run ``cosmic-ray operators``. 85 | 86 | cr-filter-pragma 87 | ---------------- 88 | 89 | The ``cr-filter-pragma`` filter looks for lines in your source code containing the comment "# pragma: no mutate". Any 90 | mutation in a session that would mutate such a line is skipped. 91 | 92 | cr-filter-git 93 | ------------- 94 | 95 | The ``cr-filter-git`` filter looks for edited or new lines from the given git branch. Any mutation in a session that 96 | would mutate other lines is skipped. 97 | 98 | By default the ``master`` branch is used, but you could define another one like this: 99 | 100 | .. code-block:: toml 101 | 102 | [cosmic-ray.filters.git-filter] 103 | branch = "rolling" 104 | 105 | External filters 106 | ================ 107 | 108 | Other filters are defined in separate projects. 109 | 110 | cosmic-ray-spor-filter 111 | ---------------------- 112 | 113 | The ``cosmic-ray-spor-filter`` filter modifies a session by skipping mutations which are indicated in a `spor 114 | `_ anchored metadata repository. In short, ``spor`` provides a way to associated 115 | arbitrary metadata with ranges of code, and this metadata is stored outside of the code. As your code changes, ``spor`` 116 | has algorithms to update the metadata (and its association with the code) automatically. 117 | 118 | Get more details at `the project page `_. 119 | 120 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | import subprocess 4 | import sys 5 | 6 | import pytest 7 | 8 | from cosmic_ray.tools.survival_rate import survival_rate 9 | from cosmic_ray.work_db import WorkDB, use_db 10 | 11 | 12 | @pytest.fixture(scope="session") 13 | def example_project_root(pytestconfig): 14 | root = pathlib.Path(str(pytestconfig.rootdir)) 15 | return root / "tests" / "resources" / "example_project" 16 | 17 | 18 | @pytest.fixture 19 | def config(tester, distributor): 20 | "Get config file name." 21 | config = f"cosmic-ray.{tester}.{distributor}.conf" 22 | return config 23 | 24 | 25 | @pytest.mark.slow 26 | def test_init_and_exec(example_project_root, config, session): 27 | subprocess.check_call( 28 | [sys.executable, "-m", "cosmic_ray.cli", "init", config, str(session)], cwd=str(example_project_root) 29 | ) 30 | 31 | subprocess.check_call( 32 | [sys.executable, "-m", "cosmic_ray.cli", "exec", config, str(session)], cwd=str(example_project_root) 33 | ) 34 | 35 | session_path = example_project_root / session 36 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 37 | rate = survival_rate(work_db) 38 | assert rate == 0.0 39 | 40 | 41 | def test_baseline_with_explicit_session_file(example_project_root, config, session): 42 | subprocess.check_call( 43 | [sys.executable, "-m", "cosmic_ray.cli", "baseline", str(config), "--session-file", str(session)], 44 | cwd=str(example_project_root), 45 | ) 46 | 47 | with use_db(str(session), WorkDB.Mode.open) as work_db: 48 | rate = survival_rate(work_db) 49 | assert rate == 100.0 50 | 51 | 52 | def test_baseline_with_temp_session_file(example_project_root, config): 53 | subprocess.check_call( 54 | [sys.executable, "-m", "cosmic_ray.cli", "baseline", str(config)], 55 | cwd=str(example_project_root), 56 | ) 57 | 58 | 59 | def test_importing(example_project_root, session): 60 | config = "cosmic-ray.import.conf" 61 | 62 | subprocess.check_call( 63 | [sys.executable, "-m", "cosmic_ray.cli", "init", config, str(session)], 64 | cwd=str(example_project_root), 65 | ) 66 | 67 | session_path = example_project_root / session 68 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 69 | rate = survival_rate(work_db) 70 | assert rate == 0.0 71 | 72 | 73 | def test_empty___init__(example_project_root, session): 74 | config = "cosmic-ray.empty.conf" 75 | 76 | subprocess.check_call( 77 | [sys.executable, "-m", "cosmic_ray.cli", "init", str(config), str(session)], 78 | cwd=str(example_project_root), 79 | ) 80 | 81 | session_path = example_project_root / session 82 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 83 | rate = survival_rate(work_db) 84 | assert rate == 0.0 85 | 86 | 87 | def test_inexisting(example_project_root, session): 88 | config = "cosmic-ray.inexisting.conf" 89 | 90 | result = subprocess.run( 91 | [sys.executable, "-m", "cosmic_ray.cli", "init", str(config), str(session)], 92 | cwd=str(example_project_root), 93 | encoding="utf-8", 94 | capture_output=True, 95 | ) 96 | 97 | # Workaround to take out new line in result.stderr value that causes error in Windows, 98 | # despite using os.linesep 99 | stripped_version = result.stderr.strip() 100 | 101 | assert result.returncode == 66 102 | assert result.stdout == "" 103 | assert stripped_version == "Could not find module path example" + os.sep + "unknown_file.py" 104 | 105 | 106 | def test_baseline_with_pytest_filter(example_project_root, session): 107 | config = "cosmic-ray.with-pytest-filter.conf" 108 | 109 | result = subprocess.run( 110 | [sys.executable, "-m", "cosmic_ray.cli", "baseline", str(config)], 111 | cwd=str(example_project_root), 112 | encoding="utf-8", 113 | capture_output=True, 114 | ) 115 | 116 | assert result.returncode == 0 117 | 118 | 119 | @pytest.mark.slow 120 | def test_reinit_session_with_results_fails(example_project_root, config, session): 121 | subprocess.check_call( 122 | [sys.executable, "-m", "cosmic_ray.cli", "init", config, str(session)], cwd=str(example_project_root) 123 | ) 124 | 125 | subprocess.check_call( 126 | [sys.executable, "-m", "cosmic_ray.cli", "exec", config, str(session)], cwd=str(example_project_root) 127 | ) 128 | 129 | session_path = example_project_root / session 130 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 131 | initial_num_work_items = work_db.num_work_items 132 | assert work_db.num_results == initial_num_work_items > 0 133 | 134 | with pytest.raises(subprocess.CalledProcessError): 135 | subprocess.check_call( 136 | [sys.executable, "-m", "cosmic_ray.cli", "init", config, str(session)], cwd=str(example_project_root) 137 | ) 138 | 139 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 140 | assert work_db.num_results == work_db.num_work_items == initial_num_work_items 141 | 142 | 143 | @pytest.mark.slow 144 | def test_force_reinit_session_with_results_succeeds(example_project_root, config, session): 145 | subprocess.check_call( 146 | [sys.executable, "-m", "cosmic_ray.cli", "init", config, str(session)], cwd=str(example_project_root) 147 | ) 148 | 149 | subprocess.check_call( 150 | [sys.executable, "-m", "cosmic_ray.cli", "exec", config, str(session)], cwd=str(example_project_root) 151 | ) 152 | 153 | session_path = example_project_root / session 154 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 155 | initial_num_work_items = work_db.num_work_items 156 | assert initial_num_work_items > 0 157 | assert work_db.num_results == work_db.num_work_items 158 | 159 | subprocess.check_call( 160 | [sys.executable, "-m", "cosmic_ray.cli", "init", config, str(session), "--force"], cwd=str(example_project_root) 161 | ) 162 | 163 | with use_db(str(session_path), WorkDB.Mode.open) as work_db: 164 | assert work_db.num_work_items == initial_num_work_items 165 | assert work_db.num_results == 0 166 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration file for the Sphinx documentation builder. 3 | # 4 | # This file does only contain a selection of the most common options. For a 5 | # full list see the documentation: 6 | # http://www.sphinx-doc.org/en/master/config 7 | 8 | # -- Path setup -------------------------------------------------------------- 9 | 10 | # If extensions (or modules to document with autodoc) are in another directory, 11 | # add these directories to sys.path here. If the directory is relative to the 12 | # documentation root, use os.path.abspath to make it absolute, like shown here. 13 | # 14 | import os 15 | import sys 16 | 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | sys.path.insert(0, os.path.abspath("../../src")) 19 | 20 | # on_rtd is whether we are on readthedocs.org 21 | on_rtd = os.environ.get("READTHEDOCS", None) == "True" 22 | 23 | # -- Project information ----------------------------------------------------- 24 | 25 | project = "Cosmic Ray" 26 | copyright = "2019, Sixty North AS" 27 | author = "Austin Bingham" 28 | 29 | # The short X.Y version 30 | version = "" 31 | # The full version, including alpha/beta/rc tags 32 | release = "" 33 | 34 | 35 | # -- General configuration --------------------------------------------------- 36 | 37 | # If your documentation needs a minimal Sphinx version, state it here. 38 | # 39 | # needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 43 | # ones. 44 | extensions = [ 45 | "sphinx.ext.autodoc", 46 | "sphinx.ext.napoleon", 47 | "sphinx_click", 48 | ] 49 | 50 | # Add any paths that contain templates here, relative to this directory. 51 | templates_path = [] # ['_templates'] 52 | 53 | # The suffix(es) of source filenames. 54 | # You can specify multiple suffix as a list of string: 55 | # 56 | # source_suffix = ['.rst', '.md'] 57 | source_suffix = ".rst" 58 | 59 | # The master toctree document. 60 | master_doc = "index" 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This pattern also affects html_static_path and html_extra_path. 72 | exclude_patterns = [] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = None 76 | 77 | 78 | # -- Options for HTML output ------------------------------------------------- 79 | 80 | # The theme to use for HTML and HTML Help pages. See the documentation for 81 | # a list of builtin themes. 82 | # 83 | if not on_rtd: # only import and set the theme if we're building docs locally 84 | import sphinx_rtd_theme 85 | 86 | html_theme = "sphinx_rtd_theme" 87 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 88 | # html_theme = 'alabaster' 89 | 90 | # Theme options are theme-specific and customize the look and feel of a theme 91 | # further. For a list of options available for each theme, see the 92 | # documentation. 93 | # 94 | # html_theme_options = {} 95 | 96 | # Add any paths that contain custom static files (such as style sheets) here, 97 | # relative to this directory. They are copied after the builtin static files, 98 | # so a file named "default.css" will overwrite the builtin "default.css". 99 | html_static_path = [] # ['_static'] 100 | 101 | # Custom sidebar templates, must be a dictionary that maps document names 102 | # to template names. 103 | # 104 | # The default sidebars (for documents that don't match any pattern) are 105 | # defined by theme itself. Builtin themes are using these templates by 106 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 107 | # 'searchbox.html']``. 108 | # 109 | # html_sidebars = {} 110 | 111 | 112 | # -- Options for HTMLHelp output --------------------------------------------- 113 | 114 | # Output file base name for HTML help builder. 115 | htmlhelp_basename = "CosmicRayDocumentationdoc" 116 | 117 | 118 | # -- Options for LaTeX output ------------------------------------------------ 119 | 120 | latex_elements = { 121 | # The paper size ('letterpaper' or 'a4paper'). 122 | # 123 | # 'papersize': 'letterpaper', 124 | # The font size ('10pt', '11pt' or '12pt'). 125 | # 126 | # 'pointsize': '10pt', 127 | # Additional stuff for the LaTeX preamble. 128 | # 129 | # 'preamble': '', 130 | # Latex figure (float) alignment 131 | # 132 | # 'figure_align': 'htbp', 133 | } 134 | 135 | # Grouping the document tree into LaTeX files. List of tuples 136 | # (source start file, target name, title, 137 | # author, documentclass [howto, manual, or own class]). 138 | latex_documents = [ 139 | (master_doc, "CosmicRayDocumentation.tex", "Cosmic Ray Documentation Documentation", "Austin Bingham", "manual"), 140 | ] 141 | 142 | 143 | # -- Options for manual page output ------------------------------------------ 144 | 145 | # One entry per manual page. List of tuples 146 | # (source start file, name, description, authors, manual section). 147 | man_pages = [(master_doc, "cosmicraydocumentation", "Cosmic Ray Documentation Documentation", [author], 1)] 148 | 149 | 150 | # -- Options for Texinfo output ---------------------------------------------- 151 | 152 | # Grouping the document tree into Texinfo files. List of tuples 153 | # (source start file, target name, title, author, 154 | # dir menu entry, description, category) 155 | texinfo_documents = [ 156 | ( 157 | master_doc, 158 | "CosmicRayDocumentation", 159 | "Cosmic Ray Documentation Documentation", 160 | author, 161 | "CosmicRayDocumentation", 162 | "One line description of project.", 163 | "Miscellaneous", 164 | ), 165 | ] 166 | 167 | 168 | # -- Options for Epub output ------------------------------------------------- 169 | 170 | # Bibliographic Dublin Core info. 171 | epub_title = project 172 | 173 | # The unique identifier of the text. This can be a ISBN number 174 | # or the project homepage. 175 | # 176 | # epub_identifier = '' 177 | 178 | # A unique identification for the text. 179 | # 180 | # epub_uid = '' 181 | 182 | # A list of files that should not be packed into the epub file. 183 | epub_exclude_files = ["search.html"] 184 | --------------------------------------------------------------------------------