├── .gitattributes ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README ├── README.md ├── demo.gif ├── fault_localization ├── __init__.py ├── display.py ├── localization.py ├── plugin.py └── tracing.py ├── setup.py ├── tests └── unit │ └── test_localization.py └── tox.ini /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # Jupyter Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # SageMath parsed files 79 | *.sage.py 80 | 81 | # Environments 82 | .env 83 | .venv 84 | env/ 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | .spyproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # mkdocs documentation 96 | /site 97 | 98 | # mypy 99 | .mypy_cache/ 100 | 101 | # Pycharm 102 | .idea/ 103 | 104 | # Tox 105 | .pytest_cache/ 106 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.6" 6 | install: pip install tox-travis 7 | script: tox 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 H. Chase Stevens 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fault-localization 2 | 3 | [![Build Status](https://travis-ci.org/hchasestevens/fault-localization.svg?branch=master)](https://travis-ci.org/hchasestevens/fault-localization) 4 | [![PyPI version](https://badge.fury.io/py/fault-localization.svg)](https://pypi.org/project/fault-localization) 5 | [![conda-forge version](https://img.shields.io/conda/vn/conda-forge/fault-localization.svg)](https://anaconda.org/conda-forge/fault-localization) 6 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/fault-localization.svg) 7 | 8 | 9 | 10 | ## What is this good for? 11 | 12 | Not all failing code raises exceptions; behavioral bugs can be the hardest to diagnose. 13 | `fault-localization` is a [pytest](https://docs.pytest.org/en/latest/) plugin that helps you identify and isolate the lines of code most 14 | likely to be causing test failure, using the simple rule-of-thumb that the most suspicious code is the 15 | code run most often in failing tests. Don't just rely on your tests to _catch_ bugs - use them to _pinpoint_ bugs. 16 | 17 | ## Installation 18 | 19 | ```bash 20 | pip install fault-localization 21 | ``` 22 | 23 | ## Usage 24 | 25 | With `fault-localization` installed, running 26 | 27 | ```bash 28 | pytest --localize {dir} [--n-hotspots {n}] [pytest args ...] 29 | ``` 30 | 31 | will highlight suspicious lines encountered within `dir` while running the test configuration specified. 32 | `--n-hotspots` can optionally be provided to show the top `n` most suspicious lines (only one line, with surrounding context, 33 | is shown by default). 34 | 35 | If you suspect multiple sources of failure, or if there are multiple tests within your suite that 36 | exercise the area of code you're interested in, using [pytest's `-k` flag](https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests) is useful for running 37 | fault localization on only a subset of your suite. 38 | 39 | Fault localization, as a technique, works best when areas of your codebase are exercised repeatedly 40 | across a bevy of differing cases and values. That's why `fault-localization` works with Python's premiere 41 | property-based testing framework, [Hypothesis](http://hypothesis.works), out of the box - which does 42 | just that. 43 | 44 | ## Contacts 45 | 46 | * Name: [H. Chase Stevens](http://www.chasestevens.com) 47 | * Twitter: [@hchasestevens](https://twitter.com/hchasestevens) 48 | -------------------------------------------------------------------------------- /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hchasestevens/fault-localization/f2935f08e967748eb502d998b73e6603450fdee8/demo.gif -------------------------------------------------------------------------------- /fault_localization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hchasestevens/fault-localization/f2935f08e967748eb502d998b73e6603450fdee8/fault_localization/__init__.py -------------------------------------------------------------------------------- /fault_localization/display.py: -------------------------------------------------------------------------------- 1 | """Display utilities.""" 2 | 3 | import os 4 | import collections 5 | from operator import attrgetter 6 | 7 | LINE_CONTEXT = 5 8 | 9 | OutputLine = collections.namedtuple( 10 | 'OutputLine', 11 | 'num content score' 12 | ) 13 | 14 | 15 | CUBE_LEVELS = (0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff,) 16 | SNAP_POINTS = tuple( 17 | (lower + upper) / 2 18 | for lower, upper in 19 | zip(CUBE_LEVELS, CUBE_LEVELS[1:]) 20 | ) 21 | 22 | 23 | def rgb_to_hex(*channels): 24 | r, g, b = ( 25 | sum( 26 | snap_point < channel 27 | for snap_point in SNAP_POINTS 28 | ) 29 | for channel in channels 30 | ) 31 | return r * 36 + g * 6 + b + 16 32 | 33 | 34 | def generate_output(line_scores, line_context=LINE_CONTEXT, n_lines=1): 35 | """Given line scores, create final localization output.""" 36 | ranked_lines = sorted(line_scores, key=line_scores.get, reverse=True) 37 | if not ranked_lines: 38 | yield "No lines from specified directory captured during test suite execution." 39 | return 40 | 41 | max_score = line_scores[ranked_lines[0]] 42 | min_score = min(line_scores.values()) 43 | 44 | out_buffer = collections.defaultdict(set) 45 | 46 | for _, (fname, line_no) in zip(range(n_lines), ranked_lines): 47 | with open(fname, 'r') as f: 48 | file_contents = f.read().splitlines() 49 | 50 | def try_get_line(line_no): 51 | try: 52 | yield file_contents[line_no] 53 | except IndexError: 54 | return 55 | 56 | output_lines = ( 57 | OutputLine( 58 | num=line_no + 1, 59 | content=line, 60 | score=round(line_scores.get((fname, line_no), 0), 2) 61 | ) 62 | for line_no in range( 63 | line_no - line_context, 64 | line_no + line_context + 1 65 | ) 66 | if line_no >= 0 67 | for line in try_get_line(line_no) 68 | ) 69 | 70 | out_buffer_key = '\x1b[1m{}\x1b[0m'.format(os.path.relpath(fname, os.getcwd())) 71 | for line in output_lines: 72 | out_buffer[out_buffer_key].add(line) 73 | 74 | out_buffer_sequence = sorted( 75 | out_buffer.items(), 76 | key=lambda k_v: max(k_v[1], key=attrgetter('score')), 77 | reverse=True 78 | ) 79 | for header, lines in out_buffer_sequence: 80 | yield header 81 | for line in sorted(lines, key=attrgetter('num')): 82 | # todo: right-justify the numeric score value (rather than tab) 83 | yield '{line.num}\t\x1b[48;5;{ansi_tag}m{line.content}\033[0m\t{line.score}'.format( 84 | line=line, 85 | ansi_tag=rgb_to_hex( 86 | int( 87 | (line.score - min_score) / (max_score - min_score) * 255 88 | ), 0, 0 89 | ) 90 | # TODO - want to actually represent this in HSV space and go from neutral green/orange to red - see also http://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html#background-colors 91 | # TODO - can we use RGB for background color rather than hex? 92 | ) 93 | -------------------------------------------------------------------------------- /fault_localization/localization.py: -------------------------------------------------------------------------------- 1 | """Fault localization calculations.""" 2 | 3 | 4 | import collections 5 | 6 | 7 | ExecutionCounts = collections.namedtuple( 8 | "ExecutionCounts", 9 | "positive_cases negative_cases" 10 | ) 11 | 12 | LINE_EXECUTIONS = collections.defaultdict( 13 | lambda: ExecutionCounts( 14 | positive_cases=0, 15 | negative_cases=0, 16 | ) 17 | ) 18 | 19 | 20 | def update_executions(lines, failed, line_executions=LINE_EXECUTIONS): 21 | """Update line executions to reflect test results.""" 22 | for line in lines: 23 | prev_executions = line_executions[line] 24 | line_executions[line] = ExecutionCounts( 25 | positive_cases=prev_executions.positive_cases + int(not failed), 26 | negative_cases=prev_executions.negative_cases + int(failed) 27 | ) 28 | 29 | 30 | PRIOR = ExecutionCounts( 31 | positive_cases=1, 32 | negative_cases=0 33 | ) 34 | 35 | 36 | def calc_scores(line_executions=LINE_EXECUTIONS, prior=PRIOR): 37 | """Return 'fault' score for each line, given prior and observations.""" 38 | return { 39 | line: float( 40 | execution_counts.negative_cases + prior.negative_cases 41 | ) / ( 42 | execution_counts.positive_cases + prior.positive_cases 43 | + execution_counts.negative_cases + prior.negative_cases 44 | ) 45 | for line, execution_counts in line_executions.items() 46 | } 47 | -------------------------------------------------------------------------------- /fault_localization/plugin.py: -------------------------------------------------------------------------------- 1 | """Pytest plugin.""" 2 | 3 | import os 4 | import sys 5 | 6 | import pytest 7 | 8 | from fault_localization.tracing import TRACER 9 | from fault_localization.localization import update_executions, calc_scores 10 | from fault_localization.display import generate_output 11 | 12 | 13 | LOCALIZATION_DIR = None 14 | N_LINES = 1 15 | 16 | 17 | def pytest_addoption(parser): 18 | group = parser.getgroup('fault-localization', 'fault localization') 19 | group.addoption('--localize', help="directory in which to localize faults") 20 | group.addoption('--n-hotspots', type=int, default=1, help="number of top-ranking lines to display") 21 | 22 | 23 | def pytest_configure(config): 24 | global LOCALIZATION_DIR 25 | global N_LINES 26 | LOCALIZATION_DIR = config.getoption('--localize') 27 | N_LINES = config.getoption('--n-hotspots') 28 | 29 | 30 | @pytest.hookimpl(hookwrapper=True) 31 | def pytest_runtest_call(item): 32 | if not LOCALIZATION_DIR: 33 | yield 34 | return 35 | 36 | sys.settrace(TRACER.trace) 37 | try: 38 | yield 39 | finally: 40 | sys.settrace(None) 41 | 42 | 43 | def pytest_runtest_makereport(item, call): 44 | failed = bool(getattr(call, 'excinfo', False)) 45 | update_executions( 46 | lines=TRACER.flush_buffer(), 47 | failed=failed 48 | ) 49 | 50 | 51 | def pytest_terminal_summary(terminalreporter): 52 | if not LOCALIZATION_DIR: 53 | return 54 | abs_localization_dir = os.path.abspath(LOCALIZATION_DIR) 55 | 56 | terminalreporter.section("Fault Localization Results") 57 | line_scores = { 58 | (path, line): score 59 | for (path, line), score in calc_scores().items() 60 | if path.startswith(abs_localization_dir) 61 | } 62 | for line in generate_output(line_scores, n_lines=N_LINES): 63 | terminalreporter.write_line(line) 64 | -------------------------------------------------------------------------------- /fault_localization/tracing.py: -------------------------------------------------------------------------------- 1 | """Line-by-line program execution tracing.""" 2 | 3 | 4 | class Tracer(object): 5 | LINE_BUFFER = [] 6 | 7 | def trace(self, frame, event, arg): 8 | if event == 'call': 9 | return self.trace 10 | elif event != 'line': 11 | return 12 | line = frame.f_code.co_filename, frame.f_lineno 13 | self.LINE_BUFFER.append(line) 14 | 15 | def flush_buffer(self): 16 | buffer, self.LINE_BUFFER = self.LINE_BUFFER, [] 17 | return buffer 18 | 19 | 20 | TRACER = Tracer() 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='fault-localization', 5 | packages=['fault_localization'], 6 | platforms='any', 7 | version='0.1.6', 8 | description='A fault localization tool for Python\'s pytest testing framework.', 9 | author='H. Chase Stevens', 10 | author_email='chase@chasestevens.com', 11 | url='https://github.com/hchasestevens/fault-localization', 12 | license='MIT', 13 | install_requires=[ 14 | 'pytest>=3.1.2', 15 | ], 16 | entry_points={ 17 | 'pytest11': [ 18 | 'fault-localization = fault_localization.plugin', 19 | ] 20 | }, 21 | classifiers=[ 22 | 'Programming Language :: Python', 23 | 'Programming Language :: Python :: 2', 24 | 'Programming Language :: Python :: 2.7', 25 | 'Programming Language :: Python :: 3', 26 | 'Programming Language :: Python :: 3.6', 27 | 'Topic :: Software Development :: Libraries :: Python Modules', 28 | 'Intended Audience :: Developers', 29 | 'Operating System :: OS Independent', 30 | 'License :: OSI Approved :: MIT License', 31 | ] 32 | ) 33 | -------------------------------------------------------------------------------- /tests/unit/test_localization.py: -------------------------------------------------------------------------------- 1 | """Unit tests for fault_localization/localization.py""" 2 | import collections 3 | 4 | import pytest 5 | 6 | from fault_localization import localization 7 | 8 | 9 | @pytest.mark.parametrize('failed,expected_output', ( 10 | (False, localization.ExecutionCounts(positive_cases=1, negative_cases=0)), 11 | (True, localization.ExecutionCounts(positive_cases=0, negative_cases=1)) 12 | )) 13 | def test_update_executions(failed, expected_output): 14 | """Ensure that line counts are incremented as expected for test result.""" 15 | executions = collections.defaultdict( 16 | lambda: localization.ExecutionCounts( 17 | positive_cases=0, 18 | negative_cases=0, 19 | ) 20 | ) 21 | localization.update_executions([1], failed, line_executions=executions) 22 | assert executions[1] == expected_output 23 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skip_missing_interpreters = True 3 | envlist = py27,py36,py36-lint 4 | 5 | [testenv] 6 | extras = dev 7 | commands = py.test ./tests 8 | --------------------------------------------------------------------------------