├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .readthedocs.yml ├── AUTHORS ├── LICENSE ├── README.rst ├── THANKS.rst ├── bin ├── ddsmt ├── ddsmt-profile └── smt2info ├── ddsmt ├── __init__.py ├── __main__.py ├── argparsemod.py ├── checker.py ├── cli.py ├── debug_utils.py ├── mutator_utils.py ├── mutators.py ├── mutators_arithmetic.py ├── mutators_boolean.py ├── mutators_bv.py ├── mutators_core.py ├── mutators_datatypes.py ├── mutators_fp.py ├── mutators_smtlib.py ├── mutators_strings.py ├── nodeio.py ├── nodes.py ├── options.py ├── progress.py ├── smtlib.py ├── strategy_ddmin.py ├── strategy_hierarchical.py ├── tests │ ├── __init__.py │ ├── test_mutators_arithmetic.py │ ├── test_mutators_boolean.py │ ├── test_mutators_bv.py │ ├── test_mutators_core.py │ ├── test_mutators_datatypes.py │ ├── test_mutators_fp.py │ ├── test_mutators_smtlib.py │ ├── test_mutators_strings.py │ ├── test_nodes.py │ ├── test_options.py │ ├── test_smtlib.py │ └── utils.py ├── tmpfiles.py └── version.py ├── docs ├── .gitignore ├── Makefile ├── conf.py ├── contributing.rst ├── example │ ├── input.smt2 │ └── solver ├── faq.rst ├── guide-developers.rst ├── guide-mutators.rst ├── guide-performance.rst ├── guide-scenarios.rst ├── guide-strategies.rst ├── guide.rst ├── img │ ├── ddmin.png │ └── hier.png ├── index.rst ├── installation.rst ├── literature.bib ├── publications │ ├── KremerNiemetzPreiner-CAV21.pdf │ └── NiemetzBiere-SMT13.pdf ├── quickstart.rst └── references.rst ├── requirements.txt ├── scripts ├── compare_time.py └── result_differs.py ├── setup.cfg ├── setup.py └── update_license_headers.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | - name: set up python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | - name: install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install flake8 pytest yapf 24 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 25 | - name: lint with flake8 26 | run: flake8 . --exit-zero 27 | - name: formatting with yapf 28 | continue-on-error: true 29 | run: yapf -r -d . 30 | - name: test with pytest 31 | run: pytest 32 | - name: collect code coverage 33 | run: | 34 | coverage run -m pytest 35 | coverage report 36 | 37 | package: 38 | if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') 39 | needs: test 40 | runs-on: ubuntu-latest 41 | 42 | steps: 43 | - uses: actions/checkout@v4 44 | with: 45 | fetch-depth: 0 46 | - name: set up python 47 | uses: actions/setup-python@v5 48 | with: 49 | python-version: '3.x' 50 | - name: install dependencies 51 | run: | 52 | python -m pip install --upgrade pip 53 | pip install setuptools wheel twine 54 | - name: build and publish package 55 | env: 56 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 57 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 58 | run: | 59 | python setup.py sdist bdist_wheel 60 | twine upload dist/* 61 | 62 | install: 63 | if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') 64 | needs: package 65 | runs-on: ubuntu-latest 66 | 67 | strategy: 68 | matrix: 69 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] 70 | 71 | steps: 72 | - name: set up python ${{ matrix.python-version }} 73 | uses: actions/setup-python@v5 74 | with: 75 | python-version: ${{ matrix.python-version }} 76 | - name: install package 77 | run: | 78 | python -m pip install --upgrade pip 79 | pip install pytest 80 | pip install --pre ddsmt 81 | - name: test package 82 | run: | 83 | pytest --pyargs ddsmt 84 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | .vscode 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.12" 7 | 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | python: 12 | install: 13 | - requirements: requirements.txt 14 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | The main authors and developers of ddSMT are: 2 | 3 | Aina Niemetz -- Johannes Kepler University, Linz (2013-2017), Stanford University (since 2017). 4 | Mathias Preiner -- Johannes Kepler University, Linz (2013-2017), Stanford University (since 2017). 5 | Gereon Kremer -- Stanford University (2020-2022). 6 | 7 | ddSMT's copyright is held by these individuals and the affiliated institutions 8 | at the time of their contributions. See the file LICENSE for details on the 9 | copyright and licensing of ddSMT. 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ddSMT is a delta debugger for input in the SMT-LIB language and its dialects 2 | and extensions. 3 | 4 | Copyright (C) 2013-2024 by its authors and contributors and their institutional 5 | affiliations as listed in file AUTHORS. 6 | 7 | MIT License 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a 10 | copy of this software and associated documentation files (the "Software"), 11 | to deal in the Software without restriction, including without limitation 12 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | and/or sell copies of the Software, and to permit persons to whom the 14 | Software is furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included 17 | in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | OTHER DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |CI Status| |Docs Status| |PyPi Status| |Python Versions| |License Badge| 2 | 3 | .. |CI Status| image:: https://github.com/ddsmt/ddSMT/actions/workflows/ci.yml/badge.svg 4 | :alt: GitHub Workflow Status 5 | :target: https://github.com/ddsmt/ddSMT/actions/workflows/ci.yml 6 | .. |Docs Status| image:: https://img.shields.io/readthedocs/ddsmt/master 7 | :alt: Read the Docs 8 | :target: https://ddsmt.readthedocs.io 9 | .. |PyPi Status| image:: https://img.shields.io/pypi/v/ddsmt 10 | :alt: PyPI 11 | :target: https://pypi.org/project/ddSMT/ 12 | .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/ddsmt 13 | :alt: PyPI - Python Version 14 | :target: https://pypi.org/project/ddSMT/ 15 | .. |License Badge| image:: https://img.shields.io/pypi/l/ddSMT 16 | :alt: PyPI - License 17 | :target: https://github.com/ddsmt/ddSMT/blob/master/LICENSE 18 | 19 | 20 | ddSMT: Delta Debugging for SMT-LIBv2 21 | ==================================== 22 | 23 | **ddSMT** is a delta debugger for input in the `SMT-LIB 24 | `_ language and its dialects and extensions. 25 | It serves as an **input minimizer** for SMT-LIB(-like) input that triggers 26 | interesting behavior for a given command. 27 | Noteworthy examples for supported extensions of the SMT-LIB language are 28 | the `SyGuS `_ input format and the extension for 29 | encoding `separation logic `_ 30 | problems. 31 | For more details on the SMT-LIB language, see http://www.smtlib.org. 32 | 33 | **ddSMT** is implemented in Python 3 and developed on a Linux OS. 34 | 35 | **ddSMT** is released under the MIT (see file `LICENSE 36 | `_). 37 | 38 | Requirements 39 | ------------ 40 | 41 | * `Python `_ version 3.6 or later 42 | 43 | Bibtex 44 | ------ 45 | 46 | For citing **ddSMT**, please use the following BibTex entry: 47 | 48 | .. code-block:: text 49 | 50 | @inproceedings{DBLP:conf/cav/KremerNP20, 51 | author = {Gereon Kremer and 52 | Aina Niemetz and 53 | Mathias Preiner}, 54 | editor = {Alexandra Silva and 55 | K. Rustan M. Leino}, 56 | title = {ddSMT 2.0: Better Delta Debugging for the SMT-LIBv2 Language and Friends}, 57 | booktitle = {Computer Aided Verification - 33rd International Conference, {CAV} 58 | 2021, Virtual Event, July 20-23, 2021, Proceedings, Part {II}}, 59 | series = {Lecture Notes in Computer Science}, 60 | volume = {12760}, 61 | pages = {231--242}, 62 | publisher = {Springer}, 63 | year = {2021}, 64 | url = {https://doi.org/10.1007/978-3-030-81688-9\_11}, 65 | doi = {10.1007/978-3-030-81688-9\_11}, 66 | timestamp = {Thu, 29 Jul 2021 13:41:58 +0200}, 67 | biburl = {https://dblp.org/rec/conf/cav/KremerNP20.bib}, 68 | bibsource = {dblp computer science bibliography, https://dblp.org} 69 | } 70 | 71 | 72 | Publications 73 | ------------ 74 | 75 | * Gereon Kremer, Aina Niemetz, Mathias Preiner. 76 | `ddSMT 2.0: Better Delta Debugging for the SMT-LIBv2 Language and Friends `_. 77 | CAV: 231-242. (2021) 78 | * Aina Niemetz, Armin Biere. 79 | `ddSMT: A Delta Debugger for the SMT-LIB v2 Format. SMT 2013 `_. 80 | (2013) 81 | 82 | Documentation 83 | ------------- 84 | 85 | Documentation for **ddSMT** is available at https://ddsmt.readthedocs.io. 86 | 87 | -------------------------------------------------------------------------------- /THANKS.rst: -------------------------------------------------------------------------------- 1 | Many thanks to our contributors: 2 | 3 | - Johannes Altmanninger 4 | 5 | - `43fafc1 `_: Several improvements to the documentation 6 | - `c675892 `_: A mutator to eliminate unused datatypes constructors 7 | 8 | - Andres Noetzli 9 | 10 | - `9a04abb `_: Fixing multiprocessing on MacOS 11 | 12 | We also thank the following people for their contributions to previous 13 | versions of ddSMT: 14 | 15 | - Jane Lange (2018) 16 | - Andres Noetzli (2018) 17 | -------------------------------------------------------------------------------- /bin/ddsmt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 4 | # 5 | # This file is part of ddSMT. 6 | # 7 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 8 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 9 | # 10 | # This file is part of ddSMT under the MIT license. See LICENSE for more 11 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 12 | 13 | import multiprocessing 14 | import os 15 | import sys 16 | 17 | if __name__ == '__main__': 18 | # Ensure that child processes are forked. Some platforms, e.g., macOS, 19 | # default to spawning child processes, which does not preserve the value of 20 | # global variables. 21 | multiprocessing.set_start_method('fork') 22 | 23 | __root_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 24 | '..') 25 | sys.path.insert(0, __root_dir) 26 | 27 | from ddsmt import __main__ # noqa: E402 28 | __main__.main() 29 | -------------------------------------------------------------------------------- /bin/ddsmt-profile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 4 | # 5 | # This file is part of ddSMT. 6 | # 7 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 8 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 9 | # 10 | # This file is part of ddSMT under the MIT license. See LICENSE for more 11 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 12 | 13 | import os 14 | import sys 15 | 16 | __root_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') 17 | sys.path.insert(0, __root_dir) 18 | 19 | from ddsmt import __main__ # noqa: E402 20 | 21 | __main__.main_profile() 22 | -------------------------------------------------------------------------------- /bin/smt2info: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 4 | # 5 | # This file is part of ddSMT. 6 | # 7 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 8 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 9 | # 10 | # This file is part of ddSMT under the MIT license. See LICENSE for more 11 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 12 | 13 | import json 14 | import os 15 | import sys 16 | import time 17 | 18 | __root_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') 19 | sys.path.insert(0, __root_dir) 20 | 21 | from ddsmt import nodeio, nodes # noqa: E402 22 | 23 | if len(sys.argv) < 2: 24 | print('usage: smt2info ') 25 | sys.exit(1) 26 | 27 | filename = sys.argv[1] 28 | if not os.path.isfile(filename): 29 | print(f'{filename} is not a file') 30 | sys.exit(1) 31 | 32 | data = {} 33 | 34 | start_time = time.time() 35 | with open(sys.argv[1], 'r') as infile: 36 | exprs = list(nodeio.parse_smtlib(infile.read())) 37 | 38 | data['parse-time'] = time.time() - start_time 39 | data['file-size'] = os.stat(filename).st_size 40 | data['num-expressions'] = nodes.count_exprs(exprs) 41 | data['num-nodes'] = nodes.count_nodes(exprs) 42 | 43 | print(json.dumps(data, indent=2)) 44 | -------------------------------------------------------------------------------- /ddsmt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddsmt/ddSMT/3efb3344a74a7b29cdf56cdd56630a97736adc21/ddsmt/__init__.py -------------------------------------------------------------------------------- /ddsmt/__main__.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import sys 13 | 14 | from . import cli 15 | from . import options 16 | from . import debug_utils 17 | 18 | 19 | def main(): 20 | """Main entry point for ddsmt.""" 21 | try: 22 | cli.ddsmt_main() 23 | if options.args().profile: 24 | debug_utils.render_profiles() 25 | return 0 26 | except MemoryError: 27 | print("[ddsmt] memory exhausted") 28 | except KeyboardInterrupt: 29 | print("[ddsmt] interrupted") 30 | except cli.DDSMTException as e: 31 | print(e) 32 | if options.args().profile: 33 | debug_utils.render_profiles() 34 | return 1 35 | 36 | 37 | if __name__ == '__main__': 38 | sys.exit(main()) 39 | -------------------------------------------------------------------------------- /ddsmt/argparsemod.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import argparse 13 | 14 | 15 | class _ModularHelpEnabler(argparse.Action): 16 | """A custom action that enables help for a single argument group.""" 17 | 18 | def __call__(self, parser, namespace, values, option_string=None): 19 | parser.enable_modular_help(self.const) 20 | 21 | 22 | class ModularArgumentParser(argparse.ArgumentParser): 23 | """A variant of argparse.ArgumentParser that allows to modularly enable and 24 | disable printing help for individual argument groups. 25 | 26 | A new variant of ``add_argument_groups()`` allows adding commands of 27 | the form ``--help-group``. 28 | """ 29 | 30 | def __init__(self, *args, **kwargs): 31 | """As ``argparse.ArgumentParser.__init__()``, additionally 32 | ``modular_action_groups = []`` can be used to specify the names of 33 | argument groups used for the custom help options.""" 34 | self._modular_action_groups = {} 35 | self._modular_active = False 36 | modular_help_groups = kwargs.pop('modular_help_groups', []) 37 | help_all = kwargs.pop('help_all', True) 38 | super().__init__(*args, **kwargs) 39 | self._modular_help_groups = {} 40 | for name in modular_help_groups: 41 | self._modular_help_groups[name] = self.add_argument_group(name) 42 | if help_all: 43 | self.add_argument('--help-all', 44 | action=_ModularHelpEnabler, 45 | nargs=0, 46 | default=argparse.SUPPRESS, 47 | const=None, 48 | help='show all help messages') 49 | 50 | def add_argument_group(self, *args, **kwargs): 51 | """As argparse.ArgumentParser.add_argument_group(), additionally 52 | help_name, help_group and help_text allow to add a new option 53 | :code:`--help-{{help_name}}.""" 54 | help_name = kwargs.pop('help_name', None) 55 | help_group = kwargs.pop('help_group', None) 56 | help_text = kwargs.pop('help_text', 'show help for {}') 57 | help_argument = kwargs.pop('help_argument', False) 58 | grp = super().add_argument_group(*args, **kwargs) 59 | if help_name is not None: 60 | self._modular_action_groups[grp] = help_name 61 | if help_argument: 62 | parser = self 63 | if help_group is not None: 64 | parser = self._modular_help_groups[help_group] 65 | parser.add_argument('--help-{}'.format(help_name), 66 | action=_ModularHelpEnabler, 67 | nargs=0, 68 | default=argparse.SUPPRESS, 69 | const=grp, 70 | help=help_text.format(help_name)) 71 | return grp 72 | 73 | def enable_modular_help(self, grp): 74 | """Remove grp from ``self._modular_action_groups`` and thereby enable 75 | printing it.""" 76 | self._modular_active = True 77 | if grp is None: 78 | self._modular_action_groups = {} 79 | else: 80 | del self._modular_action_groups[grp] 81 | 82 | def parse_args(self, *args, **kwargs): 83 | res = super().parse_args(*args, **kwargs) 84 | if self._modular_active: 85 | self.parse_args(['--help']) 86 | return res 87 | 88 | def error(self, message): 89 | if not self._modular_active: 90 | super().error(message) 91 | 92 | def format_help(self): 93 | """Removes all items from ``self._action_groups`` that are still in 94 | ``self._modular_action_groups``.""" 95 | self._action_groups = [ 96 | ag for ag in self._action_groups 97 | if ag not in self._modular_action_groups 98 | ] 99 | return super().format_help() 100 | -------------------------------------------------------------------------------- /ddsmt/checker.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import collections 13 | import logging 14 | import math 15 | import resource 16 | import subprocess 17 | import sys 18 | import time 19 | 20 | from . import nodeio 21 | from . import options 22 | from . import tmpfiles 23 | 24 | RunInfo = collections.namedtuple("RunInfo", ["exit", "out", "err", "runtime"]) 25 | 26 | __GOLDEN = None 27 | __GOLDEN_CC = None 28 | 29 | 30 | def limit_resources(timeout, pid=None): 31 | """Apply resource limit given by ``--memout`` and timeout arguments.""" 32 | if pid: 33 | setlimit = lambda *args: resource.prlimit(pid, *args) # noqa: E731 34 | else: 35 | setlimit = lambda *args: resource.setrlimit(*args) # noqa: E731 36 | 37 | if options.args().memout: 38 | setlimit(resource.RLIMIT_AS, 39 | (options.args().memout * 1024 * 1024, resource.RLIM_INFINITY)) 40 | if timeout: 41 | timeout = math.ceil(timeout) 42 | setlimit(resource.RLIMIT_CPU, (timeout, timeout)) 43 | 44 | 45 | def execute(cmd, filename, timeout): 46 | """Execute the command on the file with a timeout and a memory limit.""" 47 | if options.args().unchecked: 48 | return RunInfo(0, "unchecked", "unchecked", 0) 49 | start = time.time() 50 | if hasattr(resource, 'prlimit'): 51 | proc = subprocess.Popen(cmd + [filename], 52 | stdout=subprocess.PIPE, 53 | stderr=subprocess.PIPE) 54 | limit_resources(timeout, proc.pid) 55 | else: 56 | proc = subprocess.Popen(cmd + [filename], 57 | stdout=subprocess.PIPE, 58 | stderr=subprocess.PIPE, 59 | preexec_fn=lambda: limit_resources(timeout)) 60 | try: 61 | out, err = proc.communicate(timeout=timeout) 62 | runtime = time.time() - start 63 | except subprocess.TimeoutExpired: 64 | proc.kill() 65 | logging.debug(f'[!!] timeout: terminated after {timeout:.2f} seconds') 66 | return RunInfo(proc.returncode, None, None, timeout) 67 | return RunInfo(proc.returncode, out.decode(), err.decode(), runtime) 68 | 69 | 70 | def matches_golden(golden, run, ignore_out, ignore_err, match_out, match_err): 71 | """Checks whether the ``run`` result matches the golden run, considering 72 | ``ignore_out``, ``ignore_err``, ``match_out`` and ``match_err``. 73 | 74 | If ``ignore_out`` and ``ignore_err`` are true, only the exit code is 75 | compared. Else, if ``ignore_out`` is true, only stderr is considered, and 76 | if ``ignore_err`` is true, only stdout is considere. If either 77 | ``match_out`` or ``match_err`` are given, they need to match. Otherwise, 78 | both stdout and stderr must be the same. 79 | """ 80 | if run.exit != golden.exit: 81 | return False 82 | 83 | if not ignore_out or not ignore_err: 84 | if not ignore_out: 85 | if match_out: 86 | if match_out not in run.out: 87 | return False 88 | else: 89 | if golden.out != run.out: 90 | return False 91 | if not ignore_err: 92 | if match_err: 93 | if match_err not in run.err: 94 | return False 95 | else: 96 | if golden.err != run.err: 97 | return False 98 | return True 99 | 100 | 101 | def check(filename): 102 | """Check whether the given file behaves as the original input. 103 | 104 | First execute the command and then call ``matches_golden``. If a 105 | cross-check command is specified, do the same for that one as well. 106 | """ 107 | ri = execute(options.args().cmd, filename, options.args().timeout) 108 | if not matches_golden( 109 | __GOLDEN, ri, 110 | options.args().ignore_output or options.args().ignore_out, 111 | options.args().ignore_output or options.args().ignore_err, 112 | options.args().match_out, 113 | options.args().match_err): 114 | return False 115 | 116 | if options.args().cmd_cc: 117 | ri = execute(options.args().cmd_cc, filename, 118 | options.args().timeout_cc) 119 | if not matches_golden(__GOLDEN_CC, ri, 120 | options.args().ignore_output_cc, 121 | options.args().ignore_output_cc, 122 | options.args().match_out_cc, 123 | options.args().match_err_cc): 124 | return False 125 | 126 | return True 127 | 128 | 129 | def check_exprs(exprs): 130 | """Run the check on the given expressions. 131 | 132 | Returns (True,runtime) if the check was successful and (False,0) 133 | otherwise. 134 | """ 135 | tmpfile = tmpfiles.get_tmp_filename() 136 | nodeio.write_smtlib_for_checking(tmpfile, exprs) 137 | return check(tmpfile) 138 | 139 | 140 | def do_golden_runs(): 141 | """Do the initial runs to obtain the golden run results.""" 142 | global __GOLDEN 143 | global __GOLDEN_CC 144 | 145 | logging.info('') 146 | if options.args().cmd_cc: 147 | logging.info('starting initial run, cross checking...') 148 | else: 149 | logging.info('starting initial run...') 150 | logging.info('') 151 | 152 | __GOLDEN = execute(options.args().cmd, 153 | options.args().infile, 154 | options.args().timeout) 155 | 156 | logging.info(f'golden exit: {__GOLDEN.exit}') 157 | logging.info(f'golden err:\n{__GOLDEN.err}') 158 | logging.info(f'golden out:\n{__GOLDEN.out}') 159 | logging.info(f'golden runtime: {__GOLDEN.runtime:.2f} seconds') 160 | if options.args().ignore_output or options.args().ignore_out: 161 | logging.info(f'ignoring stdout') 162 | if options.args().ignore_output or options.args().ignore_err: 163 | logging.info(f'ignoring stderr') 164 | if options.args().match_out: 165 | logging.info(f'match (stdout): "{options.args().match_out}"') 166 | if options.args().match_out not in __GOLDEN.out: 167 | logging.error( 168 | f'Expected stdout to match "{options.args().match_out}"') 169 | sys.exit(1) 170 | if options.args().match_err: 171 | logging.info(f'match (stderr): "{options.args().match_err}"') 172 | if options.args().match_err not in __GOLDEN.err: 173 | logging.error( 174 | f'Expected stderr to match "{options.args().match_err}"') 175 | sys.exit(1) 176 | 177 | if options.args().timeout is None: 178 | options.args().timeout = round((__GOLDEN.runtime + 1) * 1.5, 2) 179 | logging.info( 180 | f'automatic timeout: {options.args().timeout:.2f} seconds') 181 | 182 | if options.args().cmd_cc: 183 | __GOLDEN_CC = execute(options.args().cmd_cc, 184 | options.args().infile, 185 | options.args().timeout_cc) 186 | 187 | logging.info("") 188 | logging.info(f'golden exit (cc): {__GOLDEN_CC.exit}') 189 | logging.info(f'golden err (cc):\n{__GOLDEN_CC.err}') 190 | logging.info(f'golden out (cc):\n{__GOLDEN_CC.out}') 191 | logging.info(f'golden runtime (cc): {__GOLDEN_CC.runtime:.2f} s') 192 | if options.args().match_out_cc: 193 | logging.info( 194 | f'match (cc) (stdout): "{options.args().match_out_cc}"') 195 | if options.args().match_err_cc: 196 | logging.info( 197 | f'match (cc) (stderr): "{options.args().match_err_cc}"') 198 | 199 | if options.args().timeout_cc is None: 200 | options.args().timeout_cc = round((__GOLDEN_CC.runtime + 1) * 1.5, 201 | 2) 202 | logging.info( 203 | f'automatic timeout (cc): {options.args().timeout_cc:.2f} s') 204 | -------------------------------------------------------------------------------- /ddsmt/cli.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # 3 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 4 | # 5 | # This file is part of ddSMT. 6 | # 7 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 8 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 9 | # 10 | # This file is part of ddSMT under the MIT license. See LICENSE for more 11 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 12 | 13 | import logging 14 | import os 15 | import sys 16 | import time 17 | 18 | from . import checker 19 | from . import strategy_ddmin 20 | from . import strategy_hierarchical 21 | from . import mutators 22 | from . import nodeio 23 | from . import nodes 24 | from . import options 25 | from . import debug_utils 26 | from . import tmpfiles 27 | 28 | 29 | class DDSMTException(Exception): 30 | 31 | def __init__(self, msg): 32 | self.__msg = msg 33 | 34 | def __str__(self): 35 | return "[ddsmt] Error: {}".format(self.__msg) 36 | 37 | 38 | def check_options(): 39 | # check input file 40 | if not os.path.isfile(options.args().infile): 41 | raise DDSMTException('input file is not a regular file') 42 | 43 | if options.args().parser_test: 44 | # only parse and print 45 | exprs = list(nodeio.parse_smtlib(open(options.args().infile).read())) 46 | print(nodeio.write_smtlib(sys.stdout, exprs)) 47 | sys.exit(0) 48 | 49 | # check executable 50 | if not options.args().cmd: 51 | raise DDSMTException('No executable was specified as command') 52 | if not os.path.isfile(options.args().cmd[0]): 53 | raise DDSMTException('Command "{}" is not a regular file'.format( 54 | options.args().cmd[0])) 55 | if not os.access(options.args().cmd[0], os.X_OK): 56 | raise DDSMTException('Command "{}" is not executable'.format( 57 | options.args().cmd[0])) 58 | 59 | 60 | def setup_logging(): 61 | logging.CHAT = 25 62 | logging.addLevelName(logging.CHAT, "CHAT") 63 | logging.chat = lambda msg, *args, **kwargs: logging.log( 64 | logging.CHAT, msg, *args, **kwargs) 65 | logging.TRACE = 5 66 | logging.addLevelName(logging.TRACE, "TRACE") 67 | logging.trace = lambda msg, *args, **kwargs: logging.log( 68 | logging.TRACE, msg, *args, **kwargs) 69 | logging.basicConfig(format='[ddSMT %(levelname)s] %(message)s') 70 | verbositymap = { 71 | -2: logging.ERROR, 72 | -1: logging.WARN, 73 | 0: logging.CHAT, 74 | 1: logging.INFO, 75 | 2: logging.DEBUG, 76 | 3: logging.TRACE, 77 | } 78 | verbosity = options.args().verbosity - options.args().quietness 79 | logging.getLogger().setLevel( 80 | level=verbositymap.get(verbosity, logging.DEBUG)) 81 | 82 | 83 | def ddsmt_main(): 84 | start_time_process = time.process_time() 85 | # general setup 86 | setup_logging() 87 | check_options() 88 | tmpfiles.init() 89 | 90 | with debug_utils.Profiler(True): 91 | # show what we are going to do 92 | logging.info("input file: '{}'".format(options.args().infile)) 93 | logging.info("output file: '{}'".format(options.args().outfile)) 94 | logging.info("command: '{}'".format(" ".join( 95 | map(str, 96 | options.args().cmd)))) 97 | if options.args().cmd_cc: 98 | logging.info("command (cc): '{}'".format(" ".join( 99 | map(str, 100 | options.args().cmd_cc)))) 101 | if options.args().timeout: 102 | logging.info("time limit: {} seconds".format( 103 | options.args().timeout)) 104 | if options.args().timeout_cc: 105 | logging.info("time limit (cc): {} seconds".format( 106 | options.args().timeout_cc)) 107 | 108 | # parse the input 109 | start_time = time.time() 110 | with open(options.args().infile, 'r') as infile: 111 | exprs = list(nodeio.parse_smtlib(infile.read())) 112 | nexprs = nodes.count_exprs(exprs) 113 | 114 | logging.debug("parsed {} s-expressions in {:.2f} seconds".format( 115 | nexprs, 116 | time.time() - start_time)) 117 | 118 | if options.args().parser_test: 119 | nodes.write_smtlib_to_file(options.args().outfile, exprs) 120 | return 121 | 122 | # disable unused theories 123 | mutators.auto_detect_theories(exprs) 124 | # copy binaries to temp folder 125 | tmpfiles.copy_binaries() 126 | # perform golden runs to see what the solver is doing 127 | checker.do_golden_runs() 128 | 129 | orig_exprs = exprs 130 | # do the reduction 131 | if options.args().strategy in ('ddmin', 'hybrid'): 132 | exprs, ntests = strategy_ddmin.reduce(exprs) 133 | if options.args().strategy in ('hierarchical', 'hybrid'): 134 | exprs, ntests = strategy_hierarchical.reduce(exprs) 135 | end_time = time.time() 136 | 137 | # show the results 138 | if exprs != orig_exprs: 139 | ifilesize = os.path.getsize(options.args().infile) 140 | ofilesize = os.path.getsize(options.args().outfile) 141 | exprcount = nodes.count_exprs(exprs) 142 | proctime = time.process_time() - start_time_process 143 | sizeperc = ofilesize / ifilesize * 100 144 | if nexprs != 0: 145 | exprperc = exprcount / nexprs * 100 146 | else: 147 | exprperc = float('inf') 148 | 149 | logging.info('') 150 | logging.info(f'runtime: {end_time - start_time:.2f} s') 151 | logging.debug(f'main process: {proctime:.2f} s') 152 | logging.info(f'tests: {ntests}') 153 | logging.info('input file:') 154 | logging.info(f' file size: {ifilesize} B') 155 | logging.info(f' s-expressions: {nexprs}') 156 | logging.info('reduced file:') 157 | logging.info(f' file size: {ofilesize} B ({sizeperc:3.1f}%)') 158 | logging.info(f' s-expressions: {exprcount} ({exprperc:3.1f}%)') 159 | else: 160 | logging.warning('unable to minimize input file') 161 | -------------------------------------------------------------------------------- /ddsmt/debug_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import cProfile 13 | import difflib 14 | import glob 15 | import io 16 | import logging 17 | import multiprocessing 18 | import os 19 | import subprocess 20 | 21 | from . import nodeio 22 | from . import options 23 | 24 | 25 | class NodeLoopChecker: 26 | 27 | def __init__(self): 28 | self.__seen = set() 29 | 30 | def add(self, exprs): 31 | if options.args().check_loops: 32 | exprt = tuple(exprs) 33 | if exprt in self.__seen: 34 | logging.error( 35 | 'The following input has already been seen before:') 36 | logging.error(f'\n{nodeio.write_smtlib_to_str(exprs)}') 37 | assert False 38 | self.__seen.add(exprt) 39 | 40 | 41 | class Profiler: 42 | """A simple wrapper around cProfile that can be used as context manager, 43 | honors the ``--profile`` option and automatically writes the results to a 44 | file. To accumulate all profiles for one process, it uses a static profiler 45 | and uses a filename pattern that includes the process id. 46 | 47 | Profiling in the main process is somewhat special, as it is enabled 48 | globally and should thus not be enabled again by a worker function 49 | that happens to be executed in the main process. Thus, to enable 50 | profiling in the main process, the Profiler needs to be created with 51 | ``is_main=True`` to confirm that the caller is aware that it is 52 | running within the main process. Except for one call in the main 53 | function, this argument should never be set to True. 54 | """ 55 | 56 | enabled = options.args().profile 57 | # static profiler, global within one process 58 | profiler = None 59 | 60 | def __init__(self, is_main=False): 61 | """Create a profiler.""" 62 | if Profiler.enabled: 63 | if multiprocessing.parent_process() is None: 64 | self.enabled = True 65 | self.filename = '.profile.prof' 66 | else: 67 | self.enabled = True 68 | self.filename = f'.profile-{os.getpid()}.prof' 69 | if Profiler.profiler is None: 70 | Profiler.profiler = cProfile.Profile() 71 | 72 | def __enter__(self): 73 | """Start profiling, if ``--profile`` was given.""" 74 | if Profiler.enabled and self.enabled: 75 | Profiler.profiler.enable() 76 | 77 | def __exit__(self, type, value, traceback): 78 | """Stop profiling and write results to file.""" 79 | if Profiler.enabled and self.enabled: 80 | try: 81 | Profiler.profiler.disable() 82 | Profiler.profiler.dump_stats(self.filename) 83 | except KeyboardInterrupt as e: 84 | logging.warn(f'Writing {self.filename} was interrupted.' 85 | 'To avoid corrupted data, we remove this file.') 86 | os.unlink(self.filename) 87 | raise e 88 | 89 | 90 | def __render_profile(infiles, dotfile, pngfile): 91 | """Helper method to render profile data from a set of input files to a 92 | png.""" 93 | cmd = [ 94 | 'gprof2dot', '--node-label=self-time-percentage', 95 | '--node-label=total-time', '--node-label=total-time-percentage', '-f', 96 | 'pstats', '-o', dotfile, *infiles 97 | ] 98 | try: 99 | subprocess.run(cmd) 100 | except FileNotFoundError: 101 | logging.warn('gprof2dot was not found. Try "pip install gprof2dot".') 102 | return False 103 | 104 | try: 105 | subprocess.run(['dot', '-Tpng', '-o', pngfile, dotfile]) 106 | except FileNotFoundError: 107 | logging.warn('dot was not found. Try "apt install graphviz".') 108 | return False 109 | return True 110 | 111 | 112 | def render_profiles(): 113 | """Convenience function to directly render profiling data to png. 114 | 115 | Splits the data into two parts, the main process and the combination 116 | of all other processes. Uses ``gprof2dot`` and ``dot``, and will try 117 | to print helpful error messages if they are not available. 118 | """ 119 | logging.debug('Rendering profile data') 120 | if __render_profile(['.profile.prof'], '.profile.dot', 'profile.png'): 121 | logging.chat('Profile data for main process is in profile.png') 122 | 123 | files = [f for f in glob.glob('.*.prof') if f != '.profile.prof'] 124 | if files: 125 | if __render_profile(files, '.profile-workder.dot', 126 | 'profile-worker.png'): 127 | logging.chat( 128 | 'Profile data for worker processes is in profile-worker.png') 129 | 130 | 131 | __DIFF_ID = 0 132 | 133 | 134 | def __render_for_diff(exprs): 135 | f = io.StringIO() 136 | for expr in exprs: 137 | nodeio.__write_smtlib_pretty(f, expr) 138 | return f.getvalue() 139 | 140 | 141 | def dump_diff(description, before, after): 142 | if not options.args().dump_diffs: 143 | return 144 | global __DIFF_ID 145 | b = __render_for_diff(before).splitlines(keepends=True) 146 | a = __render_for_diff(after).splitlines(keepends=True) 147 | 148 | diff = difflib.unified_diff(b, a) 149 | 150 | __DIFF_ID += 1 151 | with open(f'.simp-{__DIFF_ID}.diff', 'w') as out: 152 | out.write(f'{description}\n') 153 | out.writelines(diff) 154 | -------------------------------------------------------------------------------- /ddsmt/mutator_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import collections 13 | from . import smtlib 14 | from . import nodes 15 | 16 | Simplification = collections.namedtuple('Simplification', 17 | ['substs', 'fresh_vars']) 18 | 19 | 20 | def apply_simp(exprs, simp): 21 | assert isinstance(simp, Simplification) 22 | mexprs = nodes.substitute(exprs, simp.substs) 23 | if mexprs is not exprs and simp.fresh_vars: 24 | assert isinstance(mexprs, list) 25 | mexprs = smtlib.introduce_variables(mexprs, simp.fresh_vars) 26 | return mexprs 27 | -------------------------------------------------------------------------------- /ddsmt/mutators.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import argparse 13 | import logging 14 | 15 | from . import mutators_arithmetic 16 | from . import mutators_bv 17 | from . import mutators_boolean 18 | from . import mutators_core 19 | from . import mutators_datatypes 20 | from . import mutators_fp 21 | from . import mutators_smtlib 22 | from . import mutators_strings 23 | from . import nodes 24 | from . import options 25 | 26 | 27 | def get_all_mutators(): 28 | """Return all available mutators, arranged by their theory.""" 29 | return { 30 | 'core': (mutators_core, mutators_core.get_mutators()), 31 | 'arithmetic': 32 | (mutators_arithmetic, mutators_arithmetic.get_mutators()), 33 | 'bv': (mutators_bv, mutators_bv.get_mutators()), 34 | 'boolean': (mutators_boolean, mutators_boolean.get_mutators()), 35 | 'datatypes': (mutators_datatypes, mutators_datatypes.get_mutators()), 36 | 'fp': (mutators_fp, mutators_fp.get_mutators()), 37 | 'smtlib': (mutators_smtlib, mutators_smtlib.get_mutators()), 38 | 'strings': (mutators_strings, mutators_strings.get_mutators()), 39 | } 40 | 41 | 42 | def get_mutators(mutators): 43 | """Return mutator instances from a list of names. 44 | 45 | For each mutator class name in ``mutators`` retrieves the proper 46 | theory, checks whether the mutator is enabled and then adds an 47 | instance of this class to the result. 48 | """ 49 | res = [] 50 | for m in mutators: 51 | for _, theory in get_all_mutators().items(): 52 | # check if this mutator belongs to this theory 53 | if m in theory[1]: 54 | attr = f'mutator_{theory[1][m].replace("-", "_")}' 55 | # check if this mutator is enabled 56 | if getattr(options.args(), attr, True): 57 | res.append(getattr(theory[0], m)()) 58 | break 59 | return res 60 | 61 | 62 | def get_initialized_mutator(name, properties): 63 | """Obtain a mutator by name and set the given properties.""" 64 | res = get_mutators([name]) 65 | if res: 66 | for p, v in properties.items(): 67 | setattr(res[0], p, v) 68 | return res 69 | 70 | 71 | def toggle_theory(namespace, theory_name, value): 72 | """Enables or disables all mutators for the given theory by setting their 73 | respective options in ``namespace``.""" 74 | setattr(namespace, f'mutators_{theory_name}', value) 75 | _, mutators = get_all_mutators()[theory_name] 76 | for _, opt in mutators.items(): 77 | setattr(namespace, f'mutator_{opt.replace("-", "_")}', value) 78 | 79 | 80 | def toggle_all_theories(namespace, value): 81 | """Enables or disables all mutators for all theories by setting their 82 | respective options in ``namespace``.""" 83 | for theory_name, data in get_all_mutators().items(): 84 | setattr(namespace, f'mutators_{theory_name}', value) 85 | for _, opt in data[1].items(): 86 | setattr(namespace, f'mutator_{opt.replace("-", "_")}', value) 87 | 88 | 89 | class TheoryToggleAction(options.ToggleAction): 90 | """A specialization of ``ToggleAction`` that disables (or enables) all 91 | mutators of a given theory.""" 92 | 93 | def __init__(self, theory, *args, **kwargs): 94 | """Expects an additional option with the name of the theory.""" 95 | self.__theory = theory 96 | super(TheoryToggleAction, self).__init__(*args, **kwargs) 97 | 98 | def __call__(self, parser, namespace, values, option_string=None): 99 | """Set the option, as well as all mutator options for this theory.""" 100 | value = self._get_value(option_string) 101 | setattr(namespace, self.dest, value) 102 | toggle_theory(namespace, self.__theory, value) 103 | 104 | 105 | class DisableAllTheoriesAction(argparse.Action): 106 | """Disables all mutators from all theories when called.""" 107 | 108 | def __call__(self, parser, namespace, values, option_string=None): 109 | setattr(namespace, self.dest, True) 110 | toggle_all_theories(namespace, False) 111 | 112 | 113 | def add_mutator_group(argparser, name): 114 | """Add a new argument group for a mutator group, including options to 115 | enable and disable the whole theory.""" 116 | res = argparser.add_argument_group('{} mutator arguments'.format(name), 117 | help_name=name, 118 | help_group='mutator help', 119 | help_text='show help for {} mutators') 120 | 121 | res._add_action( 122 | TheoryToggleAction(name, 123 | name, 124 | default=None, 125 | dest=f'mutators_{name}', 126 | help=f'{name} theory')) 127 | return res 128 | 129 | 130 | def collect_mutator_options(argparser): 131 | """Adds all options related to mutators to the given argument parser.""" 132 | argparser.add_argument('--disable-all', 133 | action=DisableAllTheoriesAction, 134 | nargs=0, 135 | help='disable all mutators') 136 | for name, tdata in get_all_mutators().items(): 137 | theory = tdata[0] 138 | # add argument group for this theory 139 | ap = add_mutator_group(argparser, name) 140 | for mname, mopt in theory.get_mutators().items(): 141 | # add options for every individual mutator 142 | mdesc = str(getattr(theory, mname)()) 143 | ap._add_action( 144 | options.ToggleAction(mopt, 145 | dest=f'mutator_{mopt.replace("-", "_")}', 146 | help=mdesc)) 147 | # add custom option, if the theory wants it 148 | if hasattr(theory, 'get_mutator_options'): 149 | theory.get_mutator_options(ap) 150 | 151 | 152 | def auto_detect_theories(exprs): 153 | for name, tdata in get_all_mutators().items(): 154 | # if a theory was explicitly enabled or disabled by the user, we don't 155 | # change it. 156 | optval = getattr(options.args(), f'mutators_{name}') 157 | if optval is not None: 158 | logging.debug(f'{name} was specified by the user: {optval}') 159 | continue 160 | 161 | theory = tdata[0] 162 | # if a theory does not implement is_relevant(), it stays enabled 163 | if not hasattr(theory, 'is_relevant'): 164 | logging.debug(f'{name} can not be disabled') 165 | continue 166 | 167 | # now check whether theory is relevant for any of the nodes 168 | enabled = False 169 | for node in nodes.dfs(exprs, max_depth=1): 170 | if theory.is_relevant(node): 171 | enabled = True 172 | break 173 | 174 | if not enabled: 175 | logging.warn(f'automatically disabling {name} mutators. ' 176 | f'Use --{name} to enable.') 177 | toggle_theory(options.args(), name, False) 178 | else: 179 | logging.debug(f'keeping {name} mutators enabled') 180 | -------------------------------------------------------------------------------- /ddsmt/mutators_arithmetic.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | from . import nodes 13 | from .smtlib import * 14 | from .mutator_utils import Simplification 15 | 16 | 17 | def is_arithmetic_relation(node): 18 | if not node.has_ident(): 19 | return False 20 | return node.get_ident() in [ 21 | '=', '<', '>', '>=', '<=', '!=', '<>', 'distinct' 22 | ] 23 | 24 | 25 | class ArithmeticNegateRelation: 26 | """Replace a negated relation by the corresponding inverse relation. 27 | 28 | For example, ``(not (< a b))`` is replaced with ``(>= a b)``. 29 | """ 30 | 31 | def filter(self, node): 32 | return is_operator_app( 33 | node, 'not') and len(node) > 1 and is_arithmetic_relation(node[1]) 34 | 35 | def mutations(self, node): 36 | negator = { 37 | '=': 'distinct', 38 | '<': '>=', 39 | '>': '<=', 40 | '>=': '<', 41 | '<=': '>', 42 | '!=': '=', 43 | '<>': '=', 44 | 'distinct': '=' 45 | } 46 | assert node[1][0] in negator 47 | return [ 48 | Simplification({node.id: Node(negator[node[1][0]], *node[1][1:])}, 49 | []) 50 | ] 51 | 52 | def __str__(self): 53 | return 'push negation into arithmetic relation' 54 | 55 | 56 | class ArithmeticSimplifyConstant: 57 | """Replace a constant with a simpler constant. 58 | 59 | We consider constants that are smaller (in value) or with fewer decimal 60 | places as `simpler`. 61 | """ 62 | 63 | def filter(self, node): 64 | return is_arith_const(node) and get_arith_const(node) not in [0, 1] 65 | 66 | def mutations(self, node): 67 | f = get_arith_const(node) 68 | if int(f) == f: 69 | i = int(f) 70 | yield from [ 71 | Simplification({node.id: Node(str(i // 2))}, []), 72 | Simplification({node.id: Node(str(i // 10))}, []) 73 | ] 74 | return 75 | yield from [ 76 | Simplification({node.id: Node(str(int(f)))}, []), 77 | Simplification({node.id: Node(node.data[:-1])}, []) 78 | ] 79 | 80 | def global_mutations(self, node, input_): 81 | yield from [ 82 | Simplification({node: simp.substs[node.id]}, []) 83 | for simp in self.mutations(node) 84 | ] 85 | 86 | def __str__(self): 87 | return 'simplify arithmetic constant' 88 | 89 | 90 | class ArithmeticSplitNaryRelation: 91 | """Split an n-ary relation into the conjunction of multiple relations. 92 | 93 | This assumes that the relation symbol is transitive. 94 | """ 95 | 96 | def filter(self, node): 97 | return is_arithmetic_relation(node) and len(node) > 3 98 | 99 | def mutations(self, node): 100 | split = [(node.get_ident(), node[i], node[i + 1]) 101 | for i in range(1, 102 | len(node) - 1)] 103 | return [Simplification({node.id: Node('and', *split)}, [])] 104 | 105 | def __str__(self): 106 | return 'split arithmetic n-ary relation' 107 | 108 | 109 | class ArithmeticStrengthenRelation: 110 | """Replace a relation by a stronger relation. 111 | 112 | For example, ``>=`` is replaced by ``>``. 113 | """ 114 | 115 | def filter(self, node): 116 | return is_arithmetic_relation(node) and node.get_ident() in [ 117 | '<', '>', '<=', '>=', 'distinct' 118 | ] 119 | 120 | def mutations(self, node): 121 | strengthen = { 122 | '<': ['='], 123 | '>': ['='], 124 | '<=': ['<', '='], 125 | '>=': ['>', '='], 126 | 'distinct': ['='] 127 | } 128 | assert node[0] in strengthen 129 | yield from [ 130 | Simplification({node.id: Node(rel, *node.data[1:])}, []) 131 | for rel in strengthen[node[0]] 132 | ] 133 | 134 | def __str__(self): 135 | return 'strengthen arithmetic relation' 136 | 137 | 138 | def get_mutators(): 139 | """Returns a mapping from mutator class names to the name of their config 140 | options.""" 141 | return { 142 | 'ArithmeticNegateRelation': 'arith-negate-relations', 143 | 'ArithmeticSimplifyConstant': 'arith-constants', 144 | 'ArithmeticSplitNaryRelation': 'arith-split-nary-relations', 145 | 'ArithmeticStrengthenRelation': 'arith-strengthen-relations', 146 | } 147 | 148 | 149 | def is_relevant(node): 150 | """Checks whether this theory might be relevant for this node.""" 151 | if node.has_ident(): 152 | if node.get_ident() in ['declare-const']: 153 | if nodes.contains(node[2], lambda t: t in ['Int', 'Real']): 154 | return True 155 | elif node.get_ident() in ['declare-fun', 'define-fun', 'define-sort']: 156 | if nodes.contains(node[3], lambda t: t in ['Int', 'Real']): 157 | return True 158 | return False 159 | -------------------------------------------------------------------------------- /ddsmt/mutators_boolean.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | from .smtlib import * 13 | from .mutator_utils import Simplification 14 | 15 | 16 | def is_quantifier(node): 17 | return node.has_ident() and node.get_ident() in ['exists', 'forall'] 18 | 19 | 20 | def make_and(children): 21 | """Return an and from arbitrary many children. 22 | 23 | Returns node as a list so that we can return the empty list if there 24 | are no children. 25 | """ 26 | if len(children) == 0: 27 | return None 28 | if len(children) == 1: 29 | return children[0] 30 | return Node('and', *children) 31 | 32 | 33 | class BoolDeMorgan: 34 | """Use de Morgans rules to push negations inside.""" 35 | 36 | def filter(self, node): 37 | return is_operator_app(node, 'not') and len(node) > 1 and ( 38 | is_operator_app(node[1], 'and') or is_operator_app(node[1], 'or')) 39 | 40 | def mutations(self, node): 41 | negated = [Node('not', t) for t in node[1][1:]] 42 | if node[1].get_ident() == 'and': 43 | return [Simplification({node.id: Node('or', *negated)}, [])] 44 | assert node[1].get_ident() == 'or' 45 | return [Simplification({node.id: Node('and', *negated)}, [])] 46 | 47 | def __str__(self): 48 | return 'push negation inside' 49 | 50 | 51 | class BoolDoubleNegation: 52 | """Eliminate double negations.""" 53 | 54 | def filter(self, node): 55 | return is_operator_app( 56 | node, 'not') and len(node) > 1 and is_operator_app(node[1], 'not') 57 | 58 | def mutations(self, node): 59 | return [Simplification({node.id: node[1][1]}, [])] 60 | 61 | def __str__(self): 62 | return 'eliminate double negation' 63 | 64 | 65 | class BoolEliminateFalseEquality: 66 | """Replace an equality with ``false`` by a negation. 67 | 68 | For example, transform ``(= false X)`` into ``(not X)``. 69 | """ 70 | 71 | def filter(self, node): 72 | return is_eq(node) and Node('false') in node 73 | 74 | def mutations(self, node): 75 | simp = make_and([Node('not', n) for n in node[1:] if n != 'false']) 76 | return [] if simp is None else [Simplification({node.id: simp}, [])] 77 | 78 | def __str__(self): 79 | return 'replace equality with false by negation' 80 | 81 | 82 | class BoolEliminateImplication: 83 | """Replace (possibly n-ary) implications by disjunctions.""" 84 | 85 | def filter(self, node): 86 | return is_operator_app(node, '=>') 87 | 88 | def mutations(self, node): 89 | simp = make_and([ 90 | Node('or', ('not', node[i]), node[i + 1]) 91 | for i in range(1, 92 | len(node) - 1) 93 | ]) 94 | return [] if simp is None else [Simplification({node.id: simp}, [])] 95 | 96 | def __str__(self): 97 | return 'eliminate implication' 98 | 99 | 100 | class BoolNegateQuantifier: 101 | """Push negations inside quantifiers.""" 102 | 103 | def filter(self, node): 104 | return is_operator_app( 105 | node, 'not') and len(node) > 1 and is_quantifier(node[1]) 106 | 107 | def mutations(self, node): 108 | if node[1].get_ident() == 'exists': 109 | return [ 110 | Simplification( 111 | {node.id: Node('forall', node[1][1], ('not', node[1][2]))}, 112 | []) 113 | ] 114 | assert node[1].get_ident() == 'forall' 115 | return [ 116 | Simplification( 117 | {node.id: Node('exists', node[1][1], ('not', node[1][2]))}, []) 118 | ] 119 | 120 | def __str__(self): 121 | return 'push negation inside of quantifier' 122 | 123 | 124 | class BoolXOREliminateBinary: 125 | """Eliminate binary ``xor`` by ``distinct``.""" 126 | 127 | def filter(self, node): 128 | return is_operator_app(node, 'xor') and len(node) == 3 129 | 130 | def mutations(self, node): 131 | return [ 132 | Simplification({node.id: Node('distinct', node[1], node[2])}, []) 133 | ] 134 | 135 | def __str__(self): 136 | return 'eliminate binary xor' 137 | 138 | 139 | class BoolXORRemoveConstant: 140 | """Eliminate constant children from ``xor``.""" 141 | 142 | def filter(self, node): 143 | return is_operator_app(node, 'xor') 144 | 145 | def mutations(self, node): 146 | if 'false' in node: 147 | yield Simplification( 148 | {node.id: Node(*[c for c in node if c != 'false'])}, []) 149 | if 'true' in node: 150 | yield Simplification( 151 | {node.id: Node(*[c for c in node if c != 'true'])}, []) 152 | yield Simplification( 153 | {node.id: Node('not', tuple(c for c in node if c != 'true'))}, 154 | []) 155 | 156 | def __str__(self): 157 | return 'remove constants from xor' 158 | 159 | 160 | def get_mutators(): 161 | """Returns a mapping from mutator class names to the name of their config 162 | options.""" 163 | return { 164 | 'BoolDeMorgan': 'bool-de-morgan', 165 | 'BoolDoubleNegation': 'bool-double-negations', 166 | 'BoolEliminateFalseEquality': 'bool-false-eq', 167 | 'BoolEliminateImplication': 'bool-implications', 168 | 'BoolNegateQuantifier': 'bool-negate-quants', 169 | 'BoolXOREliminateBinary': 'bool-xor-binary', 170 | 'BoolXORRemoveConstant': 'bool-xor-constants', 171 | } 172 | -------------------------------------------------------------------------------- /ddsmt/mutators_core.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | from .nodes import Node, count_nodes 13 | from .smtlib import * 14 | from . import options 15 | from .mutator_utils import Simplification 16 | 17 | 18 | class BinaryReduction: 19 | """Binary reduction on the top level node. 20 | 21 | This mutator mimics what strategy ``ddmin`` achieves when applying 22 | mutator :class:`ddsmt.mutators_core.EraseNode`. 23 | 24 | If ``self.ident`` is set, only nodes with the identifier specified as 25 | ``self.ident`` are considered. 26 | 27 | .. note:: 28 | This mutator is always disabled for strategy ``ddmin``. 29 | """ 30 | 31 | def mutations(self, node): 32 | if node.is_leaf() or len(node) < 8: 33 | return 34 | for sec in nodes.binary_search(len(node)): 35 | yield Simplification({node.id: node[:sec[0]] + node[sec[1]:]}, []) 36 | 37 | def global_mutations(self, node, input_): 38 | if node != input_[0]: 39 | return [] 40 | # generate all sublists as generated by binary-search in bfs order 41 | # let den be the denominator of the list length (the tree level) 42 | # let num be the numerator within the current tree level 43 | if hasattr(self, 'ident'): 44 | ids = [ 45 | node.id for node in input_ 46 | if is_operator_app(node, self.ident) 47 | ] 48 | else: 49 | ids = [node.id for node in input_] 50 | for sec in nodes.binary_search(len(ids)): 51 | yield Simplification( 52 | {node_id: None 53 | for node_id in ids[sec[0]:sec[1]]}, []) 54 | 55 | def __str__(self): 56 | if hasattr(self, 'ident'): 57 | return f'binary reduction ({self.ident})' 58 | return 'binary reduction' 59 | 60 | 61 | class Constants: 62 | """Replace given node with a default value of the same sort. 63 | 64 | Default values are defined in :func:`ddsmt.smtlib.get_default_constants`. 65 | """ 66 | 67 | # Requires that global information has been populated via 68 | # ``collect_information``. 69 | def filter(self, node): 70 | return not is_definition_node(node) and get_sort(node) is not None 71 | 72 | def mutations(self, node): 73 | t = get_sort(node) 74 | if t is None: 75 | return [] 76 | res = get_default_constants(t) 77 | if node in res: 78 | return [] 79 | for c in res: 80 | yield Simplification({node.id: c}, []) 81 | 82 | def __str__(self): 83 | return 'substitute by a constant' 84 | 85 | 86 | class EraseNode: 87 | """Erase given node. 88 | 89 | If ``self.ident`` is set, only nodes with the identifier specified as 90 | ``self.ident`` are considered. 91 | """ 92 | 93 | def filter(self, node): 94 | if not hasattr(self, 'ident'): 95 | return True 96 | return node.has_ident() and node.get_ident() == self.ident 97 | 98 | def mutations(self, node): 99 | return [Simplification({node.id: None}, [])] 100 | 101 | def __str__(self): 102 | if hasattr(self, 'ident'): 103 | return f'erase node ({self.ident})' 104 | return 'erase node' 105 | 106 | 107 | class MergeWithChildren: 108 | """Merge given node with one of its children. 109 | 110 | This mutator can only be applied to n-ary operations like ``and`` or ``+``. 111 | """ 112 | 113 | def filter(self, node): 114 | return has_nary_operator(node) 115 | 116 | def mutations(self, node): 117 | for cid, child in enumerate(node): 118 | if child.has_ident() and node.get_ident() == child.get_ident(): 119 | simp = Node(*node[:cid], *node[cid][1:], *node[cid + 1:]) 120 | yield Simplification({node.id: simp}, []) 121 | 122 | def __str__(self): 123 | return 'merge with child' 124 | 125 | 126 | class ReplaceByChild: 127 | """Replace given node with one of its children.""" 128 | 129 | def filter(self, node): 130 | return not is_leaf(node) and not is_operator_app(node, 'let') 131 | 132 | def mutations(self, node): 133 | sort = get_sort(node) 134 | for n in node[1:]: 135 | if get_sort(n) == sort: 136 | yield Simplification({node.id: n}, []) 137 | 138 | def __str__(self): 139 | return 'replace by a child' 140 | 141 | 142 | class ReplaceByVariable: 143 | """Replace given node with an existing variable of the same type. 144 | 145 | .. note:: 146 | Replacing variables with other variables **potentially introduces 147 | cycles**. 148 | To avoid this, for leaf nodes we only allow substitutions with 149 | variables that are lexicographically **smaller** (default) 150 | or **larger** than the leaf mode. 151 | Configure with option ``--replace-by-variable-mode`` (use ``inc`` for 152 | substitution with smaller, and ``dec`` for substition with larger 153 | variables). 154 | """ 155 | 156 | # Requires that global information has been populated via 157 | # ``collect_information``. 158 | def filter(self, node): 159 | return not is_const(node) and not is_definition_node( 160 | node) and get_sort(node) is not None 161 | 162 | def mutations(self, node): 163 | if not hasattr(self, 'repl_mode'): 164 | self.repl_mode = options.args().replace_by_variable_mode 165 | 166 | ret_sort = get_sort(node) 167 | if ret_sort is None: 168 | return [] 169 | variables = get_variables_with_sort(ret_sort) 170 | # Replacing by a defined variable may loop with inlining 171 | variables = filter(lambda v: not is_defined_fun(v), variables) 172 | if is_leaf(node): 173 | if self.repl_mode == 'inc': 174 | variables = [v for v in variables if v > node.data] 175 | else: 176 | variables = [v for v in variables if v < node.data] 177 | yield from [Simplification({node.id: Node(v)}, []) for v in variables] 178 | 179 | def __str__(self): 180 | if hasattr(self, 'repl_mode'): 181 | return f'substitute by existing variable ({self.repl_mode})' 182 | return 'substitute by existing variable' 183 | 184 | 185 | class SortChildren: 186 | """Sort the children of a given node by size (count of sub-nodes).""" 187 | 188 | def filter(self, node): 189 | return not is_leaf(node) 190 | 191 | def mutations(self, node): 192 | s = nodes.Node(*sorted(node, key=count_nodes)) 193 | if s != node: 194 | return [Simplification({node.id: s}, [])] 195 | return [] 196 | 197 | def __str__(self): 198 | return 'sort children' 199 | 200 | 201 | def get_mutator_options(argparser): 202 | """Add additional options to the argument parser.""" 203 | argparser.add_argument( 204 | '--replace-by-variable-mode', 205 | choices=['inc', 'dec'], 206 | default='inc', 207 | help='replace with existing variables that are larger or smaller') 208 | 209 | 210 | def get_mutators(): 211 | """Return a mapping from mutator class names to the name of their config 212 | options.""" 213 | return { 214 | 'BinaryReduction': 'binary-reduction', 215 | 'Constants': 'constants', 216 | 'EraseNode': 'erase-node', 217 | 'MergeWithChildren': 'merge-children', 218 | 'ReplaceByChild': 'substitute-children', 219 | 'ReplaceByVariable': 'replace-by-variable', 220 | 'SortChildren': 'sort-children', 221 | } 222 | -------------------------------------------------------------------------------- /ddsmt/mutators_datatypes.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | from .smtlib import * 13 | from .mutator_utils import Simplification 14 | 15 | 16 | class RemoveConstructor: 17 | """Remove a single datatype constructor.""" 18 | 19 | def filter(self, node): 20 | return is_operator_app(node, 'declare-datatype') or is_operator_app( 21 | node, 'declare-datatypes') 22 | 23 | def mutations(self, node): 24 | if is_operator_app(node, 'declare-datatype'): 25 | if len(node) != 3: 26 | return 27 | for cons in node[2]: 28 | yield Simplification({cons.id: None}, []) 29 | elif is_operator_app(node, 'declare-datatypes'): 30 | if len(node) != 3: 31 | return 32 | for type in node[2]: 33 | for cons in type: 34 | yield Simplification({cons.id: None}, []) 35 | 36 | def __str__(self): 37 | return 'remove datatype constructor' 38 | 39 | 40 | class RemoveDatatype: 41 | """Remove a datatype from a recursive datatype declaration.""" 42 | 43 | def filter(self, node): 44 | return is_operator_app(node, 'declare-datatypes') 45 | 46 | def mutations(self, node): 47 | if len(node) != 3 or len(node[1]) != len(node[2]): 48 | return 49 | for i in range(len(node[1])): 50 | yield Simplification({ 51 | node[1][i].id: None, 52 | node[2][i].id: None 53 | }, []) 54 | 55 | def __str__(self): 56 | return 'remove datatype' 57 | 58 | 59 | class RemoveDatatypeIdentity: 60 | """Remove a datatype constructor and selector that are nested to function 61 | as the identify. For ``(declare-datatype (A 0) ((C (s A))))`` we have the 62 | datatype ``A`` with a constructor ``C`` that has the selector ``s``. 63 | Nesting ``C`` and ``s`` like ``(s (C x))`` returns ``x`` unchanged and we 64 | can thus usually replace this expression by ``x``. 65 | """ 66 | 67 | def filter(self, node): 68 | return len(node) == 2 and is_dt_selector(node) and is_dt_constructor( 69 | node[1]) and get_dt_selector(node)[0] == node[1].get_ident() 70 | 71 | def mutations(self, node): 72 | selid = get_dt_selector(node)[1] 73 | if len(node[1]) <= selid + 1: 74 | return 75 | yield Simplification({node.id: node[1][selid + 1]}, []) 76 | 77 | def __str__(self): 78 | return 'remove datatype identity' 79 | 80 | 81 | def get_mutators(): 82 | """Returns a mapping from mutator class names to the name of their config 83 | options.""" 84 | return { 85 | 'RemoveConstructor': 'dt-rm-constructor', 86 | 'RemoveDatatype': 'dt-rm-datatype', 87 | 'RemoveDatatypeIdentity': 'dt-rm-identity', 88 | } 89 | 90 | 91 | def is_relevant(node): 92 | """Checks whether this theory might be relevant for this node.""" 93 | if node.has_ident(): 94 | return node.get_ident() in ['declare-datatypes', 'declare-datatype'] 95 | return False 96 | -------------------------------------------------------------------------------- /ddsmt/mutators_fp.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | from . import nodes 13 | from .nodes import Node 14 | from .smtlib import * 15 | from .mutator_utils import Simplification 16 | 17 | 18 | class FPShortSort: 19 | """Replace long sort names with short names. 20 | 21 | .. code-block:: common-lisp 22 | 23 | (_ FloatingPoint 5 11) --> Float16 24 | (_ FloatingPoint 8 24) --> Float32 25 | (_ FloatingPoint 11 53) --> Float64 26 | (_ FloatingPoint 15 113) --> Float128 27 | """ 28 | 29 | def filter(self, node): 30 | return is_fp_sort(node) and len(node) == 4 31 | 32 | def mutations(self, node): 33 | if node[2] == '5' and node[3] == '11': 34 | return [Simplification({node.id: Node('Float16')}, [])] 35 | if node[2] == '8' and node[3] == '24': 36 | return [Simplification({node.id: Node('Float32')}, [])] 37 | if node[2] == '11' and node[3] == '53': 38 | return [Simplification({node.id: Node('Float64')}, [])] 39 | if node[2] == '15' and node[3] == '113': 40 | return [Simplification({node.id: Node('Float128')}, [])] 41 | return [] 42 | 43 | def __str__(self): 44 | return 'replace long FP sort name with short name' 45 | 46 | 47 | def get_mutators(): 48 | """Return mapping from mutator class names to the name of their config 49 | options.""" 50 | return { 51 | 'FPShortSort': 'fp-short-sort', 52 | } 53 | 54 | 55 | def is_relevant(node): 56 | """Checks whether this theory might be relevant for this node.""" 57 | if node.has_ident(): 58 | if node.get_ident() in ['declare-const']: 59 | if nodes.contains(node[2], is_fp_sort) or nodes.contains( 60 | node[2], is_rm_sort): 61 | return True 62 | elif node.get_ident() in ['declare-fun', 'define-fun', 'define-sort']: 63 | if nodes.contains(node[3], is_fp_sort) or nodes.contains( 64 | node[3], is_rm_sort): 65 | return True 66 | return False 67 | -------------------------------------------------------------------------------- /ddsmt/mutators_smtlib.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import re 13 | 14 | from .smtlib import * 15 | from .mutator_utils import Simplification 16 | 17 | 18 | class CheckSatAssuming: 19 | """Replace ``check-sat-assuming`` with a regular ``check-sat``. 20 | 21 | This discards any assumptions provided to ``check-sat-assuming``. 22 | """ 23 | 24 | def filter(self, node): 25 | return node.has_ident() and node.get_ident() == 'check-sat-assuming' 26 | 27 | def mutations(self, node): 28 | return [Simplification({node.id: Node(Node('check-sat'))}, [])] 29 | 30 | def __str__(self): 31 | return 'substitute check-sat-assuming by check-sat' 32 | 33 | 34 | class EliminateVariable: 35 | """Eliminate variables that occur as children of equalities. 36 | 37 | Globally replace every leaf node from an equality with every 38 | other node from the same equality. For example, for 39 | ``(= x (* a b) (+ c d))``, ``x`` is replaced with either 40 | ``(* a b)`` or ``(+ c d)``. 41 | """ 42 | 43 | def filter(self, node): 44 | return is_eq(node) and len(node) > 1 and any( 45 | map(lambda n: n.is_leaf(), node[1:])) 46 | 47 | def global_mutations(self, node, input_): 48 | ops = node[1:] 49 | targets = list(filter(lambda n: n.is_leaf(), ops)) 50 | for t in targets: 51 | for c in ops: 52 | if c == t: 53 | continue 54 | if t in nodes.dfs(c): 55 | # Avoid cycles (for example with core.ReplaceByChild) 56 | continue 57 | substs = {} 58 | for n in nodes.dfs(input_): 59 | if n == t and not is_definition_node(n): 60 | substs[n.id] = c 61 | if substs: 62 | yield Simplification(substs, []) 63 | 64 | def __str__(self): 65 | return 'eliminate variable from equality' 66 | 67 | 68 | class InlineDefinedFuns: 69 | """Explicitly inline a defined function.""" 70 | 71 | def filter(self, node): 72 | return is_defined_fun(node) 73 | 74 | def mutations(self, node): 75 | if node.id in map(lambda n: n.id, nodes.dfs(get_defined_fun(node))): 76 | # we are about to inline the function into its own body 77 | return [] 78 | if is_definition_node(node): 79 | # we are about to inline the function into its own name 80 | return [] 81 | res = get_defined_fun(node) 82 | if res == node: 83 | return [] 84 | return [Simplification({node.id: res}, [])] 85 | 86 | def __str__(self): 87 | return 'inline defined function' 88 | 89 | 90 | class IntroduceFreshVariable: 91 | """Replace a term by a fresh variable of the same sort.""" 92 | 93 | def filter(self, node): 94 | # introducing a fresh variable may lead to cycles in various cases, 95 | # thus we exclude a variety of nodes here: 96 | # - constants, leaf nodes 97 | # - BV terms with a single variable and smaller bit-width. 98 | if is_const(node) or node.is_leaf() or is_definition_node(node): 99 | return False 100 | sort = get_sort(node) 101 | if sort is None: 102 | return False 103 | if is_bv_sort(sort): 104 | found = False 105 | for n in nodes.dfs(node): 106 | if is_var(n): 107 | if found: 108 | return True 109 | found = True 110 | bw = get_bv_width(n) 111 | if bw > int(sort[2].data): 112 | return True 113 | return False 114 | return True 115 | 116 | def global_mutations(self, node, input_): 117 | varname = Node(f'x{node.id}__fresh') 118 | if is_var(varname): 119 | return [] 120 | var = Node('declare-const', varname, get_sort(node)) 121 | return [Simplification({node.id: varname}, [var])] 122 | 123 | def __str__(self): 124 | return 'introduce fresh variable' 125 | 126 | 127 | class LetElimination: 128 | """Substitute a ``let`` expression with its body.""" 129 | 130 | def filter(self, node): 131 | return is_operator_app(node, 'let') 132 | 133 | def mutations(self, node): 134 | if len(node) <= 2: 135 | return [] 136 | return [Simplification({node.id: node[2]}, [])] 137 | 138 | def __str__(self): 139 | return 'eliminate let binder' 140 | 141 | 142 | class LetSubstitution: 143 | """Substitute a variable bound by a ``let`` binder into the nested term.""" 144 | 145 | def filter(self, node): 146 | return is_operator_app(node, 'let') 147 | 148 | def mutations(self, node): 149 | if len(node) <= 2: 150 | return [] 151 | for var in node[1]: 152 | if any(n == var[0] for n in nodes.dfs(node[2])): 153 | subs = nodes.substitute(node[2], {var[0]: var[1]}) 154 | yield Simplification({node.id: Node(node[0], node[1], subs)}, 155 | []) 156 | 157 | def __str__(self): 158 | return 'substitute variable into let body' 159 | 160 | 161 | class RemoveAnnotation: 162 | """Remove an annotation ``(! t ...)`` from a term.""" 163 | 164 | def filter(self, node): 165 | return is_operator_app(node, '!') 166 | 167 | def mutations(self, node): 168 | yield Simplification({node.id: node[1]}, []) 169 | 170 | def __str__(self): 171 | return 'remove annotation' 172 | 173 | 174 | class RemoveRecursiveFunction: 175 | """Remove a function from a recursive function definition.""" 176 | 177 | def filter(self, node): 178 | return is_operator_app(node, 'define-funs-rec') 179 | 180 | def mutations(self, node): 181 | if len(node) != 3 or len(node[1]) != len(node[2]): 182 | return 183 | for i in range(len(node[1])): 184 | yield Simplification({ 185 | node[1][i].id: None, 186 | node[2][i].id: None 187 | }, []) 188 | 189 | def __str__(self): 190 | return 'remove recursive function' 191 | 192 | 193 | class SimplifyLogic: 194 | """Replace the logic specified in ``(check-logic ...)`` with a simpler one. 195 | """ 196 | 197 | def filter(self, node): 198 | return node.has_ident() and node.get_ident() == 'set-logic' 199 | 200 | def mutations(self, node): 201 | logic = node[1] 202 | cands = [] 203 | repls = { 204 | 'BV': '', 205 | 'FP': '', 206 | 'UF': '', 207 | 'S': '', 208 | 'T': '', 209 | 'NRA': 'LRA', 210 | 'LRA': '', 211 | 'NIA': 'LIA', 212 | 'LIA': '', 213 | 'NIRA': 'LIRA', 214 | 'LIRA': 'LRA' 215 | } 216 | for r in repls: 217 | assert logic.is_leaf() 218 | if r in logic.data: 219 | cands.append(logic.data.replace(r, repls[r])) 220 | yield from [ 221 | Simplification({node.id: Node('set-logic', c)}, []) for c in cands 222 | ] 223 | 224 | def __str__(self): 225 | return 'simplify logic' 226 | 227 | 228 | class SimplifyQuotedSymbols: 229 | """Turn a quoted symbol into a simple symbol.""" 230 | 231 | def filter(self, node): 232 | return is_piped_symbol(node) and re.match( 233 | '\\|[a-zA-Z0-9~!@$%^&*_+=<>.?/-]+\\|', node.data) is not None 234 | 235 | def mutations(self, node): 236 | return [Simplification({node.id: get_piped_symbol(node)}, [])] 237 | 238 | def global_mutations(self, node, input_): 239 | return [Simplification({node: get_piped_symbol(node)}, [])] 240 | 241 | def __str__(self): 242 | return 'simplify quoted symbol' 243 | 244 | 245 | class SimplifySymbolNames: 246 | """Simplify variable names. 247 | 248 | This only has cosmetic impact except for cases where variable names 249 | are compared/ordered lexicographically (it may enable, for example, 250 | the application of mutator :class:`ddsmt.mutators_core.ReplaceByVariable`). 251 | """ 252 | 253 | def filter(self, node): 254 | # check for is_const(node[1]) to avoid x -> false -> fals -> false 255 | # if the variable is irrelevant, false may be accepted by the solver 256 | return node.has_ident() and len(node) >= 2 and node.get_ident() in [ 257 | 'declare-const', 'declare-datatype', 'declare-datatypes', 258 | 'declare-fun', 'declare-sort', 'define-fun', 'exists', 'forall' 259 | ] and not is_const(node[1]) 260 | 261 | def global_mutations(self, node, input_): 262 | if node.get_ident() in ['declare-datatype', 'declare-datatypes']: 263 | for c in self.__flatten(Node(node[1:])): 264 | yield from self.__mutate_symbol(c, input_) 265 | return 266 | if node.get_ident() in ['exists', 'forall']: 267 | for v in node[1]: 268 | yield from self.__mutate_symbol(v[0], input_) 269 | return 270 | yield from self.__mutate_symbol(node[1], input_) 271 | 272 | def __flatten(self, n): 273 | """Yield given node as flattened sequence.""" 274 | if n.is_leaf(): 275 | yield n 276 | else: 277 | for e in n.data: 278 | yield from self.__flatten(e) 279 | 280 | def __mutate_symbol(self, symbol, input_): 281 | """Return a list of mutations of input_ based on simpler versions of 282 | symbol.""" 283 | if is_piped_symbol(symbol): 284 | for s in self.__simpler(get_piped_symbol(symbol)): 285 | if not is_var(Node('|' + s + '|')): 286 | yield Simplification({symbol: Node('|' + s + '|')}, []) 287 | else: 288 | for s in self.__simpler(symbol): 289 | if not is_var(Node(s)): 290 | yield Simplification({symbol: Node(s)}, []) 291 | 292 | def __simpler(self, symbol): 293 | """Return a list of simpler versions of the given symbol.""" 294 | sym = symbol.data 295 | if len(sym) > 3: 296 | yield sym[:len(sym) // 2] 297 | if len(sym) > 1: 298 | yield sym[:-1] 299 | yield sym[1:] 300 | 301 | def __str__(self): 302 | return 'simplify symbol name' 303 | 304 | 305 | def get_mutators(): 306 | """Returns a mapping from mutator class names to the name of their config 307 | options.""" 308 | return { 309 | 'CheckSatAssuming': 'check-sat-assuming', 310 | 'EliminateVariable': 'eliminate-variables', 311 | 'InlineDefinedFuns': 'inline-functions', 312 | 'IntroduceFreshVariable': 'introduce-fresh-variables', 313 | 'LetElimination': 'let-elimination', 314 | 'LetSubstitution': 'let-substitution', 315 | 'RemoveAnnotation': 'remove-annotation', 316 | 'RemoveRecursiveFunction': 'remove-recursive-function', 317 | 'SimplifyLogic': 'simplify-logic', 318 | 'SimplifyQuotedSymbols': 'simplify-quoted-symbols', 319 | 'SimplifySymbolNames': 'simplify-symbol-names', 320 | } 321 | -------------------------------------------------------------------------------- /ddsmt/mutators_strings.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | from . import nodes 13 | from .smtlib import * 14 | from .mutator_utils import Simplification 15 | 16 | 17 | class SeqNthUnit: 18 | """Simplify ``(seq.nth (seq.unit t) 0)`` to ``t``.""" 19 | 20 | def filter(self, node): 21 | return is_operator_app( 22 | node, 'seq.nth') and len(node) > 1 and is_operator_app( 23 | node[1], 'seq.unit') 24 | 25 | def mutations(self, node): 26 | yield Simplification({node.id: node[1][1]}, []) 27 | 28 | def __str__(self): 29 | return 'remove nth of unit seq' 30 | 31 | 32 | class StringSimplifyConstant: 33 | """Replace a string constant by a shorter constant.""" 34 | 35 | def __fix_escape_sequences(self, s, end): 36 | id = s.rfind('\\', end - 8, end) 37 | if id != -1: 38 | return id 39 | return end 40 | 41 | def filter(self, node): 42 | return is_string_const(node) and node != '""' 43 | 44 | def mutations(self, node): 45 | yield Simplification({node.id: Node('""')}, []) 46 | content = node[1:-1] 47 | for sec in nodes.binary_search(len(content)): 48 | start = self.__fix_escape_sequences(content, sec[0]) 49 | yield Simplification( 50 | {node.id: Node(f'"{content[:start]}{content[sec[1]:]}"')}, []) 51 | yield Simplification({node.id: Node(f'"{content[1:]}"')}, []) 52 | yield Simplification({node.id: Node(f'"{content[:-1]}"')}, []) 53 | 54 | def global_mutations(self, node, input_): 55 | for simp in self.mutations(node): 56 | yield Simplification({node: simp.substs[node.id]}, []) 57 | 58 | def __str__(self): 59 | return 'simplify string constant' 60 | 61 | 62 | class StringReplaceAll: 63 | """Replace ``str.replace_all`` by ``str.replace``.""" 64 | 65 | def filter(self, node): 66 | return node.has_ident() and node.get_ident() == 'str.replace_all' 67 | 68 | def mutations(self, node): 69 | return [Simplification({node.id: Node('str.replace', *node[1:])}, [])] 70 | 71 | def __str__(self): 72 | return 'eliminate str.replace_all' 73 | 74 | 75 | class StringIndexOfNotFound: 76 | """Replace ``str.indexof`` by special value ``(- 1)`` indicating that the 77 | substring was not found.""" 78 | 79 | def filter(self, node): 80 | return node.has_ident() and node.get_ident() == 'str.indexof' 81 | 82 | def mutations(self, node): 83 | return [Simplification({node.id: Node('-', '1')}, [])] 84 | 85 | def __str__(self): 86 | return 'eliminate str.indexof' 87 | 88 | 89 | class StringContainsToConcat: 90 | """Replace ``str.contains`` by concatenation.""" 91 | 92 | def filter(self, node): 93 | return node.has_ident() and node.get_ident() == 'str.contains' 94 | 95 | def global_mutations(self, node, input_): 96 | var = node[1] 97 | k1 = f'{var}_prefix' 98 | k2 = f'{var}_suffix' 99 | vars = [ 100 | Node('declare-const', k1, 'String'), 101 | Node('declare-const', k2, 'String'), 102 | ] 103 | eq = Node('=', var, ('str.++', k1, node[2], k2)) 104 | return [Simplification({node: eq}, vars)] 105 | 106 | def __str__(self): 107 | return 'eliminate str.contains by ++' 108 | 109 | 110 | def get_mutators(): 111 | """Returns a mapping from mutator class names to the name of their config 112 | options.""" 113 | return { 114 | 'SeqNthUnit': 'seq-nth-unit', 115 | 'StringContainsToConcat': 'str-contains-to-concat', 116 | 'StringIndexOfNotFound': 'str-index-not-found', 117 | 'StringReplaceAll': 'str-replace-all', 118 | 'StringSimplifyConstant': 'str-constants', 119 | } 120 | 121 | 122 | def is_relevant(node): 123 | """Checks whether this theory might be relevant for this node.""" 124 | if node.has_ident(): 125 | if node.get_ident() in ['declare-const']: 126 | if nodes.contains(node[2], lambda t: t == 'String'): 127 | return True 128 | if nodes.contains(node[2], is_seq_type): 129 | return True 130 | elif node.get_ident() in ['declare-fun', 'define-fun', 'define-sort']: 131 | if nodes.contains(node[3], lambda t: t == 'String'): 132 | return True 133 | if nodes.contains(node[3], is_seq_type): 134 | return True 135 | return False 136 | -------------------------------------------------------------------------------- /ddsmt/nodeio.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import io 13 | import textwrap 14 | import typing 15 | 16 | from .nodes import Node 17 | 18 | 19 | def parse_smtlib(text: str): # noqa: C901 20 | """Parse SMT-LIB input to list of ``Node`` objects. 21 | 22 | Every node represents an s-expression. This generator yields top- 23 | level s-expressions (commands) or comments. 24 | """ 25 | exprs = [] 26 | cur_expr = None 27 | 28 | pos = 0 29 | size = len(text) 30 | while pos < size: 31 | char = text[pos] 32 | pos += 1 33 | 34 | # String literals/quoted symbols 35 | if char in ('"', '|'): 36 | first_char = char 37 | literal = [char] 38 | # Read until terminating " or | 39 | while True: 40 | if pos >= size: 41 | return 42 | char = text[pos] 43 | pos += 1 44 | literal.append(char) 45 | if char == first_char: 46 | # Check is quote is escaped "a "" b" is one string literal 47 | if char == '"' and pos < size and text[pos] == '"': 48 | literal.append(text[pos]) 49 | pos += 1 50 | continue 51 | break 52 | cur_expr.append(Node(''.join(literal))) 53 | 54 | # Comments 55 | elif char == ';': 56 | comment = [char] 57 | # Read until newline 58 | while pos < size: 59 | char = text[pos] 60 | pos += 1 61 | comment.append(char) 62 | if char == '\n': 63 | break 64 | comment = ''.join(comment) 65 | if cur_expr: 66 | cur_expr.append(Node(comment)) 67 | else: 68 | yield Node(comment) 69 | 70 | # Open s-expression 71 | elif char == '(': 72 | cur_expr = [] 73 | exprs.append(cur_expr) 74 | 75 | # Close s-expression 76 | elif char == ')': 77 | cur_expr = exprs.pop() 78 | 79 | # Do we have nested s-expressions? 80 | if exprs: 81 | exprs[-1].append(Node(*cur_expr)) 82 | cur_expr = exprs[-1] 83 | else: 84 | yield Node(*cur_expr) 85 | cur_expr = None 86 | 87 | # Identifier 88 | elif char not in (' ', '\t', '\n'): 89 | token = [char] 90 | while True: 91 | if pos >= size: 92 | return 93 | char = text[pos] 94 | pos += 1 95 | if char in (' ', '\t', '\n'): 96 | break 97 | if char in ('(', ')', ';'): 98 | pos -= 1 99 | break 100 | token.append(char) 101 | 102 | token = Node(''.join(token)) 103 | 104 | # Append to current s-expression 105 | if cur_expr is not None: 106 | cur_expr.append(token) 107 | else: 108 | yield token 109 | 110 | 111 | def __write_smtlib(file: typing.TextIO, expr: Node): 112 | """Write the given smtlib expression in one line into the file object.""" 113 | visit = [expr] 114 | needs_space = False 115 | while visit: 116 | ex = visit.pop() 117 | if ex is None: 118 | file.write(')') 119 | needs_space = True 120 | continue 121 | 122 | if needs_space: 123 | file.write(' ') 124 | 125 | if ex.is_leaf(): 126 | if ex.data == '': 127 | continue 128 | if ex.data[0] == ';': 129 | file.write(f'\n{ex.data}\n') 130 | else: 131 | file.write(ex.data) 132 | needs_space = True 133 | continue 134 | 135 | file.write('(') 136 | needs_space = False 137 | visit.append(None) 138 | visit.extend(x for x in reversed(ex.data)) 139 | 140 | 141 | def __write_smtlib_pretty(file: typing.TextIO, expr: Node): 142 | """Write the given smtlib expression in one line into the file object.""" 143 | visit = [(expr, False)] 144 | indent = '' 145 | while visit: 146 | ex, visited = visit.pop() 147 | if ex.is_leaf(): 148 | assert isinstance(ex.data, str) 149 | if ex.data == '': 150 | continue 151 | if ex.data[0] == ';': 152 | file.write(f'\n{ex.data}\n') 153 | else: 154 | file.write(f'{indent}{ex.data}\n') 155 | continue 156 | 157 | if visited: 158 | indent = indent[:-2] 159 | file.write(f'{indent})\n') 160 | else: 161 | if all(map(lambda n: n.is_leaf(), ex.data)): 162 | file.write(f'{indent}{ex}\n') 163 | else: 164 | visit.append((ex, True)) 165 | if ex.has_ident(): 166 | file.write(f'{indent}({ex.data[0]}\n') 167 | visit.extend((x, False) for x in reversed(ex.data[1:])) 168 | else: 169 | file.write(f'{indent}(\n') 170 | visit.extend((x, False) for x in reversed(ex.data)) 171 | indent += ' ' 172 | 173 | 174 | def __write_smtlib_str(expr: Node): 175 | """Write the given smtlib expression in one line to a string.""" 176 | f = io.StringIO() 177 | __write_smtlib(f, expr) 178 | return f.getvalue() 179 | 180 | 181 | def __write_smtlib_pretty_str(expr: Node): 182 | """Write the given smtlib expression to a formatted string.""" 183 | f = io.StringIO() 184 | __write_smtlib_pretty(f, expr) 185 | return f.getvalue() 186 | 187 | 188 | def write_smtlib(file: typing.TextIO, exprs: typing.List[Node]): 189 | """Write the given expressions to the given file object 190 | Honor options to wrap lines or pretty-print.""" 191 | from . import options 192 | if options.args().pretty_print: 193 | # pretty print 194 | for expr in exprs: 195 | __write_smtlib_pretty(file, expr) 196 | else: 197 | # regular writeing 198 | lines = [__write_smtlib_str(expr) for expr in exprs] 199 | if options.args().wrap_lines: 200 | # wrap every line 201 | lines = map( 202 | lambda line: textwrap.wrap( 203 | line, width=78, subsequent_indent=' '), lines) 204 | # and flatten the list 205 | lines = [sub for line in lines for sub in line] 206 | for line in lines: 207 | file.write(line) 208 | file.write('\n') 209 | 210 | 211 | def write_smtlib_to_file(filename: str, exprs: typing.List[Node]): 212 | """Use ``write_smtlib`` to write to a filename.""" 213 | with open(filename, 'w') as file: 214 | write_smtlib(file, exprs) 215 | 216 | 217 | def write_smtlib_to_str(exprs: typing.List[Node]): 218 | """Return result of ``write_smtlib`` as a string.""" 219 | f = io.StringIO() 220 | write_smtlib(f, exprs) 221 | return f.getvalue() 222 | 223 | 224 | def write_smtlib_for_checking(filename: str, exprs: typing.List[Node]): 225 | """Slightly faster writing without wrapping or pretty-printing during 226 | checking.""" 227 | with open(filename, 'w') as file: 228 | for expr in exprs: 229 | __write_smtlib(file, expr) 230 | -------------------------------------------------------------------------------- /ddsmt/options.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import argparse 13 | import os 14 | 15 | from . import argparsemod 16 | from . import version 17 | 18 | 19 | class ToggleAction(argparse.Action): 20 | '''A simple ``argparse.Action`` class that is used for option pairs of the 21 | form ``--option`` and ``--no-option``.''' 22 | 23 | def __init__(self, opt_name, default=True, dest=None, help=None): 24 | super(ToggleAction, 25 | self).__init__([f'--{opt_name}', f'--no-{opt_name}'], 26 | dest=dest, 27 | nargs=0, 28 | help=help, 29 | default=default) 30 | 31 | def _get_value(self, option_string): 32 | '''Figure out whether the option shall be true or false.''' 33 | return not option_string.startswith('--no-') 34 | 35 | def __call__(self, parser, namespace, values, option_string=None): 36 | '''Set the option depending on which option was used.''' 37 | setattr(namespace, self.dest, self._get_value(option_string)) 38 | 39 | 40 | class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, 41 | argparse.HelpFormatter): 42 | '''A custom formatter for printing the commandline help. 43 | 44 | Uses ``argparse.ArgumentDefaultsHelpFormatter`` to print default 45 | values, but avoids printing for ``ToggleAction`` options and 46 | ``None`` defaults. Furthermore uses the ``argparse.HelpFormatter``, 47 | to slightly increase the width reserved for the options. 48 | ''' 49 | 50 | def __init__(self, *args, **kwargs): 51 | super().__init__(*args, **kwargs, max_help_position=35) 52 | 53 | def _get_help_string(self, action): 54 | '''Render the help string. 55 | 56 | Appends the default, if needed. 57 | ''' 58 | if action.default is None or isinstance(action, ToggleAction): 59 | return action.help 60 | return super(CustomFormatter, self)._get_help_string(action) 61 | 62 | def _format_action_invocation(self, action): 63 | '''Formats the options for the help page, uses a special formatting for 64 | ``ToggleAction`` options.''' 65 | if isinstance(action, ToggleAction): 66 | name = action.option_strings[0][2:] 67 | return ', '.join([f'--[no-]{name}'] + action.option_strings[2:]) 68 | return super(CustomFormatter, self)._format_action_invocation(action) 69 | 70 | 71 | class DumpConfigAction(argparse.Action): 72 | '''Dump the current config using ``pprint``.''' 73 | 74 | def __call__(self, parser, namespace, values, option_string=None): 75 | setattr(namespace, self.dest, True) 76 | import pprint 77 | pprint.pprint(vars(namespace)) 78 | 79 | 80 | def parse_options(mutators, cmdlineoptions=None): 81 | '''Configures the commandline parse and then parse the commandline options. 82 | 83 | ``cmdlineoptions`` can be used to override the contents of 84 | ``sys.argv``, but is mostly useful for unit tests. 85 | ''' 86 | 87 | usage = 'ddsmt [] []' 88 | ap = argparsemod.ModularArgumentParser( 89 | usage=usage, 90 | formatter_class=CustomFormatter, 91 | modular_help_groups=['mutator help']) 92 | ap.add_argument('infile', help='the input file (in SMT-LIB v2 format)') 93 | ap.add_argument('outfile', help='the output file') 94 | ap.add_argument('cmd', 95 | nargs=argparse.REMAINDER, 96 | help='the command (with optional arguments)') 97 | 98 | ap.add_argument('--version', action='version', version=version.VERSION) 99 | ap.add_argument('-v', 100 | action='count', 101 | dest='verbosity', 102 | default=0, 103 | help='increase verbosity') 104 | ap.add_argument('-q', 105 | action='count', 106 | dest='quietness', 107 | default=0, 108 | help='decrease verbosity') 109 | ap.add_argument('--dump-config', 110 | nargs=0, 111 | action=DumpConfigAction, 112 | help='dump configuration') 113 | ap.add_argument('--parser-test', 114 | action='store_true', 115 | help='only test the parser') 116 | ap.add_argument('--pretty-print', 117 | action='store_true', 118 | default=False, 119 | help='pretty-print to output file') 120 | ap.add_argument('--wrap-lines', 121 | action='store_true', 122 | default=False, 123 | help='wrap lines in output file') 124 | ap.add_argument('--strategy', 125 | choices=['ddmin', 'hierarchical', 'hybrid'], 126 | default='hybrid', 127 | help='minimization strategy') 128 | 129 | apcheck = ap.add_argument_group('checker arguments') 130 | apcheck.add_argument('--unchecked', 131 | action='store_true', 132 | help='assume every change is okay without checking') 133 | apcheck.add_argument('-j', 134 | '--jobs', 135 | type=int, 136 | metavar='n', 137 | default=1, 138 | help='number of parallel checks') 139 | apcheck.add_argument('--memout', 140 | type=int, 141 | metavar='megabytes', 142 | help='memout for individual checks') 143 | apcheck.add_argument('--timeout', 144 | metavar='timeout', 145 | type=float, 146 | help='timeout for test runs in seconds ' 147 | '(default: 1.5 * golden runtime)') 148 | apcheck.add_argument( 149 | '--ignore-output', 150 | action='store_true', 151 | help='ignore stdout and stderr, only consider exit code') 152 | apcheck.add_argument('--ignore-out', 153 | action='store_true', 154 | help='ignore stdout') 155 | apcheck.add_argument('--ignore-err', 156 | action='store_true', 157 | help='ignore stderr') 158 | apcheck.add_argument( 159 | '--match-err', 160 | metavar='str', 161 | help='match string in stderr to identify failing input') 162 | apcheck.add_argument( 163 | '--match-out', 164 | metavar='str', 165 | help='match string in stdout to identify failing input') 166 | 167 | apcheck.add_argument('-c', 168 | '--cross-check', 169 | metavar='cmd-cc', 170 | dest='cmd_cc', 171 | help='cross check command') 172 | apcheck.add_argument( 173 | '--timeout-cc', 174 | metavar='timeout', 175 | type=float, 176 | help='timeout for test runs of the cross check in seconds ' 177 | '(default: 1.5 * golden runtime)') 178 | apcheck.add_argument( 179 | '--ignore-output-cc', 180 | action='store_true', 181 | help='ignore stdout and stderr, only consider exit code for cross check' 182 | ) 183 | apcheck.add_argument( 184 | '--match-err-cc', 185 | metavar='str', 186 | help='match string in stderr to identify failing input for cross check' 187 | ) 188 | apcheck.add_argument( 189 | '--match-out-cc', 190 | metavar='str', 191 | help='match string in stdout to identify failing input for cross check' 192 | ) 193 | 194 | apdebug = ap.add_argument_group('debug arguments') 195 | apdebug.add_argument('--check-loops', 196 | action='store_true', 197 | default=False, 198 | help='check for loops in the minimization process') 199 | apdebug.add_argument('--profile', 200 | action='store_true', 201 | default=False, 202 | help='use cProfile for profiling') 203 | apdebug.add_argument('--dump-diffs', 204 | action='store_true', 205 | default=False, 206 | help='dump a diff of every simplification') 207 | 208 | mutators.collect_mutator_options(ap) 209 | 210 | res = ap.parse_args(cmdlineoptions) 211 | 212 | if res.cmd_cc: 213 | res.cmd_cc = res.cmd_cc.split() 214 | 215 | return res 216 | 217 | 218 | __PARSED_ARGS = None 219 | 220 | 221 | def args(cmdlineoptions=None): 222 | '''Returns the commandline options. 223 | 224 | Calls ``parse_options()`` if parsing has not yet happened. The 225 | ``cmdlineoptions`` is passed on to ``parse_options`` in this case. 226 | ''' 227 | global __PARSED_ARGS 228 | if __PARSED_ARGS is None: 229 | # make sure that options as a whole does not depend on other modules 230 | from . import mutators 231 | __PARSED_ARGS = parse_options(mutators, cmdlineoptions) 232 | return __PARSED_ARGS 233 | -------------------------------------------------------------------------------- /ddsmt/progress.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import logging 13 | import sys 14 | import itertools 15 | 16 | __MAX_VAL = None 17 | __CUR_VAL = 0 18 | __SPINNER = itertools.cycle(['-', '/', '|', '\\']) 19 | 20 | 21 | def __print(): 22 | global __CUR_VAL, __MAX_VAL 23 | print('{} {} / {}'.format(next(__SPINNER), __CUR_VAL, __MAX_VAL), 24 | end='\r', 25 | flush=True, 26 | file=sys.stderr) 27 | 28 | 29 | def start(max): 30 | """Initialize a new progress bar for at most ``max`` steps. 31 | 32 | Only initialize if the current log level is at least 33 | ``logging.INFO``. 34 | """ 35 | global __CUR_VAL, __MAX_VAL 36 | if not sys.stdout.isatty(): 37 | return 38 | if logging.getLogger().level > logging.INFO: 39 | return 40 | __MAX_VAL = max 41 | __CUR_VAL = 0 42 | __print() 43 | 44 | 45 | def update(newval=None): 46 | """Update the current progress bar. 47 | 48 | The value is incremented by one, or set to newval if newval is not 49 | ``None``. 50 | """ 51 | global __CUR_VAL, __MAX_VAL 52 | if __MAX_VAL is not None: 53 | if __CUR_VAL < __MAX_VAL: 54 | if newval is not None: 55 | if __CUR_VAL + newval >= __MAX_VAL: 56 | __CUR_VAL = __MAX_VAL 57 | else: 58 | __CUR_VAL += newval 59 | else: 60 | __CUR_VAL += 1 61 | __print() 62 | 63 | 64 | def finish(): 65 | """Stop the current progress bar. 66 | 67 | Delete the object and write a newline. 68 | """ 69 | global __MAX_VAL 70 | if __MAX_VAL is not None: 71 | __MAX_VAL = None 72 | sys.stdout.write('\n') 73 | -------------------------------------------------------------------------------- /ddsmt/strategy_hierarchical.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import collections 13 | import logging 14 | import multiprocessing 15 | import pickle 16 | import sys 17 | import time 18 | import traceback 19 | 20 | from . import checker 21 | from . import mutators 22 | from . import nodeio 23 | from . import nodes 24 | from . import options 25 | from . import debug_utils 26 | from . import progress 27 | from . import smtlib 28 | from .mutator_utils import Simplification, apply_simp 29 | 30 | 31 | def get_passes(): 32 | """Returns a list of passes, each pass being a list of mutators. 33 | 34 | The list is ordered so that earlier passes are more promising for a 35 | quick reduction. 36 | """ 37 | prelude = [ 38 | (mutators.get_initialized_mutator('BinaryReduction', 39 | {'ident': 'assert'}), { 40 | 'max_depth': 1 41 | }), 42 | mutators.get_mutators(['Constants', 'BinaryReduction']), 43 | mutators.get_mutators([ 44 | 'EraseNode', 45 | 'Constants', 46 | 'IntroduceFreshVariable', 47 | 'LetElimination', 48 | 'LetSubstitution', 49 | 'ReplaceByChild', 50 | 'BinaryReduction', 51 | 'CheckSatAssuming', 52 | ]), 53 | ] 54 | 55 | late = [ # Usually only have cosmetic impact 56 | 'SimplifyQuotedSymbols', 57 | 'SimplifySymbolNames', 58 | 'StringSimplifyConstant', 59 | ] 60 | main = [] 61 | for _, theory in mutators.get_all_mutators().items(): 62 | for mname in theory[1]: 63 | if mname not in late: 64 | main.append(mname) 65 | 66 | return prelude + [ 67 | mutators.get_mutators(main), 68 | mutators.get_mutators(late + main), 69 | ] 70 | 71 | 72 | def get_pass(passes, id): 73 | """Return the current pass, add empty parameter dict if none is given 74 | explicitly.""" 75 | res = passes[id] 76 | if isinstance(res, tuple): 77 | return res 78 | return res, {} 79 | 80 | 81 | # nodeid: id of the mutated node in bfs order. Only used for progress indication 82 | # name: name of the mutator 83 | # exprs: the current input 84 | # simp: the substitution to be checked 85 | # runtime: time needed to check this task 86 | Task = collections.namedtuple('Task', 87 | ['nodeid', 'name', 'exprs', 'simp', 'runtime']) 88 | 89 | 90 | class Producer: 91 | """Manages the generation of candidates that shall be checked. 92 | 93 | Performs a walk through the current input and applies the 94 | ``mutators`` to every node. Supports skipping the first ``skip`` 95 | nodes. As soon as ``abort_flag`` is triggered, stops generation as 96 | soon as possible. 97 | """ 98 | 99 | def __init__(self, mutators, abort_flag, original): 100 | self.__node_count = 0 101 | self.__mutators = mutators 102 | self.__abort = abort_flag 103 | self.__original = original 104 | self.__pickled = pickle.dumps(original) 105 | 106 | def __mutate_node(self, count, linput): 107 | """Apply all mutators to the given node. 108 | 109 | Returns a list of all possible mutations as ``Task`` objects. 110 | """ 111 | for m in self.__mutators: 112 | if self.__abort.is_set(): 113 | break 114 | try: 115 | if hasattr(m, 'filter') and not m.filter(linput): 116 | continue 117 | if hasattr(m, 'mutations'): 118 | for x in m.mutations(linput): 119 | if self.__abort.is_set(): 120 | break 121 | assert isinstance(x, Simplification) 122 | yield Task(count, str(m), self.__pickled, 123 | pickle.dumps(x), None) 124 | if hasattr(m, 'global_mutations'): 125 | for x in m.global_mutations(linput, self.__original): 126 | if self.__abort.is_set(): 127 | break 128 | assert isinstance(x, Simplification) 129 | yield Task(count, f'(global) {m}', self.__pickled, 130 | pickle.dumps(x), None) 131 | except Exception as e: 132 | logging.info(f'{type(e)} in application of {m}: {e}') 133 | exc_type, exc_value, exc_traceback = sys.exc_info() 134 | traceback.print_tb(exc_traceback, limit=10, file=sys.stderr) 135 | 136 | def generate(self, skip, params): 137 | """A generator that produces all possible mutations as ``Task`` from 138 | the given original.""" 139 | count = 0 140 | for node in nodes.bfs(self.__original, params.get('max_depth', None)): 141 | count += 1 142 | if skip < count: 143 | yield from self.__mutate_node(count, node) 144 | if self.__abort.is_set(): 145 | break 146 | 147 | 148 | class Consumer: 149 | """Calls the ``checker`` on individual tasks to figure out whether they are 150 | valid simplifications. 151 | 152 | Uses the ``abort_flag`` to stop as soon as a valid simplification 153 | has been found. 154 | """ 155 | 156 | def __init__(self, abort_flag): 157 | self.__abort = abort_flag 158 | 159 | def check(self, task): 160 | with debug_utils.Profiler(): 161 | abortres = pickle.dumps( 162 | (False, Task(task.nodeid, task.name, None, None, None))) 163 | if self.__abort.is_set(): 164 | return abortres 165 | try: 166 | start = time.time() 167 | simp = pickle.loads(task.simp) 168 | assert isinstance(simp, Simplification) 169 | if self.__abort.is_set(): 170 | return abortres 171 | exprs = apply_simp(pickle.loads(task.exprs), simp) 172 | 173 | if self.__abort.is_set(): 174 | return abortres 175 | res = checker.check_exprs(exprs) 176 | runtime = time.time() - start 177 | if self.__abort.is_set(): 178 | return abortres 179 | if res: 180 | return pickle.dumps((True, 181 | Task(task.nodeid, task.name, exprs, 182 | None, runtime))) 183 | return pickle.dumps( 184 | (False, Task(task.nodeid, task.name, None, None, runtime))) 185 | except Exception as e: 186 | logging.info(f'{type(e)} in check of {task.name}: {e}') 187 | exc_type, exc_value, exc_traceback = sys.exc_info() 188 | traceback.print_tb(exc_traceback, limit=10, file=sys.stderr) 189 | return abortres 190 | 191 | 192 | class MutatorStats: 193 | """Gather information about the performance of the individual mutators.""" 194 | 195 | def __init__(self): 196 | self.data = {} 197 | self.__enabled = logging.getLogger().isEnabledFor(logging.INFO) 198 | 199 | def add(self, success, task, original): 200 | """Add results from one check.""" 201 | if task.runtime is None or not self.__enabled: 202 | return 203 | d = self.data.setdefault(task.name, { 204 | 'tests': 0, 205 | 'success': 0, 206 | 'diff': 0, 207 | 'runtime': 0 208 | }) 209 | d['tests'] += 1 210 | if success: 211 | d['success'] += 1 212 | d['diff'] += nodes.count_exprs( 213 | task.exprs) - nodes.count_exprs(original) 214 | d['runtime'] += task.runtime 215 | 216 | def print(self): 217 | """Print cumulative results.""" 218 | if self.__enabled: 219 | for name, data in sorted(self.data.items()): 220 | logging.info(f'{name}: diff {data["diff"]:+} expressions, ' 221 | f'{data["tests"]} tests ({data["success"]}), ' 222 | f'{data["runtime"]:.1f}s') 223 | 224 | 225 | def reduce(exprs): 226 | """Reduces the input given in ``exprs`` as good as possible in a fixed- 227 | point loop.""" 228 | passes = get_passes() 229 | passid = 0 230 | 231 | nchecks = 0 232 | nreduce = 0 233 | stats = MutatorStats() 234 | loop_checker = debug_utils.NodeLoopChecker() 235 | loop_checker.add(exprs) 236 | 237 | # use one pool for the whole reduction 238 | with multiprocessing.Pool(options.args().jobs) as pool: 239 | # abort flag is passed to both producer and consumer 240 | # important: the pool will break if the abort_flag is destroyed early 241 | abort_flag = multiprocessing.Manager().Event() 242 | # iterate over all passes 243 | for passid in range(len(passes)): 244 | cur_passes, params = get_pass(passes, passid) 245 | if not cur_passes: 246 | # no passes enabled 247 | continue 248 | skip = 0 249 | fresh_run = True 250 | # repeat until no further reduction is found 251 | while True: 252 | reduction = False 253 | start = time.time() 254 | smtlib.collect_information(exprs) 255 | cnt = nodes.count_nodes(exprs) 256 | progress.start(cnt) 257 | progress.update(min(cnt, skip)) 258 | abort_flag.clear() 259 | prod = Producer(cur_passes, abort_flag, exprs) 260 | cons = Consumer(abort_flag) 261 | for result in pool.imap_unordered(cons.check, 262 | prod.generate(skip, params)): 263 | nchecks += 1 264 | success, task = pickle.loads(result) 265 | if abort_flag.is_set(): 266 | # skip remaining results if we had a success 267 | skip = min(skip, task.nodeid - 1) 268 | continue 269 | progress.update(task.nodeid) 270 | stats.add(success, task, exprs) 271 | if success: 272 | # trigger abort, then process the result 273 | abort_flag.set() 274 | progress.finish() 275 | nreduce += 1 276 | runtime = time.time() - start 277 | logging.chat( 278 | f'#{nreduce}: {task.name} ({runtime:.2f}s, ' 279 | f'{nodes.count_nodes(task.exprs)} expressions)') 280 | reduction = True 281 | debug_utils.dump_diff(task.name, exprs, task.exprs) 282 | exprs = nodes.reduplicate(task.exprs) 283 | loop_checker.add(exprs) 284 | skip = task.nodeid - 1 285 | fresh_run = False 286 | nodeio.write_smtlib_to_file(options.args().outfile, 287 | exprs) 288 | if not reduction: 289 | if fresh_run: 290 | # this was a fresh run, continue with next pass 291 | break 292 | progress.finish() 293 | logging.info('Starting over') 294 | skip = 0 295 | fresh_run = True 296 | logging.info('No further simplification found') 297 | 298 | stats.print() 299 | 300 | return exprs, nchecks 301 | -------------------------------------------------------------------------------- /ddsmt/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | if not hasattr(logging, 'trace'): 4 | logging.TRACE = 5 5 | logging.addLevelName(logging.TRACE, "TRACE") 6 | logging.trace = lambda msg, *args, **kwargs: logging.log( 7 | logging.TRACE, msg, *args, **kwargs) 8 | 9 | if not hasattr(logging, 'chat'): 10 | logging.CHAT = 25 11 | logging.addLevelName(logging.CHAT, "CHAT") 12 | logging.chat = lambda msg, *args, **kwargs: logging.log( 13 | logging.CHAT, msg, *args, **kwargs) 14 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_arithmetic.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_arithmetic 3 | from .utils import * 4 | 5 | 6 | def test_arith_get_mutators(): 7 | d = mutators_arithmetic.get_mutators() 8 | assert isinstance(d, dict) 9 | assert len(d) == 4 10 | 11 | 12 | def test_arith_is_relevant(): 13 | assert mutators_arithmetic.is_relevant(Node('declare-const', 'x', 'Real')) 14 | assert mutators_arithmetic.is_relevant(Node('declare-fun', 'x', (), 15 | 'Real')) 16 | assert not mutators_arithmetic.is_relevant( 17 | Node('declare-const', 'x', 'Bool')) 18 | assert not mutators_arithmetic.is_relevant( 19 | Node('declare-fun', 'x', (), 'String')) 20 | assert not mutators_arithmetic.is_relevant(Node()) 21 | assert not mutators_arithmetic.is_relevant(Node('assert', 'x')) 22 | 23 | 24 | def test_arith_negate_relation(): 25 | m = mutators_arithmetic.ArithmeticNegateRelation() 26 | assert isinstance(str(m), str) 27 | notfilter = [ 28 | Node('>', 'x', 'y'), 29 | Node('=', '1', 'y'), 30 | Node('distinct', '1', 'y'), 31 | Node('not', ('and', 'y', 'x')), 32 | ] 33 | exprs = { 34 | Node('not', ('=', 'x', 'y')): [Node('distinct', 'x', 'y')], 35 | Node('not', ('>', '1', 'y')): [Node('<=', '1', 'y')], 36 | Node('not', ('distinct', 'x', 'y')): [Node('=', 'x', 'y')], 37 | Node('not', ('<>', 'x', 'y')): [Node('=', 'x', 'y')], 38 | Node('not', ('!=', 'x', 'y')): [Node('=', 'x', 'y')], 39 | } 40 | for e in notfilter: 41 | assert not m.filter(e) 42 | for k, v in exprs.items(): 43 | assert m.filter(k) 44 | assert check_mutations(m, k, v) 45 | 46 | 47 | def test_arith_simplify_constant(): 48 | m = mutators_arithmetic.ArithmeticSimplifyConstant() 49 | assert isinstance(str(m), str) 50 | notfilter = [ 51 | Node('x'), 52 | Node('+', '1.0', 'x'), 53 | Node('0'), 54 | Node('0.0'), 55 | Node('1'), 56 | Node('1.0') 57 | ] 58 | exprs = { 59 | Node('2'): [Node('1'), Node('0')], 60 | Node('13'): [Node('6'), Node('1')], 61 | Node('20'): [Node('10'), Node('2')], 62 | Node('50'): [Node('25'), Node('5')], 63 | Node('1.5'): [Node('1'), Node('1.')], 64 | Node('3.1415'): [Node('3'), Node('3.141')], 65 | } 66 | for e in notfilter: 67 | assert not m.filter(e) 68 | for k, v in exprs.items(): 69 | assert m.filter(k) 70 | assert check_mutations(m, k, v) 71 | 72 | exprs = [ 73 | Node('assert', ('=', '13', 'y')), 74 | Node('assert', ('>', '13', 'z')), 75 | Node('assert', ('<', 'y', 'z')), 76 | ] 77 | x = Node('13') 78 | assert m.filter(x) 79 | assert check_global_mutations(m, x, exprs, [ 80 | [ 81 | Node('assert', ('=', '6', 'y')), 82 | Node('assert', ('>', '6', 'z')), 83 | Node('assert', ('<', 'y', 'z')), 84 | ], 85 | [ 86 | Node('assert', ('=', '1', 'y')), 87 | Node('assert', ('>', '1', 'z')), 88 | Node('assert', ('<', 'y', 'z')), 89 | ], 90 | ]) 91 | 92 | 93 | def test_arith_split_nary_relations(): 94 | m = mutators_arithmetic.ArithmeticSplitNaryRelation() 95 | assert isinstance(str(m), str) 96 | notfilter = [Node('>'), Node('>', 'x', 'y'), Node('+', 'x', 'y')] 97 | exprs = { 98 | Node('=', 'a', 'b', 'c'): 99 | [Node('and', ('=', 'a', 'b'), ('=', 'b', 'c'))], 100 | Node('>', 'a', 'b', 'c'): 101 | [Node('and', ('>', 'a', 'b'), ('>', 'b', 'c'))], 102 | Node('<', 'a', 'b', 'c', 'd'): 103 | [Node('and', ('<', 'a', 'b'), ('<', 'b', 'c'), ('<', 'c', 'd'))], 104 | } 105 | for e in notfilter: 106 | assert not m.filter(e) 107 | for k, v in exprs.items(): 108 | assert m.filter(k) 109 | assert check_mutations(m, k, v) 110 | 111 | 112 | def test_arith_strengthen_relations(): 113 | m = mutators_arithmetic.ArithmeticStrengthenRelation() 114 | assert isinstance(str(m), str) 115 | notfilter = [Node('=', 'x', 'y')] 116 | exprs = { 117 | Node('<', 'a', 'b'): [Node('=', 'a', 'b')], 118 | Node('>', 'a', 'b'): [Node('=', 'a', 'b')], 119 | Node('<=', 'a', 'b'): [Node('<', 'a', 'b'), 120 | Node('=', 'a', 'b')], 121 | Node('>=', 'a', 'b'): [Node('>', 'a', 'b'), 122 | Node('=', 'a', 'b')], 123 | Node('distinct', 'x', 'y'): [Node('=', 'x', 'y')], 124 | } 125 | for e in notfilter: 126 | assert not m.filter(e) 127 | for k, v in exprs.items(): 128 | assert m.filter(k) 129 | assert check_mutations(m, k, v) 130 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_boolean.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_boolean 3 | from .utils import * 4 | 5 | 6 | def test_bool_get_mutators(): 7 | d = mutators_boolean.get_mutators() 8 | assert isinstance(d, dict) 9 | assert len(d) == 7 10 | 11 | 12 | def test_bool_de_morgan(): 13 | m = mutators_boolean.BoolDeMorgan() 14 | assert isinstance(str(m), str) 15 | notfilter = [ 16 | Node('=', '1', 'y'), 17 | Node('and', 'x', 'y'), 18 | Node('or', 'x', 'y'), 19 | ] 20 | exprs = { 21 | Node('not', ('and', 'x', 'y', 'z')): 22 | [Node('or', ('not', 'x'), ('not', 'y'), ('not', 'z'))], 23 | Node('not', ('or', 'x', 'y')): 24 | [Node('and', ('not', 'x'), ('not', 'y'))], 25 | } 26 | for e in notfilter: 27 | assert not m.filter(e) 28 | for k, v in exprs.items(): 29 | assert m.filter(k) 30 | assert check_mutations(m, k, v) 31 | 32 | 33 | def test_bool_double_negation(): 34 | m = mutators_boolean.BoolDoubleNegation() 35 | assert isinstance(str(m), str) 36 | notfilter = [ 37 | Node('not', ('=', '1', 'y')), 38 | Node('not', ('and', 'x', 'y')), 39 | Node('not', ('or', 'x', 'y')), 40 | ] 41 | exprs = { 42 | Node('not', ('not', 'x')): [Node('x')], 43 | Node('not', ('not', ('or', 'x', 'y'))): [Node('or', 'x', 'y')], 44 | } 45 | for e in notfilter: 46 | assert not m.filter(e) 47 | for k, v in exprs.items(): 48 | assert m.filter(k) 49 | assert check_mutations(m, k, v) 50 | 51 | 52 | def test_bool_eliminate_false_equality(): 53 | m = mutators_boolean.BoolEliminateFalseEquality() 54 | assert isinstance(str(m), str) 55 | notfilter = [ 56 | Node('not', ('=', '1', 'y')), 57 | Node('=', 'true', 'x', 'y'), 58 | Node('not', ('=', 'false', 'y')), 59 | ] 60 | exprs = { 61 | Node('=', 'false'): [], 62 | Node('=', 'false', 'x'): [Node('not', 'x')], 63 | Node('=', 'x', 'false', 'y'): 64 | [Node('and', ('not', 'x'), ('not', 'y'))], 65 | } 66 | for e in notfilter: 67 | assert not m.filter(e) 68 | for k, v in exprs.items(): 69 | assert m.filter(k) 70 | assert check_mutations(m, k, v) 71 | 72 | 73 | def test_bool_eliminate_implication(): 74 | m = mutators_boolean.BoolEliminateImplication() 75 | assert isinstance(str(m), str) 76 | notfilter = [ 77 | Node('or', 'true', 'y'), 78 | Node('not', ('=>', 'false', 'y')), 79 | ] 80 | exprs = { 81 | Node('=>', 'x'): [], 82 | Node('=>', 'x', 'y'): [Node('or', ('not', 'x'), 'y')], 83 | Node('=>', 'x', 'y', 'z'): 84 | [Node('and', ('or', ('not', 'x'), 'y'), ('or', ('not', 'y'), 'z'))], 85 | } 86 | for e in notfilter: 87 | assert not m.filter(e) 88 | for k, v in exprs.items(): 89 | assert m.filter(k) 90 | assert check_mutations(m, k, v) 91 | 92 | 93 | def test_bool_negate_quantifier(): 94 | m = mutators_boolean.BoolNegateQuantifier() 95 | assert isinstance(str(m), str) 96 | notfilter = [ 97 | Node('forall', ('Real', 'x'), 'y'), 98 | Node('not', ('or', 'x', 'y')), 99 | ] 100 | exprs = { 101 | Node('not', ('forall', ('Real', 'x'), 'y')): 102 | [Node('exists', ('Real', 'x'), ('not', 'y'))], 103 | Node('not', ('exists', ('Real', 'x'), 'y')): 104 | [Node('forall', ('Real', 'x'), ('not', 'y'))], 105 | } 106 | for e in notfilter: 107 | assert not m.filter(e) 108 | for k, v in exprs.items(): 109 | assert m.filter(k) 110 | assert check_mutations(m, k, v) 111 | 112 | 113 | def test_bool_xor_eliminate_binary(): 114 | m = mutators_boolean.BoolXOREliminateBinary() 115 | assert isinstance(str(m), str) 116 | notfilter = [ 117 | Node('or', 'x', 'y'), 118 | Node('xor', 'x', 'y', 'z'), 119 | ] 120 | exprs = { 121 | Node('xor', 'x', 'y'): [Node('distinct', 'x', 'y')], 122 | } 123 | for e in notfilter: 124 | assert not m.filter(e) 125 | for k, v in exprs.items(): 126 | assert m.filter(k) 127 | assert check_mutations(m, k, v) 128 | 129 | 130 | def test_bool_xor_remove_constant(): 131 | m = mutators_boolean.BoolXORRemoveConstant() 132 | assert isinstance(str(m), str) 133 | notfilter = [ 134 | Node('or', 'x', 'y', 'false'), 135 | Node('and', 'true', 'z'), 136 | ] 137 | exprs = { 138 | Node('xor', 'x', 'y'): [], 139 | Node('xor', 'false', 'y'): [Node('xor', 'y')], 140 | Node('xor', 'false', 'y', 'z', 'false'): [Node('xor', 'y', 'z')], 141 | Node('xor', 'true', 'y'): 142 | [Node('xor', 'y'), Node('not', ('xor', 'y'))], 143 | Node('xor', 'true', 'y', 'z', 'true'): 144 | [Node('xor', 'y', 'z'), 145 | Node('not', ('xor', 'y', 'z'))], 146 | } 147 | for e in notfilter: 148 | assert not m.filter(e) 149 | for k, v in exprs.items(): 150 | assert m.filter(k) 151 | assert check_mutations(m, k, v) 152 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_core.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_core 3 | from .. import options 4 | from .. import smtlib 5 | from .utils import * 6 | 7 | 8 | def test_binary_reduction(): 9 | m = mutators_core.BinaryReduction() 10 | assert isinstance(str(m), str) 11 | assert not hasattr(m, 'filter') 12 | a = Node('a') 13 | b = Node('b') 14 | c = Node('c') 15 | d = Node('d') 16 | e = Node('e') 17 | f = Node('f') 18 | g = Node('g') 19 | h = Node('h') 20 | i = Node('i') 21 | exprs = [a, b, c, d, e, f, g, h, i] 22 | mut = [[a, b, c, d], [e, f, g, h, i], [a, b, c, d, e, f], 23 | [a, b, c, d, g, h, i], [a, b, e, f, g, h, i], [c, d, e, f, g, h, i]] 24 | assert check_global_mutations(m, b, exprs, []) 25 | assert check_global_mutations(m, a, exprs, mut) 26 | 27 | assert check_mutations(m, Node('and', 'a', 'b', 'c'), []) 28 | n = Node('and', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j') 29 | assert check_mutations(m, n, [ 30 | Node('and', 'a', 'b', 'c', 'd'), 31 | Node('e', 'f', 'g', 'h', 'i', 'j'), 32 | Node('and', 'a', 'b', 'c', 'd', 'e', 'f', 'g'), 33 | Node('and', 'a', 'b', 'c', 'd', 'h', 'i', 'j'), 34 | Node('and', 'a', 'e', 'f', 'g', 'h', 'i', 'j'), 35 | Node('b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'), 36 | ]) 37 | 38 | 39 | def test_binary_reduction_named(): 40 | m = mutators_core.BinaryReduction() 41 | m.ident = 'assert' 42 | assert isinstance(str(m), str) 43 | assert not hasattr(m, 'filter') 44 | a = Node('a') 45 | b = Node('assert', 'b') 46 | c = Node('c') 47 | d = Node('assert', 'd') 48 | e = Node('e') 49 | f = Node('assert', 'f') 50 | g = Node('g') 51 | h = Node('assert', 'h') 52 | i = Node('assert', 'i') 53 | exprs = [a, b, c, d, e, f, g, h, i] 54 | mut = [[a, b, c, d, e, g], [a, c, e, f, g, h, i]] 55 | assert check_global_mutations(m, a, exprs, mut) 56 | 57 | 58 | def test_constants(): 59 | smtlib.reset_information() 60 | m = mutators_core.Constants() 61 | assert isinstance(str(m), str) 62 | exprs = [Node('declare-const', 'x', 'Real'), Node('#b1011')] 63 | assert not m.filter(Node('x')) 64 | assert list(m.mutations(Node('x'))) == [] 65 | smtlib.collect_information(exprs) 66 | assert m.filter(Node('x')) 67 | assert m.filter(Node('#b1011')) 68 | assert m.filter(Node('=', 'x', '1')) 69 | assert not m.filter(Node('y')) 70 | 71 | # can't handle this case without BV const normalization 72 | # else this should be an empty set 73 | assert check_mutations( 74 | m, Node('#b0'), 75 | [Node('_', 'bv0', 1), Node('_', 'bv1', 1)]) 76 | assert check_mutations(m, Node('_', 'bv0', 1), []) 77 | 78 | assert check_mutations( 79 | m, Node('#b1011'), 80 | [Node('_', 'bv0', 4), Node('_', 'bv1', 4)]) 81 | assert check_mutations(m, Node('x'), [Node('0.0'), Node('1.0')]) 82 | assert check_mutations(m, Node('=', 'x', '1'), 83 | [Node('false'), Node('true')]) 84 | 85 | 86 | def test_erase_node(): 87 | m = mutators_core.EraseNode() 88 | assert isinstance(str(m), str) 89 | assert m.filter(Node('x')) 90 | assert check_mutations(m, Node('x'), [None]) 91 | 92 | 93 | def test_erase_named_node(): 94 | m = mutators_core.EraseNode() 95 | m.ident = 'assert' 96 | assert isinstance(str(m), str) 97 | assert m.filter(Node('assert', 'x')) 98 | assert not m.filter(Node('x')) 99 | assert check_mutations(m, Node(Node('assert'), Node('true')), [None]) 100 | 101 | 102 | def test_merge_with_children(): 103 | node = Node(Node('+'), Node('x'), Node(Node('+'), Node('x'), Node('y'))) 104 | m = mutators_core.MergeWithChildren() 105 | assert isinstance(str(m), str) 106 | assert m.filter(node) 107 | assert check_mutations(m, node, 108 | [Node(Node('+'), Node('x'), Node('x'), Node('y'))]) 109 | 110 | 111 | def test_replace_by_child(): 112 | smtlib.collect_information([ 113 | Node('declare-const', 'x', 'Int'), 114 | Node('declare-const', 'y', 'Int'), 115 | ]) 116 | node = Node('+', 'x', 'y') 117 | m = mutators_core.ReplaceByChild() 118 | assert isinstance(str(m), str) 119 | assert m.filter(node) 120 | assert not m.filter(Node('x')) 121 | assert check_mutations(m, node, ['x', 'y']) 122 | smtlib.reset_information() 123 | 124 | 125 | def test_replace_by_variable(): 126 | smtlib.reset_information() 127 | options.args(['in', 'out', 'bin']) 128 | 129 | x = Node('x') 130 | c = Node('42') 131 | node = Node(Node('+'), x, c) 132 | m = mutators_core.ReplaceByVariable() 133 | assert isinstance(str(m), str) 134 | assert check_mutations(m, Node('x'), []) 135 | 136 | m.repl_mode = 'inc' 137 | assert isinstance(str(m), str) 138 | assert check_mutations(m, node, []) 139 | 140 | v1 = Node('v1') 141 | v3 = Node('v3') 142 | exprs = [ 143 | Node(Node('declare-const'), v1, Node('Int')), 144 | Node(Node('declare-const'), x, Node('Int')), 145 | Node(Node('declare-const'), Node('v2'), Node('Bool')), 146 | Node(Node('declare-const'), v3, Node('Int')) 147 | ] 148 | 149 | smtlib.collect_information(exprs) 150 | assert m.filter(node) 151 | # we should not replace the x from the declare-const 152 | assert not m.filter(x) 153 | # but we should replace another x from somewhere else 154 | assert m.filter(Node('x')) 155 | assert not m.filter(c) 156 | assert check_mutations(m, node, [v1, x, v3]) 157 | assert check_mutations(m, x, []) 158 | assert check_mutations(m, v1, [x, v3]) 159 | assert check_mutations(m, v3, [x]) 160 | 161 | m.repl_mode = 'dec' 162 | assert check_mutations(m, node, [v1, x, v3]) 163 | assert check_mutations(m, x, [v1, v3]) 164 | assert check_mutations(m, v1, []) 165 | assert check_mutations(m, v3, [v1]) 166 | 167 | 168 | def test_sort_children(): 169 | node = Node(Node('+'), Node('123'), Node(Node('*'), Node('y'), Node('2')), 170 | Node('x')) 171 | expected = Node(Node('+'), Node('123'), Node('x'), 172 | Node(Node('*'), Node('y'), Node('2'))) 173 | 174 | m = mutators_core.SortChildren() 175 | assert isinstance(str(m), str) 176 | assert not m.filter(Node('x')) 177 | assert not m.filter(Node('123')) 178 | assert m.filter(node) 179 | assert m.filter(expected) 180 | assert check_mutations(m, node, [expected]) 181 | assert check_mutations(m, expected, []) 182 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_datatypes.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_datatypes 3 | from .. import smtlib 4 | from ..mutator_utils import * 5 | from .utils import * 6 | 7 | 8 | def test_datatypes_get_mutators(): 9 | d = mutators_datatypes.get_mutators() 10 | assert isinstance(d, dict) 11 | assert len(d) == 3 12 | 13 | 14 | def test_strings_is_relevant(): 15 | color_dt = (Node('Color 0'), Node('red', 'green', 'blue')) 16 | assert mutators_datatypes.is_relevant(Node('declare-datatypes', color_dt)) 17 | assert mutators_datatypes.is_relevant(Node('declare-datatype', color_dt)) 18 | assert not mutators_datatypes.is_relevant( 19 | Node('declare-const', 'x', 'Real')) 20 | assert not mutators_datatypes.is_relevant( 21 | Node('declare-fun', 'x', (), 'Real')) 22 | assert not mutators_datatypes.is_relevant(Node()) 23 | assert not mutators_datatypes.is_relevant(Node('assert', 'x')) 24 | 25 | 26 | def test_datatypes_remove_constructor(): 27 | m = mutators_datatypes.RemoveConstructor() 28 | assert isinstance(str(m), str) 29 | assert not m.filter(Node('x')) 30 | 31 | n = Node( 32 | 'declare-datatypes', 33 | (('Yin', '0'), ('Yang', '0')), 34 | (('yin', ), ('yang', )), 35 | ) 36 | assert m.filter(n) 37 | assert check_mutations(m, n, [ 38 | ('declare-datatypes', (('Yin', '0'), ('Yang', '0')), ( 39 | (), 40 | ('yang', ), 41 | )), 42 | ('declare-datatypes', (('Yin', '0'), ('Yang', '0')), (('yin', ), ())), 43 | ]) 44 | 45 | 46 | def test_datatypes_remove_datatype(): 47 | m = mutators_datatypes.RemoveDatatype() 48 | assert isinstance(str(m), str) 49 | assert not m.filter(Node('x')) 50 | 51 | n = Node( 52 | 'declare-datatypes', 53 | ( 54 | ('Yin', '0'), 55 | ('Yang', '0'), 56 | ), 57 | ( 58 | ('yin', ), 59 | ('yang', ), 60 | ), 61 | ) 62 | assert m.filter(n) 63 | assert check_mutations(m, n, [ 64 | ('declare-datatypes', (('Yang', '0'), ), (('yang', ), )), 65 | ('declare-datatypes', (('Yin', '0'), ), (('yin', ), )), 66 | ]) 67 | 68 | 69 | def test_datatypes_remove_datatype_identity(): 70 | smtlib.reset_information() 71 | m = mutators_datatypes.RemoveDatatypeIdentity() 72 | assert isinstance(str(m), str) 73 | assert not m.filter(Node('x')) 74 | 75 | decl = Node( 76 | 'declare-datatypes', 77 | (('a', '0'), ), 78 | (( 79 | ('r', ), 80 | ('b', ('c', ('_', 'BitVec', '3')), ('d', 'a')), 81 | ), ), 82 | ) 83 | smtlib.collect_information([decl]) 84 | n = Node('d', ('b', 'y', 'z')) 85 | assert m.filter(n) 86 | assert check_mutations(m, n, ['z']) 87 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_fp.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_fp 3 | from .utils import * 4 | 5 | 6 | def test_fp_get_mutators(): 7 | d = mutators_fp.get_mutators() 8 | assert isinstance(d, dict) 9 | assert len(d) == 1 10 | 11 | 12 | def test_fp_is_relevant(): 13 | assert mutators_fp.is_relevant(Node('declare-const', 'x', 'Float16')) 14 | assert mutators_fp.is_relevant( 15 | Node('declare-fun', 'x', (), ('_', 'FloatingPoint', '5', '11'))) 16 | assert not mutators_fp.is_relevant(Node('declare-const', 'x', 'Real')) 17 | assert not mutators_fp.is_relevant(Node('declare-fun', 'x', (), 'Bool')) 18 | assert not mutators_fp.is_relevant(Node()) 19 | assert not mutators_fp.is_relevant(Node('assert', 'x')) 20 | 21 | 22 | def test_fp_short_sort(): 23 | m = mutators_fp.FPShortSort() 24 | assert isinstance(str(m), str) 25 | assert not m.filter(Node('x')) 26 | assert not m.filter( 27 | Node('declare-const', 'x', ('_', 'FloatingPoint', 3, 5))) 28 | assert not m.filter(Node('Float16')) 29 | assert not m.filter(Node('_', 'FloatingPoint', 5)) 30 | assert not m.filter(Node('_', 'FloatingPoin', 5, 11)) 31 | assert m.filter(Node('_', 'FloatingPoint', 3, 5)) 32 | assert check_mutations(m, Node('_', 'FloatingPoint', 3, 5), []) 33 | assert check_mutations(m, Node('_', 'FloatingPoint', 5, 11), 34 | [Node('Float16')]) 35 | assert check_mutations(m, Node('_', 'FloatingPoint', 8, 24), 36 | [Node('Float32')]) 37 | assert check_mutations(m, Node('_', 'FloatingPoint', 11, 53), 38 | [Node('Float64')]) 39 | assert check_mutations(m, Node('_', 'FloatingPoint', 15, 113), 40 | [Node('Float128')]) 41 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_smtlib.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_smtlib 3 | from .. import smtlib 4 | from ..mutator_utils import * 5 | from .utils import * 6 | 7 | 8 | def test_smtlib_get_mutators(): 9 | d = mutators_smtlib.get_mutators() 10 | assert isinstance(d, dict) 11 | assert len(d) == 11 12 | 13 | 14 | def test_smtlib_checksat_assuming(): 15 | m = mutators_smtlib.CheckSatAssuming() 16 | assert isinstance(str(m), str) 17 | assert not m.filter(Node(Node('check-sat'))) 18 | assert not m.filter(Node('assert', 'true')) 19 | assert m.filter(Node('check-sat-assuming', 'true')) 20 | assert m.filter(Node('check-sat-assuming', ('xor', 'x', 'y'))) 21 | 22 | assert check_mutations(m, Node('check-sat-assuming', 'true'), 23 | [Node(Node('check-sat'))]) 24 | assert check_mutations(m, Node('check-sat-assuming', ('xor', 'x', 'y')), 25 | [Node(Node('check-sat'))]) 26 | 27 | 28 | def test_smtlib_eliminate_variable(): 29 | m = mutators_smtlib.EliminateVariable() 30 | assert isinstance(str(m), str) 31 | # special case where 'x' occurs in the replacement 32 | eq = Node('=', 'x', ('*', 'x', 'y')) 33 | assert m.filter(eq) 34 | assert check_global_mutations(m, eq, eq, []) 35 | eq = Node('=', 'x', ('*', 'y', 'y')) 36 | exprs = [ 37 | Node('declare-const', 'x', 'Real'), 38 | Node('define-fun', 'y', (), 'Real', ('*', 'x', 'x')), 39 | Node('define-fun', 'z', (), 'Real', ('*', 'x', 'y')), 40 | Node('assert', eq), 41 | Node('assert', ('>', 'x', 'y')), 42 | ] 43 | smtlib.collect_information(exprs) 44 | assert m.filter(eq) 45 | assert check_global_mutations(m, eq, exprs, [[ 46 | Node('declare-const', 'x', 'Real'), 47 | Node('define-fun', 'y', (), 'Real', 48 | ('*', ('*', 'y', 'y'), ('*', 'y', 'y'))), 49 | Node('define-fun', 'z', (), 'Real', ('*', ('*', 'y', 'y'), 'y')), 50 | Node('assert', ('=', ('*', 'y', 'y'), ('*', 'y', 'y'))), 51 | Node('assert', ('>', ('*', 'y', 'y'), 'y')), 52 | ]]) 53 | 54 | eq = Node('=', 'x', 'y', 'a', ('*', 'y', 'y'), ('+', 'a', 'b')) 55 | exprs = [Node('assert', ('>', 'x', 'y'))] 56 | smtlib.collect_information(exprs) 57 | assert m.filter(eq) 58 | assert check_global_mutations(m, eq, exprs, [ 59 | [Node('assert', ('>', 'y', 'y'))], 60 | [Node('assert', ('>', 'a', 'y'))], 61 | [Node('assert', ('>', ('*', 'y', 'y'), 'y'))], 62 | [Node('assert', ('>', ('+', 'a', 'b'), 'y'))], 63 | [Node('assert', ('>', 'x', 'x'))], 64 | [Node('assert', ('>', 'x', 'a'))], 65 | [Node('assert', ('>', 'x', ('+', 'a', 'b')))], 66 | ]) 67 | 68 | 69 | def test_smtlib_inline_define_fun(): 70 | smtlib.reset_information() 71 | m = mutators_smtlib.InlineDefinedFuns() 72 | assert isinstance(str(m), str) 73 | # special case where we should not attempt to inline into itself 74 | inner = Node('x') 75 | n = Node('define-fun', 'x', (), 'Bool', ('not', inner)) 76 | smtlib.collect_information([n]) 77 | assert m.filter(inner) 78 | assert check_mutations(m, inner, []) 79 | 80 | n = Node('define-fun', 'x', (), 'Bool', inner) 81 | smtlib.collect_information([n]) 82 | assert m.mutations(inner) == [] 83 | 84 | inner = Node('x') 85 | expr = Node(inner, 'a', 'b') 86 | smtlib.collect_information([ 87 | Node('define-fun', 'x', (('y', 'Bool'), ('z', 'Bool')), 'Bool', 88 | ('and', 'y', 'z')) 89 | ]) 90 | assert m.filter(expr) 91 | assert check_mutations(m, expr, [Node('and', 'a', 'b')]) 92 | smtlib.reset_information() 93 | 94 | 95 | def test_smtlib_introduce_fresh_variable(): 96 | smtlib.reset_information() 97 | m = mutators_smtlib.IntroduceFreshVariable() 98 | assert isinstance(str(m), str) 99 | 100 | decl = [ 101 | Node('declare-const', 'x', 'Bool'), 102 | Node('declare-const', 'y', ('_', 'BitVec', 8)), 103 | Node('declare-const', 'z', ('_', 'BitVec', 8)), 104 | ] 105 | smtlib.collect_information(decl) 106 | term = Node('not', 'x') 107 | exprs = [*decl, Node('assert', term)] 108 | assert not m.filter(Node('false')) 109 | assert not m.filter(Node('x')) 110 | assert m.filter(term) 111 | assert check_global_mutations(m, term, exprs, [[ 112 | Node('declare-const', f'x{term.id}__fresh', 'Bool'), *decl, 113 | Node('assert', f'x{term.id}__fresh') 114 | ]]) 115 | 116 | # actually, it is much more important to check that this mutator 117 | # does *not* apply for various cases, as it quickly leads to cycles. 118 | assert not m.filter(Node('false')) 119 | assert not m.filter(Node('1.0')) 120 | assert not m.filter(Node('_', 'bv123', '8')) 121 | assert not m.filter(Node('y')) 122 | assert not m.filter(Node(('_', 'zero_extend', '8'), 'y')) 123 | assert not m.filter(Node(('_', 'zero_extend', '0'), 'y')) 124 | assert m.filter(Node('bvor', 'y', 'z')) 125 | assert m.filter(Node('bvor', 'y', 'y')) 126 | 127 | 128 | def test_smtlib_let_elimination(): 129 | m = mutators_smtlib.LetElimination() 130 | assert isinstance(str(m), str) 131 | assert not m.filter(Node('and', 'x', 'y')) 132 | 133 | n = Node('let', (('a', 'x'), ('b', 'y')), ('and', 'a', 'b')) 134 | assert m.filter(n) 135 | assert check_mutations(m, n, [('and', 'a', 'b')]) 136 | 137 | n = Node( 138 | 'let', 139 | (('a', 'x'), ('b', 'y')), 140 | ) 141 | assert m.filter(n) 142 | assert check_mutations(m, n, []) 143 | 144 | 145 | def test_smtlib_let_substitution(): 146 | m = mutators_smtlib.LetSubstitution() 147 | assert isinstance(str(m), str) 148 | assert not m.filter(Node('and', 'x', 'y')) 149 | n = Node('let', (('a', 'x'), ('b', 'y')), ('and', 'a', 'b')) 150 | assert m.filter(n) 151 | assert check_mutations(m, n, [('let', (('a', 'x'), ('b', 'y')), 152 | ('and', 'x', 'b')), 153 | ('let', (('a', 'x'), ('b', 'y')), 154 | ('and', 'a', 'y'))]) 155 | n = Node('let', (('a', 'x'), ('b', 'y')), ('and', 'abc', 'def')) 156 | assert m.filter(n) 157 | assert check_mutations(m, n, []) 158 | 159 | n = Node('let', (('a', 'x'), ('b', 'y'))) 160 | assert m.filter(n) 161 | assert check_mutations(m, n, []) 162 | 163 | 164 | def test_smtlib_remove_annotation(): 165 | m = mutators_smtlib.RemoveAnnotation() 166 | assert isinstance(str(m), str) 167 | 168 | assert not m.filter(Node('x')) 169 | assert not m.filter(Node('and', 'x', 'y')) 170 | assert m.filter(Node('!', 'x', ':foo')) 171 | assert m.filter(Node('!', ('and', 'x', 'y'), ':foo', 'bar')) 172 | assert check_mutations(m, Node('!', 'x', ':foo'), [('x')]) 173 | assert check_mutations(m, Node('!', ('and', 'x', 'y'), ':foo', 'bar'), 174 | [('and', 'x', 'y')]) 175 | 176 | 177 | def test_smtlib_simplify_logic(): 178 | m = mutators_smtlib.SimplifyLogic() 179 | assert isinstance(str(m), str) 180 | 181 | assert not m.filter(Node('assert', 'x')) 182 | assert m.filter(Node('set-logic', 'QF_NRA')) 183 | assert check_mutations(m, Node('set-logic', 'QF_NRA'), 184 | [('set-logic', 'QF_LRA')]) 185 | assert check_mutations(m, Node('set-logic', 'QF_BVFPLRAS'), [ 186 | ('set-logic', 'QF_FPLRAS'), 187 | ('set-logic', 'QF_BVLRAS'), 188 | ('set-logic', 'QF_BVFPLRA'), 189 | ('set-logic', 'QF_BVFPS'), 190 | ]) 191 | 192 | 193 | def test_smtlib_quoted_symbols(): 194 | m = mutators_smtlib.SimplifyQuotedSymbols() 195 | assert isinstance(str(m), str) 196 | assert not m.filter(Node('x')) 197 | assert m.filter(Node('|x|')) 198 | assert not m.filter(Node('|"x|')) 199 | assert not m.filter(Node('and', 'x', 'y')) 200 | assert not m.filter(Node('and', '|x|', 'y')) 201 | assert check_mutations(m, Node('|x|'), ['x']) 202 | assert check_global_mutations(m, Node('|x|'), Node('assert', '|x|'), 203 | [Node('assert', 'x')]) 204 | 205 | 206 | def test_smtlib_simplify_symbol_names(): 207 | m = mutators_smtlib.SimplifySymbolNames() 208 | assert isinstance(str(m), str) 209 | assert m.filter(Node('declare-const', 'x', 'Real')) 210 | assert not m.filter(Node('declare-const', 'false', 'Real')) 211 | assert m.filter(Node('declare-const', 'x', 'Real')) 212 | exprs = [ 213 | Node('declare-const', 'abcdef', 'Bool'), 214 | Node('declare-datatype', 'Color', (('red', ), ('|green|', ))), 215 | Node('assert', 'abcdef'), 216 | Node('assert', 217 | ('exists', (('xy', 'Bool'), ('z', 'Bool')), ('and', 'xy', 'z'))), 218 | ] 219 | smtlib.collect_information(exprs) 220 | assert check_global_mutations(m, exprs[0], exprs, [ 221 | apply_simp(exprs, Simplification({Node('abcdef'): Node('abc')}, [])), 222 | apply_simp(exprs, Simplification({Node('abcdef'): Node('abcde')}, [])), 223 | apply_simp(exprs, Simplification({Node('abcdef'): Node('bcdef')}, [])) 224 | ]) 225 | assert check_global_mutations(m, exprs[1], exprs, [ 226 | apply_simp(exprs, Simplification({Node('Color'): Node('Co')}, [])), 227 | apply_simp(exprs, Simplification({Node('Color'): Node('Colo')}, [])), 228 | apply_simp(exprs, Simplification({Node('Color'): Node('olor')}, [])), 229 | apply_simp(exprs, Simplification({Node('red'): Node('re')}, [])), 230 | apply_simp(exprs, Simplification({Node('red'): Node('ed')}, [])), 231 | apply_simp(exprs, Simplification({Node('|green|'): Node('|gr|')}, [])), 232 | apply_simp(exprs, Simplification({Node('|green|'): Node('|gree|')}, 233 | [])), 234 | apply_simp(exprs, Simplification({Node('|green|'): Node('|reen|')}, 235 | [])) 236 | ]) 237 | assert check_global_mutations(m, exprs[3][1], exprs, [ 238 | apply_simp(exprs, Simplification({Node('xy'): Node('x')}, [])), 239 | apply_simp(exprs, Simplification({Node('xy'): Node('y')}, [])) 240 | ]) 241 | 242 | 243 | def test_smtlib_remove_recursive_function(): 244 | m = mutators_smtlib.RemoveRecursiveFunction() 245 | assert isinstance(str(m), str) 246 | assert not m.filter(Node('x')) 247 | n = Node( 248 | 'define-funs-rec', 249 | ( 250 | ('a', (), 'Bool'), 251 | ('b', (), 'Bool'), 252 | ), 253 | ( 254 | 'true', 255 | 'a', 256 | ), 257 | ) 258 | assert m.filter(n) 259 | assert check_mutations(m, n, [ 260 | Node('define-funs-rec', (('b', (), 'Bool'), ), ('a', )), 261 | Node('define-funs-rec', (('a', (), 'Bool'), ), ('true', )), 262 | ]) 263 | -------------------------------------------------------------------------------- /ddsmt/tests/test_mutators_strings.py: -------------------------------------------------------------------------------- 1 | from ..nodes import Node 2 | from .. import mutators_strings 3 | from .utils import * 4 | from ..mutator_utils import * 5 | 6 | 7 | def test_strings_get_mutators(): 8 | d = mutators_strings.get_mutators() 9 | assert isinstance(d, dict) 10 | assert len(d) == 5 11 | 12 | 13 | def test_strings_is_relevant(): 14 | assert mutators_strings.is_relevant(Node('declare-const', 'x', 'String')) 15 | assert mutators_strings.is_relevant(Node('declare-fun', 'x', (), 'String')) 16 | assert not mutators_strings.is_relevant(Node('declare-const', 'x', 'Real')) 17 | assert not mutators_strings.is_relevant( 18 | Node('declare-fun', 'x', (), 'Real')) 19 | assert not mutators_strings.is_relevant(Node()) 20 | assert not mutators_strings.is_relevant(Node('assert', 'x')) 21 | 22 | 23 | def test_strings_seqnthunit(): 24 | expr = Node('seq.nth', ('seq.unit', 'x'), 'y') 25 | target = Node('x') 26 | m = mutators_strings.SeqNthUnit() 27 | assert isinstance(str(m), str) 28 | assert m.filter(expr) 29 | assert check_mutations(m, expr, [target]) 30 | 31 | 32 | def test_strings_simplify_constant(): 33 | expr = Node('"abc123"') 34 | gexpr = Node('define-fun', 's', (), 'String', '"abc123"') 35 | m = mutators_strings.StringSimplifyConstant() 36 | assert isinstance(str(m), str) 37 | assert m.filter(expr) 38 | assert check_mutations(m, expr, [ 39 | Node('""'), 40 | Node('"abc"'), 41 | Node('"123"'), 42 | Node('"bc123"'), 43 | Node('"abc12"') 44 | ]) 45 | 46 | assert check_global_mutations(m, expr, gexpr, [ 47 | apply_simp(gexpr, Simplification({Node('"abc123"'): Node('""')}, [])), 48 | apply_simp(gexpr, Simplification({Node('"abc123"'): Node('"abc"')}, 49 | [])), 50 | apply_simp(gexpr, Simplification({Node('"abc123"'): Node('"123"')}, 51 | [])), 52 | apply_simp(gexpr, 53 | Simplification({Node('"abc123"'): Node('"bc123"')}, [])), 54 | apply_simp(gexpr, 55 | Simplification({Node('"abc123"'): Node('"abc12"')}, [])) 56 | ]) 57 | 58 | assert check_mutations(m, Node('"abcdefg\\uab"'), [ 59 | Node('""'), 60 | Node('"abcde"'), 61 | Node('"fg\\uab"'), 62 | Node('"abcdefg"'), 63 | Node('"abcdeuab"'), 64 | Node('"abfg\\uab"'), 65 | Node('"cdefg\\uab"'), 66 | Node('"bcdefg\\uab"'), 67 | Node('"abcdefg\\ua"'), 68 | ]) 69 | 70 | 71 | def test_strings_replace_all(): 72 | expr = Node('str.replace_all', 'x', 'from', 'to') 73 | target = Node('str.replace', 'x', 'from', 'to') 74 | m = mutators_strings.StringReplaceAll() 75 | assert isinstance(str(m), str) 76 | assert m.filter(expr) 77 | assert check_mutations(m, expr, [target]) 78 | 79 | 80 | def test_strings_indexof_not_found(): 81 | expr = Node('str.indexof', 'x', '0', '10') 82 | target = Node('-', '1') 83 | m = mutators_strings.StringIndexOfNotFound() 84 | assert isinstance(str(m), str) 85 | assert m.filter(expr) 86 | assert check_mutations(m, expr, [target]) 87 | 88 | 89 | def test_strings_contains_to_concat(): 90 | expr = Node('str.contains', 'x', 'y') 91 | exprs = [ 92 | Node('set-logic', 'QF_S'), 93 | Node('declare-const', 'x', 'String'), 94 | Node('declare-fun', 'y', (), 'String'), 95 | Node('assert', expr), 96 | ] 97 | target = [ 98 | Node('set-logic', 'QF_S'), 99 | Node('declare-const', 'x_prefix', 'String'), 100 | Node('declare-const', 'x_suffix', 'String'), 101 | Node('declare-const', 'x', 'String'), 102 | Node('declare-fun', 'y', (), 'String'), 103 | Node('assert', ('=', 'x', ('str.++', 'x_prefix', 'y', 'x_suffix'))), 104 | ] 105 | m = mutators_strings.StringContainsToConcat() 106 | assert isinstance(str(m), str) 107 | assert m.filter(expr) 108 | assert check_global_mutations(m, expr, exprs, [target]) 109 | -------------------------------------------------------------------------------- /ddsmt/tests/test_nodes.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from .. import nodeio 4 | from .. import nodes 5 | from ..nodes import Node 6 | 7 | 8 | def test_constructor(): 9 | assert Node().data == () 10 | assert Node(15).data == '15' 11 | assert Node('foobar').data == 'foobar' 12 | assert Node('a', 15, ('b', 13)).data == ('a', '15', ('b', '13')) 13 | assert Node(()).data == ((), ) 14 | 15 | 16 | def test_eq(): 17 | assert not Node('x') == 'y' 18 | assert not Node('declare-const', 'x', 19 | 'Int') == ('declare-const', 'y', 'Int') 20 | assert not Node('x') == Node('y') 21 | assert Node('x') == 'x' 22 | assert Node('declare-const', 'x', 'Int') == ('declare-const', 'x', 'Int') 23 | assert Node('x') == Node('x') 24 | assert Node('declare-const', 'x', 'Int') == Node('declare-const', 'x', 25 | 'Int') 26 | assert Node(Node('pop')) == (Node('pop'), ) 27 | assert Node('pop', 1) == ('pop', 1) 28 | 29 | assert not Node('#b101') == Node('#b100') 30 | assert not Node('#b101') == 'x' 31 | assert not Node('#b101') == ('x', ) 32 | assert not Node('#b101') == Node('x') 33 | 34 | assert Node('#b101') == '#b101' 35 | assert not Node('#b101') == (Node('#b101'), ) 36 | assert Node('#b101') == Node('#b101') 37 | assert Node('#x1af') == '#x1af' 38 | assert not Node('#x1af') == (Node('#x1af'), ) 39 | assert Node('#x1af') == Node('#x1af') 40 | 41 | assert not Node('#x1af') == '(_ bv431 12)' 42 | assert not Node('#b0101') == ('_', 'bv5', 4) 43 | assert not Node('_', 'bv5', 4) == ('#b0101', ) 44 | assert not Node('_', 'bv431', 12) == ('#x1af', ) 45 | 46 | assert not Node('#b110101111') == '#x1af' 47 | assert not Node('#b000110101111') == ('#x1af', ) 48 | assert not Node('#x1af') == ('#b000110101111', ) 49 | 50 | # can't handle any of these cases without BV const normalization 51 | # each of these denotes equal values 52 | assert not Node('#b0101') == '(_ bv5 4)' 53 | assert not Node('#b0101') == Node('_', 'bv5', 4) 54 | assert not Node('#x1af') == ('_', 'bv431', 12) 55 | assert not Node('#x1af') == Node('_', 'bv431', 12) 56 | assert not Node('_', 'bv0', '4') == '#b0000' 57 | assert not Node(Node('_'), Node('bv5'), Node('4')) == '#b0101' 58 | assert not Node('_', 'bv5', 4) == Node('#b0101') 59 | assert not Node('_', 'bv431', 12) == '#x1af' 60 | assert not Node('_', 'bv431', 12) == Node('#x1af') 61 | assert not [Node('_', 'bv5', 4)] == ['#b0101'] 62 | assert not Node('#b000110101111') == '#x1af' 63 | assert not Node('#b000110101111') == Node('#x1af') 64 | assert not Node('#x1af') == '#b000110101111' 65 | assert not Node('#x1af') == Node('#b000110101111') 66 | 67 | 68 | def test_is_leaf(): 69 | assert Node('x').is_leaf() 70 | assert not Node('not', 'x').is_leaf() 71 | 72 | 73 | def test_has_ident(): 74 | assert not Node('x').has_ident() 75 | assert Node('_', 'BitVec', 3).has_ident() 76 | assert Node('assert', 'x', 'x').has_ident() 77 | 78 | 79 | def test_get_ident(): 80 | with pytest.raises(AssertionError): 81 | Node('x').get_ident() 82 | assert Node('_', 'BitVec', 3).get_ident() == '_' 83 | assert Node('assert', 'x', 'x').get_ident() == 'assert' 84 | 85 | 86 | def test_pickle(): 87 | import pickle 88 | # test __getstate__ and __setstate__ which are used by pickle 89 | n = Node('not', ('and', 'x', 'y')) 90 | assert pickle.loads(pickle.dumps(n)) == n 91 | 92 | 93 | def test_parse_smtlib(): 94 | assert list(nodeio.parse_smtlib('(reset)')) == [Node(Node('reset'))] 95 | assert list(nodeio.parse_smtlib('(assert (> x y))')) == [ 96 | Node('assert', ('>', 'x', 'y')) 97 | ] 98 | input = """ 99 | (set-logic QF_NRA) 100 | (set-option :source |just for testing|) 101 | (define-fun x () String "abc""123") 102 | ; now some other nonsense 103 | (declare-const y Real 104 | ; some other comment 105 | ) 106 | (assert (= (* y y) y)) 107 | (check-sat) 108 | """ 109 | assert list(nodeio.parse_smtlib(input)) == [ 110 | Node('set-logic', 'QF_NRA'), 111 | Node('set-option', ':source', '|just for testing|'), 112 | Node('define-fun', 'x', (), 'String', '"abc""123"'), 113 | Node('; now some other nonsense\n'), 114 | Node('declare-const', 'y', 'Real', '; some other comment\n'), 115 | Node('assert', ('=', ('*', 'y', 'y'), 'y')), 116 | Node(Node('check-sat')), 117 | ] 118 | 119 | assert list( 120 | nodeio.parse_smtlib("(set-option :source |just for testing")) == [] 121 | assert list(nodeio.parse_smtlib("(set-option :source testing")) == [] 122 | 123 | 124 | def test_dfs(): 125 | exprs = [ 126 | Node('assert', ('>', 'x', 'y')), 127 | Node('assert', ('=', ('*', 'y', 'y'), 'y')), 128 | ] 129 | assert list(nodes.dfs(exprs)) == [('assert', ('>', 'x', 'y')), 'assert', 130 | ('>', 'x', 'y'), '>', 'x', 'y', 131 | ('assert', ('=', ('*', 'y', 'y'), 'y')), 132 | 'assert', ('=', ('*', 'y', 'y'), 'y'), 133 | '=', ('*', 'y', 'y'), '*', 'y', 'y', 'y'] 134 | assert list(nodes.dfs(exprs, max_depth=2)) == [ 135 | ('assert', ('>', 'x', 'y')), 'assert', ('>', 'x', 'y'), 136 | ('assert', ('=', ('*', 'y', 'y'), 'y')), 'assert', 137 | ('=', ('*', 'y', 'y'), 'y') 138 | ] 139 | 140 | 141 | def test_substitute(): 142 | x = Node('x') 143 | expr = Node('assert', ('>', x, 'y')) 144 | exprs = [expr, Node('assert', ('=', ('*', 'x', 'y'), 'y'))] 145 | 146 | assert nodes.substitute(x, {x: Node('z')}) == Node('z') 147 | assert nodes.substitute(x, {x.id: Node('z')}) == Node('z') 148 | assert nodes.substitute(x, {x.id: None}) is None 149 | assert nodes.substitute(x, {Node('y'): None}) == x 150 | 151 | assert nodes.substitute(expr, 152 | {x: Node('z')}) == Node('assert', ('>', 'z', 'y')) 153 | assert nodes.substitute(expr, 154 | {x.id: Node('z')}) == Node('assert', 155 | ('>', 'z', 'y')) 156 | assert nodes.substitute(expr, {x.id: None}) == Node('assert', ('>', 'y')) 157 | assert nodes.substitute(expr, {expr.id: Node('z')}) == Node('z') 158 | 159 | assert nodes.substitute(exprs, {x: Node('z')}) == [ 160 | Node('assert', ('>', 'z', 'y')), 161 | Node('assert', ('=', ('*', 'z', 'y'), 'y')), 162 | ] 163 | assert nodes.substitute(exprs, {x.id: Node('z')}) == [ 164 | Node('assert', ('>', 'z', 'y')), 165 | Node('assert', ('=', ('*', 'x', 'y'), 'y')), 166 | ] 167 | assert nodes.substitute(exprs, {x.id: None}) == [ 168 | Node('assert', ('>', 'y')), 169 | Node('assert', ('=', ('*', 'x', 'y'), 'y')), 170 | ] 171 | 172 | 173 | def test_render_smtlib_expression(): 174 | expr = Node('x') 175 | assert nodeio.__write_smtlib_str(expr) == 'x' 176 | assert nodeio.__write_smtlib_pretty_str(expr) == 'x\n' 177 | 178 | expr = Node('; foo') 179 | assert nodeio.__write_smtlib_str(expr) == '\n; foo\n' 180 | assert nodeio.__write_smtlib_pretty_str(expr) == '\n; foo\n' 181 | 182 | expr = Node('declare-const', 'x', 'Real') 183 | assert nodeio.__write_smtlib_str(expr) == '(declare-const x Real)' 184 | assert nodeio.__write_smtlib_pretty_str(expr) == '(declare-const x Real)\n' 185 | 186 | expr = Node('assert', ('and', )) 187 | assert nodeio.__write_smtlib_str(expr) == '(assert (and))' 188 | assert nodeio.__write_smtlib_pretty_str(expr) == '(assert\n (and)\n)\n' 189 | 190 | expr = Node('assert', ('>', 'x', 'y'), ()) 191 | assert nodeio.__write_smtlib_str(expr) == '(assert (> x y) ())' 192 | assert nodeio.__write_smtlib_pretty_str( 193 | expr) == '(assert\n (> x y)\n ()\n)\n' 194 | 195 | 196 | def test_binary_search(): 197 | assert list(nodes.binary_search(4)) == [(2, 4), (0, 2)] 198 | assert list(nodes.binary_search(15)) == [ 199 | (7, 15), 200 | (0, 7), 201 | (11, 15), 202 | (7, 11), 203 | (3, 7), 204 | (0, 3), 205 | ] 206 | -------------------------------------------------------------------------------- /ddsmt/tests/test_options.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from .. import mutators 4 | from .. import options 5 | 6 | 7 | def test_help(capsys): 8 | options.__PARSED_ARGS = None 9 | with pytest.raises(SystemExit): 10 | options.parse_options(mutators, ['--help']) 11 | captured = capsys.readouterr() 12 | assert ('usage: ddsmt [] []' 13 | in captured.out) 14 | 15 | 16 | def test_help_all(capsys): 17 | options.__PARSED_ARGS = None 18 | with pytest.raises(SystemExit): 19 | options.parse_options(mutators, ['--help-all']) 20 | captured = capsys.readouterr() 21 | assert '--[no-]core' in captured.out 22 | 23 | 24 | def test_dump_config(capsys): 25 | options.__PARSED_ARGS = None 26 | with pytest.raises(SystemExit): 27 | options.parse_options(mutators, ['--dump-config']) 28 | captured = capsys.readouterr() 29 | assert "'dump_config': True," in captured.out 30 | 31 | 32 | def test_no_options(capsys): 33 | options.__PARSED_ARGS = None 34 | with pytest.raises(SystemExit): 35 | options.parse_options(mutators, []) 36 | captured = capsys.readouterr() 37 | assert ('error: the following arguments are required: infile, outfile, cmd' 38 | in captured.err) 39 | 40 | 41 | def test_basic(capsys): 42 | options.__PARSED_ARGS = None 43 | options.parse_options(mutators, ['input.smt2', 'output.smt2', 'binary']) 44 | options.parse_options(mutators, 45 | ['--no-core', 'input.smt2', 'output.smt2', 'binary']) 46 | options.parse_options( 47 | mutators, 48 | ['--no-core', '--constants', 'input.smt2', 'output.smt2', 'binary']) 49 | 50 | 51 | def test_cmd_cc(capsys): 52 | options.__PARSED_ARGS = None 53 | options.parse_options( 54 | mutators, 55 | ['-c', 'other-binary', 'input.smt2', 'output.smt2', 'binary']) 56 | 57 | 58 | def test_args(capsys): 59 | options.__PARSED_ARGS = None 60 | with pytest.raises(SystemExit): 61 | options.args() 62 | opts = options.args(['--no-core', 'input.smt2', 'output.smt2', 'binary']) 63 | assert options.args() == opts 64 | -------------------------------------------------------------------------------- /ddsmt/tests/utils.py: -------------------------------------------------------------------------------- 1 | from .. import mutator_utils 2 | 3 | 4 | def check_mutations(mut, node, expected): 5 | assert hasattr(mut, 'mutations') 6 | 7 | mutations = list(mut.mutations(node)) 8 | assert len(mutations) == len(expected) 9 | 10 | for res, exp in zip(mutations, expected): 11 | simp = mutator_utils.apply_simp(node, res) 12 | if simp != exp: 13 | return False 14 | 15 | return True 16 | 17 | 18 | def check_global_mutations(mut, node, exprs, expected): 19 | assert hasattr(mut, 'global_mutations') 20 | 21 | mutations = list(mut.global_mutations(node, exprs)) 22 | assert len(mutations) == len(expected) 23 | 24 | for res, exp in zip(mutations, expected): 25 | simp = mutator_utils.apply_simp(exprs, res) 26 | if simp != exp: 27 | return False 28 | 29 | return True 30 | -------------------------------------------------------------------------------- /ddsmt/tmpfiles.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import os 13 | import shutil 14 | import tempfile 15 | import threading 16 | 17 | from . import options 18 | 19 | # The temporary directory. Automatically deleted upon termination. 20 | __TMPDIR = None 21 | __BINARY = None 22 | __BINARY_CC = None 23 | __FILEEXT = None 24 | 25 | 26 | def init(): 27 | """Create temporary directory, set filenames.""" 28 | global __TMPDIR 29 | global __BINARY 30 | global __BINARY_CC 31 | global __FILEEXT 32 | __TMPDIR = tempfile.TemporaryDirectory(prefix="ddsmt-") 33 | __BINARY = os.path.join(__TMPDIR.name, 'binary') 34 | __BINARY_CC = os.path.join(__TMPDIR.name, 'binary_cc') 35 | __FILEEXT = os.path.splitext(options.args().infile)[1] 36 | 37 | 38 | def copy_binaries(): 39 | """Copy the solver binary from ``cmd`` to our temporary directory. 40 | 41 | If a cross check is defined, also copies this binary. After copying, 42 | it modifies the ``cmd`` argument to use the copied binary. 43 | """ 44 | shutil.copy(options.args().cmd[0], __BINARY) 45 | options.args().cmd[0] = __BINARY 46 | 47 | if options.args().cmd_cc: 48 | shutil.copy(options.args().cmd_cc[0], __BINARY_CC) 49 | options.args().cmd_cc[0] = __BINARY_CC 50 | 51 | 52 | def get_tmp_filename(): 53 | """Return a filename within our temporary directory based on the pid.""" 54 | return os.path.join( 55 | __TMPDIR.name, 56 | f'ddsmt-tmp-{os.getpid()}-{threading.get_ident()}{__FILEEXT}') 57 | 58 | 59 | def copy_to_tmp_file(source): 60 | """Copy the given source file to a temporary file as returned by 61 | ``get_tmp_filename()``.""" 62 | res = get_tmp_filename() 63 | shutil.copy(source, res.name) 64 | return res 65 | -------------------------------------------------------------------------------- /ddsmt/version.py: -------------------------------------------------------------------------------- 1 | # 2 | # ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 3 | # 4 | # This file is part of ddSMT. 5 | # 6 | # Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 7 | # at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 8 | # 9 | # This file is part of ddSMT under the MIT license. See LICENSE for more 10 | # information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 11 | 12 | import os 13 | 14 | 15 | def version_from_git(gitdir): 16 | """Tries to obtain a version string from git based on the last tag.""" 17 | import re 18 | import subprocess 19 | try: 20 | GIT_VERSION = subprocess.check_output( 21 | ['git', 'describe', '--tags'], cwd=gitdir, 22 | stderr=subprocess.PIPE).decode('utf8').strip() 23 | except subprocess.CalledProcessError: 24 | cnum = subprocess.check_output(['git', 'rev-list', 'HEAD', '--count'], 25 | cwd=gitdir).decode('utf8').strip() 26 | cid = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], 27 | cwd=gitdir).decode('utf8').strip() 28 | GIT_VERSION = 'v0.0-{}-g{}'.format(cnum, cid) 29 | if re.match('^v[0-9.]+$', GIT_VERSION) is not None: 30 | return GIT_VERSION[1:] 31 | else: 32 | m = re.match('v([0-9]+)\\.([0-9]+)\\.([0-9]+)-([0-9]+)-g[a-z0-9]+', 33 | GIT_VERSION) 34 | if m is not None: 35 | return '{}.{}.{}.dev{}'.format(m.group(1), m.group(2), 36 | int(m.group(3)) + 1, m.group(4)) 37 | return GIT_VERSION 38 | 39 | 40 | def version_from_package_metadata(): 41 | """Tries to obtain a version string from git based on the pypi package 42 | information.""" 43 | try: 44 | from importlib import metadata 45 | except ImportError: 46 | import importlib_metadata as metadata 47 | return metadata.version('ddSMT') 48 | 49 | 50 | __dot_git = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 51 | '.git') 52 | if os.path.isdir(__dot_git) or os.path.isfile(__dot_git): 53 | VERSION = version_from_git(os.path.split(__dot_git)[0]) 54 | else: 55 | VERSION = version_from_package_metadata() 56 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | build -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | import inspect 4 | import os 5 | import subprocess 6 | import sys 7 | 8 | # add project root path 9 | __root_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') 10 | sys.path.insert(0, __root_dir) 11 | 12 | import ddsmt.version # noqa: E402 13 | 14 | # make sure we can see the last tag 15 | subprocess.call(['git', 'fetch']) 16 | 17 | # basic project information 18 | project = 'ddSMT' 19 | copyright = '2013-2021, the authors of ddSMT' 20 | author = 'The authors of ddSMT' 21 | version = ddsmt.version.VERSION 22 | release = ddsmt.version.VERSION 23 | 24 | extensions = [ 25 | 'sphinx_rtd_theme', 26 | 'sphinx.ext.autodoc', 27 | 'sphinx.ext.autosectionlabel', 28 | 'sphinxcontrib.bibtex', 29 | 'sphinxcontrib.programoutput', 30 | ] 31 | 32 | bibtex_bibfiles = ['literature.bib'] 33 | 34 | autodoc_default_options = { 35 | 'members': None, 36 | 'member-order': 'bysource', 37 | } 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['templates'] 41 | 42 | # The suffix(es) of source filenames. 43 | # You can specify multiple suffix as a list of string: 44 | # 45 | # source_suffix = ['.rst', '.md'] 46 | source_suffix = '.rst' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # The language for content autogenerated by Sphinx. Refer to documentation 52 | # for a list of supported languages. 53 | # 54 | # This is also used if you do content translation via gettext catalogs. 55 | # Usually you set "language" from the command line for these cases. 56 | language = None 57 | 58 | # List of patterns, relative to source directory, that match files and 59 | # directories to ignore when looking for source files. 60 | # This pattern also affects html_static_path and html_extra_path. 61 | exclude_patterns = ['build', 'Thumbs.db', '.DS_Store'] 62 | 63 | # The name of the Pygments (syntax highlighting) style to use. 64 | pygments_style = None 65 | 66 | # -- Options for HTML output ------------------------------------------------- 67 | 68 | # The theme to use for HTML and HTML Help pages. See the documentation for 69 | # a list of builtin themes. 70 | # 71 | html_theme = 'sphinx_rtd_theme' 72 | 73 | # Theme options are theme-specific and customize the look and feel of a theme 74 | # further. For a list of options available for each theme, see the 75 | # documentation. 76 | # 77 | html_theme_options = { 78 | 'display_version': True, 79 | 'collapse_navigation': False, 80 | 'sticky_navigation': True, 81 | } 82 | 83 | # Add any paths that contain custom static files (such as style sheets) here, 84 | # relative to this directory. They are copied after the builtin static files, 85 | # so a file named "default.css" will overwrite the builtin "default.css". 86 | html_static_path = [] 87 | 88 | # Custom sidebar templates, must be a dictionary that maps document names 89 | # to template names. 90 | # 91 | # The default sidebars (for documents that don't match any pattern) are 92 | # defined by theme itself. Builtin themes are using these templates by 93 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 94 | # 'searchbox.html']``. 95 | # 96 | # html_sidebars = {} 97 | 98 | # -- Options for HTMLHelp output --------------------------------------------- 99 | 100 | # Output file base name for HTML help builder. 101 | htmlhelp_basename = 'pydoc' 102 | 103 | 104 | def autodoc_mutators_skip_member(app, what, name, obj, skip, options): 105 | if what == 'module': 106 | if hasattr( 107 | obj, 108 | '__module__') and obj.__module__.startswith('ddsmt.mutators_'): 109 | return not inspect.isclass(obj) 110 | return skip 111 | 112 | 113 | def setup(app): 114 | app.connect('autodoc-skip-member', autodoc_mutators_skip_member) 115 | 116 | 117 | # Allow to use "ddsmt" in "command-output" directives. 118 | os.environ['PATH'] = os.path.join(__root_dir, 119 | "bin") + os.pathsep + os.environ['PATH'] 120 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | We very much welcome any external contribution, for example to improve the 5 | documentation to help other users or to add new mutators that allow for novel 6 | simplifications. 7 | 8 | The easiest way to submit your contribution is to fork `ddSMT on github 9 | `_ and create a pull request from a branch that 10 | contains your changes. 11 | 12 | .. include:: ../THANKS.rst -------------------------------------------------------------------------------- /docs/example/solver: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | grep "declare-const" $2 > /dev/null 2>&1 4 | if [[ $? -eq 0 ]] 5 | then 6 | echo "printing something to stdout" 7 | >&2 echo "error: this message is printed to stderr" 8 | exit 1 9 | fi 10 | exit 0 11 | -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | FAQ 2 | === 3 | 4 | Delta debugging is slow 5 | Read the section about :ref:`how to speed up delta debugging `. 7 | 8 | ddSMT seems to be stuck in an infinite loop 9 | To ensure termination of the minimization procedure, we require that it is 10 | impossible to get into a cycle between applications of mutators. 11 | For this, we need to guarantee that all mutations applied to the input by 12 | **ddSMT** result in a "simpler" input. 13 | Formally, this would require to define a ranking over possible 14 | simplifications and exclude mutations that do not simplify the input 15 | according to this ranking in order to break cycles. 16 | In practice, however, defining such a ranking is difficult. 17 | If you think that **ddSMT** got into a cycle, try option 18 | :code:`--check-loops`. 19 | -------------------------------------------------------------------------------- /docs/guide-developers.rst: -------------------------------------------------------------------------------- 1 | Developer Guide 2 | =============== 3 | 4 | 5 | Performance Profiling 6 | --------------------- 7 | 8 | **ddSMT** provides option :code:`--profile`, which uses :code:`cProfile` to 9 | collect profiling data. 10 | Since :code:`cProfile` has no support for parallelism, **ddSMT** collects 11 | separate profiles for every process and combines them accordingly: 12 | it generates one profile for the main process, and one combined profile for all 13 | subprocesses. 14 | 15 | 16 | Reading profiling data 17 | ^^^^^^^^^^^^^^^^^^^^^^ 18 | 19 | **ddSMT** dumps :code:`cProfile` data as :code:`.profile.prof` (for the main 20 | process) (and :code:`.profile-.prof` for the subprocesses). 21 | **ddSMT** renders this data using :code:`gprof2dot` and :code:`dot` into 22 | :code:`.png` images :code:`profile.png` and :code:`profile-workers.png`. 23 | These images usually give a pretty good intuition about the performance profile. 24 | 25 | .. note:: 26 | Strategy :ref:`ddmin ` sometimes does not delegate work 27 | to worker processes, but decides to run in sequential mode. 28 | If this is the case, the main profile will contain work that you might expect 29 | to actually have been delegated. 30 | 31 | The "solver side" mostly consists of the :code:`checker.check()` function, 32 | which makes the actual call to the command under test. 33 | Calls to :code:`subprocess.communicate()` and :code:`select.poll()` are usually 34 | responsible for the largest chunk of the run time spent on the "ddSMT side". 35 | 36 | The "ddSMT side" is pretty much everything else. 37 | The most expensive parts are usually 38 | 39 | * applying a simplification (:code:`mutator_utils.apply_simp` and :code:`nodes.substitute()`) 40 | * writing inputs to files (:code:`nodeio.write_smtlib_for_checking`) 41 | * pickling (:code:`nodes.__getstate__` and :code:`nodes.__setstate__`) 42 | -------------------------------------------------------------------------------- /docs/guide-mutators.rst: -------------------------------------------------------------------------------- 1 | Mutators 2 | ==================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | Possible input simplifications are generated by so-called **mutators**, 9 | the drivers of input minimization. 10 | Given an S-expression, they may perform small `local changes`, or introduce 11 | `global minimizations` on the input based on that expression. 12 | 13 | Mutator classes must provide the following interface: 14 | 15 | .. code-block:: python3 16 | 17 | class Mutator: 18 | """Some simplification.""" 19 | def filter(self, node): 20 | """Check if mutator can be applied to the given node. 21 | Default: ``True``. 22 | """ 23 | return True 24 | 25 | def mutations(self, node): 26 | """Create a list of local mutations of the given node. 27 | Default: [] 28 | """ 29 | return [] 30 | 31 | def global_mutations(self, node, input_): 32 | """Create a list of mutations of the whole input. 33 | Default: [] 34 | """ 35 | return [] 36 | 37 | def __str__(self): 38 | """Return a description of this mutator. 39 | This is (also) used for the command line help message. 40 | """ 41 | return "some simplification" 42 | 43 | Each mutator implements a :code:`filter` method, which checks if the 44 | mutator is applicable to the given S-expression. 45 | If it is, the mutator can be queried to suggest (a list of) possible local 46 | (:code:`mutations()`) and global simplifications (:code:`global_mutations()`). 47 | 48 | .. note:: 49 | Mutators are not required to be equivalence or satisfiability 50 | preserving. 51 | They may extract semantic information from the input when needed, e.g., 52 | to infer the sort of a term, to query the set of declared or defined 53 | symbols, to extract indices of indexed operators, and more. 54 | 55 | Method :code:`mutations()` constructs **local** replacements for a given 56 | **node**. 57 | Method :code:`global_mutations()` constructs **global** replacements for the 58 | **whole input**, given both a specific node and the current input. 59 | The concept of `local mutations` is pretty much self-explanatory, a 60 | :code:`node` is replaced with a simpler node, for example, :code:`(neg (neg 61 | a))` is replaced with :code:`a`. 62 | For `global mutations`, on the other hand, some :code:`node` triggers a 63 | simplification that needs to be applied to the whole input at once. 64 | For example, when renaming variables or replacing constants that occur 65 | multiple times, or when simplifications require adding declarations of 66 | new symbols. 67 | 68 | .. note:: 69 | Please refer to section :ref:`writing new mutators` for details on **how to 70 | implement new mutators**. 71 | 72 | Below follows a list of all available mutators, grouped by their main concern: 73 | :ref:`generic mutators ` that apply mutations based on the 74 | node structure, :ref:`SMT-LIB mutators ` that deal with 75 | certain SMT-LIB constructs, and mutators for individual SMT-LIB theories 76 | (:ref:`arithmetic `, :ref:`bit-vectors `, :ref:`boolean `, :ref:`floating-point arithmetic 78 | ` and :ref:`strings `). 79 | 80 | How to Enable and Disable Mutators 81 | ---------------------------------- 82 | 83 | By default, all mutators are enabled. 84 | 85 | Mutators can be explicitly enabled with command line options of the form 86 | :code:`--`, and disabled with options of the form 87 | :code:`--no-`, for example, :code:`--no-eliminate-variables`. 88 | 89 | All mutators from a specific group can by enabled and disabled similarly with 90 | :code:`--` and :code:`--no-`, for example, :code:`--no-strings`. 91 | 92 | If you want to enable a single mutator or mutator group, 93 | combine disabling all mutators (:code:`--disable-all`) with enabling a single 94 | mutator or group. 95 | For example, :code:`--disable-all --bool-de-morgan` or :code:`--disable-all 96 | --strings`). 97 | 98 | Generic Mutators 99 | ---------------- 100 | .. automodule:: ddsmt.mutators_core 101 | 102 | SMT-LIB Mutators 103 | ---------------- 104 | .. automodule:: ddsmt.mutators_smtlib 105 | 106 | Arithmetic Mutators 107 | ------------------- 108 | .. automodule:: ddsmt.mutators_arithmetic 109 | 110 | Bit-vector Mutators 111 | ------------------- 112 | .. automodule:: ddsmt.mutators_bv 113 | 114 | Boolean Mutators 115 | ---------------- 116 | .. automodule:: ddsmt.mutators_boolean 117 | 118 | Datatype Mutators 119 | ----------------- 120 | .. automodule:: ddsmt.mutators_datatypes 121 | 122 | Floating-Point mutators 123 | ----------------------- 124 | .. automodule:: ddsmt.mutators_fp 125 | 126 | String mutators 127 | --------------- 128 | .. automodule:: ddsmt.mutators_strings 129 | 130 | 131 | Writing New Mutators 132 | -------------------- 133 | 134 | If you need a certain simplification that is not covered by existing mutators, 135 | it's easy to add your own mutator. 136 | If it is of general interest, we'd be happy if you contributed it back to 137 | **ddSMT**. 138 | The following instructions aim to provide a guide on what needs to be done and 139 | what you should consider when adding a new mutator. 140 | 141 | Generally, mutators are organized into mutators for general purpose 142 | (:ref:`generic ` and :ref:`SMT-LIB ` 143 | mutators), and mutators for specific theories 144 | (:ref:`arithmetic `, :ref:`bit-vectors `, :ref:`boolean `, :ref:`floating-point arithmetic 146 | ` and :ref:`strings `). 147 | 148 | 1. Identify into which **group** your new mutator fits best. 149 | 150 | 2. Determine if you need to have a **global** view on the input, or if 151 | **local** mutations of a single node suffice. Prefer local mutations over 152 | global mutations if possible. 153 | 154 | 3. Implement the mutator following these guidelines: 155 | 156 | - Use the above :code:`Mutator` class as a **template**. 157 | - **Add** your code to the corresponding :code:`mutators_.py` file. 158 | - **Register** your mutator in the :code:`get_mutators()` function towards 159 | the end of the file. 160 | - Make sure your mutator is reasonably **fast**. 161 | - Make sure that your mutator (mostly) generates **valid** SMT-LIB constructs. 162 | - Make sure to return a list of :code:`nodes.Node` objects. 163 | - If your mutator returns a large number of candidates, do not return them 164 | as a list from :code:`mutations()` and :code:`global_mutations()`. Instead 165 | use :code:`yield` to turn your mutator into a **generator**. 166 | - Add some **unit tests** in :code:`ddsmt/tests/`). This way you can document 167 | what your mutator does, and ensure that it keeps working in the future. 168 | Also, test that it does not apply to unrelated nodes (i.e., 169 | :code:`filter()` returns :code:`False`). 170 | 171 | .. note:: 172 | Try do identify and **avoid** possible mutation **cycles**. Introducing such 173 | cycles may result in **non-termination**. 174 | -------------------------------------------------------------------------------- /docs/guide-performance.rst: -------------------------------------------------------------------------------- 1 | How to Speed Up Delta Debugging 2 | =============================== 3 | 4 | Overall run time of **ddSMT** can be split into two parts: the run time 5 | contributed by **ddSMT** code itself (the :ref:`"ddSMT side" `), and the run time of the calls to the command under test (the 7 | :ref:`"solver side" `). 8 | Depending on which side is the bottleneck, there are different strategies 9 | to speed up **ddSMT** delta debugging sessions. 10 | 11 | How to Identify Which Side is The Bottleneck 12 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 13 | 14 | Most of the time, which side is the bottleneck can be identified by looking at 15 | the CPU usage of the 16 | **ddSMT** processes when run with option :code:`-jn`. 17 | If the parent process is using a lot of CPU but the child processes are not, 18 | the :ref:`ddSMT side is slow `. 19 | If the parent process is mostly idle but the child processes are at full 20 | capacity, the :ref:`solver side is slow `. 21 | If you are not sure, you may want to look at :ref:`profiling data `. 23 | 24 | 25 | The Solver Side is Slow 26 | ^^^^^^^^^^^^^^^^^^^^^^^ 27 | 28 | If the bottleneck is on the solver side, there are a several options: 29 | 30 | Make the solver faster 31 | Sometimes, it is simple to make the solver faster, for example when the 32 | binary was compiled in debug mode, but release mode would also work. 33 | 34 | Increase parallelism 35 | **ddSMT** supports parallel checks via the :code:`-jn` option. 36 | Increase ``n`` to run more checks in parallel, if your machine has unused 37 | cores. 38 | 39 | Reduce the time limit 40 | The time limit for executing the command under test is calculated once, 41 | based on the run time of the golden run. 42 | It often happens that the solver run time improves after some initial 43 | simplifications. 44 | If the solver is already faster on intermediate output, restart **ddSMT** 45 | on this output and the new time limit may be significantly slower. 46 | 47 | Call the solver less often 48 | There are several ways to avoid calls to the solver that may not yield 49 | simplifications anyway. 50 | For example, you can disable certain mutators, or you may want to initially 51 | disable most theories (options :code:`--no-arithmetic`, :code:`--no-bv` and 52 | friends), since theory-specific simplifications are usually less powerful 53 | in the initial simplification phase. 54 | The **ddSMT** strategies already implement some staging to prioritize 55 | mutators with more impact in the initial phases, but there may be room 56 | for improvement in some cases, depending on the input. 57 | 58 | 59 | The ddSMT Side is Slow 60 | ^^^^^^^^^^^^^^^^^^^^^^ 61 | 62 | If the bottleneck is on the **ddSMT** side, the following options may help: 63 | 64 | Use `PyPy` 65 | Running `CPython` (the standard Python implementation) is known to be 66 | comparably slow, as it is interpreted at runtime. 67 | `PyPy `_ is an alternative to `CPython` that uses a 68 | just-in-time compiler which can speed up the execution of Python code 69 | drastically. 70 | 71 | Decrease parallelism 72 | There is a certain overhead to running checks in parallel. 73 | Decreasing the number of processes may help. 74 | 75 | Combine ddSMT with alternative delta debugging tools 76 | Sometimes, delta debugging tools that are more general purpose can help 77 | to get an initial reduction faster, which can then be further simplified 78 | with **ddSMT**. 79 | In particular, it may be worthwhile to try `linedd 80 | `_ or `ddsexpr 81 | `_ which both have a significantly lower 82 | overhead. 83 | -------------------------------------------------------------------------------- /docs/guide-scenarios.rst: -------------------------------------------------------------------------------- 1 | Common Scenarios 2 | ================ 3 | 4 | Within the SMT community, the notion of **failure** is generally defined as 5 | one of the following: 6 | 7 | * **abnormal termination** or **crashes** (including segmentation 8 | faults and assertion failures) 9 | * **performance regressions** (solver performs significantly worse on an 10 | input than reference solver) 11 | * **unsoundness** (solver answers `sat` instead of `unsat` and vice versa) 12 | * **model unsoundness** (the determined satisfiability model is incorrect) 13 | * **conflict unsoundness** (the determined proof of unsatisfiability is 14 | incorrect) 15 | 16 | `Abnormal termination` and `crashes` are the most common and the default 17 | use case of **ddSMT**. 18 | 19 | `Model unsoundness` and `conflict unsoundness` issues, on the other hand, are 20 | more involved since they typically require some checking mechanism to determine 21 | if a generated model or proof is incorrect. 22 | Most SMT solvers implement such mechanisms and will throw an assertion failure 23 | in debug mode when such a failure is detected, and in that case, these failures 24 | fall into the first category. 25 | However, if they don't, external checking tools are required. 26 | **ddSMT** does not provide such tools. 27 | 28 | For `perfomance regressions` and `unsoundness` issues, **ddSMT** provides 29 | easy-to-use and easy-to-adapt wrapper scripts in directory :code:`scripts`. 30 | In this guide, we show how to debug issues that fall into these two categories 31 | and address how to approach other common scenarios users may face. 32 | 33 | 34 | Debugging Unsoundness 35 | --------------------- 36 | 37 | Unsoundness issues are issues where a solver terminates successfully, but 38 | with the wrong answer (`sat` instead of `unsat` and vice versa). 39 | This is a particular nasty case for delta debugging if the solver cannot detect 40 | the issue (for example, by checking the generated model or proof). 41 | Some solvers check their answer against the expected status of an SMT-LIB input 42 | if it is provided via :code:`(set-info :status ...)`, but for delta debugging 43 | purposes, it is a bad idea to rely on this status to check for soundness since 44 | mutations can flip this status but still reveal the underlying problem. 45 | 46 | A better approach is to check the answer against the answer of a reference 47 | solver. 48 | This can be achieved by either using the builtin ``--cross-check`` option, or 49 | the provided :download:`scripts/results_differs.py 50 | <../scripts/result_differs.py>` script. 51 | 52 | Using Option ``--cross-check`` 53 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 54 | If ``--cross-check`` is given, **ddSMT** checks whether the command 55 | under test and the reference command behave the same as their respective golden 56 | runs (see :ref:`How Behavior is Compared with the Golden Run`), 57 | As with the default use case, cross checking can be refined in terms of what 58 | output to consider via 59 | :code:`--ignore-output-cc` (both `stdout` and `stderr` channels are ignored), 60 | :code:`--match-err-cc` (pattern match `stderr` against a given string) 61 | and :code:`--match-out-cc` (pattern match `stdout` against a given string). 62 | 63 | Using Wrapper Script ``result_differs.py`` 64 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 65 | For cases where the ``--cross-check`` option is not flexible enough, 66 | **ddSMT** provides a wrapper script :download:`scripts/result_differs.py 67 | <../scripts/result_differs.py>`. 68 | The script runs two solvers :code:`A` and :code:`B`, expects them to output 69 | `sat` or `unsat` and compares their output. 70 | This can be useful if you want to allow that the input flips from `sat` 71 | to `unsat` (or vice versa) as long as the solver under test still disagrees. 72 | 73 | This script can be adapted to any other kind of further analysis that you 74 | might need but is not supported by the cross check option, for example, if you 75 | want to compare the output of the two solvers and want to allow any output to 76 | compare models or error messages. 77 | 78 | .. literalinclude:: ../scripts/result_differs.py 79 | :language: python3 80 | :linenos: 81 | 82 | 83 | Debugging Performance Issues 84 | ---------------------------- 85 | 86 | Performance issues are issues where a solver performs significantly worse 87 | than a reference solver. 88 | Typically, this means that the solver under test terminates successfully but 89 | with a significantly higher run time, or doesn't terminate at all within a 90 | given time limit. 91 | Performance issues are similarly nasty to delta debug as unsoundness issues. 92 | And as with unsoundness, it usually only makes sense to debug performance 93 | issues with respect to a reference solver. 94 | 95 | **ddSMT** provides a wrapper script 96 | :download:`scripts/compare_time.py <../scripts/compare_time.py>` 97 | for debugging performance issues. 98 | It runs two solvers :code:`A` and :code:`B` and checks whether the slower one 99 | is (still) much slower than the faster one. 100 | Since it outputs some information about the run time of the two solvers, it 101 | should be used in combination with option :code:`--ignore-output`. 102 | 103 | .. literalinclude:: ../scripts/compare_time.py 104 | :language: python3 105 | :linenos: 106 | 107 | 108 | Eliminate :code:`let` Binders 109 | ----------------------------- 110 | 111 | SMT-LIB :code:`let` binders are notorious for impeding simplification during 112 | delta debugging. 113 | While the mutators that target :code:`let` binders 114 | (:class:`ddsmt.mutators_smtlib.LetElimination` and 115 | :class:`ddsmt.mutators_smtlib.LetSubstitution`) are applied in an early stage 116 | for both strategies **ddmin** and **hierarchical**, 117 | it may still take a while until they are taken care of. 118 | If the input contains a lot of :code:`let` binders, it is sometimes useful to 119 | first eliminate them by using only these two mutators, and then delta debug the 120 | resulting files without let binders. 121 | 122 | .. code-block:: bash 123 | 124 | $ ddsmt --disable-all --let-elimination --let-substitution 125 | 126 | If you are reasonably sure that the issue is unrelated to let binders itself, 127 | you can use option :code:`--unchecked` to not even execute the command under 128 | test. 129 | Note that you should do two separate runs for :code:`let` substitution and 130 | :code:`let` elimination in this case: 131 | 132 | .. code-block:: bash 133 | 134 | $ ddsmt --unchecked --disable-all --let-substitution 135 | $ ddsmt --unchecked --disable-all --let-elimination 136 | 137 | -------------------------------------------------------------------------------- /docs/guide-strategies.rst: -------------------------------------------------------------------------------- 1 | Minimization Strategies 2 | ======================= 3 | 4 | **ddSMT** implements two fundamentally different strategies for input 5 | minimization (**ddmin** and **hierarchical**), which can be combined 6 | into the best of both worlds, strategy **hybrid**. 7 | 8 | Strategy **ddmin** implements a variant of the minimization strategy from 9 | :cite:`Zeller1999`, and is the direct successor of what the original **ddSMT** 10 | from :cite:`Niemetz2013` implemented: it tries to perform mutations on multiple 11 | S-expressions in the input in parallel. 12 | Strategy **hierarchical** performs a simple breadth-first traversal through the 13 | input and applies mutations to every S-expression, one mutation at a time 14 | (similar to what is presented as "hierarchical delta-debugging" in 15 | :cite:`Brummayer2009`). 16 | Strategy **hybrid** is the **default** strategy of **ddSMT**. 17 | It first applies **ddmin** until a fixed point is reached, and then calls 18 | strategy **hierarchical** on the simplified input. 19 | 20 | For a more in-depth analysis of these minimization strategies, see 21 | :cite:`Kremer2021`. 22 | 23 | .. _ddmin-strategy: 24 | 25 | The **ddmin** Strategy 26 | ---------------------- 27 | 28 | Strategy **ddmin** tries to perform simplifications on multiple S-expressions 29 | in the input in parallel. 30 | The algorithm below shows the main loop of **ddmin** in pseudo code. 31 | For each active mutator :code:`M`, the algorithm first collects all 32 | S-expressions in the input that can be simplified by :code:`M` (Line 4). 33 | Simplifications are applied and checked in a fashion similar to the original 34 | `ddmin` algorithm :cite:`Zeller2002`: 35 | 36 | * the set of S-expressions :code:`sexprs` is partitioned into subsets of size 37 | :code:`size` 38 | * each S-expression :code:`e` from :code:`subset` is substituted in the current 39 | :code:`input` (Line 7) with a simplification suggested by mutator :code:`M` 40 | * the resulting simplified input candidate is then checked if it still triggers 41 | the original behavior from the golden run (Line 8) 42 | 43 | Once all subsets of a given size are checked, :code:`sexprs` is updated based 44 | on the current input and partitioned into smaller subsets. 45 | As soon as all subsets of size 1 were checked, these steps are repeated 46 | with the next mutator. 47 | This loop is run until the input cannot be further simplified. 48 | 49 | **ddmin** applies mutators in two stages. The first stage targets 50 | top-level S-expressions (e.g., specific kinds of SMT-LIB commands) until a 51 | fixed point to aggressively simplify the input before applying more expensive 52 | mutators in the second stage. 53 | 54 | .. image:: img/ddmin.png 55 | :alt: Pseydo-code of the main algorithm of the ddmin strategy. 56 | 57 | 58 | The **hierarchical** Strategy 59 | ----------------------------- 60 | 61 | The main loop of the **hierarchical** strategy (as shown in pseudo code below) 62 | performs a simple breadth-first traversal 63 | of the S-expressions in the input, and applies all enabled mutators 64 | to every S-expression. 65 | Once a simplification is found (Line 7), all pending checks for the current 66 | S-expression are aborted and the breadth-first traversal continues with the 67 | simplified S-expression :code:`sexpr` (Line 9). 68 | This process is repeated until further simplifications are found. 69 | 70 | The main simplification loop (Line 3) is applied multiple times, with varying 71 | sets of mutators. 72 | In the initial stages, **hierarchical** aims for aggressive minimization 73 | using only a small set of selected mutators, in the next-to-last stage it 74 | employs all but a few mutators that usually only have cosmetic impact, and in 75 | the last stage it includes all mutators. 76 | 77 | Breadth-first traversal yields significantly better results than a depth-first 78 | traversal, most probably since it tends to favor simplifications on larger 79 | subtrees of the input. 80 | 81 | .. image:: img/hier.png 82 | :alt: Pseydo-code of the main algorithm of the hierarchical strategy. 83 | 84 | 85 | The **hybrid** Strategy 86 | ----------------------- 87 | 88 | Strategy **hybrid** is the **default** strategy of **ddSMT**. 89 | It combines strategies **ddmin** and **hierarchical** in a sequential 90 | portfolio manner for a best of both worlds. 91 | 92 | Strategy **hybrid** first applies **ddmin** until a fixed point is reached, and 93 | then calls strategy **hierarchical** on the simplified input. 94 | This order of strategies is due to our observation that **ddmin** 95 | is usually faster in simplifying input, while **hierarchical** often 96 | yields smaller inputs. 97 | -------------------------------------------------------------------------------- /docs/guide.rst: -------------------------------------------------------------------------------- 1 | User Guide 2 | ========== 3 | 4 | This guide covers some more advanced topics of using **ddSMT**. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | guide-strategies 10 | guide-mutators 11 | guide-scenarios 12 | guide-performance 13 | -------------------------------------------------------------------------------- /docs/img/ddmin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddsmt/ddSMT/3efb3344a74a7b29cdf56cdd56630a97736adc21/docs/img/ddmin.png -------------------------------------------------------------------------------- /docs/img/hier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddsmt/ddSMT/3efb3344a74a7b29cdf56cdd56630a97736adc21/docs/img/hier.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | 2 | .. include:: ../README.rst 3 | 4 | .. toctree:: 5 | :hidden: 6 | 7 | self 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | installation 13 | quickstart 14 | guide 15 | guide-developers 16 | contributing 17 | faq 18 | references 19 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ==================================== 3 | 4 | There are several ways to obtain and install **ddSMT**. 5 | If you just want to use a stable version of **ddSMT** use 6 | 7 | .. code-block:: bash 8 | 9 | $ pip3 install ddsmt 10 | $ ddsmt --version 11 | 12 | If you are interested in a specific version of **ddSMT** (including development 13 | versions) use the following commands to list available versions and install a 14 | specific one: 15 | 16 | .. code-block:: bash 17 | 18 | $ pip3 install ddsmt== 19 | $ pip3 install ddsmt== 20 | 21 | To install the latest version of **ddSMT**, corresponding to the latest version 22 | from master (whose CI builds worked), use ``pip`` as follows: 23 | 24 | .. code-block:: bash 25 | 26 | $ pip3 install --force-reinstall --pre ddsmt 27 | 28 | If you want to experiment with **ddSMT**, for example to test :ref:`custom 29 | mutators `, clone **ddSMT** from github and run it from 30 | your working copy: 31 | 32 | .. code-block:: bash 33 | 34 | $ git clone https://github.com/ddsmt/ddSMT.git 35 | $ ddSMT/bin/ddsmt --version 36 | 37 | -------------------------------------------------------------------------------- /docs/literature.bib: -------------------------------------------------------------------------------- 1 | @techreport{Barrett2017, 2 | author = {Clark Barrett and Pascal Fontaine and Cesare Tinelli}, 3 | title = {{The SMT-LIB Standard: Version 2.6}}, 4 | institution = {Department of Computer Science, The University of Iowa}, 5 | year = 2017, 6 | note = {Available at {\tt www.SMT-LIB.org}} 7 | } 8 | 9 | @inproceedings{Brummayer2009, 10 | title={Fuzzing and delta-debugging {SMT} solvers}, 11 | author={Brummayer, Robert and Biere, Armin}, 12 | booktitle={Proceedings of the 7th International Workshop on Satisfiability Modulo Theories}, 13 | pages={1--5}, 14 | year={2009} 15 | } 16 | 17 | @inproceedings{Kremer2021, 18 | title = {{ddSMT 2.0: Better Delta Debugging for the SMT-LIBv2 Language and 19 | Friends}}, 20 | author = {Kremer, Gereon and Niemetz, Aina and Preiner, Mathias}, 21 | booktitle = {to appear}, 22 | year = {2021}, 23 | } 24 | 25 | @inproceedings{Misherghi2006, 26 | title={HDD: hierarchical delta debugging}, 27 | author={Misherghi, Ghassan and Su, Zhendong}, 28 | booktitle={Proceedings of the 28th international conference on Software engineering}, 29 | pages={142--151}, 30 | year={2006} 31 | } 32 | 33 | @inproceedings{Niemetz2013, 34 | title={{ddSMT}: a delta debugger for the {SMT-LIB} v2 format}, 35 | author={Niemetz, Aina and Biere, Armin}, 36 | booktitle={Proceedings of the 11th International Workshop on Satisfiability Modulo Theories, SMT}, 37 | pages={8--9}, 38 | year={2013} 39 | } 40 | 41 | @inproceedings{Regehr2012, 42 | title={Test-case reduction for C compiler bugs}, 43 | author={Regehr, John and Chen, Yang and Cuoq, Pascal and Eide, Eric and Ellison, Chucky and Yang, Xuejun}, 44 | booktitle={Proceedings of the 33rd ACM SIGPLAN conference on Programming Language Design and Implementation}, 45 | pages={335--346}, 46 | year={2012} 47 | } 48 | 49 | @article{Zeller1999, 50 | title={Yesterday, my program worked. Today, it does not. Why?}, 51 | author={Zeller, Andreas}, 52 | journal={ACM SIGSOFT Software engineering notes}, 53 | volume={24}, 54 | number={6}, 55 | pages={253--267}, 56 | year={1999}, 57 | publisher={ACM New York, NY, USA} 58 | } 59 | 60 | @article{Zeller2002, 61 | author = {Andreas Zeller and 62 | Ralf Hildebrandt}, 63 | title = {Simplifying and Isolating Failure-Inducing Input}, 64 | journal = {{IEEE} Trans. Software Eng.}, 65 | volume = {28}, 66 | number = {2}, 67 | pages = {183--200}, 68 | year = {2002}, 69 | doi = {10.1109/32.988498}, 70 | timestamp = {Wed, 14 Nov 2018 10:49:20 +0100}, 71 | biburl = {https://dblp.org/rec/journals/tse/ZellerH02.bib}, 72 | bibsource = {dblp computer science bibliography, https://dblp.org} 73 | } 74 | -------------------------------------------------------------------------------- /docs/publications/KremerNiemetzPreiner-CAV21.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddsmt/ddSMT/3efb3344a74a7b29cdf56cdd56630a97736adc21/docs/publications/KremerNiemetzPreiner-CAV21.pdf -------------------------------------------------------------------------------- /docs/publications/NiemetzBiere-SMT13.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddsmt/ddSMT/3efb3344a74a7b29cdf56cdd56630a97736adc21/docs/publications/NiemetzBiere-SMT13.pdf -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | The main purpose of **ddSMT** is to *minimize* an input that triggers some 5 | (erroneous) behavior when fed to a given command. 6 | **ddSMT** first executes the command on the given input and records its behavior 7 | (exit code, standard output and error channels). 8 | After that, it mutates the input while *preserving the behavior* of the given 9 | command until no further mutations are possible. 10 | 11 | **ddSMT** aims at extracting a *minimal working example*, that is, an input 12 | that is *as small as possible* but still triggers the original faulty behavior 13 | of the executed command. 14 | Such a command is usually a call to an SMT solver executable with command line 15 | options (but can be a call to any tool that expects the SMT-LIB language or an 16 | extension/dialect of the language as input). 17 | 18 | The initial execution of the given command (the *golden run*) records its 19 | behavior as a combination of 20 | 21 | * the standard output 22 | * the standard error output 23 | * the exit code 24 | 25 | For any mutations that are accepted, the command must exit with the same 26 | behavior. 27 | Additionally, **ddSMT** imposes a time limit for executing the command that is 28 | by default computed based on the run time of the golden run. 29 | Optionally, **ddSMT** allows to configure this time limit, and to ignore 30 | output channels. 31 | 32 | **ddSMT** implements several :ref:`minimization strategies ` to derive inputs by applying all enabled mutators. 34 | After the golden run, it generates mutated inputs and executes the given 35 | command on these inputs. 36 | The first mutated input on which the command behaves the same as on the 37 | original input is accepted, and new mutations are tested based on 38 | this new input. 39 | When no further mutations are possible, **ddSMT** terminates with the final 40 | output written to the specified output file. 41 | 42 | .. note:: 43 | **ddSMT** writes (intermediate) accepted mutations to the specified output 44 | file. This allows to already use intermediate minimizations even when 45 | **ddSMT** is not yet done minimizing. 46 | 47 | 48 | How to run ddSMT 49 | ---------------- 50 | 51 | Given an input file :download:`example/input.smt2` and a call to 52 | :download:`example/solver` with command line option :code:`--option`, **ddSMT** 53 | is executed to write minimized input to :code:`output.smt2` as follows: 54 | 55 | .. code-block:: bash 56 | 57 | $ ddsmt input.smt2 output.smt2 solver --option 58 | 59 | **ddSMT** provides many options that cover common use cases. 60 | Consult the :ref:`full option listing ` for an exhaustive 61 | list. 62 | The most commonly used options are: 63 | 64 | * ``--jobs`` (or ``-j``) 65 | to set the number of processes allowed to run in parallel (uses the number of 66 | CPU cores minus two by default). 67 | * ``--memout`` 68 | to impose a memory limit (in MB), otherwise memory usage is not limited. 69 | * ``--timeout`` 70 | to impose a custom time limit (in seconds), otherwise the time limit is 71 | computed based on the run time of the initial run. 72 | * ``--strategy`` 73 | to select the minimization strategy (see :ref:`minimization strategies`). 74 | * ``--ignore-output`` 75 | to ignore output on standard output and error channels. 76 | * ``--match-out`` and ``--match-err`` 77 | to define pattern matches against on the standard channels (see :ref:`How 78 | Behavior is Compared with the Golden Run`). 79 | 80 | How Behavior is Compared with the Golden Run 81 | -------------------------------------------- 82 | 83 | By default, a run on mutated input is considered equivalent in behavior to the 84 | golden run if the exit code, the standard output and the error output are the 85 | same as for the golden run. 86 | 87 | For example, assume that our example from above produces the following output: 88 | 89 | .. code-block:: bash 90 | 91 | $ example/solver --option example/input.smt2 92 | printing something to stdout 93 | error: this message is printed to stderr 94 | 95 | $ echo $? # the exit code 96 | 1 97 | 98 | By default 99 | **ddSMT** will exactly match 100 | :code:`printing something to stdout` against what is printed on `stdout` and 101 | :code:`error: this message is printed to stderr` against what is printed on 102 | `stderr` for mutated inputs, and check if the exit code is :code:`1`. 103 | When increasing the verbosity level with :code:`-v`, it will print some 104 | information about the golden run and progress: 105 | 106 | .. command-output:: ddsmt -v example/input.smt2 example/output.smt2 example/solver --option 107 | :ellipsis: 17 108 | 109 | Now, let's assume we want to ignore output on `stdout` and `stderr`, we enable 110 | option :code:`--ignore-output`: 111 | 112 | .. code:: bash 113 | 114 | $ ddsmt -v --ignore-output example/input.smt2 example/output.smt2 example/solver --option 115 | 116 | .. note:: 117 | 118 | With option :code:`--ignore-output` enabled, **ddSMT** will still report 119 | output on both channels for the golden run with option :code:`-v`. 120 | When comparing behavior, however, this output is ignored, and only the 121 | exit code is matched. 122 | 123 | Now, let's assume our command yields output on `stdout` or `stderr` that 124 | contains information that depends on the actual execution, e.g., a stack trace. 125 | In this case, matching against the full output will never be successful, 126 | and we rather only want to check if a phrase occurs in the output. 127 | We can achieve this with options :code:`--match-out` (for `stdout`) and 128 | :code:`--match-err` (for `stderr`) as follows: 129 | 130 | .. code:: bash 131 | 132 | $ ddsmt -v --match-out things example/input.smt2 example/output.smt2 example/solver --option 133 | 134 | $ ddsmt -v --match-err error example/input.smt2 example/output.smt2 example/solver --option 135 | 136 | In case you are wondering how the comparison of a run on mutated input with the 137 | golden run is implemented, this is the actual code that is implemented in 138 | **ddSMT**: 139 | 140 | .. literalinclude:: ../ddsmt/checker.py 141 | :language: python3 142 | :linenos: 143 | :pyobject: matches_golden 144 | 145 | 146 | Full Option Listing 147 | ------------------- 148 | 149 | .. command-output:: ddsmt --help-all 150 | -------------------------------------------------------------------------------- /docs/references.rst: -------------------------------------------------------------------------------- 1 | References 2 | ==================================== 3 | 4 | .. bibliography:: 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # for regular operation 2 | gprof2dot>=2019.11 3 | importlib-metadata>=1.7 ; python_version<"3.8" 4 | # for CI 5 | coverage>=4.5 6 | docformatter>=1.4 7 | flake8>=3.8 8 | yapf>=0.30 9 | # for documentation 10 | sphinx-rtd-theme>=0.4.3 11 | # there is a weird issue with 2.1.4 that leads to a PicklingError. We just stick to 2.1.0 for now 12 | sphinxcontrib-bibtex==2.1.0 13 | sphinxcontrib-programoutput>=0.15 14 | -------------------------------------------------------------------------------- /scripts/compare_time.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import subprocess 4 | import sys 5 | import time 6 | 7 | filename = sys.argv[1] 8 | 9 | solvers = { 10 | 'A': ['solverA', '--option'], 11 | 'B': ['solverB', '--option'], 12 | } 13 | # minimal factor between the faster and the slower solver 14 | threshold = 10 15 | 16 | slow = 'A' 17 | 18 | dur = {} 19 | 20 | CUR_PROC = None 21 | 22 | 23 | def stop_process(proc): 24 | proc.terminate() 25 | try: 26 | proc.wait(timeout=0.5) 27 | except subprocess.TimeoutExpired: 28 | proc.kill() 29 | try: 30 | proc.wait(timeout=0.5) 31 | except subprocess.TimeoutExpired: 32 | print(f'Killing pid {proc.pid} failed. ' 33 | 'Please check manually to avoid memory exhaustion.') 34 | 35 | 36 | def handler(sig, frame): 37 | if CUR_PROC is not None: 38 | stop_process(CUR_PROC) 39 | 40 | 41 | for s in solvers: 42 | before = time.perf_counter() 43 | try: 44 | CUR_PROC = subprocess.Popen(solvers[s] + [filename], 45 | stdout=subprocess.PIPE, 46 | stderr=subprocess.PIPE) 47 | CUR_PROC.communicate(timeout=10) 48 | except subprocess.TimeoutExpired: 49 | stop_process(CUR_PROC) 50 | duration = time.perf_counter() - before 51 | print('{} -> {}'.format(s, duration)) 52 | dur[s] = duration 53 | 54 | factor = max(dur.values()) / min(dur.values()) 55 | print('Factor: {}'.format(factor)) 56 | 57 | # discard if the slower solver became too fast 58 | if dur[slow] < 3: 59 | print('-> 0') 60 | sys.exit(0) 61 | 62 | if factor > threshold: 63 | # still much slower 64 | print('-> 1') 65 | sys.exit(1) 66 | # not that much slower 67 | print('-> 0') 68 | sys.exit(0) 69 | -------------------------------------------------------------------------------- /scripts/result_differs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import signal 4 | import subprocess 5 | import sys 6 | 7 | 8 | def handler(sig, frame): 9 | print("Aborted") 10 | sys.exit(1) 11 | 12 | 13 | signal.signal(signal.SIGTERM, handler) 14 | 15 | 16 | def run(cmd): 17 | cmd = cmd + [sys.argv[1]] 18 | res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 19 | return res.stdout.decode('utf8').strip() 20 | 21 | 22 | A = run(['solverA', '--option']) 23 | B = run(['solverB', '--option']) 24 | 25 | if A not in ['sat', 'unsat']: 26 | print(f'Unexpected output for A: {A}') 27 | sys.exit(2) 28 | 29 | if B not in ['sat', 'unsat']: 30 | print(f'Unexpected output for B: {B}') 31 | sys.exit(2) 32 | 33 | print(f'{A} / {B}') 34 | if A == B: 35 | sys.exit(0) 36 | else: 37 | sys.exit(1) 38 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = ddSMT 3 | version = attr: ddsmt.version.VERSION 4 | url = https://github.com/ddsmt/ddSMT 5 | download_url = https://github.com/ddsmt/ddSMT 6 | project_urls = 7 | Bug Tracker = https://github.com/ddsmt/ddSMT/issues 8 | Documentation = https://ddsmt.readthedocs.io/ 9 | Source Code = https://github.com/ddsmt/ddSMT 10 | author = The authors of ddSMT 11 | license = MIT 12 | license_file = LICENSE 13 | description = A delta debugger for SMT-LIBv2 files 14 | long_description = file: README.rst 15 | long_description_content_type = text/x-rst 16 | classifiers = 17 | Intended Audience :: Science/Research 18 | License :: OSI Approved :: MIT License 19 | Programming Language :: Python :: 3.6 20 | Programming Language :: Python :: 3.7 21 | Programming Language :: Python :: 3.8 22 | Programming Language :: Python :: 3.9 23 | Programming Language :: Python :: 3.10 24 | Programming Language :: Python :: 3.11 25 | Programming Language :: Python :: 3.12 26 | 27 | [options] 28 | packages = find: 29 | install_requires = 30 | gprof2dot>=2019.11 31 | importlib-metadata>=1.7 ; python_version<"3.8" 32 | python_requires = >=3.6 33 | scripts = 34 | bin/smt2info 35 | 36 | [options.entry_points] 37 | console_scripts = 38 | ddsmt = ddsmt:__main__.main 39 | 40 | [coverage:run] 41 | branch = True 42 | include = 43 | ddsmt/* 44 | 45 | [coverage:report] 46 | show_missing = True 47 | 48 | [flake8] 49 | count = True 50 | extend-exclude = build/ 51 | ignore = F403,F405,W503 52 | max-complexity = 15 53 | max-line-length = 80 54 | show-source = True 55 | statistics = True 56 | 57 | [yapf] 58 | based_on_style = pep8 59 | split_before_arithmetic_operator = true 60 | split_before_bitwise_operator = true 61 | split_before_logical_operator = true 62 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup() 4 | -------------------------------------------------------------------------------- /update_license_headers.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import argparse 4 | import glob 5 | import os 6 | import textwrap 7 | 8 | HEADER_TEMPLATE = """ 9 | ddSMT: A delta debugger for SMT benchmarks in SMT-Lib v2 format. 10 | 11 | This file is part of ddSMT. 12 | 13 | Copyright (C) 2013-2024 by the authors listed in the AUTHORS file 14 | at https://github.com/ddsmt/ddSMT/blob/master/AUTHORS. 15 | 16 | This file is part of ddSMT under the MIT license. See LICENSE for more 17 | information at https://github.com/ddsmt/ddSMT/blob/master/LICENSE. 18 | """ 19 | 20 | 21 | def parse_options(): 22 | """Parse command line options into ``global ARGS``.""" 23 | global ARGS 24 | ap = argparse.ArgumentParser( 25 | usage='update_license_headers.py [] ') 26 | ap.add_argument('basedir', help='base directory of the ddSMT directory') 27 | ARGS = ap.parse_args() 28 | 29 | 30 | def prepare_header(): 31 | """Return new header as list of strings.""" 32 | return [ 33 | s.strip() for s in textwrap.indent(HEADER_TEMPLATE, '# ', 34 | lambda line: True).split('\n') 35 | ] 36 | 37 | 38 | def list_files(): 39 | """Iterable over all files that should get the copyright header.""" 40 | yield f'{ARGS.basedir}/bin/ddsmt' 41 | yield f'{ARGS.basedir}/bin/ddsmt-profile' 42 | yield f'{ARGS.basedir}/bin/smt2info' 43 | yield from glob.iglob(f'{ARGS.basedir}/ddsmt/*.py') 44 | 45 | 46 | def update_file(filename, new_header): 47 | """Read the filename and add or update the copyright header. 48 | 49 | Keep an existing shebang in place 50 | """ 51 | stat = os.stat(filename) 52 | cur = open(filename).read().split('\n') 53 | 54 | start = 0 55 | try: 56 | start = next(x for x in enumerate(cur) 57 | if x[1].strip() != '' and not x[1].startswith('#'))[0] 58 | except StopIteration: 59 | # No actual code was found, probably an __init__.py 60 | pass 61 | 62 | new = [] 63 | if cur[0].startswith('#!'): 64 | new.append(cur[0]) 65 | 66 | new.extend(new_header) 67 | new.extend(cur[start:]) 68 | 69 | open(filename, 'w').write('\n'.join(new)) 70 | os.utime(filename, ns=(stat.st_atime_ns, stat.st_mtime_ns)) 71 | 72 | 73 | if __name__ == '__main__': 74 | parse_options() 75 | header = prepare_header() 76 | for filename in list_files(): 77 | if filename.endswith('__init__.py'): 78 | continue 79 | 80 | print(filename) 81 | update_file(filename, header) 82 | --------------------------------------------------------------------------------