├── .editorconfig ├── .flake8 ├── .gitattributes ├── .github └── FUNDING.yml ├── .gitignore ├── LICENSE ├── README.md ├── demo.png ├── notes ├── no-caching.md ├── parallelism.md └── pumps-and-passes.md ├── noxfile.py ├── poetry.lock ├── pyproject.toml ├── pytest.ini ├── src └── shrinkray │ ├── __init__.py │ ├── __main__.py │ ├── passes │ ├── __init__.py │ ├── bytes.py │ ├── clangdelta.py │ ├── definitions.py │ ├── genericlanguages.py │ ├── json.py │ ├── patching.py │ ├── python.py │ ├── sat.py │ └── sequences.py │ ├── problem.py │ ├── py.typed │ ├── reducer.py │ └── work.py └── tests ├── __init__.py ├── conftest.py ├── helpers.py ├── test_byte_reduction_passes.py ├── test_clang_delta.py ├── test_generic_language.py ├── test_generic_shrinking_properties.py ├── test_json_passes.py ├── test_main.py ├── test_misc_reduction_performance.py ├── test_python_reducers.py ├── test_sat.py └── test_work.py /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | 9 | [*.{py,toml}] 10 | indent_style = space 11 | indent_size = 4 12 | 13 | [*.yml,yaml,json] 14 | indent_style = space 15 | indent_size = 2 16 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | select = B,B9,C,D,DAR,E,F,N,RST,W 3 | ignore = E203,E501,RST201,RST203,RST301,W503,D100,D101,D102,D103,D104,D105,C901,B902,B950,N804,N818 4 | enable = TRIO910, TRIO911 5 | max-line-length = 88 6 | max-complexity = 10 7 | docstring-convention = google 8 | rst-roles = class,const,func,meth,mod,ref 9 | rst-directives = deprecated 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: DRMacIver 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .mypy_cache/ 2 | /.coverage 3 | /.coverage.* 4 | /.nox/ 5 | /.python-version 6 | /.pytype/ 7 | /dist/ 8 | /docs/_build/ 9 | /src/*.egg-info/ 10 | __pycache__/ 11 | .vscode 12 | scratch 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright © 2023 David R. MacIver 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shrink Ray 2 | 3 | Shrink Ray is a modern multiformat test-case reducer. 4 | 5 | ## What is test-case reduction? 6 | 7 | Test-case reduction is the process of automatically taking a *test case* and *reducing* it to something close to a [minimal reproducible example](https://en.wikipedia.org/wiki/Minimal_reproducible_example). 8 | 9 | That is, you have some file that has some interesting property (usually that it triggers a bug in some software), 10 | but it is large and complicated and as a result you can't figure out what about the file actually matters. 11 | You want to be able to trigger the bug with a small, simple, version of it that contains only the features of interest. 12 | 13 | For example, the following is some Python code that [triggered a bug in libcst](https://github.com/Instagram/LibCST/issues/1061): 14 | 15 | ```python 16 | () if 0 else(lambda:()) 17 | ``` 18 | 19 | This was extracted from a large Python file (probably several thousand lines of code) and systematically reduced down to this example. 20 | 21 | You would obtain this by running `shrinkray breakslibcst.py mytestcase.py`, where `breakslibcst.py` looks something like this: 22 | 23 | ```python 24 | import libcst 25 | import sys 26 | 27 | if __name__ == '__main__': 28 | try: 29 | libcst.parse_module(sys.stdin.read()) 30 | except TypeError: 31 | sys.exit(0) 32 | sys.exit(1) 33 | ``` 34 | 35 | This script exits with 0 if the code passed to it on standard input triggers the relevant bug (that libcst raises a TypeError when parsing this code), and with a non-zero exit code otherwise. 36 | 37 | shrinkray (or any other test-case reducer) then systematically tries smaller and simpler variants of your original source file until it reduces it to something as small as it can manage. 38 | 39 | While it runs, you will see the following user interface: 40 | 41 | ![Demo of shrink ray running](demo.png) 42 | 43 | When it finishes you will be left with the reduced test case in `mytestcase.py`. 44 | 45 | Test-case reducers are useful for any tools that handle files with complex formats that can trigger bugs in them. Historically this has been particularly useful for compilers and other programming tools, but in principle it can be used for anything. 46 | 47 | Most test-case reducers only work well on a few formats. Shrink Ray is designed to be able to support a wide variety of formats, including binary ones, although it's currently best tuned for "things that look like programming languages". 48 | 49 | ## What makes Shrink Ray distinctive? 50 | 51 | It's designed to be highly parallel, and work with a very wide variety of formats, through a mix of good generic algorithms and format-specific reduction passes. 52 | 53 | Currently shrink ray is a "prerelease" version in the sense that there is no official release yet and you're expected to just run off main (don't worry this is easy to do), as it's a bit experimental. 54 | 55 | That being said this probably doesn't matter that much for the question of whether to use it. It's in the nature of test-case reduction that it doesn't matter all that much if it's bad, because it's still going to do a bunch of work that you didn't have to do by hand. Try it out, see if it works. If it doesn't, please tell me and I'll make it work better. 56 | 57 | ## Installation 58 | 59 | Shrink Ray requires Python 3.12 or later, and can be installed using pip. 60 | 61 | There is currently no official release for shrink ray, and I recommend running off main. You can install it as follows: 62 | 63 | ``` 64 | pipx install git+https://github.com/DRMacIver/shrinkray.git 65 | ``` 66 | 67 | (if you don't have or want [pipx](https://pypa.github.io/pipx/) you could also do this with pip and it would work fine) 68 | 69 | Shrink Ray requires Python 3.12 or later and won't work on earlier versions. If everything is working correctly, it should refuse to install 70 | on versions it's incompatible with. If you do not have Python 3.12 installed, I recommend [pyenv](https://github.com/pyenv/pyenv) for managing 71 | Python installs. 72 | 73 | If you want to use it from the git repo directly, you can do the following: 74 | 75 | ``` 76 | git clone https://github.com/DRMacIver/shrinkray.git 77 | cd shrinkray 78 | python -m venv .venv 79 | .venv/bin/pip install -e . 80 | ``` 81 | 82 | You will now have a shrinkray executable in .venv/bin, which you can also put on your path by running `source .venv/bin/activate`. 83 | 84 | ## Usage 85 | 86 | Shrink Ray is run as follows: 87 | 88 | ``` 89 | shrinkray is_interesting.sh my-test-case 90 | ``` 91 | 92 | Where `my-test-case` is some file you want to reduce and `is_interesting.sh` can be any executable that exits with `0` when a test case passed to it is interesting and non-zero otherwise. 93 | 94 | Variant test cases are passed to the interestingness test both on STDIN and as a file name passed as an argument. Additionally for creduce compatibility, the file has the same base name as the original test case and is in the current working directory the script is run with. This behaviour can be customised with the `--input-type` argument. 95 | 96 | `shrinkray --help` will give more usage instructions. 97 | 98 | ## Supported formats 99 | 100 | Shrink Ray is fully generic in the sense that it will work with literally any file you give it in any format. However, some formats will work a lot better than others. 101 | 102 | It has a generic reduction algorithm that should work pretty well with any textual format, and an architecture that is designed to make it easy to add specialised support for specific formats as needed. 103 | 104 | Additionally, Shrink Ray has special support for the following formats: 105 | 106 | * C and C++ (via `clang_delta`, which you will have if creduce is installed) 107 | * Python 108 | * JSON 109 | * Dimacs CNF format for SAT problems 110 | 111 | Most of this support is quite basic and is just designed to deal with specific cases that the generic logic is known 112 | not to handle well, but it's easy to extend with additional transformations. 113 | It is also fairly easy to add support for new formats as needed. 114 | 115 | If you run into a test case and interestingness test that you care about that shrink ray handles badly please let me know and I'll likely see about improving its handling of that format. 116 | 117 | ## Parallelism 118 | 119 | You can control the number of parallel tasks shrinkray will run with the `--parallelism` flag. By default this will be the number of CPU cores you have available 120 | 121 | Shrink Ray is designed to be able to run heavily in parallel, with a basic heuristic of aiming to be embarrassingly parallel when making no progress, mostly sequential when making progress, and smoothly scaling in between the two. It mostly succeeds at this. 122 | 123 | Currently the bottleneck on scaling to a very large number of cores is how fast the controlling Python program can generate variant test cases to try and pass them to the interestingness test. This isn't well optimised at present and I don't currently have good benchmarks for it, but I'd expect you to be able to get linear speedups on most workflows while running 10-20 test cases in parallel, and to start to struggle past that. 124 | 125 | This also depends on the performance of the interestingness test - the slower your test is to run, the more you'll be able to scale linearly with the number of cores available. 126 | 127 | I'm quite interested in getting this part to scale well, so please let me know if you find examples where it doesn't seem to work. 128 | 129 | ## Bug Reports 130 | 131 | Shrink Ray is still pretty new and under-tested software, so it definitely has bugs. If you run into any, [please file an issue](https://github.com/DRMacIver/shrinkray/issues). 132 | 133 | As well as obvious bugs (crashes, etc) I'm also very interested in hearing about usability issues and cases where the reduced test case isn't very good. 134 | 135 | Requests for new features, new supported formats, etc. also welcome although I'm less likely to jump right on them. 136 | 137 | ## Sponsorship 138 | 139 | Shrink Ray is something of a labour of love - I wanted to have a tool that actually put into practice many of my ideas about test-case reduction, as I think the previous state of the art was well behind where I'd like it to be. 140 | 141 | That being said, it is first and foremost designed to be a useful tool for practical engineering problems. 142 | If you find it useful as such, please [consider sponsoring my development of it](https://github.com/sponsors/DRMacIver). 143 | -------------------------------------------------------------------------------- /demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DRMacIver/shrinkray/3b4d19902c0dbd1912c8825ca9dba16d4344cf02/demo.png -------------------------------------------------------------------------------- /notes/no-caching.md: -------------------------------------------------------------------------------- 1 | # Caching in Shrink Ray 2 | 3 | Historically most test-case reducers implement caching of their interestingness test, 4 | so that if you generate the same test case variant multiple times you don't need to 5 | call the underlying interestingness test multiple times. 6 | 7 | Shrink Ray doesn't do this, because it doesn't seem worth it. In fairly natural test 8 | cases I was noticing cache hit rates of literally 0%. 9 | 10 | I suspect caching in general is not that useful for test-case reduction, but this is 11 | likely particularly the case with Shrink Ray which has a very large number of fine-grained 12 | transformations, many of which it tries in a random order, so it's actually quite unlikely 13 | to hit duplicates. 14 | 15 | I suspect the popularity of caching with test-case reduction is a historical artefact from 16 | delta debugging, which has a very high chance of generating duplicates due to the way 17 | its coarse grained passes decompose into multiple operations from its fine grained passes. 18 | Shrink Ray basically never does that (I don't think it actually works) so has few opportunities 19 | to generate duplicates. -------------------------------------------------------------------------------- /notes/parallelism.md: -------------------------------------------------------------------------------- 1 | # Parallelism in Shrink Ray 2 | 3 | How to do highly parallel test-case reduction is one of the active research problems I'm working on in Shrink Ray, 4 | so I'm still experimenting with it and changing it. I think the current approach works very well[^1], but it lacks a good unifying 5 | abstraction, and it will doubtless change a lot as the reducer develops. 6 | 7 | This document describes the approach taken at the time of writing, but I can't promise it's always 100% up to date. 8 | 9 | ## The problem to be solved 10 | 11 | At its most basic, test-case reduction consists of repeatedly running the following algorithm: 12 | 13 | 1. Take your current best test-case and generate `N` smaller variants of it. 14 | 2. Find one of those `N` variants that is still interesting, and replace your current best test-case with it. 15 | 16 | It is hopefully clear that step 2 can be run in an embarrassingly parallel manner, making use of as many threads 17 | as you want to give it. 18 | 19 | The problem is that you can only really take advantage of this parallelism in the case where the test-case reducer 20 | isn't making progress. The best case scenario for it is when the reducer is done and you just have to evaluate that 21 | none of the `N` test cases are interesting, this will be able to take full advantage of parallelism. In the case 22 | where the reducer is making easy progress, you end up not being able to take much advantage of parallelism, because 23 | you search a few of the variants, find a successful reduction, and then immediately have to start the process again. 24 | The result is that you end up mostly sequential, or at least significantly less parallel than you could be.[^2] 25 | 26 | When the variants found are *much* smaller (as in the early steps of a classic delta debugging[^3]), this is largely 27 | fine and you end up in a situation where you're making large progress (and thus fast) or you're highly parallel 28 | (and thus still fast). The big problem is when you've got to the point where there are many small reductions that 29 | can be made but few large ones. At that point this sort of algorithm does not manage to take as much advantage of 30 | parallelism as one might hope. 31 | 32 | ## Shrink Ray's solution 33 | 34 | Note: This section makes it sound like this approach is better abstracted in Shrink Ray than it actually is right now. 35 | In reality, there are at least two independent implementations of this sort of idea in Shrink Ray that work subtly 36 | differently in important ways. This is more of a high level description of the idea than the actual implementation. 37 | 38 | The essential idea is this: Rather than generating a list of variants, generate a list of edits. For example, 39 | "delete this region of the test case". Now, in parallel, try applying each of these edits with the following 40 | algorithm: 41 | 42 | 1. If the edit is redundant or conflicts[^4] with edits already successfully applied, skip it. 43 | 2. Check if applying this edit on top of the already applied edits would lead to an interesting test case. 44 | a. If it would not, then skip this edit and return False. 45 | b. If it *would* then add it to the merge queue. 46 | c. If nobody else is processing the merge queue, switch into merging mode. Otherwise wait until 47 | this patch has either been merged or discarded and return true if it was merged. 48 | 49 | Merging mode consists of: 50 | 51 | 1. Looking at the current sequence of patches in the queue. 52 | 2. Using an adaptive merge strategy to apply as many of them as possible. 53 | 3. Update the current patch with the successfully applied patches. 54 | 55 | This happens under a lock, ensuring that only a single worker can update the current patch at once. 56 | 57 | In the cases where successful variants are *very* common this will still not be fully parallel, 58 | because it needs to wait on or perform the merge, but merges will tend to be relatively fast (if 59 | there are no conflicts, they require only one call to the interestingness test for the entire queue. 60 | If there area lot of conflicts you may end up having a 50% slowdown as you have to call the interestingness 61 | test for every patch). 62 | 63 | ## Bounding the amount of parallelism 64 | 65 | Shrink Ray gives you an option of how much parallelism to use, though defaults to the number of cores available. 66 | 67 | The way that this is implemented is that internally Shrink Ray is written as if it had access to unlimited 68 | parallelism and just spawns as many tasks (using trio, so these are light weight tasks rather than OS threads), 69 | which rather than calling the underlying interestingness test directly instead communicate with a number of 70 | worker tasks, each of which reads a result off the queue, runs the underlying interestingness test, and then 71 | replies. This means that most of Shrink Ray can be written in a way that is agnostic to the amount of parallelism 72 | it's actually been given (although it does have access to this information when useful). 73 | 74 | 75 | [^1]: It is possibly currently the best test-case reducer for making effective use of parallelism. It should be, 76 | but I don't have benchmarks to back this up at present and it's very likely I've screwed something up or missed 77 | some of the existing state of the art, so you should not believe this claim right now even though I think it's probably true. 78 | 79 | [^2]: Another problem with this is that if generating variants is expensive (which it can be - some of Shrink Ray's reduction passes 80 | certainly have an expensive variant generation step), you end up spending a lot of time on that. As a result the 81 | algorithms described should generally be faster even with no parallelism. 82 | 83 | [^3]: Although, I don't actually think these steps usually work. They're fine for very forgiving parsers, such as HTML, 84 | but for something with quite strict syntactic correctness requirements like a programming language, I think the 85 | early steps largely just waste time. 86 | 87 | [^4]: In fact currently Shrink Ray has no passes where conflicts are possible, but redundancy still is. -------------------------------------------------------------------------------- /notes/pumps-and-passes.md: -------------------------------------------------------------------------------- 1 | # Pumps and Passes 2 | 3 | Shrink Ray makes a distinction between two different types of transformation: A reduction pass, and a reduction pump. 4 | 5 | A reduction pass is a set of transformations of the current test case of a reduction problem which is designed to produce a smaller interesting test case. Reduction passes are always greedy, in the sense that you only care about transforming the test case into a strictly better one. 6 | 7 | A reduction pump on the other hand is any transformation of the current test case into another interesting test case. It can, and typically will, 8 | result in a test case larger than the starting one. 9 | 10 | The general idea of a reduction pump is that you only run it when you're stuck. If you hit a point where none of the reduction passes are making progress, you start running reduction pumps, which may unlock further progress. For example, if you inline a function, this will typically increase the size of the test case. However, this may allow significantly more deletions, as the function definition may now be deleted, and each individual 11 | call site may be reduced independently. -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | """Nox sessions.""" 2 | 3 | import os 4 | import shlex 5 | import shutil 6 | import sys 7 | from glob import glob 8 | from pathlib import Path 9 | from textwrap import dedent 10 | 11 | import nox 12 | 13 | try: 14 | from nox_poetry import Session, session 15 | except ImportError: 16 | message = f"""\ 17 | Nox failed to import the 'nox-poetry' package. 18 | 19 | Please install it using the following command: 20 | 21 | {sys.executable} -m pip install nox-poetry""" 22 | raise SystemExit(dedent(message)) from None 23 | 24 | 25 | package = "shrinkray" 26 | python_versions = ["3.12"] 27 | nox.needs_version = ">= 2021.6.6" 28 | nox.options.sessions = ( 29 | "pre-commit", 30 | "safety", 31 | "mypy", 32 | "tests", 33 | "typeguard", 34 | "xdoctest", 35 | "docs-build", 36 | ) 37 | 38 | 39 | def activate_virtualenv_in_precommit_hooks(session: Session) -> None: 40 | """Activate virtualenv in hooks installed by pre-commit. 41 | 42 | This function patches git hooks installed by pre-commit to activate the 43 | session's virtual environment. This allows pre-commit to locate hooks in 44 | that environment when invoked from git. 45 | 46 | Args: 47 | session: The Session object. 48 | """ 49 | assert session.bin is not None # noqa: S101 50 | 51 | # Only patch hooks containing a reference to this session's bindir. Support 52 | # quoting rules for Python and bash, but strip the outermost quotes so we 53 | # can detect paths within the bindir, like /python. 54 | bindirs = [ 55 | bindir[1:-1] if bindir[0] in "'\"" else bindir 56 | for bindir in (repr(session.bin), shlex.quote(session.bin)) 57 | ] 58 | 59 | virtualenv = session.env.get("VIRTUAL_ENV") 60 | if virtualenv is None: 61 | return 62 | 63 | headers = { 64 | # pre-commit < 2.16.0 65 | "python": f"""\ 66 | import os 67 | os.environ["VIRTUAL_ENV"] = {virtualenv!r} 68 | os.environ["PATH"] = os.pathsep.join(( 69 | {session.bin!r}, 70 | os.environ.get("PATH", ""), 71 | )) 72 | """, 73 | # pre-commit >= 2.16.0 74 | "bash": f"""\ 75 | VIRTUAL_ENV={shlex.quote(virtualenv)} 76 | PATH={shlex.quote(session.bin)}"{os.pathsep}$PATH" 77 | """, 78 | # pre-commit >= 2.17.0 on Windows forces sh shebang 79 | "/bin/sh": f"""\ 80 | VIRTUAL_ENV={shlex.quote(virtualenv)} 81 | PATH={shlex.quote(session.bin)}"{os.pathsep}$PATH" 82 | """, 83 | } 84 | 85 | hookdir = Path(".git") / "hooks" 86 | if not hookdir.is_dir(): 87 | return 88 | 89 | for hook in hookdir.iterdir(): 90 | if hook.name.endswith(".sample") or not hook.is_file(): 91 | continue 92 | 93 | if not hook.read_bytes().startswith(b"#!"): 94 | continue 95 | 96 | text = hook.read_text() 97 | 98 | if not any( 99 | Path("A") == Path("a") and bindir.lower() in text.lower() or bindir in text 100 | for bindir in bindirs 101 | ): 102 | continue 103 | 104 | lines = text.splitlines() 105 | 106 | for executable, header in headers.items(): 107 | if executable in lines[0].lower(): 108 | lines.insert(1, dedent(header)) 109 | hook.write_text("\n".join(lines)) 110 | break 111 | 112 | 113 | @session(name="lint", python=python_versions[0]) 114 | def precommit(session: Session) -> None: 115 | session.install( 116 | "flake8", 117 | "flake8-bandit", 118 | "flake8-bugbear", 119 | "flake8-rst-docstrings", 120 | "flake8-trio", 121 | ) 122 | 123 | session.run("flake8", "src", "tests") 124 | session.run("flake8-trio", *PY_FILES) 125 | 126 | 127 | PY_FILES = glob("src/**/*.py", recursive=True) + glob("test/**/*.py", recursive=True) 128 | assert PY_FILES 129 | 130 | 131 | @session(python=python_versions[0]) 132 | def format(session: Session) -> None: 133 | session.install( 134 | "shed==2023.6.1", 135 | ) 136 | session.run("shed", "--refactor", *PY_FILES) 137 | 138 | 139 | @session(python=python_versions[0]) 140 | def safety(session: Session) -> None: 141 | """Scan dependencies for insecure packages.""" 142 | requirements = session.poetry.export_requirements() 143 | session.install("safety") 144 | session.run("safety", "check", "--full-report", f"--file={requirements}") 145 | 146 | 147 | @session(python=python_versions) 148 | def mypy(session: Session) -> None: 149 | """Type-check using mypy.""" 150 | args = session.posargs or ["src", "tests"] 151 | session.install(".") 152 | session.install("mypy", "pytest", "trio-typing", "types-tqdm") 153 | session.run("mypy", *args) 154 | if not session.posargs: 155 | session.run("mypy", f"--python-executable={sys.executable}", "noxfile.py") 156 | 157 | 158 | @session(python=python_versions, reuse_venv=True) 159 | def tests(session: Session) -> None: 160 | """Run the test suite.""" 161 | session.install(".") 162 | session.run_always("poetry", "install", external=True) 163 | session.install("coverage[toml]", "pytest", "pytest-trio", "pygments") 164 | try: 165 | session.run("coverage", "run", "--parallel", "-m", "pytest", *session.posargs) 166 | finally: 167 | if session.interactive and not session.posargs: 168 | session.notify("coverage", posargs=[]) 169 | 170 | 171 | @session(python=python_versions[0]) 172 | def coverage(session: Session) -> None: 173 | """Produce the coverage report.""" 174 | args = session.posargs or ["report"] 175 | 176 | session.install("coverage[toml]") 177 | 178 | if not session.posargs and any(Path().glob(".coverage.*")): 179 | session.run("coverage", "combine") 180 | 181 | session.run("coverage", *args) 182 | 183 | 184 | @session(python=python_versions[0]) 185 | def typeguard(session: Session) -> None: 186 | """Runtime type checking using Typeguard.""" 187 | session.install(".") 188 | session.install("pytest", "typeguard", "pygments") 189 | session.run("pytest", f"--typeguard-packages={package}", *session.posargs) 190 | 191 | 192 | @session(python=python_versions) 193 | def xdoctest(session: Session) -> None: 194 | """Run examples with xdoctest.""" 195 | if session.posargs: 196 | args = [package, *session.posargs] 197 | else: 198 | args = [f"--modname={package}", "--command=all"] 199 | if "FORCE_COLOR" in os.environ: 200 | args.append("--colored=1") 201 | 202 | session.install(".") 203 | session.install("xdoctest[colors]") 204 | session.run("python", "-m", "xdoctest", *args) 205 | 206 | 207 | @session(name="docs-build", python=python_versions[0]) 208 | def docs_build(session: Session) -> None: 209 | """Build the documentation.""" 210 | args = session.posargs or ["docs", "docs/_build"] 211 | if not session.posargs and "FORCE_COLOR" in os.environ: 212 | args.insert(0, "--color") 213 | 214 | session.install(".") 215 | session.install("sphinx", "sphinx-click", "furo", "myst-parser") 216 | 217 | build_dir = Path("docs", "_build") 218 | if build_dir.exists(): 219 | shutil.rmtree(build_dir) 220 | 221 | session.run("sphinx-build", *args) 222 | 223 | 224 | @session(python=python_versions[0]) 225 | def docs(session: Session) -> None: 226 | """Build and serve the documentation with live reloading on file changes.""" 227 | args = session.posargs or ["--open-browser", "docs", "docs/_build"] 228 | session.install(".") 229 | session.install("sphinx", "sphinx-autobuild", "sphinx-click", "furo", "myst-parser") 230 | 231 | build_dir = Path("docs", "_build") 232 | if build_dir.exists(): 233 | shutil.rmtree(build_dir) 234 | 235 | session.run("sphinx-autobuild", *args) 236 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. 2 | 3 | [[package]] 4 | name = "attrs" 5 | version = "24.3.0" 6 | description = "Classes Without Boilerplate" 7 | optional = false 8 | python-versions = ">=3.8" 9 | files = [ 10 | {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, 11 | {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, 12 | ] 13 | 14 | [package.extras] 15 | benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] 16 | cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] 17 | dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] 18 | docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] 19 | tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] 20 | tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] 21 | 22 | [[package]] 23 | name = "binaryornot" 24 | version = "0.4.4" 25 | description = "Ultra-lightweight pure Python package to check if a file is binary or text." 26 | optional = false 27 | python-versions = "*" 28 | files = [ 29 | {file = "binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4"}, 30 | {file = "binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061"}, 31 | ] 32 | 33 | [package.dependencies] 34 | chardet = ">=3.0.2" 35 | 36 | [[package]] 37 | name = "cffi" 38 | version = "1.17.1" 39 | description = "Foreign Function Interface for Python calling C code." 40 | optional = false 41 | python-versions = ">=3.8" 42 | files = [ 43 | {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, 44 | {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, 45 | {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, 46 | {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, 47 | {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, 48 | {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, 49 | {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, 50 | {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, 51 | {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, 52 | {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, 53 | {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, 54 | {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, 55 | {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, 56 | {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, 57 | {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, 58 | {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, 59 | {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, 60 | {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, 61 | {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, 62 | {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, 63 | {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, 64 | {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, 65 | {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, 66 | {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, 67 | {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, 68 | {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, 69 | {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, 70 | {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, 71 | {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, 72 | {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, 73 | {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, 74 | {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, 75 | {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, 76 | {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, 77 | {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, 78 | {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, 79 | {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, 80 | {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, 81 | {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, 82 | {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, 83 | {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, 84 | {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, 85 | {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, 86 | {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, 87 | {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, 88 | {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, 89 | {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, 90 | {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, 91 | {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, 92 | {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, 93 | {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, 94 | {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, 95 | {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, 96 | {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, 97 | {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, 98 | {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, 99 | {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, 100 | {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, 101 | {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, 102 | {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, 103 | {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, 104 | {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, 105 | {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, 106 | {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, 107 | {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, 108 | {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, 109 | {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, 110 | ] 111 | 112 | [package.dependencies] 113 | pycparser = "*" 114 | 115 | [[package]] 116 | name = "chardet" 117 | version = "5.2.0" 118 | description = "Universal encoding detector for Python 3" 119 | optional = false 120 | python-versions = ">=3.7" 121 | files = [ 122 | {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, 123 | {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, 124 | ] 125 | 126 | [[package]] 127 | name = "click" 128 | version = "8.1.8" 129 | description = "Composable command line interface toolkit" 130 | optional = false 131 | python-versions = ">=3.7" 132 | files = [ 133 | {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, 134 | {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, 135 | ] 136 | 137 | [package.dependencies] 138 | colorama = {version = "*", markers = "platform_system == \"Windows\""} 139 | 140 | [[package]] 141 | name = "colorama" 142 | version = "0.4.6" 143 | description = "Cross-platform colored terminal text." 144 | optional = false 145 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" 146 | files = [ 147 | {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, 148 | {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, 149 | ] 150 | 151 | [[package]] 152 | name = "exceptiongroup" 153 | version = "1.2.2" 154 | description = "Backport of PEP 654 (exception groups)" 155 | optional = false 156 | python-versions = ">=3.7" 157 | files = [ 158 | {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, 159 | {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, 160 | ] 161 | 162 | [package.extras] 163 | test = ["pytest (>=6)"] 164 | 165 | [[package]] 166 | name = "humanize" 167 | version = "4.11.0" 168 | description = "Python humanize utilities" 169 | optional = false 170 | python-versions = ">=3.9" 171 | files = [ 172 | {file = "humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0"}, 173 | {file = "humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"}, 174 | ] 175 | 176 | [package.extras] 177 | tests = ["freezegun", "pytest", "pytest-cov"] 178 | 179 | [[package]] 180 | name = "hypothesis" 181 | version = "6.123.2" 182 | description = "A library for property-based testing" 183 | optional = false 184 | python-versions = ">=3.9" 185 | files = [ 186 | {file = "hypothesis-6.123.2-py3-none-any.whl", hash = "sha256:0a8bf07753f1436f1b8697a13ea955f3fef3ef7b477c2972869b1d142bcdb30e"}, 187 | {file = "hypothesis-6.123.2.tar.gz", hash = "sha256:02c25552783764146b191c69eef69d8375827b58a75074055705ab8fdbc95fc5"}, 188 | ] 189 | 190 | [package.dependencies] 191 | attrs = ">=22.2.0" 192 | lark = {version = ">=0.10.1", optional = true, markers = "extra == \"lark\""} 193 | sortedcontainers = ">=2.1.0,<3.0.0" 194 | 195 | [package.extras] 196 | all = ["black (>=19.10b0)", "click (>=7.0)", "crosshair-tool (>=0.0.78)", "django (>=4.2)", "dpcontracts (>=0.4)", "hypothesis-crosshair (>=0.0.18)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.19.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2024.2)"] 197 | cli = ["black (>=19.10b0)", "click (>=7.0)", "rich (>=9.0.0)"] 198 | codemods = ["libcst (>=0.3.16)"] 199 | crosshair = ["crosshair-tool (>=0.0.78)", "hypothesis-crosshair (>=0.0.18)"] 200 | dateutil = ["python-dateutil (>=1.4)"] 201 | django = ["django (>=4.2)"] 202 | dpcontracts = ["dpcontracts (>=0.4)"] 203 | ghostwriter = ["black (>=19.10b0)"] 204 | lark = ["lark (>=0.10.1)"] 205 | numpy = ["numpy (>=1.19.3)"] 206 | pandas = ["pandas (>=1.1)"] 207 | pytest = ["pytest (>=4.6)"] 208 | pytz = ["pytz (>=2014.1)"] 209 | redis = ["redis (>=3.0.0)"] 210 | zoneinfo = ["tzdata (>=2024.2)"] 211 | 212 | [[package]] 213 | name = "hypothesmith" 214 | version = "0.3.3" 215 | description = "Hypothesis strategies for generating Python programs, something like CSmith" 216 | optional = false 217 | python-versions = ">=3.8" 218 | files = [ 219 | {file = "hypothesmith-0.3.3-py3-none-any.whl", hash = "sha256:fdb0172f9de97d09450da40da7da083fdd118bcd2f88b1a2289413d2d496b1b1"}, 220 | {file = "hypothesmith-0.3.3.tar.gz", hash = "sha256:96c14802d6c8e85d8975264176878db54b28d2ed921fdbfedc2e6b8ce3c81716"}, 221 | ] 222 | 223 | [package.dependencies] 224 | hypothesis = {version = ">=6.93.0", extras = ["lark"]} 225 | libcst = ">=1.0.1" 226 | 227 | [[package]] 228 | name = "idna" 229 | version = "3.10" 230 | description = "Internationalized Domain Names in Applications (IDNA)" 231 | optional = false 232 | python-versions = ">=3.6" 233 | files = [ 234 | {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, 235 | {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, 236 | ] 237 | 238 | [package.extras] 239 | all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] 240 | 241 | [[package]] 242 | name = "lark" 243 | version = "1.2.2" 244 | description = "a modern parsing library" 245 | optional = false 246 | python-versions = ">=3.8" 247 | files = [ 248 | {file = "lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c"}, 249 | {file = "lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80"}, 250 | ] 251 | 252 | [package.extras] 253 | atomic-cache = ["atomicwrites"] 254 | interegular = ["interegular (>=0.3.1,<0.4.0)"] 255 | nearley = ["js2py"] 256 | regex = ["regex"] 257 | 258 | [[package]] 259 | name = "libcst" 260 | version = "1.5.1" 261 | description = "A concrete syntax tree with AST-like properties for Python 3.0 through 3.13 programs." 262 | optional = false 263 | python-versions = ">=3.9" 264 | files = [ 265 | {file = "libcst-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab83633e61ee91df575a3838b1e73c371f19d4916bf1816554933235553d41ea"}, 266 | {file = "libcst-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b58a49895d95ec1fd34fad041a142d98edf9b51fcaf632337c13befeb4d51c7c"}, 267 | {file = "libcst-1.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d9ec764aa781ef35ab96b693569ac3dced16df9feb40ee6c274d13e86a1472e"}, 268 | {file = "libcst-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99bbffd8596d192bc0e844a4cf3c4fc696979d4e20ab1c0774a01768a59b47ed"}, 269 | {file = "libcst-1.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec6ee607cfe4cc4cc93e56e0188fdb9e50399d61a1262d58229752946f288f5e"}, 270 | {file = "libcst-1.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:72132756f985a19ef64d702a821099d4afc3544974662772b44cbc55b7279727"}, 271 | {file = "libcst-1.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:40b75bf2d70fc0bc26b1fa73e61bdc46fef59f5c71aedf16128e7c33db8d5e40"}, 272 | {file = "libcst-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:56c944acaa781b8e586df3019374f5cf117054d7fc98f85be1ba84fe810005dc"}, 273 | {file = "libcst-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:db7711a762b0327b581be5a963908fecd74412bdda34db34553faa521563c22d"}, 274 | {file = "libcst-1.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa524bd012aaae1f485fd44490ef5abf708b14d2addc0f06b28de3e4585c4b9e"}, 275 | {file = "libcst-1.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ffb8135c09e41e8cf710b152c33e9b7f1d0d0b9f242bae0c502eb082fdb1fb"}, 276 | {file = "libcst-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76a8ac7a84f9b6f678a668bff85b360e0a93fa8d7f25a74a206a28110734bb2a"}, 277 | {file = "libcst-1.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:89c808bdb5fa9ca02df41dd234cbb0e9de0d2e0c029c7063d5435a9f6781cc10"}, 278 | {file = "libcst-1.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40fbbaa8b839bfbfa5b300623ca2b6b0768b58bbc31b341afbc99110c9bee232"}, 279 | {file = "libcst-1.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c7021e3904d8d088c369afc3fe17c279883e583415ef07edacadba76cfbecd27"}, 280 | {file = "libcst-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f053a5deb6a214972dbe9fa26ecd8255edb903de084a3d7715bf9e9da8821c50"}, 281 | {file = "libcst-1.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:666813950b8637af0c0e96b1ca46f5d5f183d2fe50bbac2186f5b283a99f3529"}, 282 | {file = "libcst-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b58b36022ae77a5a00002854043ae95c03e92f6062ad08473eff326f32efa0"}, 283 | {file = "libcst-1.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eeb13d7c598fe9a798a1d22eae56ab3d3d599b38b83436039bd6ae229fc854d7"}, 284 | {file = "libcst-1.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5987daff8389b0df60b5c20499ff4fb73fc03cb3ae1f6a746eefd204ed08df85"}, 285 | {file = "libcst-1.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00f3d2f32ee081bad3394546b0b9ac5e31686d3b5cfe4892d716d2ba65f9ec08"}, 286 | {file = "libcst-1.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ff21005c33b634957a98db438e882522febf1cacc62fa716f29e163a3f5871a"}, 287 | {file = "libcst-1.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:15697ea9f1edbb9a263364d966c72abda07195d1c1a6838eb79af057f1040770"}, 288 | {file = "libcst-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:cedd4c8336e01c51913113fbf5566b8f61a86d90f3d5cc5b1cb5049575622c5f"}, 289 | {file = "libcst-1.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:06a9b4c9b76da4a7399e6f1f3a325196fb5febd3ea59fac1f68e2116f3517cd8"}, 290 | {file = "libcst-1.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:940ec4c8db4c2d620a7268d6c83e64ff646e4afd74ae5183d0f0ef3b80e05be0"}, 291 | {file = "libcst-1.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fbccb016b1ac6d892344300dcccc8a16887b71bb7f875ba56c0ed6c1a7ade8be"}, 292 | {file = "libcst-1.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c615af2117320e9a218083c83ec61227d3547e38a0de80329376971765f27a9e"}, 293 | {file = "libcst-1.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02b38fa4d9f13e79fe69e9b5407b9e173557bcfb5960f7866cf4145af9c7ae09"}, 294 | {file = "libcst-1.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3334afe9e7270e175de01198f816b0dc78dda94d9d72152b61851c323e4e741e"}, 295 | {file = "libcst-1.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26c804fa8091747128579013df0b5f8e6b0c7904d9c4ee83841f136f53e18684"}, 296 | {file = "libcst-1.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:b5a0d3c632aa2b21c5fa145e4e8dbf86f45c9b37a64c0b7221a5a45caf58915a"}, 297 | {file = "libcst-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1cc7393aaac733e963f0ee00466d059db74a38e15fc7e6a46dddd128c5be8d08"}, 298 | {file = "libcst-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bbaf5755be50fa9b35a3d553d1e62293fbb2ee5ce2c16c7e7ffeb2746af1ab88"}, 299 | {file = "libcst-1.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e397f5b6c0fc271acea44579f154b0f3ab36011050f6db75ab00cef47441946"}, 300 | {file = "libcst-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1947790a4fd7d96bcc200a6ecaa528045fcb26a34a24030d5859c7983662289e"}, 301 | {file = "libcst-1.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:697eabe9f5ffc40f76d6d02e693274e0a382826d0cf8183bd44e7407dfb0ab90"}, 302 | {file = "libcst-1.5.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dc06b7c60d086ef1832aebfd31b64c3c8a645adf0c5638d6243e5838f6a9356e"}, 303 | {file = "libcst-1.5.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:19e39cfef4316599ca20d1c821490aeb783b52e8a8543a824972a525322a85d0"}, 304 | {file = "libcst-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:01e01c04f0641188160d3b99c6526436e93a3fbf9783dba970f9885a77ec9b38"}, 305 | {file = "libcst-1.5.1.tar.gz", hash = "sha256:71cb294db84df9e410208009c732628e920111683c2f2b2e0c5b71b98464f365"}, 306 | ] 307 | 308 | [package.dependencies] 309 | pyyaml = ">=5.2" 310 | 311 | [package.extras] 312 | dev = ["Sphinx (>=5.1.1)", "black (==24.8.0)", "build (>=0.10.0)", "coverage[toml] (>=4.5.4)", "fixit (==2.1.0)", "flake8 (==7.1.1)", "hypothesis (>=4.36.0)", "hypothesmith (>=0.0.4)", "jinja2 (==3.1.4)", "jupyter (>=1.0.0)", "maturin (>=1.7.0,<1.8)", "nbsphinx (>=0.4.2)", "prompt-toolkit (>=2.0.9)", "pyre-check (==0.9.18)", "setuptools-rust (>=1.5.2)", "setuptools-scm (>=6.0.1)", "slotscheck (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "ufmt (==2.7.3)", "usort (==1.0.8.post1)"] 313 | 314 | [[package]] 315 | name = "outcome" 316 | version = "1.3.0.post0" 317 | description = "Capture the outcome of Python function calls." 318 | optional = false 319 | python-versions = ">=3.7" 320 | files = [ 321 | {file = "outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b"}, 322 | {file = "outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8"}, 323 | ] 324 | 325 | [package.dependencies] 326 | attrs = ">=19.2.0" 327 | 328 | [[package]] 329 | name = "pycparser" 330 | version = "2.22" 331 | description = "C parser in Python" 332 | optional = false 333 | python-versions = ">=3.8" 334 | files = [ 335 | {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, 336 | {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, 337 | ] 338 | 339 | [[package]] 340 | name = "pyyaml" 341 | version = "6.0.2" 342 | description = "YAML parser and emitter for Python" 343 | optional = false 344 | python-versions = ">=3.8" 345 | files = [ 346 | {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, 347 | {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, 348 | {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, 349 | {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, 350 | {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, 351 | {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, 352 | {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, 353 | {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, 354 | {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, 355 | {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, 356 | {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, 357 | {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, 358 | {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, 359 | {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, 360 | {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, 361 | {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, 362 | {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, 363 | {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, 364 | {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, 365 | {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, 366 | {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, 367 | {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, 368 | {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, 369 | {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, 370 | {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, 371 | {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, 372 | {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, 373 | {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, 374 | {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, 375 | {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, 376 | {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, 377 | {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, 378 | {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, 379 | {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, 380 | {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, 381 | {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, 382 | {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, 383 | {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, 384 | {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, 385 | {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, 386 | {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, 387 | {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, 388 | {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, 389 | {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, 390 | {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, 391 | {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, 392 | {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, 393 | {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, 394 | {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, 395 | {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, 396 | {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, 397 | {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, 398 | {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, 399 | ] 400 | 401 | [[package]] 402 | name = "sniffio" 403 | version = "1.3.1" 404 | description = "Sniff out which async library your code is running under" 405 | optional = false 406 | python-versions = ">=3.7" 407 | files = [ 408 | {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, 409 | {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, 410 | ] 411 | 412 | [[package]] 413 | name = "sortedcontainers" 414 | version = "2.4.0" 415 | description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" 416 | optional = false 417 | python-versions = "*" 418 | files = [ 419 | {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, 420 | {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, 421 | ] 422 | 423 | [[package]] 424 | name = "trio" 425 | version = "0.28.0" 426 | description = "A friendly Python library for async concurrency and I/O" 427 | optional = false 428 | python-versions = ">=3.9" 429 | files = [ 430 | {file = "trio-0.28.0-py3-none-any.whl", hash = "sha256:56d58977acc1635735a96581ec70513cc781b8b6decd299c487d3be2a721cd94"}, 431 | {file = "trio-0.28.0.tar.gz", hash = "sha256:4e547896fe9e8a5658e54e4c7c5fa1db748cbbbaa7c965e7d40505b928c73c05"}, 432 | ] 433 | 434 | [package.dependencies] 435 | attrs = ">=23.2.0" 436 | cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""} 437 | idna = "*" 438 | outcome = "*" 439 | sniffio = ">=1.3.0" 440 | sortedcontainers = "*" 441 | 442 | [[package]] 443 | name = "typing-extensions" 444 | version = "4.12.2" 445 | description = "Backported and Experimental Type Hints for Python 3.8+" 446 | optional = false 447 | python-versions = ">=3.8" 448 | files = [ 449 | {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, 450 | {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, 451 | ] 452 | 453 | [[package]] 454 | name = "urwid" 455 | version = "2.6.16" 456 | description = "A full-featured console (xterm et al.) user interface library" 457 | optional = false 458 | python-versions = ">3.7" 459 | files = [ 460 | {file = "urwid-2.6.16-py3-none-any.whl", hash = "sha256:de14896c6df9eb759ed1fd93e0384a5279e51e0dde8f621e4083f7a8368c0797"}, 461 | {file = "urwid-2.6.16.tar.gz", hash = "sha256:93ad239939e44c385e64aa00027878b9e5c486d59e855ec8ab5b1e1adcdb32a2"}, 462 | ] 463 | 464 | [package.dependencies] 465 | typing-extensions = "*" 466 | wcwidth = "*" 467 | 468 | [package.extras] 469 | curses = ["windows-curses"] 470 | glib = ["PyGObject"] 471 | lcd = ["pyserial"] 472 | serial = ["pyserial"] 473 | tornado = ["tornado (>=5.0)"] 474 | trio = ["exceptiongroup", "trio (>=0.22.0)"] 475 | twisted = ["twisted"] 476 | zmq = ["zmq"] 477 | 478 | [[package]] 479 | name = "wcwidth" 480 | version = "0.2.13" 481 | description = "Measures the displayed width of unicode strings in a terminal" 482 | optional = false 483 | python-versions = "*" 484 | files = [ 485 | {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, 486 | {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, 487 | ] 488 | 489 | [metadata] 490 | lock-version = "2.0" 491 | python-versions = ">=3.12, <4.0" 492 | content-hash = "12f9acbdde7e4a59dfdcb16be7bd2ca35bd2e1b70ded201ac99026d0c011d68a" 493 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "shrinkray" 3 | version = "0.0.0" 4 | description = "Shrink Ray" 5 | authors = ["David R. MacIver "] 6 | license = "MIT" 7 | readme = "README.md" 8 | homepage = "https://github.com/DRMacIver/shrinkray" 9 | repository = "https://github.com/DRMacIver/shrinkray" 10 | documentation = "https://shrinkray.readthedocs.io" 11 | classifiers = [ 12 | "Development Status :: 3 - Alpha", 13 | ] 14 | 15 | [tool.poetry.urls] 16 | Changelog = "https://github.com/DRMacIver/shrinkray/releases" 17 | 18 | [tool.poetry.dependencies] 19 | python = ">=3.12, <4.0" 20 | click = ">=8.0.1" 21 | chardet = "^5.2.0" 22 | trio = "^0.28.0" 23 | urwid = "^2.2.3" 24 | humanize = "^4.9.0" 25 | libcst = "^1.1.0" 26 | exceptiongroup = "^1.2.0" 27 | binaryornot = "^0.4.4" 28 | 29 | [tool.poetry.scripts] 30 | shrinkray = "shrinkray.__main__:main" 31 | 32 | [tool.poetry.group.dev.dependencies] 33 | hypothesis = "^6.92.1" 34 | hypothesmith = "^0.3.1" 35 | 36 | [tool.coverage.paths] 37 | source = ["src", "*/site-packages"] 38 | tests = ["tests", "*/tests"] 39 | 40 | [tool.coverage.run] 41 | branch = true 42 | source = ["shrinkray", "tests"] 43 | 44 | [tool.coverage.report] 45 | show_missing = true 46 | fail_under = 100 47 | 48 | [tool.isort] 49 | profile = "black" 50 | force_single_line = false 51 | lines_after_imports = 2 52 | 53 | [tool.mypy] 54 | strict = true 55 | warn_unreachable = true 56 | pretty = true 57 | show_column_numbers = true 58 | show_error_context = true 59 | disable_error_code = ["import-untyped"] 60 | 61 | 62 | [build-system] 63 | requires = ["poetry-core>=1.0.0"] 64 | build-backend = "poetry.core.masonry.api" 65 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | trio_mode = true 3 | -------------------------------------------------------------------------------- /src/shrinkray/__init__.py: -------------------------------------------------------------------------------- 1 | """Shrink Ray.""" 2 | -------------------------------------------------------------------------------- /src/shrinkray/passes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DRMacIver/shrinkray/3b4d19902c0dbd1912c8825ca9dba16d4344cf02/src/shrinkray/passes/__init__.py -------------------------------------------------------------------------------- /src/shrinkray/passes/bytes.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict, deque 2 | from typing import Sequence 3 | 4 | from attrs import define 5 | 6 | from shrinkray.passes.definitions import Format, ReductionProblem 7 | from shrinkray.passes.patching import Cuts, Patches, apply_patches 8 | 9 | 10 | @define(frozen=True) 11 | class Encoding(Format[bytes, str]): 12 | encoding: str 13 | 14 | def __repr__(self) -> str: 15 | return f"Encoding({repr(self.encoding)})" 16 | 17 | @property 18 | def name(self) -> str: 19 | return self.encoding 20 | 21 | def parse(self, input: bytes) -> str: 22 | return input.decode(self.encoding) 23 | 24 | def dumps(self, input: str) -> bytes: 25 | return input.encode(self.encoding) 26 | 27 | 28 | @define(frozen=True) 29 | class Split(Format[bytes, list[bytes]]): 30 | splitter: bytes 31 | 32 | def __repr__(self) -> str: 33 | return f"Split({repr(self.splitter)})" 34 | 35 | @property 36 | def name(self) -> str: 37 | return f"split({repr(self.splitter)})" 38 | 39 | def parse(self, input: bytes) -> list[bytes]: 40 | return input.split(self.splitter) 41 | 42 | def dumps(self, input: list[bytes]) -> bytes: 43 | return self.splitter.join(input) 44 | 45 | 46 | def find_ngram_endpoints(value: bytes) -> list[tuple[int, list[int]]]: 47 | if len(set(value)) <= 1: 48 | return [] 49 | queue: deque[tuple[int, Sequence[int]]] = deque([(0, range(len(value)))]) 50 | results: list[tuple[int, list[int]]] = [] 51 | 52 | while queue and len(results) < 10000: 53 | k, indices = queue.popleft() 54 | 55 | if k > 1: 56 | normalized: list[int] = [] 57 | for i in indices: 58 | if not normalized or i >= normalized[-1] + k: 59 | normalized.append(i) 60 | indices = normalized 61 | 62 | while ( 63 | indices[-1] + k < len(value) and len({value[i + k] for i in indices}) == 1 64 | ): 65 | k += 1 66 | 67 | if k > 0 and (indices[0] == 0 or len({value[i - 1] for i in indices}) > 1): 68 | assert isinstance(indices, list), value 69 | results.append((k, indices)) 70 | 71 | split: dict[int, list[int]] = defaultdict(list) 72 | for i in indices: 73 | try: 74 | split[value[i + k]].append(i) 75 | except IndexError: 76 | pass 77 | queue.extend([(k + 1, v) for v in split.values() if len(v) > 1]) 78 | 79 | return results 80 | 81 | 82 | def tokenize(text: bytes) -> list[bytes]: 83 | result: list[bytes] = [] 84 | i = 0 85 | while i < len(text): 86 | c = bytes([text[i]]) 87 | j = i + 1 88 | if b"A" <= c <= b"z": 89 | while j < len(text) and ( 90 | b"A"[0] <= text[j] <= b"z"[0] 91 | or text[j] == b"_"[0] 92 | or b"0"[0] <= text[j] <= b"9"[0] 93 | ): 94 | j += 1 95 | elif b"0" <= c <= b"9": 96 | while j < len(text) and ( 97 | text[j] == b"."[0] or b"0"[0] <= text[j] <= b"9"[0] 98 | ): 99 | j += 1 100 | elif c == b" ": 101 | while j < len(text) and (text[j] == b" "[0]): 102 | j += 1 103 | result.append(text[i:j]) 104 | i = j 105 | assert b"".join(result) == text 106 | return result 107 | 108 | 109 | MAX_DELETE_INTERVAL = 8 110 | 111 | 112 | async def lexeme_based_deletions( 113 | problem: ReductionProblem[bytes], min_size: int = 8 114 | ) -> None: 115 | intervals_by_k: dict[int, set[tuple[int, int]]] = defaultdict(set) 116 | 117 | for k, endpoints in find_ngram_endpoints(problem.current_test_case): 118 | intervals_by_k[k].update(zip(endpoints, endpoints[1:])) 119 | 120 | intervals_to_delete = [ 121 | t 122 | for _, intervals in sorted(intervals_by_k.items(), reverse=True) 123 | for t in sorted(intervals, key=lambda t: (t[1] - t[0], t[0]), reverse=True) 124 | if t[1] - t[0] >= min_size 125 | ] 126 | 127 | await delete_intervals(problem, intervals_to_delete, shuffle=True) 128 | 129 | 130 | async def delete_intervals( 131 | problem: ReductionProblem[bytes], 132 | intervals_to_delete: list[tuple[int, int]], 133 | shuffle: bool = False, 134 | ) -> None: 135 | await apply_patches(problem, Cuts(), [[t] for t in intervals_to_delete]) 136 | 137 | 138 | def brace_intervals(target: bytes, brace: bytes) -> list[tuple[int, int]]: 139 | open, close = brace 140 | intervals: list[tuple[int, int]] = [] 141 | stack: list[int] = [] 142 | for i, c in enumerate(target): 143 | if c == open: 144 | stack.append(i) 145 | elif c == close and stack: 146 | start = stack.pop() + 1 147 | end = i 148 | if end > start: 149 | intervals.append((start, end)) 150 | return intervals 151 | 152 | 153 | async def debracket(problem: ReductionProblem[bytes]) -> None: 154 | cuts = [ 155 | [(u - 1, u), (v, v + 1)] 156 | for brackets in [b"{}", b"()", b"[]"] 157 | for u, v in brace_intervals(problem.current_test_case, brackets) 158 | ] 159 | await apply_patches( 160 | problem, 161 | Cuts(), 162 | cuts, 163 | ) 164 | 165 | 166 | def quote_intervals(target: bytes) -> list[tuple[int, int]]: 167 | indices: dict[int, list[int]] = defaultdict(list) 168 | for i, c in enumerate(target): 169 | indices[c].append(i) 170 | 171 | intervals: list[tuple[int, int]] = [] 172 | for quote in b"\"'": 173 | xs = indices[quote] 174 | for u, v in zip(xs, xs[1:], strict=False): 175 | if u + 1 < v: 176 | intervals.append((u + 1, v)) 177 | return intervals 178 | 179 | 180 | async def hollow(problem: ReductionProblem[bytes]) -> None: 181 | target = problem.current_test_case 182 | intervals: list[tuple[int, int]] = [] 183 | for b in [ 184 | quote_intervals(target), 185 | brace_intervals(target, b"[]"), 186 | brace_intervals(target, b"{}"), 187 | ]: 188 | b.sort(key=lambda t: (t[1] - t[0], t[0])) 189 | intervals.extend(b) 190 | await delete_intervals( 191 | problem, 192 | intervals, 193 | ) 194 | 195 | 196 | async def short_deletions(problem: ReductionProblem[bytes]) -> None: 197 | target = problem.current_test_case 198 | await delete_intervals( 199 | problem, 200 | [ 201 | (i, j) 202 | for i in range(len(target)) 203 | for j in range(i + 1, min(i + 11, len(target) + 1)) 204 | ], 205 | ) 206 | 207 | 208 | async def lift_braces(problem: ReductionProblem[bytes]) -> None: 209 | target = problem.current_test_case 210 | 211 | open_brace, close_brace = b"{}" 212 | start_stack: list[int] = [] 213 | child_stack: list[list[tuple[int, int]]] = [] 214 | 215 | results: list[tuple[int, int, list[tuple[int, int]]]] = [] 216 | 217 | for i, c in enumerate(target): 218 | if c == open_brace: 219 | start_stack.append(i) 220 | child_stack.append([]) 221 | elif c == close_brace and start_stack: 222 | start = start_stack.pop() + 1 223 | end = i 224 | children = child_stack.pop() 225 | if child_stack: 226 | child_stack[-1].append((start, end)) 227 | if end > start: 228 | results.append((start, end, children)) 229 | 230 | cuts: list[list[tuple[int, int]]] = [] 231 | for start, end, children in results: 232 | for child_start, child_end in children: 233 | cuts.append([(start, child_start), (child_end, end)]) 234 | 235 | await apply_patches(problem, Cuts(), cuts) 236 | 237 | 238 | @define(frozen=True) 239 | class Tokenize(Format[bytes, list[bytes]]): 240 | def __repr__(self) -> str: 241 | return "tokenize" 242 | 243 | @property 244 | def name(self) -> str: 245 | return "tokenize" 246 | 247 | def parse(self, input: bytes) -> list[bytes]: 248 | return tokenize(input) 249 | 250 | def dumps(self, input: list[bytes]) -> bytes: 251 | return b"".join(input) 252 | 253 | 254 | async def delete_byte_spans(problem: ReductionProblem[bytes]) -> None: 255 | indices: dict[int, list[int]] = defaultdict(list) 256 | target = problem.current_test_case 257 | for i, c in enumerate(target): 258 | indices[c].append(i) 259 | 260 | spans: list[tuple[int, int]] = [] 261 | 262 | for c, ix in sorted(indices.items()): 263 | if len(ix) > 1: 264 | spans.append((0, ix[0] + 1)) 265 | spans.extend(zip(ix, ix[1:])) 266 | spans.append((ix[-1], len(target))) 267 | 268 | await apply_patches(problem, Cuts(), [[s] for s in spans]) 269 | 270 | 271 | async def remove_indents(problem: ReductionProblem[bytes]) -> None: 272 | target = problem.current_test_case 273 | spans: list[list[tuple[int, int]]] = [] 274 | 275 | newline = ord(b"\n") 276 | space = ord(b" ") 277 | 278 | for i, c in enumerate(target): 279 | if c == newline: 280 | j = i + 1 281 | while j < len(target) and target[j] == space: 282 | j += 1 283 | 284 | if j > i + 1: 285 | spans.append([(i + 1, j)]) 286 | 287 | await apply_patches(problem, Cuts(), spans) 288 | 289 | 290 | async def remove_whitespace(problem: ReductionProblem[bytes]) -> None: 291 | target = problem.current_test_case 292 | spans: list[list[tuple[int, int]]] = [] 293 | 294 | for i, c in enumerate(target): 295 | char = bytes([c]) 296 | if char.isspace(): 297 | j = i + 1 298 | while j < len(target) and target[j : j + 1].isspace(): 299 | j += 1 300 | 301 | if j > i + 1: 302 | spans.append([(i, j)]) 303 | if j > i + 2: 304 | spans.append([(i + 1, j)]) 305 | 306 | await apply_patches(problem, Cuts(), spans) 307 | 308 | 309 | class NewlineReplacer(Patches[frozenset[int], bytes]): 310 | @property 311 | def empty(self) -> frozenset[int]: 312 | return frozenset() 313 | 314 | def combine(self, *patches: frozenset[int]) -> frozenset[int]: 315 | result: set[int] = set() 316 | for p in patches: 317 | result.update(p) 318 | return frozenset(result) 319 | 320 | def apply(self, patch: frozenset[int], target: bytes) -> bytes: 321 | result = bytearray() 322 | 323 | for i, c in enumerate(target): 324 | if i in patch: 325 | result.extend(b"\n") 326 | else: 327 | result.append(c) 328 | return bytes(result) 329 | 330 | def size(self, patch: frozenset[int]) -> int: 331 | return len(patch) 332 | 333 | 334 | async def replace_space_with_newlines(problem: ReductionProblem[bytes]) -> None: 335 | await apply_patches( 336 | problem, 337 | NewlineReplacer(), 338 | [ 339 | frozenset({i}) 340 | for i, c in enumerate(problem.current_test_case) 341 | if c in b" \t" 342 | ], 343 | ) 344 | 345 | 346 | ReplacementPatch = dict[int, int] 347 | 348 | 349 | class ByteReplacement(Patches[ReplacementPatch, bytes]): 350 | @property 351 | def empty(self) -> ReplacementPatch: 352 | return {} 353 | 354 | def combine(self, *patches: ReplacementPatch) -> ReplacementPatch: 355 | result = {} 356 | for p in patches: 357 | for k, v in p.items(): 358 | if k not in result: 359 | result[k] = v 360 | else: 361 | result[k] = min(result[k], v) 362 | return result 363 | 364 | def apply(self, patch: ReplacementPatch, target: bytes) -> bytes: 365 | result = bytearray() 366 | for c in target: 367 | result.append(patch.get(c, c)) 368 | return bytes(result) 369 | 370 | def size(self, patch: ReplacementPatch) -> int: 371 | return 0 372 | 373 | 374 | async def lower_bytes(problem: ReductionProblem[bytes]) -> None: 375 | sources = sorted(set(problem.current_test_case)) 376 | 377 | patches = [ 378 | {c: r} 379 | for c in sources 380 | for r in sorted({0, 1, c // 2, c - 1} | set(b" \t\r\n")) 381 | if r < c and r >= 0 382 | ] + [ 383 | {c: r, d: r} 384 | for c in sources 385 | for d in sources 386 | if c != d 387 | for r in sorted({0, 1, c // 2, c - 1, d // 2, d - 1} | set(b" \t\r\n")) 388 | if (r < c or r < d) and r >= 0 389 | ] 390 | 391 | await apply_patches(problem, ByteReplacement(), patches) 392 | 393 | 394 | class IndividualByteReplacement(Patches[ReplacementPatch, bytes]): 395 | @property 396 | def empty(self) -> ReplacementPatch: 397 | return {} 398 | 399 | def combine(self, *patches: ReplacementPatch) -> ReplacementPatch: 400 | result = {} 401 | for p in patches: 402 | for k, v in p.items(): 403 | if k not in result: 404 | result[k] = v 405 | else: 406 | result[k] = min(result[k], v) 407 | return result 408 | 409 | def apply(self, patch: ReplacementPatch, target: bytes) -> bytes: 410 | result = bytearray() 411 | for i, c in enumerate(target): 412 | result.append(patch.get(i, c)) 413 | return bytes(result) 414 | 415 | def size(self, patch: ReplacementPatch) -> int: 416 | return 0 417 | 418 | 419 | async def lower_individual_bytes(problem: ReductionProblem[bytes]) -> None: 420 | initial = problem.current_test_case 421 | patches = [ 422 | {i: r} 423 | for i, c in enumerate(initial) 424 | for r in sorted({0, 1, c // 2, c - 1} | set(b" \t\r\n")) 425 | if r < c and r >= 0 426 | ] + [ 427 | {i - 1: initial[i - 1] - 1, i: 255} 428 | for i, c in enumerate(initial) 429 | if i > 0 and initial[i - 1] > 0 and c == 0 430 | ] 431 | await apply_patches(problem, IndividualByteReplacement(), patches) 432 | 433 | 434 | RegionReplacementPatch = list[tuple[int, int, int]] 435 | 436 | 437 | class RegionReplacement(Patches[ReplacementPatch, bytes]): 438 | @property 439 | def empty(self) -> ReplacementPatch: 440 | return [] 441 | 442 | def combine(self, *patches: ReplacementPatch) -> ReplacementPatch: 443 | result = [] 444 | for p in patches: 445 | result.extend(p) 446 | return result 447 | 448 | def apply(self, patch: ReplacementPatch, target: bytes) -> bytes: 449 | result = bytearray(target) 450 | for i, j, d in patch: 451 | if d < result[i]: 452 | for k in range(i, j): 453 | result[k] = d 454 | return bytes(result) 455 | 456 | def size(self, patch: ReplacementPatch) -> int: 457 | return 0 458 | 459 | 460 | async def short_replacements(problem: ReductionProblem[bytes]) -> None: 461 | target = problem.current_test_case 462 | patches = [ 463 | [(i, j, c)] 464 | for c in [0, 1] + list(b"01 \t\n\r.") 465 | for i in range(len(target)) 466 | if target[i] > c 467 | for j in range(i + 1, min(i + 5, len(target) + 1)) 468 | ] 469 | 470 | await apply_patches(problem, RegionReplacement(), patches) 471 | 472 | 473 | WHITESPACE = b" \t\r\n" 474 | 475 | 476 | async def sort_whitespace(problem: ReductionProblem[bytes]) -> None: 477 | """NB: This is a stupid pass that we only really need for artificial 478 | test cases, but it's helpful for allowing those artificial test cases 479 | to expose other issues.""" 480 | 481 | whitespace_up_to = 0 482 | while ( 483 | whitespace_up_to < len(problem.current_test_case) 484 | and problem.current_test_case[whitespace_up_to] not in WHITESPACE 485 | ): 486 | whitespace_up_to += 1 487 | while ( 488 | whitespace_up_to < len(problem.current_test_case) 489 | and problem.current_test_case[whitespace_up_to] in WHITESPACE 490 | ): 491 | whitespace_up_to += 1 492 | 493 | # If the initial whitespace ends with a newline we want to keep it doing 494 | # that. This is mostly for Python purposes. 495 | if ( 496 | whitespace_up_to > 0 497 | and problem.current_test_case[whitespace_up_to - 1] == b"\n"[0] 498 | ): 499 | whitespace_up_to -= 1 500 | 501 | i = whitespace_up_to + 1 502 | 503 | while i < len(problem.current_test_case): 504 | if problem.current_test_case[i] not in WHITESPACE: 505 | i += 1 506 | continue 507 | 508 | async def can_move_to_whitespace(k): 509 | if i + k > len(problem.current_test_case): 510 | return False 511 | 512 | base = problem.current_test_case 513 | target = base[i : i + k] 514 | 515 | if any(c not in WHITESPACE for c in target): 516 | return False 517 | 518 | prefix = base[:whitespace_up_to] 519 | attempt = prefix + target + base[whitespace_up_to:i] + base[i + k :] 520 | return await problem.is_interesting(attempt) 521 | 522 | k = await problem.work.find_large_integer(can_move_to_whitespace) 523 | whitespace_up_to += k 524 | i += k + 1 525 | test_case = problem.current_test_case 526 | await problem.is_interesting( 527 | bytes(sorted(test_case[:whitespace_up_to])) + test_case[whitespace_up_to:] 528 | ) 529 | 530 | 531 | # These are some cheat substitutions that are sometimes helpful, but mostly 532 | # for passing stupid tests. 533 | STANDARD_SUBSTITUTIONS = [(b"\0\0", b"\1"), (b"\0\0", b"\xff")] 534 | 535 | 536 | async def standard_substitutions(problem: ReductionProblem[bytes]): 537 | i = 0 538 | while i < len(problem.current_test_case): 539 | for k, v in STANDARD_SUBSTITUTIONS: 540 | x = problem.current_test_case 541 | if i + len(k) <= len(x) and x[i : i + len(k)] == k: 542 | attempt = x[:i] + v + x[i + len(k) :] 543 | if await problem.is_interesting(attempt): 544 | assert problem.current_test_case == attempt 545 | break 546 | else: 547 | i += 1 548 | -------------------------------------------------------------------------------- /src/shrinkray/passes/clangdelta.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from glob import glob 4 | from shutil import which 5 | from tempfile import NamedTemporaryFile 6 | 7 | import trio 8 | 9 | from shrinkray.passes.definitions import ReductionPump 10 | from shrinkray.problem import ReductionProblem 11 | from shrinkray.work import NotFound 12 | 13 | C_FILE_EXTENSIONS = (".c", ".cpp", ".h", ".hpp", ".cxx", ".cc") 14 | 15 | 16 | def find_clang_delta(): 17 | clang_delta = which("clang_delta") or "" 18 | if not clang_delta: 19 | possible_locations = glob( 20 | "/opt/homebrew//Cellar/creduce/*/libexec/clang_delta" 21 | ) + glob("/usr/libexec/clang_delta") 22 | if possible_locations: 23 | clang_delta = max(possible_locations) 24 | return clang_delta 25 | 26 | 27 | TRANSFORMATIONS: list[str] = [ 28 | "aggregate-to-scalar", 29 | "binop-simplification", 30 | "callexpr-to-value", 31 | "class-template-to-class", 32 | "combine-global-var", 33 | "combine-local-var", 34 | "copy-propagation", 35 | "empty-struct-to-int", 36 | "expression-detector", 37 | "instantiate-template-param", 38 | "instantiate-template-type-param-to-int", 39 | "lift-assignment-expr", 40 | "local-to-global", 41 | "move-function-body", 42 | "move-global-var", 43 | "param-to-global", 44 | "param-to-local", 45 | "reduce-array-dim", 46 | "reduce-array-size", 47 | "reduce-class-template-param", 48 | "reduce-pointer-level", 49 | "reduce-pointer-pairs", 50 | "remove-addr-taken", 51 | "remove-array", 52 | "remove-base-class", 53 | "remove-ctor-initializer", 54 | "remove-enum-member-value", 55 | "remove-namespace", 56 | "remove-nested-function", 57 | "remove-pointer", 58 | "remove-trivial-base-template", 59 | "remove-unresolved-base", 60 | "remove-unused-enum-member", 61 | "remove-unused-field", 62 | "remove-unused-function", 63 | "remove-unused-outer-class", 64 | "remove-unused-var", 65 | "rename-class", 66 | "rename-cxx-method", 67 | "rename-fun", 68 | "rename-param", 69 | "rename-var", 70 | "replace-array-access-with-index", 71 | "replace-array-index-var", 72 | "replace-callexpr", 73 | "replace-class-with-base-template-spec", 74 | "replace-dependent-name", 75 | "replace-dependent-typedef", 76 | "replace-derived-class", 77 | "replace-function-def-with-decl", 78 | "replace-one-level-typedef-type", 79 | "replace-simple-typedef", 80 | "replace-undefined-function", 81 | "return-void", 82 | "simple-inliner", 83 | "simplify-callexpr", 84 | "simplify-comma-expr", 85 | "simplify-dependent-typedef", 86 | "simplify-if", 87 | "simplify-nested-class", 88 | "simplify-recursive-template-instantiation", 89 | "simplify-struct", 90 | "simplify-struct-union-decl", 91 | "template-arg-to-int", 92 | "template-non-type-arg-to-int", 93 | "unify-function-decl", 94 | "union-to-struct", 95 | "vector-to-array", 96 | ] 97 | 98 | 99 | class ClangDelta: 100 | def __init__(self, path: str): 101 | self.path_to_exec = path 102 | 103 | self.transformations: list[str] = TRANSFORMATIONS 104 | 105 | def __validate_transformation(self, transformation: str) -> None: 106 | if transformation not in self.transformations: 107 | raise ValueError(f"Invalid transformation {transformation}") 108 | 109 | async def query_instances(self, transformation: str, data: bytes) -> int: 110 | self.__validate_transformation(transformation) 111 | with NamedTemporaryFile(suffix=".cpp", delete_on_close=False) as tmp: 112 | tmp.write(data) 113 | tmp.close() 114 | 115 | try: 116 | results = ( 117 | await trio.run_process( 118 | [ 119 | self.path_to_exec, 120 | f"--query-instances={transformation}", 121 | tmp.name, 122 | ], 123 | capture_stdout=True, 124 | capture_stderr=True, 125 | ) 126 | ).stdout 127 | except subprocess.CalledProcessError as e: 128 | msg = (e.stdout + e.stderr).strip() 129 | if msg == b"Error: Unsupported file type!": 130 | raise ValueError("Not a C or C++ test case") 131 | elif b"Assertion failed" in msg: 132 | return 0 133 | else: 134 | raise ClangDeltaError(msg) 135 | finally: 136 | os.unlink(tmp.name) 137 | 138 | prefix = b"Available transformation instances:" 139 | assert results.startswith(prefix) 140 | return int(results[len(prefix) :].strip().decode("ascii")) 141 | 142 | async def apply_transformation( 143 | self, transformation: str, counter: int, data: bytes 144 | ) -> bytes: 145 | self.__validate_transformation(transformation) 146 | with NamedTemporaryFile(suffix=".cpp", delete_on_close=False) as tmp: 147 | tmp.write(data) 148 | tmp.close() 149 | 150 | try: 151 | return ( 152 | await trio.run_process( 153 | [ 154 | self.path_to_exec, 155 | f"--transformation={transformation}", 156 | f"--counter={int(counter)}", 157 | tmp.name, 158 | ], 159 | capture_stdout=True, 160 | capture_stderr=True, 161 | ) 162 | ).stdout 163 | except subprocess.CalledProcessError as e: 164 | if e.stdout.strip() == b"Error: Unsupported file type!": 165 | raise ValueError("Not a C or C++ test case") 166 | elif ( 167 | e.stdout.strip() 168 | == b"Error: No modification to the transformed program!" 169 | ): 170 | return data 171 | elif b"Assertion failed" in e.stderr.strip(): 172 | return data 173 | else: 174 | raise ClangDeltaError(e.stdout + e.stderr) 175 | finally: 176 | os.unlink(tmp.name) 177 | 178 | 179 | class ClangDeltaError(Exception): 180 | def __init__(self, message): 181 | assert b"Assertion failed" not in message, message 182 | super().__init__(message) 183 | 184 | 185 | def clang_delta_pump( 186 | clang_delta: ClangDelta, transformation: str 187 | ) -> ReductionPump[bytes]: 188 | async def apply(problem: ReductionProblem[bytes]) -> bytes: 189 | target = problem.current_test_case 190 | assert target is not None 191 | try: 192 | n = await clang_delta.query_instances(transformation, target) 193 | except ValueError: 194 | import traceback 195 | 196 | traceback.print_exc() 197 | return target 198 | i = 1 199 | while i <= n: 200 | 201 | async def can_apply(j: int) -> bool: 202 | attempt = await clang_delta.apply_transformation( 203 | transformation, j, target 204 | ) 205 | assert attempt is not None 206 | if attempt == target: 207 | return False 208 | return await problem.is_interesting(attempt) 209 | 210 | try: 211 | i = await problem.work.find_first_value(range(i, n + 1), can_apply) 212 | except NotFound: 213 | break 214 | except ClangDeltaError as e: 215 | # Clang delta has a large number of internal assertions that you can trigger 216 | # if you feed it bad enough C++. We solve this problem by ignoring it. 217 | if b"Assertion failed" in e.args[0]: 218 | return target 219 | 220 | target = await clang_delta.apply_transformation(transformation, i, target) 221 | assert target is not None 222 | n = await clang_delta.query_instances(transformation, target) 223 | return target 224 | 225 | apply.__name__ = f"clang_delta({transformation})" 226 | 227 | return apply 228 | 229 | 230 | def clang_delta_pumps(clang_delta: ClangDelta) -> list[ReductionPump[bytes]]: 231 | return [ 232 | clang_delta_pump(clang_delta, transformation) 233 | for transformation in clang_delta.transformations 234 | ] 235 | -------------------------------------------------------------------------------- /src/shrinkray/passes/definitions.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from functools import wraps 3 | from typing import Awaitable, Callable, Generic, TypeVar 4 | 5 | from shrinkray.problem import ReductionProblem 6 | 7 | S = TypeVar("S") 8 | T = TypeVar("T") 9 | 10 | 11 | ReductionPass = Callable[[ReductionProblem[T]], Awaitable[None]] 12 | ReductionPump = Callable[[ReductionProblem[T]], Awaitable[T]] 13 | 14 | 15 | class ParseError(Exception): 16 | pass 17 | 18 | 19 | class Format(Generic[S, T], ABC): 20 | @property 21 | def name(self) -> str: 22 | return repr(self) 23 | 24 | @abstractmethod 25 | def parse(self, input: S) -> T: ... 26 | 27 | def is_valid(self, input: S) -> bool: 28 | try: 29 | self.parse(input) 30 | return True 31 | except ParseError: 32 | return False 33 | 34 | @abstractmethod 35 | def dumps(self, input: T) -> S: ... 36 | 37 | 38 | def compose(format: Format[S, T], reduction_pass: ReductionPass[T]) -> ReductionPass[S]: 39 | @wraps(reduction_pass) 40 | async def wrapped_pass(problem: ReductionProblem[S]) -> None: 41 | view = problem.view(format) 42 | 43 | try: 44 | view.current_test_case 45 | except ParseError: 46 | return 47 | 48 | await reduction_pass(view) 49 | 50 | wrapped_pass.__name__ = f"{format.name}/{reduction_pass.__name__}" 51 | 52 | return wrapped_pass 53 | -------------------------------------------------------------------------------- /src/shrinkray/passes/genericlanguages.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module of reduction passes designed for "things that look like programming languages". 3 | """ 4 | 5 | import re 6 | from functools import wraps 7 | from string import ascii_lowercase, ascii_uppercase 8 | from typing import AnyStr, Callable 9 | 10 | import trio 11 | from attr import define 12 | 13 | from shrinkray.passes.bytes import ByteReplacement, delete_intervals 14 | from shrinkray.passes.definitions import Format, ParseError, ReductionPass 15 | from shrinkray.passes.patching import PatchApplier, Patches, apply_patches 16 | from shrinkray.problem import BasicReductionProblem, ReductionProblem 17 | from shrinkray.work import NotFound 18 | 19 | 20 | @define(frozen=True) 21 | class Substring(Format[AnyStr, AnyStr]): 22 | prefix: AnyStr 23 | suffix: AnyStr 24 | 25 | @property 26 | def name(self) -> str: 27 | return f"Substring({len(self.prefix)}, {len(self.suffix)})" 28 | 29 | def parse(self, input: AnyStr) -> AnyStr: 30 | if input.startswith(self.prefix) and input.endswith(self.suffix): 31 | return input[len(self.prefix) : len(input) - len(self.suffix)] 32 | else: 33 | raise ParseError() 34 | 35 | def dumps(self, input: AnyStr) -> AnyStr: 36 | return self.prefix + input + self.suffix 37 | 38 | 39 | class RegionReplacingPatches(Patches[dict[int, AnyStr], AnyStr]): 40 | def __init__(self, regions): 41 | assert regions 42 | for (_, v), (u, _) in zip(regions, regions[1:]): 43 | assert v <= u 44 | self.regions = regions 45 | 46 | @property 47 | def empty(self): 48 | return {} 49 | 50 | def combine(self, *patches): 51 | result = {} 52 | for p in patches: 53 | result.update(p) 54 | return result 55 | 56 | def apply(self, patch, target): 57 | empty = target[:0] 58 | parts = [] 59 | prev = 0 60 | for j, (u, v) in enumerate(self.regions): 61 | assert v <= len(target) 62 | parts.append(target[prev:u]) 63 | try: 64 | parts.append(patch[j]) 65 | except KeyError: 66 | parts.append(target[u:v]) 67 | prev = v 68 | parts.append(target[prev:]) 69 | return empty.join(parts) 70 | 71 | def size(self, patch): 72 | total = 0 73 | for i, s in patch.items(): 74 | u, v = self.regions[i] 75 | return v - u - len(s) 76 | 77 | 78 | def regex_pass( 79 | pattern: AnyStr | re.Pattern[AnyStr], 80 | flags: re.RegexFlag = 0, 81 | ) -> Callable[[ReductionPass[AnyStr]], ReductionPass[AnyStr]]: 82 | if not isinstance(pattern, re.Pattern): 83 | pattern = re.compile(pattern, flags=flags) 84 | 85 | def inner(fn: ReductionPass[AnyStr]) -> ReductionPass[AnyStr]: 86 | @wraps(fn) 87 | async def reduction_pass(problem: ReductionProblem[AnyStr]) -> None: 88 | matching_regions = [] 89 | initial_values_for_regions = [] 90 | 91 | i = 0 92 | while i < len(problem.current_test_case): 93 | search = pattern.search(problem.current_test_case, i) 94 | if search is None: 95 | break 96 | 97 | u, v = search.span() 98 | matching_regions.append((u, v)) 99 | initial_values_for_regions.append(problem.current_test_case[u:v]) 100 | 101 | i = v 102 | 103 | if not matching_regions: 104 | return 105 | 106 | patches = RegionReplacingPatches(matching_regions) 107 | 108 | patch_applier = PatchApplier(patches, problem) 109 | 110 | async with trio.open_nursery() as nursery: 111 | 112 | async def reduce_region(i: int) -> None: 113 | async def is_interesting(s): 114 | return await patch_applier.try_apply_patch({i: s}) 115 | 116 | subproblem = BasicReductionProblem( 117 | initial_values_for_regions[i], 118 | is_interesting, 119 | work=problem.work, 120 | ) 121 | nursery.start_soon(fn, subproblem) 122 | 123 | for i in range(len(matching_regions)): 124 | await reduce_region(i) 125 | 126 | return reduction_pass 127 | 128 | return inner 129 | 130 | 131 | async def reduce_integer(problem: ReductionProblem[int]) -> None: 132 | assert problem.current_test_case >= 0 133 | 134 | if await problem.is_interesting(0): 135 | return 136 | 137 | lo = 0 138 | hi = problem.current_test_case 139 | 140 | while lo + 1 < hi: 141 | mid = (lo + hi) // 2 142 | if await problem.is_interesting(mid): 143 | hi = mid 144 | else: 145 | lo = mid 146 | 147 | if await problem.is_interesting(hi - 1): 148 | hi -= 1 149 | 150 | if await problem.is_interesting(lo + 1): 151 | return 152 | else: 153 | lo += 1 154 | 155 | 156 | class IntegerFormat(Format[bytes, int]): 157 | def parse(self, input: bytes) -> int: 158 | try: 159 | return int(input.decode("ascii")) 160 | except (ValueError, UnicodeDecodeError): 161 | raise ParseError() 162 | 163 | def dumps(self, input: int) -> bytes: 164 | return str(input).encode("ascii") 165 | 166 | 167 | @regex_pass(b"[0-9]+") 168 | async def reduce_integer_literals(problem: ReductionProblem[bytes]) -> None: 169 | await reduce_integer(problem.view(IntegerFormat())) 170 | 171 | 172 | @regex_pass(rb"[0-9]+ [*+-/] [0-9]+") 173 | async def combine_expressions(problem: ReductionProblem[bytes]) -> None: 174 | try: 175 | # NB: Use of eval is safe, as everything passed to this is a simple 176 | # arithmetic expression. Would ideally replace with a guaranteed 177 | # safe version though. 178 | await problem.is_interesting( 179 | str(eval(problem.current_test_case)).encode("ascii") 180 | ) 181 | except ArithmeticError: 182 | pass 183 | 184 | 185 | @regex_pass(rb'([\'"])\s*\1') 186 | async def merge_adjacent_strings(problem: ReductionProblem[bytes]) -> None: 187 | await problem.is_interesting(b"") 188 | 189 | 190 | @regex_pass(rb"''|\"\"|false|\(\)|\[\]", re.IGNORECASE) 191 | async def replace_falsey_with_zero(problem: ReductionProblem[bytes]) -> None: 192 | await problem.is_interesting(b"0") 193 | 194 | 195 | async def simplify_brackets(problem: ReductionProblem[bytes]) -> None: 196 | bracket_types = [b"[]", b"{}", b"()"] 197 | 198 | patches = [dict(zip(u, v)) for u in bracket_types for v in bracket_types if u > v] 199 | 200 | await apply_patches(problem, ByteReplacement(), patches) 201 | 202 | 203 | IDENTIFIER = re.compile(rb"(\b[A-Za-z][A-Za-z0-9_]*\b)|([0-9]+)") 204 | 205 | 206 | def shortlex(s): 207 | return (len(s), s) 208 | 209 | 210 | async def normalize_identifiers(problem: ReductionProblem[bytes]) -> None: 211 | identifiers = {m.group(0) for m in IDENTIFIER.finditer(problem.current_test_case)} 212 | replacements = set(identifiers) 213 | 214 | for char_type in [ascii_lowercase, ascii_uppercase]: 215 | for cc in char_type.encode("ascii"): 216 | c = bytes([cc]) 217 | if c not in replacements: 218 | replacements.add(c) 219 | break 220 | 221 | replacements = sorted(replacements, key=shortlex) 222 | targets = sorted(identifiers, key=shortlex, reverse=True) 223 | 224 | # TODO: This could use better parallelisation. 225 | for t in targets: 226 | pattern = re.compile(rb"\b" + t + rb"\b") 227 | source = problem.current_test_case 228 | if not pattern.search(source): 229 | continue 230 | 231 | async def can_replace(r): 232 | if shortlex(r) >= shortlex(t): 233 | return False 234 | attempt = pattern.sub(r, source) 235 | assert attempt != source 236 | return await problem.is_interesting(attempt) 237 | 238 | try: 239 | await problem.work.find_first_value(replacements, can_replace) 240 | except NotFound: 241 | pass 242 | 243 | 244 | def iter_indices(s, substring): 245 | try: 246 | i = s.index(substring) 247 | yield i 248 | while True: 249 | i = s.index(substring, i + 1) 250 | yield i 251 | except ValueError: 252 | return 253 | 254 | 255 | async def cut_comments(problem: ReductionProblem[bytes], start, end, include_end=True): 256 | cuts = [] 257 | target = problem.current_test_case 258 | # python comments 259 | for i in iter_indices(target, start): 260 | try: 261 | j = target.index(end, i + 1) 262 | except ValueError: 263 | if include_end: 264 | continue 265 | j = len(target) 266 | if include_end: 267 | cuts.append((i, j + len(end))) 268 | else: 269 | cuts.append((i, j)) 270 | await delete_intervals(problem, cuts) 271 | 272 | 273 | async def cut_comment_like_things(problem: ReductionProblem[bytes]): 274 | await cut_comments(problem, b"#", b"\n", include_end=False) 275 | await cut_comments(problem, b"//", b"\n", include_end=False) 276 | await cut_comments(problem, b'"""', b'"""') 277 | await cut_comments(problem, b"/*", b"*/") 278 | -------------------------------------------------------------------------------- /src/shrinkray/passes/json.py: -------------------------------------------------------------------------------- 1 | import json 2 | from copy import deepcopy 3 | from typing import Any 4 | 5 | from attrs import define 6 | 7 | from shrinkray.passes.definitions import Format, ParseError, ReductionPass 8 | from shrinkray.passes.patching import Patches, apply_patches 9 | from shrinkray.problem import ReductionProblem 10 | 11 | 12 | def is_json(s: bytes) -> bool: 13 | try: 14 | json.loads(s) 15 | return True 16 | except ValueError: 17 | return False 18 | 19 | 20 | @define(frozen=True) 21 | class _JSON(Format[bytes, Any]): 22 | def __repr__(self) -> str: 23 | return "JSON" 24 | 25 | @property 26 | def name(self) -> str: 27 | return "JSON" 28 | 29 | def parse(self, input: bytes) -> Any: 30 | try: 31 | return json.loads(input) 32 | except (json.JSONDecodeError, UnicodeDecodeError) as e: 33 | raise ParseError(*e.args) 34 | 35 | def dumps(self, input: Any) -> bytes: 36 | return json.dumps(input).encode("utf-8") 37 | 38 | 39 | JSON = _JSON() 40 | 41 | 42 | def gather_identifiers(value: Any) -> set[str]: 43 | result = set() 44 | stack = [value] 45 | while stack: 46 | target = stack.pop() 47 | if isinstance(target, dict): 48 | result.update(target.keys()) 49 | stack.extend(target.values()) 50 | elif isinstance(target, list): 51 | stack.extend(target) 52 | return result 53 | 54 | 55 | class DeleteIdentifiers(Patches[frozenset[str], Any]): 56 | @property 57 | def empty(self) -> frozenset[str]: 58 | return frozenset() 59 | 60 | def combine(self, *patches: frozenset[str]) -> frozenset[str]: 61 | result = set() 62 | for p in patches: 63 | result.update(p) 64 | return frozenset(result) 65 | 66 | def apply(self, patch: frozenset[str], target: Any) -> Any: 67 | target = deepcopy(target) 68 | stack = [target] 69 | while stack: 70 | value = stack.pop() 71 | if isinstance(value, dict): 72 | for k in patch: 73 | value.pop(k, None) 74 | stack.extend(value.values()) 75 | elif isinstance(value, list): 76 | stack.extend(value) 77 | return target 78 | 79 | def size(self, patch: frozenset[str]) -> int: 80 | return len(patch) 81 | 82 | 83 | async def delete_identifiers(problem: ReductionProblem[Any]): 84 | identifiers = gather_identifiers(problem.current_test_case) 85 | 86 | await apply_patches( 87 | problem, DeleteIdentifiers(), [frozenset({id}) for id in identifiers] 88 | ) 89 | 90 | 91 | JSON_PASSES: list[ReductionPass[Any]] = [delete_identifiers] 92 | -------------------------------------------------------------------------------- /src/shrinkray/passes/patching.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from enum import Enum 3 | from random import Random 4 | from typing import Any, Callable, Generic, Iterable, Sequence, TypeVar, cast 5 | 6 | import trio 7 | 8 | from shrinkray.problem import ReductionProblem 9 | 10 | Seq = TypeVar("Seq", bound=Sequence[Any]) 11 | T = TypeVar("T") 12 | 13 | PatchType = TypeVar("PatchType") 14 | TargetType = TypeVar("TargetType") 15 | 16 | 17 | class Conflict(Exception): 18 | pass 19 | 20 | 21 | class Patches(Generic[PatchType, TargetType], ABC): 22 | @property 23 | @abstractmethod 24 | def empty(self) -> PatchType: ... 25 | 26 | @abstractmethod 27 | def combine(self, *patches: PatchType) -> PatchType: ... 28 | 29 | @abstractmethod 30 | def apply(self, patch: PatchType, target: TargetType) -> TargetType: ... 31 | 32 | @abstractmethod 33 | def size(self, patch: PatchType) -> int: ... 34 | 35 | 36 | class SetPatches(Patches[frozenset[T], TargetType]): 37 | def __init__(self, apply: Callable[[frozenset[T], TargetType], TargetType]): 38 | self.__apply = apply 39 | 40 | @property 41 | def empty(self): 42 | return frozenset() 43 | 44 | def combine(self, *patches: frozenset[T]) -> frozenset[T]: 45 | result = set() 46 | for p in patches: 47 | result.update(p) 48 | return frozenset(result) 49 | 50 | def apply(self, patch: frozenset[T], target: TargetType) -> TargetType: 51 | return self.__apply(patch, target) 52 | 53 | def size(self, patch: frozenset[T]) -> int: 54 | return len(patch) 55 | 56 | 57 | class ListPatches(Patches[list[T], TargetType]): 58 | def __init__(self, apply: Callable[[list[T], TargetType], TargetType]): 59 | self.__apply = apply 60 | 61 | @property 62 | def empty(self): 63 | return [] 64 | 65 | def combine(self, *patches: list[T]) -> list[T]: 66 | result = [] 67 | for p in patches: 68 | result.extend(p) 69 | return result 70 | 71 | def apply(self, patch: list[T], target: TargetType) -> TargetType: 72 | return self.__apply(patch, target) 73 | 74 | def size(self, patch: list[T]) -> int: 75 | return len(patch) 76 | 77 | 78 | class PatchApplier(Generic[PatchType, TargetType], ABC): 79 | def __init__( 80 | self, 81 | patches: Patches[PatchType, TargetType], 82 | problem: ReductionProblem[TargetType], 83 | ): 84 | self.__patches = patches 85 | self.__problem = problem 86 | 87 | self.__tick = 0 88 | self.__merge_queue = [] 89 | self.__merge_lock = trio.Lock() 90 | 91 | self.__current_patch = self.__patches.empty 92 | self.__initial_test_case = problem.current_test_case 93 | 94 | async def try_apply_patch(self, patch: PatchType) -> bool: 95 | initial_patch = self.__current_patch 96 | try: 97 | combined_patch = self.__patches.combine(initial_patch, patch) 98 | except Conflict: 99 | return False 100 | if combined_patch == self.__current_patch: 101 | return True 102 | with_patch_applied = self.__patches.apply( 103 | combined_patch, self.__initial_test_case 104 | ) 105 | if with_patch_applied == self.__problem.current_test_case: 106 | return True 107 | if not await self.__problem.is_interesting(with_patch_applied): 108 | return False 109 | send_merge_result, receive_merge_result = trio.open_memory_channel(1) 110 | 111 | sort_key = (self.__tick, self.__problem.sort_key(with_patch_applied)) 112 | self.__tick += 1 113 | 114 | self.__merge_queue.append((sort_key, patch, send_merge_result)) 115 | 116 | async with self.__merge_lock: 117 | if ( 118 | self.__current_patch == initial_patch 119 | and len(self.__merge_queue) == 1 120 | and self.__merge_queue[0][1] == patch 121 | and self.__problem.sort_key(with_patch_applied) 122 | <= self.__problem.sort_key(self.__problem.current_test_case) 123 | ): 124 | self.__current_patch = combined_patch 125 | self.__merge_queue.clear() 126 | return True 127 | 128 | while self.__merge_queue: 129 | base_patch = self.__current_patch 130 | to_merge = len(self.__merge_queue) 131 | 132 | async def can_merge(k): 133 | if k > to_merge: 134 | return False 135 | try: 136 | attempted_patch = self.__patches.combine( 137 | base_patch, *[p for _, p, _ in self.__merge_queue[:k]] 138 | ) 139 | except Conflict: 140 | return False 141 | if attempted_patch == base_patch: 142 | return True 143 | with_patch_applied = self.__patches.apply( 144 | attempted_patch, self.__initial_test_case 145 | ) 146 | if await self.__problem.is_reduction(with_patch_applied): 147 | self.__current_patch = attempted_patch 148 | return True 149 | else: 150 | return False 151 | 152 | if await can_merge(to_merge): 153 | merged = to_merge 154 | else: 155 | merged = await self.__problem.work.find_large_integer(can_merge) 156 | 157 | for _, _, send_result in self.__merge_queue[:merged]: 158 | send_result.send_nowait(True) 159 | 160 | assert merged <= to_merge 161 | if merged < to_merge: 162 | self.__merge_queue[merged][-1].send_nowait(False) 163 | del self.__merge_queue[: merged + 1] 164 | else: 165 | del self.__merge_queue[:to_merge] 166 | 167 | # This should always have been populated during the previous merge, 168 | # either by us or someone else merging. 169 | return receive_merge_result.receive_nowait() 170 | 171 | 172 | class Direction(Enum): 173 | LEFT = 0 174 | RIGHT = 1 175 | 176 | 177 | class Completed(Exception): 178 | pass 179 | 180 | 181 | async def apply_patches( 182 | problem: ReductionProblem[TargetType], 183 | patch_info: Patches[PatchType, TargetType], 184 | patches: Iterable[PatchType], 185 | ) -> None: 186 | if await problem.is_interesting( 187 | patch_info.apply(patch_info.combine(*patches), problem.current_test_case) 188 | ): 189 | return 190 | 191 | applier = PatchApplier(patch_info, problem) 192 | 193 | send_patches, receive_patches = trio.open_memory_channel(float("inf")) 194 | 195 | patches = list(patches) 196 | problem.work.random.shuffle(patches) 197 | patches.sort(key=patch_info.size, reverse=True) 198 | for patch in patches: 199 | send_patches.send_nowait(patch) 200 | send_patches.close() 201 | 202 | async with trio.open_nursery() as nursery: 203 | for _ in range(problem.work.parallelism): 204 | 205 | @nursery.start_soon 206 | async def _(): 207 | while True: 208 | try: 209 | patch = await receive_patches.receive() 210 | except trio.EndOfChannel: 211 | break 212 | await applier.try_apply_patch(patch) 213 | 214 | 215 | class LazyMutableRange: 216 | def __init__(self, n: int): 217 | self.__size = n 218 | self.__mask: dict[int, int] = {} 219 | 220 | def __getitem__(self, i: int) -> int: 221 | return self.__mask.get(i, i) 222 | 223 | def __setitem__(self, i: int, v: int) -> None: 224 | self.__mask[i] = v 225 | 226 | def __len__(self) -> int: 227 | return self.__size 228 | 229 | def pop(self) -> int: 230 | i = len(self) - 1 231 | result = self[i] 232 | self.__size = i 233 | self.__mask.pop(i, None) 234 | return result 235 | 236 | 237 | def lazy_shuffle(seq: Sequence[T], rnd: Random) -> Iterable[T]: 238 | indices = LazyMutableRange(len(seq)) 239 | while indices: 240 | j = len(indices) - 1 241 | i = rnd.randrange(0, len(indices)) 242 | indices[i], indices[j] = indices[j], indices[i] 243 | yield seq[indices.pop()] 244 | 245 | 246 | CutPatch = list[tuple[int, int]] 247 | 248 | 249 | class Cuts(Patches[CutPatch, Seq]): 250 | @property 251 | def empty(self) -> CutPatch: 252 | return [] 253 | 254 | def combine(self, *patches: CutPatch) -> CutPatch: 255 | all_cuts: CutPatch = [] 256 | for p in patches: 257 | all_cuts.extend(p) 258 | all_cuts.sort() 259 | normalized: list[list[int]] = [] 260 | for start, end in all_cuts: 261 | if normalized and normalized[-1][-1] >= start: 262 | normalized[-1][-1] = max(normalized[-1][-1], end) 263 | else: 264 | normalized.append([start, end]) 265 | return [cast(tuple[int, int], tuple(x)) for x in normalized] 266 | 267 | def apply(self, patch: CutPatch, target: Seq) -> Seq: 268 | result: list[Any] = [] 269 | prev = 0 270 | total_deleted = 0 271 | for start, end in patch: 272 | total_deleted += end - start 273 | result.extend(target[prev:start]) 274 | prev = end 275 | result.extend(target[prev:]) 276 | assert len(result) + total_deleted == len(target) 277 | return type(target)(result) # type: ignore 278 | 279 | def size(self, patch: CutPatch) -> int: 280 | return sum(v - u for u, v in patch) 281 | -------------------------------------------------------------------------------- /src/shrinkray/passes/python.py: -------------------------------------------------------------------------------- 1 | from typing import Any, AnyStr, Callable 2 | 3 | import libcst 4 | import libcst.matchers as m 5 | from libcst import CSTNode, codemod 6 | 7 | from shrinkray.problem import ReductionProblem 8 | from shrinkray.work import NotFound 9 | 10 | 11 | def is_python(source: AnyStr) -> bool: 12 | try: 13 | libcst.parse_module(source) 14 | return True 15 | except (SyntaxError, UnicodeDecodeError, libcst.ParserSyntaxError, Exception): 16 | return False 17 | 18 | 19 | Replacement = CSTNode | libcst.RemovalSentinel | libcst.FlattenSentinel[Any] 20 | 21 | 22 | async def libcst_transform( 23 | problem: ReductionProblem[bytes], 24 | matcher: m.BaseMatcherNode, 25 | transformer: Callable[ 26 | [CSTNode], 27 | Replacement, 28 | ], 29 | ) -> None: 30 | class CM(codemod.VisitorBasedCodemodCommand): 31 | def __init__( 32 | self, context: codemod.CodemodContext, start_index: int, end_index: int 33 | ): 34 | super().__init__(context) 35 | self.start_index = start_index 36 | self.end_index = end_index 37 | self.current_index = 0 38 | self.fired = False 39 | 40 | # We have to have an ignore on the return type because if we don't LibCST 41 | # will do some stupid bullshit with checking if the return type is correct 42 | # and we use this generically in a way that makes it hard to type correctly. 43 | @m.leave(matcher) 44 | def maybe_change_node(self, _, updated_node): # type: ignore 45 | if self.start_index <= self.current_index < self.end_index: 46 | self.fired = True 47 | self.current_index += 1 48 | return transformer(updated_node) 49 | else: 50 | self.current_index += 1 51 | return updated_node 52 | 53 | try: 54 | module = libcst.parse_module(problem.current_test_case) 55 | except Exception: 56 | return 57 | 58 | context = codemod.CodemodContext() 59 | 60 | counting_mod = CM(context, -1, -1) 61 | counting_mod.transform_module(module) 62 | 63 | n = counting_mod.current_index + 1 64 | 65 | async def can_apply(start: int, end: int) -> bool: 66 | nonlocal n 67 | if start >= n: 68 | return False 69 | initial_test_case = problem.current_test_case 70 | try: 71 | module = libcst.parse_module(initial_test_case) 72 | except libcst.ParserSyntaxError: 73 | n = 0 74 | return False 75 | 76 | codemod_i = CM(context, start, end) 77 | try: 78 | transformed = codemod_i.transform_module(module) 79 | except libcst.CSTValidationError: 80 | return False 81 | except TypeError as e: 82 | if "does not allow for it" in e.args[0]: 83 | return False 84 | raise 85 | 86 | if not codemod_i.fired: 87 | n = start 88 | return False 89 | 90 | transformed_test_case = transformed.code.encode(transformed.encoding) 91 | 92 | if problem.sort_key(transformed_test_case) >= problem.sort_key( 93 | initial_test_case 94 | ): 95 | return False 96 | 97 | return await problem.is_interesting(transformed_test_case) 98 | 99 | i = 0 100 | while i < n: 101 | try: 102 | i = await problem.work.find_first_value( 103 | range(i, n), lambda i: can_apply(i, i + 1) 104 | ) 105 | await problem.work.find_large_integer(lambda k: can_apply(i + 1, i + 1 + k)) 106 | i += 1 107 | except NotFound: 108 | break 109 | 110 | 111 | async def lift_indented_constructs(problem: ReductionProblem[bytes]) -> None: 112 | await libcst_transform( 113 | problem, 114 | m.OneOf(m.While(), m.If(), m.Try()), 115 | lambda x: x.with_changes(orelse=None), 116 | ) 117 | 118 | await libcst_transform( 119 | problem, 120 | m.OneOf(m.While(), m.If(), m.Try(), m.With()), 121 | lambda x: libcst.FlattenSentinel(x.body.body), # type: ignore 122 | ) 123 | 124 | 125 | async def delete_statements(problem: ReductionProblem[bytes]) -> None: 126 | await libcst_transform( 127 | problem, 128 | m.SimpleStatementLine(), 129 | lambda x: libcst.RemoveFromParent(), # type: ignore 130 | ) 131 | 132 | 133 | async def replace_statements_with_pass(problem: ReductionProblem[bytes]) -> None: 134 | await libcst_transform( 135 | problem, 136 | m.SimpleStatementLine(), 137 | lambda x: x.with_changes(body=[libcst.Pass()]), # type: ignore 138 | ) 139 | 140 | 141 | ELLIPSIS_STATEMENT = libcst.parse_statement("...") 142 | 143 | 144 | async def replace_bodies_with_ellipsis(problem: ReductionProblem[bytes]) -> None: 145 | await libcst_transform( 146 | problem, 147 | m.IndentedBlock(), 148 | lambda x: x.with_changes(body=[ELLIPSIS_STATEMENT]), # type: ignore 149 | ) 150 | 151 | 152 | async def strip_annotations(problem: ReductionProblem[bytes]) -> None: 153 | await libcst_transform( 154 | problem, 155 | m.FunctionDef(), 156 | lambda x: x.with_changes(returns=None), 157 | ) 158 | await libcst_transform( 159 | problem, 160 | m.Param(), 161 | lambda x: x.with_changes(annotation=None), 162 | ) 163 | await libcst_transform( 164 | problem, 165 | m.AnnAssign(), 166 | lambda x: ( 167 | libcst.Assign( 168 | targets=[libcst.AssignTarget(target=x.target)], 169 | value=x.value, 170 | semicolon=x.semicolon, 171 | ) 172 | if x.value 173 | else libcst.RemoveFromParent() 174 | ), 175 | ) 176 | 177 | 178 | PYTHON_PASSES = [ 179 | replace_bodies_with_ellipsis, 180 | strip_annotations, 181 | lift_indented_constructs, 182 | delete_statements, 183 | replace_statements_with_pass, 184 | ] 185 | -------------------------------------------------------------------------------- /src/shrinkray/passes/sat.py: -------------------------------------------------------------------------------- 1 | from shrinkray.passes.definitions import Format, ParseError, ReductionPass 2 | from shrinkray.passes.patching import SetPatches, apply_patches 3 | from shrinkray.passes.sequences import delete_elements 4 | from shrinkray.problem import ReductionProblem 5 | 6 | Clause = list[int] 7 | SAT = list[Clause] 8 | 9 | 10 | class _DimacsCNF(Format[bytes, SAT]): 11 | @property 12 | def name(self) -> str: 13 | return "DimacsCNF" 14 | 15 | def parse(self, input: bytes) -> SAT: 16 | try: 17 | contents = input.decode("utf-8") 18 | except UnicodeDecodeError as e: 19 | raise ParseError(*e.args) 20 | clauses = [] 21 | for line in contents.splitlines(): 22 | line = line.strip() 23 | if line.startswith("c"): 24 | continue 25 | if line.startswith("p"): 26 | continue 27 | if not line.strip(): 28 | continue 29 | try: 30 | clause = list(map(int, line.strip().split())) 31 | except ValueError as e: 32 | raise ParseError(*e.args) 33 | if clause[-1] != 0: 34 | raise ParseError(f"{line} did not end with 0") 35 | clause.pop() 36 | clauses.append(clause) 37 | if not clauses: 38 | raise ParseError("No clauses found") 39 | return clauses 40 | 41 | def dumps(self, input: SAT) -> bytes: 42 | n_variables = max(abs(literal) for clause in input for literal in clause) 43 | 44 | parts = [f"p cnf {n_variables} {len(input)}"] 45 | 46 | for c in input: 47 | parts.append(" ".join(map(repr, list(c) + [0]))) 48 | 49 | return "\n".join(parts).encode("utf-8") 50 | 51 | 52 | DimacsCNF = _DimacsCNF() 53 | 54 | 55 | async def renumber_variables(problem: ReductionProblem[SAT]): 56 | renumbering = {} 57 | 58 | def renumber(l): 59 | if l < 0: 60 | return -renumber(-l) 61 | try: 62 | return renumbering[l] 63 | except KeyError: 64 | pass 65 | result = len(renumbering) + 1 66 | renumbering[l] = result 67 | return result 68 | 69 | renumbered = [ 70 | [renumber(literal) for literal in clause] 71 | for clause in problem.current_test_case 72 | ] 73 | 74 | await problem.is_interesting(renumbered) 75 | 76 | 77 | async def flip_literal_signs(problem: ReductionProblem[SAT]): 78 | seen_variables = set() 79 | target = problem.current_test_case 80 | for i in range(len(target)): 81 | for j, v in enumerate(target[i]): 82 | if abs(v) not in seen_variables and v < 0: 83 | attempt = [] 84 | for clause in target: 85 | new_clause = [] 86 | for literal in clause: 87 | if abs(literal) == abs(v): 88 | new_clause.append(-literal) 89 | else: 90 | new_clause.append(literal) 91 | attempt.append(new_clause) 92 | if await problem.is_interesting(attempt): 93 | target = attempt 94 | seen_variables.add(abs(v)) 95 | 96 | 97 | async def remove_redundant_clauses(problem: ReductionProblem[SAT]): 98 | attempt = [] 99 | seen = set() 100 | for clause in problem.current_test_case: 101 | if len(set(map(abs, clause))) < len(set(clause)): 102 | continue 103 | key = tuple(clause) 104 | if key in seen: 105 | continue 106 | seen.add(key) 107 | attempt.append(clause) 108 | await problem.is_interesting(attempt) 109 | 110 | 111 | def literals_in(sat: SAT) -> frozenset[int]: 112 | return frozenset({literal for clause in sat for literal in clause}) 113 | 114 | 115 | async def delete_literals(problem: ReductionProblem[SAT]): 116 | def remove_literals(literals: frozenset[int], sat: SAT) -> SAT: 117 | result = [] 118 | for clause in sat: 119 | new_clause = [v for v in clause if v not in literals] 120 | if new_clause: 121 | result.append(new_clause) 122 | return result 123 | 124 | await apply_patches( 125 | problem, 126 | SetPatches(remove_literals), 127 | [frozenset({v}) for v in literals_in(problem.current_test_case)], 128 | ) 129 | 130 | 131 | async def merge_variables(problem: ReductionProblem[SAT]): 132 | i = 0 133 | j = 1 134 | while True: 135 | variables = sorted({abs(l) for c in problem.current_test_case for l in c}) 136 | if j >= len(variables): 137 | i += 1 138 | j = i + 1 139 | if j >= len(variables): 140 | return 141 | 142 | target = variables[i] 143 | to_replace = variables[j] 144 | 145 | new_clauses = [] 146 | for c in problem.current_test_case: 147 | c = set(c) 148 | if to_replace in c: 149 | c.discard(to_replace) 150 | c.add(target) 151 | if -to_replace in c: 152 | c.discard(-to_replace) 153 | c.add(-target) 154 | if len(set(map(abs, c))) < len(c): 155 | continue 156 | new_clauses.append(sorted(c)) 157 | 158 | assert new_clauses != problem.current_test_case 159 | await problem.is_interesting(new_clauses) 160 | if new_clauses != problem.current_test_case: 161 | j += 1 162 | 163 | 164 | async def sort_clauses(problem: ReductionProblem[SAT]): 165 | await problem.is_interesting(sorted(map(sorted, problem.current_test_case))) 166 | 167 | 168 | SAT_PASSES: list[ReductionPass[SAT]] = [ 169 | sort_clauses, 170 | renumber_variables, 171 | flip_literal_signs, 172 | remove_redundant_clauses, 173 | delete_elements, 174 | delete_literals, 175 | merge_variables, 176 | ] 177 | -------------------------------------------------------------------------------- /src/shrinkray/passes/sequences.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from typing import Any, Sequence, TypeVar 3 | 4 | from shrinkray.passes.definitions import ReductionPass 5 | from shrinkray.passes.patching import CutPatch, Cuts, apply_patches 6 | from shrinkray.problem import ReductionProblem 7 | 8 | Seq = TypeVar("Seq", bound=Sequence[Any]) 9 | 10 | 11 | async def delete_elements(problem: ReductionProblem[Seq]) -> None: 12 | await apply_patches( 13 | problem, Cuts(), [[(i, i + 1)] for i in range(len(problem.current_test_case))] 14 | ) 15 | 16 | 17 | def merged_intervals(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]: 18 | normalized: list[list[int]] = [] 19 | for start, end in sorted(map(tuple, intervals)): 20 | if normalized and normalized[-1][-1] >= start: 21 | normalized[-1][-1] = max(normalized[-1][-1], end) 22 | else: 23 | normalized.append([start, end]) 24 | return list(map(tuple, normalized)) # type: ignore 25 | 26 | 27 | def with_deletions(target: Seq, deletions: list[tuple[int, int]]) -> Seq: 28 | result: list[Any] = [] 29 | prev = 0 30 | total_deleted = 0 31 | for start, end in deletions: 32 | total_deleted += end - start 33 | result.extend(target[prev:start]) 34 | prev = end 35 | result.extend(target[prev:]) 36 | assert len(result) + total_deleted == len(target) 37 | return type(target)(result) # type: ignore 38 | 39 | 40 | def block_deletion(min_block: int, max_block: int) -> ReductionPass[Seq]: 41 | async def apply(problem: ReductionProblem[Seq]) -> None: 42 | n = len(problem.current_test_case) 43 | if n <= min_block: 44 | return 45 | blocks = [ 46 | [(i, i + block_size)] 47 | for block_size in range(min_block, max_block + 1) 48 | for offset in range(block_size) 49 | for i in range(offset, n, block_size) 50 | if i + block_size <= n 51 | ] 52 | await apply_patches(problem, Cuts(), blocks) 53 | 54 | apply.__name__ = f"block_deletion({min_block}, {max_block})" 55 | return apply 56 | 57 | 58 | async def delete_duplicates(problem: ReductionProblem[Seq]) -> None: 59 | index: dict[int, list[int]] = defaultdict(list) 60 | 61 | for i, c in enumerate(problem.current_test_case): 62 | index[c].append(i) 63 | 64 | cuts: list[CutPatch] = [] 65 | 66 | for ix in index.values(): 67 | if len(ix) > 1: 68 | cuts.append([(i, i + 1) for i in ix]) 69 | await apply_patches(problem, Cuts(), cuts) 70 | -------------------------------------------------------------------------------- /src/shrinkray/problem.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import time 3 | from abc import ABC, abstractmethod, abstractproperty 4 | from datetime import timedelta 5 | from typing import Any, Awaitable, Callable, Generic, Optional, TypeVar, cast 6 | 7 | import attrs 8 | import trio 9 | from attrs import define 10 | from humanize import naturalsize, precisedelta 11 | 12 | from shrinkray.work import WorkContext 13 | 14 | S = TypeVar("S") 15 | T = TypeVar("T") 16 | 17 | 18 | def shortlex(value: Any) -> Any: 19 | return (len(value), value) 20 | 21 | 22 | def default_sort_key(value: Any): 23 | if isinstance(value, (str, bytes)): 24 | return shortlex(value) 25 | else: 26 | return shortlex(repr(value)) 27 | 28 | 29 | def default_display(value: Any) -> str: 30 | r = repr(value) 31 | if len(r) < 50: 32 | return f"{r} (size {len(value)})" 33 | return f"value of size {len(value)}" 34 | 35 | 36 | def default_size(value: Any) -> int: 37 | try: 38 | return len(value) 39 | except TypeError: 40 | return 0 41 | 42 | 43 | @define 44 | class ReductionStats: 45 | reductions: int = 0 46 | failed_reductions: int = 0 47 | 48 | calls: int = 0 49 | interesting_calls: int = 0 50 | wasted_interesting_calls: int = 0 51 | 52 | time_of_last_reduction: float = 0.0 53 | start_time: float = attrs.Factory(time.time) 54 | 55 | initial_test_case_size: int = 0 56 | current_test_case_size: int = 0 57 | 58 | def time_since_last_reduction(self) -> float: 59 | return time.time() - self.time_of_last_reduction 60 | 61 | def display_stats(self) -> str: 62 | runtime = time.time() - self.start_time 63 | if self.reductions > 0: 64 | reduction_percentage = ( 65 | 1.0 - self.current_test_case_size / self.initial_test_case_size 66 | ) * 100 67 | reduction_rate = ( 68 | self.initial_test_case_size - self.current_test_case_size 69 | ) / runtime 70 | reduction_msg = ( 71 | f"Current test case size: {naturalsize(self.current_test_case_size)} " 72 | f"({reduction_percentage:.2f}% reduction, {naturalsize(reduction_rate)} / second)" 73 | ) 74 | else: 75 | reduction_msg = ( 76 | f"Current test case size: {self.current_test_case_size} bytes" 77 | ) 78 | 79 | return "\n".join( 80 | [ 81 | reduction_msg, 82 | f"Total runtime: {precisedelta(timedelta(seconds=runtime))}", 83 | ( 84 | ( 85 | f"Calls to interestingness test: {self.calls} ({self.calls / runtime:.2f} calls / second, " 86 | f"{self.interesting_calls / self.calls * 100.0:.2f}% interesting, " 87 | f"{self.wasted_interesting_calls / self.calls * 100:.2f}% wasted)" 88 | ) 89 | if self.calls > 0 90 | else "Not yet called interestingness test" 91 | ), 92 | ( 93 | f"Time since last reduction: {self.time_since_last_reduction():.2f}s ({self.reductions / runtime:.2f} reductions / second)" 94 | if self.reductions 95 | else "No reductions yet" 96 | ), 97 | ] 98 | ) 99 | 100 | 101 | @define(slots=False) 102 | class ReductionProblem(Generic[T], ABC): 103 | work: WorkContext 104 | 105 | def __attrs_post_init__(self) -> None: 106 | self.__view_cache: dict[Any, ReductionProblem[Any]] = {} 107 | 108 | def view( 109 | self, format: "Format[T, S] | type[Format[T, S]]" 110 | ) -> "ReductionProblem[S]": 111 | try: 112 | return cast(ReductionProblem[S], self.__view_cache[format]) 113 | except KeyError: 114 | pass 115 | 116 | concrete_format: Format[T, S] = format() if isinstance(format, type) else format 117 | 118 | result: View[T, S] = View( 119 | problem=self, 120 | work=self.work, 121 | dump=concrete_format.dumps, 122 | parse=concrete_format.parse, 123 | ) 124 | 125 | return cast(ReductionProblem[S], self.__view_cache.setdefault(format, result)) 126 | 127 | async def setup(self) -> None: 128 | pass 129 | 130 | @abstractproperty 131 | def current_test_case(self) -> T: ... 132 | 133 | @abstractmethod 134 | async def is_interesting(self, test_case: T) -> bool: 135 | pass 136 | 137 | async def is_reduction(self, test_case: T) -> bool: 138 | if test_case == self.current_test_case: 139 | return True 140 | if self.sort_key(test_case) > self.sort_key(self.current_test_case): 141 | return False 142 | return await self.is_interesting(test_case) 143 | 144 | @abstractmethod 145 | def sort_key(self, test_case: T) -> Any: ... 146 | 147 | @abstractmethod 148 | def size(self, test_case: T) -> int: 149 | return len(test_case) # type: ignore 150 | 151 | @property 152 | def current_size(self) -> int: 153 | return self.size(self.current_test_case) 154 | 155 | @abstractmethod 156 | def display(self, value: T) -> str: ... 157 | 158 | def backtrack(self, new_test_case: T) -> "ReductionProblem[T]": 159 | return BasicReductionProblem( 160 | initial=new_test_case, 161 | is_interesting=self.is_interesting, 162 | work=self.work, 163 | sort_key=self.sort_key, 164 | size=self.size, 165 | display=self.display, 166 | ) 167 | 168 | 169 | class InvalidInitialExample(ValueError): 170 | pass 171 | 172 | 173 | def default_cache_key(value: Any) -> str: 174 | if not isinstance(value, bytes): 175 | if not isinstance(value, str): 176 | value = repr(value) 177 | value = value.encode("utf-8") 178 | 179 | hex = hashlib.sha1(value).hexdigest()[:8] 180 | return f"{len(value)}:{hex}" 181 | 182 | 183 | class BasicReductionProblem(ReductionProblem[T]): 184 | def __init__( 185 | self, 186 | initial: T, 187 | is_interesting: Callable[[T], Awaitable[bool]], 188 | work: WorkContext, 189 | sort_key: Callable[[T], Any] = default_sort_key, 190 | size: Callable[[T], int] = default_size, 191 | display: Callable[[T], str] = default_display, 192 | stats: Optional[ReductionStats] = None, 193 | cache_key: Callable[[Any], str] = default_cache_key, 194 | ): 195 | super().__init__(work=work) 196 | self.__current = initial 197 | self.__sort_key = sort_key 198 | self.__size = size 199 | self.__display = display 200 | if stats is None: 201 | self.stats = ReductionStats() 202 | self.stats.initial_test_case_size = self.size(initial) 203 | self.stats.current_test_case_size = self.size(initial) 204 | else: 205 | self.stats = stats 206 | 207 | self.__is_interesting_cache: dict[str, bool] = {} 208 | self.__cache_key = cache_key 209 | self.__is_interesting = is_interesting 210 | self.__on_reduce_callbacks: list[Callable[[T], Awaitable[None]]] = [] 211 | self.__current = initial 212 | self.__has_set_up = False 213 | 214 | async def setup(self) -> None: 215 | if self.__has_set_up: 216 | return 217 | self.__has_set_up = True 218 | if not await self.__is_interesting(self.current_test_case): 219 | raise InvalidInitialExample( 220 | f"Initial example ({self.display(self.current_test_case)}) does not satisfy interestingness test." 221 | ) 222 | 223 | def display(self, value: T) -> str: 224 | return self.__display(value) 225 | 226 | def sort_key(self, test_case: T) -> Any: 227 | return self.__sort_key(test_case) 228 | 229 | def size(self, test_case: T) -> int: 230 | return self.__size(test_case) 231 | 232 | def on_reduce(self, callback: Callable[[T], Awaitable[None]]) -> None: 233 | """Every time `is_interesting` is called with a successful reduction, 234 | call `fn` with the new value. Note that these are called outside the lock.""" 235 | self.__on_reduce_callbacks.append(callback) 236 | 237 | async def is_interesting(self, value: T) -> bool: 238 | """Returns true if this value is interesting.""" 239 | await trio.lowlevel.checkpoint() 240 | if value == self.current_test_case: 241 | return True 242 | cache_key = self.__cache_key(value) 243 | try: 244 | return self.__is_interesting_cache[cache_key] 245 | except KeyError: 246 | pass 247 | result = await self.__is_interesting(value) 248 | self.__is_interesting_cache[cache_key] = result 249 | self.stats.failed_reductions += 1 250 | self.stats.calls += 1 251 | if result: 252 | self.stats.interesting_calls += 1 253 | if self.sort_key(value) < self.sort_key(self.current_test_case): 254 | self.__is_interesting_cache.clear() 255 | self.stats.failed_reductions -= 1 256 | self.stats.reductions += 1 257 | self.stats.time_of_last_reduction = time.time() 258 | self.stats.current_test_case_size = self.size(value) 259 | self.__current = value 260 | for f in self.__on_reduce_callbacks: 261 | await f(value) 262 | else: 263 | self.stats.wasted_interesting_calls += 1 264 | return result 265 | 266 | @property 267 | def current_test_case(self) -> T: 268 | return self.__current 269 | 270 | 271 | class View(ReductionProblem[T], Generic[S, T]): 272 | def __init__( 273 | self, 274 | problem: ReductionProblem[S], 275 | parse: Callable[[S], T], 276 | dump: Callable[[T], S], 277 | work: Optional[WorkContext] = None, 278 | sort_key: Optional[Callable[[T], Any]] = None, 279 | ): 280 | super().__init__(work=work or problem.work) 281 | self.__problem = problem 282 | self.__parse = parse 283 | self.__dump = dump 284 | self.__sort_key = sort_key 285 | 286 | current = problem.current_test_case 287 | self.__prev = current 288 | self.__current = parse(current) 289 | 290 | def display(self, value: T) -> str: 291 | return default_display(value) 292 | 293 | @property 294 | def stats(self) -> ReductionStats: 295 | return self.__problem.stats # type: ignore 296 | 297 | @property 298 | def current_test_case(self) -> T: 299 | current = self.__problem.current_test_case 300 | if current != self.__prev: 301 | self.__prev = current 302 | new_value = self.__parse(current) 303 | if self.__sort_key is None or self.__sort_key(new_value) < self.__sort_key( 304 | self.__current 305 | ): 306 | self.__current = new_value 307 | return self.__current 308 | 309 | async def is_interesting(self, test_case: T) -> bool: 310 | return await self.__problem.is_interesting(self.__dump(test_case)) 311 | 312 | def sort_key(self, test_case: T) -> Any: 313 | if self.__sort_key is not None: 314 | return self.__sort_key(test_case) 315 | return self.__problem.sort_key(self.__dump(test_case)) 316 | 317 | def size(self, test_case: T) -> int: 318 | return self.__problem.size(self.__dump(test_case)) 319 | -------------------------------------------------------------------------------- /src/shrinkray/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DRMacIver/shrinkray/3b4d19902c0dbd1912c8825ca9dba16d4344cf02/src/shrinkray/py.typed -------------------------------------------------------------------------------- /src/shrinkray/reducer.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from collections.abc import Generator 3 | from contextlib import contextmanager 4 | from typing import Any, Generic, Iterable, Optional, TypeVar 5 | 6 | import attrs 7 | import trio 8 | from attrs import define 9 | 10 | from shrinkray.passes.bytes import ( 11 | Split, 12 | Tokenize, 13 | debracket, 14 | delete_byte_spans, 15 | hollow, 16 | lexeme_based_deletions, 17 | lift_braces, 18 | lower_bytes, 19 | lower_individual_bytes, 20 | remove_indents, 21 | remove_whitespace, 22 | replace_space_with_newlines, 23 | short_deletions, 24 | standard_substitutions, 25 | ) 26 | from shrinkray.passes.clangdelta import C_FILE_EXTENSIONS, ClangDelta, clang_delta_pumps 27 | from shrinkray.passes.definitions import Format, ReductionPass, ReductionPump, compose 28 | from shrinkray.passes.genericlanguages import ( 29 | combine_expressions, 30 | cut_comment_like_things, 31 | merge_adjacent_strings, 32 | normalize_identifiers, 33 | reduce_integer_literals, 34 | replace_falsey_with_zero, 35 | simplify_brackets, 36 | ) 37 | from shrinkray.passes.json import JSON, JSON_PASSES 38 | from shrinkray.passes.patching import PatchApplier, Patches 39 | from shrinkray.passes.python import PYTHON_PASSES, is_python 40 | from shrinkray.passes.sat import SAT_PASSES, DimacsCNF 41 | from shrinkray.passes.sequences import block_deletion, delete_duplicates 42 | from shrinkray.problem import ReductionProblem, shortlex 43 | 44 | S = TypeVar("S") 45 | T = TypeVar("T") 46 | 47 | 48 | @define 49 | class Reducer(Generic[T], ABC): 50 | target: ReductionProblem[T] 51 | 52 | @contextmanager 53 | def backtrack(self, restart: T) -> Generator[None, None, None]: 54 | current = self.target 55 | try: 56 | self.target = self.target.backtrack(restart) 57 | yield 58 | finally: 59 | self.target = current 60 | 61 | @abstractmethod 62 | async def run(self) -> None: ... 63 | 64 | @property 65 | def status(self) -> str: 66 | return "" 67 | 68 | 69 | @define 70 | class BasicReducer(Reducer[T]): 71 | reduction_passes: Iterable[ReductionPass[T]] 72 | pumps: Iterable[ReductionPump[T]] = () 73 | status: str = "Starting up" 74 | 75 | def __attrs_post_init__(self) -> None: 76 | self.reduction_passes = list(self.reduction_passes) 77 | 78 | async def run_pass(self, rp: ReductionPass[T]) -> None: 79 | await rp(self.target) 80 | 81 | async def run(self) -> None: 82 | await self.target.setup() 83 | 84 | while True: 85 | prev = self.target.current_test_case 86 | for rp in self.reduction_passes: 87 | self.status = f"Running reduction pass {rp.__name__}" 88 | await self.run_pass(rp) 89 | for pump in self.pumps: 90 | self.status = f"Pumping with {pump.__name__}" 91 | pumped = await pump(self.target) 92 | if pumped != self.target.current_test_case: 93 | with self.backtrack(pumped): 94 | for rp in self.reduction_passes: 95 | self.status = f"Running reduction pass {rp.__name__} under pump {pump.__name__}" 96 | await self.run_pass(rp) 97 | if prev == self.target.current_test_case: 98 | return 99 | 100 | 101 | class RestartPass(Exception): 102 | pass 103 | 104 | 105 | @define 106 | class ShrinkRay(Reducer[bytes]): 107 | clang_delta: Optional[ClangDelta] = None 108 | 109 | current_reduction_pass: Optional[ReductionPass[bytes]] = None 110 | current_pump: Optional[ReductionPump[bytes]] = None 111 | 112 | unlocked_ok_passes: bool = False 113 | 114 | initial_cuts: list[ReductionPass[bytes]] = attrs.Factory( 115 | lambda: [ 116 | cut_comment_like_things, 117 | hollow, 118 | compose(Split(b"\n"), delete_duplicates), 119 | compose(Split(b"\n"), block_deletion(10, 100)), 120 | lift_braces, 121 | remove_indents, 122 | remove_whitespace, 123 | ] 124 | ) 125 | 126 | great_passes: list[ReductionPass[bytes]] = attrs.Factory( 127 | lambda: [ 128 | compose(Split(b"\n"), delete_duplicates), 129 | compose(Split(b"\n"), block_deletion(1, 10)), 130 | compose(Split(b";"), block_deletion(1, 10)), 131 | remove_indents, 132 | hollow, 133 | lift_braces, 134 | delete_byte_spans, 135 | debracket, 136 | ] 137 | ) 138 | 139 | ok_passes: list[ReductionPass[bytes]] = attrs.Factory( 140 | lambda: [ 141 | compose(Split(b"\n"), block_deletion(11, 20)), 142 | remove_indents, 143 | remove_whitespace, 144 | compose(Tokenize(), block_deletion(1, 20)), 145 | reduce_integer_literals, 146 | replace_falsey_with_zero, 147 | combine_expressions, 148 | merge_adjacent_strings, 149 | lexeme_based_deletions, 150 | short_deletions, 151 | normalize_identifiers, 152 | ] 153 | ) 154 | 155 | last_ditch_passes: list[ReductionPass[bytes]] = attrs.Factory( 156 | lambda: [ 157 | compose(Split(b"\n"), block_deletion(21, 100)), 158 | replace_space_with_newlines, 159 | delete_byte_spans, 160 | lower_bytes, 161 | lower_individual_bytes, 162 | simplify_brackets, 163 | standard_substitutions, 164 | # This is in last ditch because it's probably not useful 165 | # to run it more than once. 166 | cut_comment_like_things, 167 | ] 168 | ) 169 | 170 | def __attrs_post_init__(self) -> None: 171 | if is_python(self.target.current_test_case): 172 | self.great_passes.extend(PYTHON_PASSES) 173 | self.initial_cuts.extend(PYTHON_PASSES) 174 | self.register_format_specific_pass(JSON, JSON_PASSES) 175 | self.register_format_specific_pass( 176 | DimacsCNF, 177 | SAT_PASSES, 178 | ) 179 | 180 | def register_format_specific_pass( 181 | self, format: Format[bytes, T], passes: Iterable[ReductionPass[T]] 182 | ): 183 | if format.is_valid(self.target.current_test_case): 184 | composed = [compose(format, p) for p in passes] 185 | self.great_passes[:0] = composed 186 | self.initial_cuts[:0] = composed 187 | 188 | @property 189 | def pumps(self) -> Iterable[ReductionPump[bytes]]: 190 | if self.clang_delta is None: 191 | return () 192 | else: 193 | return clang_delta_pumps(self.clang_delta) 194 | 195 | @property 196 | def status(self) -> str: 197 | if self.current_pump is None: 198 | if self.current_reduction_pass is not None: 199 | return f"Running reduction pass {self.current_reduction_pass.__name__}" 200 | else: 201 | return "Selecting reduction pass" 202 | else: 203 | if self.current_reduction_pass is not None: 204 | return f"Running reduction pass {self.current_reduction_pass.__name__} under pump {self.current_pump.__name__}" 205 | else: 206 | return f"Running reduction pump {self.current_pump.__name__}" 207 | 208 | async def run_pass(self, rp: ReductionPass[bytes]) -> None: 209 | try: 210 | assert self.current_reduction_pass is None 211 | self.current_reduction_pass = rp 212 | await rp(self.target) 213 | finally: 214 | self.current_reduction_pass = None 215 | 216 | async def pump(self, rp: ReductionPump[bytes]) -> None: 217 | try: 218 | assert self.current_pump is None 219 | self.current_pump = rp 220 | pumped = await rp(self.target) 221 | current = self.target.current_test_case 222 | if pumped == current: 223 | return 224 | with self.backtrack(pumped): 225 | for f in [ 226 | self.run_great_passes, 227 | self.run_ok_passes, 228 | self.run_last_ditch_passes, 229 | ]: 230 | await f() 231 | if self.target.sort_key( 232 | self.target.current_test_case 233 | ) < self.target.sort_key(current): 234 | break 235 | 236 | finally: 237 | self.current_pump = None 238 | 239 | async def run_great_passes(self) -> None: 240 | for rp in self.great_passes: 241 | await self.run_pass(rp) 242 | 243 | async def run_ok_passes(self) -> None: 244 | for rp in self.ok_passes: 245 | await self.run_pass(rp) 246 | 247 | async def run_last_ditch_passes(self) -> None: 248 | for rp in self.last_ditch_passes: 249 | await self.run_pass(rp) 250 | 251 | async def run_some_passes(self) -> None: 252 | prev = self.target.current_test_case 253 | await self.run_great_passes() 254 | if prev != self.target.current_test_case and not self.unlocked_ok_passes: 255 | return 256 | self.unlocked_ok_passes = True 257 | await self.run_ok_passes() 258 | if prev != self.target.current_test_case: 259 | return 260 | await self.run_last_ditch_passes() 261 | 262 | async def initial_cut(self) -> None: 263 | while True: 264 | prev = self.target.current_size 265 | for rp in self.initial_cuts: 266 | async with trio.open_nursery() as nursery: 267 | 268 | @nursery.start_soon 269 | async def _() -> None: 270 | """ 271 | Watcher task that cancels the current reduction pass as 272 | soon as it stops looking like a good idea to keep running 273 | it. Current criteria: 274 | 275 | 1. If it's been more than 5s since the last successful reduction. 276 | 2. If the reduction rate of the task has dropped under 50% of its 277 | best so far. 278 | """ 279 | iters = 0 280 | initial_size = self.target.current_size 281 | best_reduction_rate: float | None = None 282 | 283 | while True: 284 | iters += 1 285 | deleted = initial_size - self.target.current_size 286 | 287 | current = self.target.current_test_case 288 | await trio.sleep(5) 289 | rate = deleted / iters 290 | 291 | if ( 292 | best_reduction_rate is None 293 | or rate > best_reduction_rate 294 | ): 295 | best_reduction_rate = rate 296 | 297 | assert best_reduction_rate is not None 298 | 299 | if ( 300 | rate < 0.5 * best_reduction_rate 301 | or current == self.target.current_test_case 302 | ): 303 | nursery.cancel_scope.cancel() 304 | break 305 | 306 | await self.run_pass(rp) 307 | nursery.cancel_scope.cancel() 308 | if self.target.current_size >= 0.99 * prev: 309 | return 310 | 311 | async def run(self) -> None: 312 | await self.target.setup() 313 | 314 | if await self.target.is_interesting(b""): 315 | return 316 | 317 | prev = 0 318 | for c in [0, 1, ord(b"\n"), ord(b"0"), ord(b"z"), 255]: 319 | if await self.target.is_interesting(bytes([c])): 320 | for i in range(c): 321 | if await self.target.is_interesting(bytes([i])): 322 | break 323 | return 324 | 325 | await self.initial_cut() 326 | 327 | while True: 328 | prev = self.target.current_test_case 329 | await self.run_some_passes() 330 | if self.target.current_test_case != prev: 331 | continue 332 | for pump in self.pumps: 333 | await self.pump(pump) 334 | if self.target.current_test_case == prev: 335 | break 336 | 337 | 338 | class UpdateKeys(Patches[dict[str, bytes], dict[str, bytes]]): 339 | @property 340 | def empty(self) -> dict[str, bytes]: 341 | return {} 342 | 343 | def combine(self, *patches: dict[str, bytes]) -> dict[str, bytes]: 344 | result = {} 345 | for p in patches: 346 | for k, v in p.items(): 347 | result[k] = v 348 | return result 349 | 350 | def apply( 351 | self, patch: dict[str, bytes], target: dict[str, bytes] 352 | ) -> dict[str, bytes]: 353 | result = target.copy() 354 | result.update(patch) 355 | return result 356 | 357 | def size(self, patch: dict[str, bytes]) -> int: 358 | return len(patch) 359 | 360 | 361 | class KeyProblem(ReductionProblem[bytes]): 362 | def __init__( 363 | self, 364 | base_problem: ReductionProblem[dict[str, bytes]], 365 | applier: PatchApplier[dict[str, bytes], dict[str, bytes]], 366 | key: str, 367 | ): 368 | super().__init__(work=base_problem.work) 369 | self.base_problem = base_problem 370 | self.applier = applier 371 | self.key = key 372 | 373 | @property 374 | def current_test_case(self) -> bytes: 375 | return self.base_problem.current_test_case[self.key] 376 | 377 | async def is_interesting(self, test_case: bytes) -> bool: 378 | return await self.applier.try_apply_patch({self.key: test_case}) 379 | 380 | def size(self, test_case: bytes) -> int: 381 | return len(test_case) 382 | 383 | def sort_key(self, test_case: bytes) -> Any: 384 | return shortlex(test_case) 385 | 386 | def display(self, value: bytes) -> str: 387 | return repr(value) 388 | 389 | 390 | @define 391 | class DirectoryShrinkRay(Reducer[dict[str, bytes]]): 392 | clang_delta: Optional[ClangDelta] = None 393 | 394 | async def run(self): 395 | prev = None 396 | while prev != self.target.current_test_case: 397 | prev = self.target.current_test_case 398 | await self.delete_keys() 399 | await self.shrink_values() 400 | 401 | async def delete_keys(self): 402 | target = self.target.current_test_case 403 | keys = list(target.keys()) 404 | keys.sort(key=lambda k: (shortlex(target[k]), shortlex(k)), reverse=True) 405 | for k in keys: 406 | attempt = self.target.current_test_case.copy() 407 | del attempt[k] 408 | await self.target.is_interesting(attempt) 409 | 410 | async def shrink_values(self): 411 | async with trio.open_nursery() as nursery: 412 | applier = PatchApplier(patches=UpdateKeys(), problem=self.target) 413 | for k in self.target.current_test_case.keys(): 414 | key_problem = KeyProblem( 415 | base_problem=self.target, 416 | applier=applier, 417 | key=k, 418 | ) 419 | if self.clang_delta is not None and any( 420 | k.endswith(s) for s in C_FILE_EXTENSIONS 421 | ): 422 | clang_delta = self.clang_delta 423 | else: 424 | clang_delta = None 425 | 426 | key_shrinkray = ShrinkRay( 427 | clang_delta=clang_delta, 428 | target=key_problem, 429 | ) 430 | nursery.start_soon(key_shrinkray.run) 431 | -------------------------------------------------------------------------------- /src/shrinkray/work.py: -------------------------------------------------------------------------------- 1 | import heapq 2 | from contextlib import asynccontextmanager 3 | from enum import IntEnum 4 | from itertools import islice 5 | from random import Random 6 | from typing import Awaitable, Callable, Optional, Sequence, TypeVar 7 | 8 | import trio 9 | 10 | 11 | class Volume(IntEnum): 12 | quiet = 0 13 | normal = 1 14 | verbose = 2 15 | debug = 3 16 | 17 | 18 | S = TypeVar("S") 19 | T = TypeVar("T") 20 | 21 | 22 | TICK_FREQUENCY = 0.05 23 | 24 | 25 | class WorkContext: 26 | """A grab bag of useful tools for 'doing work'. Manages randomness, 27 | logging, concurrency.""" 28 | 29 | def __init__( 30 | self, 31 | random: Optional[Random] = None, 32 | parallelism: int = 1, 33 | volume: Volume = Volume.normal, 34 | ): 35 | self.random = random or Random(0) 36 | self.parallelism = parallelism 37 | self.volume = volume 38 | self.last_ticked = float("-inf") 39 | 40 | @asynccontextmanager 41 | async def map(self, ls: Sequence[T], f: Callable[[T], Awaitable[S]]): 42 | """Lazy parallel map. 43 | 44 | Does a reasonable amount of fine tuning so that it doesn't race 45 | ahead of the current point of iteration and will generallly have 46 | prefetched at most as many values as you've already read. This 47 | is especially important for its use in implementing `find_first`, 48 | which we want to avoid doing redundant work when there are lots of 49 | reduction opportunities. 50 | """ 51 | 52 | async with trio.open_nursery() as nursery: 53 | send, receive = trio.open_memory_channel(self.parallelism + 1) 54 | 55 | @nursery.start_soon 56 | async def do_map(): 57 | if self.parallelism > 1: 58 | it = iter(ls) 59 | 60 | for x in it: 61 | await send.send(await f(x)) 62 | break 63 | else: 64 | return 65 | 66 | n = 2 67 | while True: 68 | values = list(islice(it, n)) 69 | if not values: 70 | send.close() 71 | return 72 | 73 | async with parallel_map( 74 | values, f, parallelism=min(self.parallelism, n) 75 | ) as result: 76 | async for v in result: 77 | await send.send(v) 78 | 79 | n *= 2 80 | else: 81 | for x in ls: 82 | await send.send(await f(x)) 83 | send.close() 84 | 85 | yield receive 86 | 87 | @asynccontextmanager 88 | async def filter(self, ls: Sequence[T], f: Callable[[T], Awaitable[bool]]): 89 | async def apply(x: T) -> tuple[T, bool]: 90 | return (x, await f(x)) 91 | 92 | async with trio.open_nursery() as nursery: 93 | send, receive = trio.open_memory_channel(float("inf")) 94 | 95 | @nursery.start_soon 96 | async def _(): 97 | async with self.map(ls, apply) as results: 98 | async for x, v in results: 99 | if v: 100 | await send.send(x) 101 | send.close() 102 | 103 | yield receive 104 | nursery.cancel_scope.cancel() 105 | 106 | async def find_first_value( 107 | self, ls: Sequence[T], f: Callable[[T], Awaitable[bool]] 108 | ) -> T: 109 | """Returns the first element of `ls` that satisfies `f`, or 110 | raises `NotFound` if no such element exists. 111 | 112 | Will run in parallel if parallelism is enabled. 113 | """ 114 | async with self.filter(ls, f) as filtered: 115 | async for x in filtered: 116 | return x 117 | raise NotFound() 118 | 119 | async def find_large_integer(self, f: Callable[[int], Awaitable[bool]]) -> int: 120 | """Finds a (hopefully large) integer n such that f(n) is True and f(n + 1) 121 | is False. Runs in O(log(n)). 122 | 123 | f(0) is assumed to be True and will not be checked. May not terminate unless 124 | f(n) is False for all sufficiently large n. 125 | """ 126 | # We first do a linear scan over the small numbers and only start to do 127 | # anything intelligent if f(4) is true. This is because it's very hard to 128 | # win big when the result is small. If the result is 0 and we try 2 first 129 | # then we've done twice as much work as we needed to! 130 | for i in range(1, 5): 131 | if not await f(i): 132 | return i - 1 133 | 134 | # We now know that f(4) is true. We want to find some number for which 135 | # f(n) is *not* true. 136 | # lo is the largest number for which we know that f(lo) is true. 137 | lo = 4 138 | 139 | # Exponential probe upwards until we find some value hi such that f(hi) 140 | # is not true. Subsequently we maintain the invariant that hi is the 141 | # smallest number for which we know that f(hi) is not true. 142 | hi = 5 143 | while await f(hi): 144 | lo = hi 145 | hi *= 2 146 | 147 | # Now binary search until lo + 1 = hi. At that point we have f(lo) and not 148 | # f(lo + 1), as desired. 149 | while lo + 1 < hi: 150 | mid = (lo + hi) // 2 151 | if await f(mid): 152 | lo = mid 153 | else: 154 | hi = mid 155 | return lo 156 | 157 | def warn(self, msg: str) -> None: 158 | self.report(msg, Volume.normal) 159 | 160 | def note(self, msg: str) -> None: 161 | self.report(msg, Volume.normal) 162 | 163 | def debug(self, msg: str) -> None: 164 | self.report(msg, Volume.debug) 165 | 166 | def report(self, msg: str, level: Volume) -> None: 167 | return 168 | 169 | 170 | class NotFound(Exception): 171 | pass 172 | 173 | 174 | @asynccontextmanager 175 | async def parallel_map( 176 | ls: Sequence[T], 177 | f: Callable[[T], Awaitable[S]], 178 | parallelism: int, 179 | ): 180 | send_out_values, receive_out_values = trio.open_memory_channel(parallelism) 181 | 182 | work = list(enumerate(ls)) 183 | work.reverse() 184 | 185 | result_heap = [] 186 | 187 | async with trio.open_nursery() as nursery: 188 | results_ready = trio.Event() 189 | 190 | for _ in range(parallelism): 191 | 192 | @nursery.start_soon 193 | async def do_work(): 194 | while work: 195 | i, x = work.pop() 196 | result = await f(x) 197 | heapq.heappush(result_heap, (i, result)) 198 | results_ready.set() 199 | 200 | @nursery.start_soon 201 | async def consolidate() -> None: 202 | i = 0 203 | 204 | while work or result_heap: 205 | while not result_heap: 206 | await results_ready.wait() 207 | assert result_heap 208 | j, x = result_heap[0] 209 | if j == i: 210 | await send_out_values.send(x) 211 | i = j + 1 212 | heapq.heappop(result_heap) 213 | else: 214 | await results_ready.wait() 215 | send_out_values.close() 216 | 217 | yield receive_out_values 218 | nursery.cancel_scope.cancel() 219 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test suite for the shrink_ray package.""" 2 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from hypothesis import settings 2 | 3 | settings.register_profile("default", settings(deadline=None)) 4 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | import random 2 | from typing import Callable, Iterable, TypeVar 3 | 4 | import trio 5 | 6 | from shrinkray.passes.definitions import ReductionPass 7 | from shrinkray.passes.python import is_python 8 | from shrinkray.problem import BasicReductionProblem, default_sort_key 9 | from shrinkray.reducer import BasicReducer, ShrinkRay 10 | from shrinkray.work import WorkContext 11 | 12 | T = TypeVar("T") 13 | 14 | 15 | def reduce_with( 16 | rp: Iterable[ReductionPass[T]], 17 | initial: T, 18 | is_interesting: Callable[[T], bool], 19 | parallelism: int = 1, 20 | ) -> T: 21 | async def acondition(x: T) -> bool: 22 | await trio.lowlevel.checkpoint() 23 | return is_interesting(x) 24 | 25 | async def calc_result() -> T: 26 | problem: BasicReductionProblem[T] = BasicReductionProblem( 27 | initial=initial, 28 | is_interesting=acondition, 29 | work=WorkContext(parallelism=parallelism), 30 | ) 31 | 32 | reducer = BasicReducer( 33 | target=problem, 34 | reduction_passes=rp, 35 | ) 36 | 37 | await reducer.run() 38 | 39 | return problem.current_test_case 40 | 41 | return trio.run(calc_result) 42 | 43 | 44 | def reduce( 45 | initial: bytes, 46 | is_interesting: Callable[[bytes], bool], 47 | parallelism: int = 1, 48 | ) -> T: 49 | async def acondition(x: bytes) -> bool: 50 | await trio.lowlevel.checkpoint() 51 | return is_interesting(x) 52 | 53 | async def calc_result() -> bytes: 54 | problem: BasicReductionProblem[bytes] = BasicReductionProblem( 55 | initial=initial, 56 | is_interesting=acondition, 57 | work=WorkContext(parallelism=parallelism), 58 | ) 59 | 60 | reducer = ShrinkRay( 61 | target=problem, 62 | ) 63 | 64 | await reducer.run() 65 | 66 | return problem.current_test_case 67 | 68 | return trio.run(calc_result) 69 | 70 | 71 | def assert_no_blockers( 72 | is_interesting: Callable[[bytes], bool], 73 | potential_blockers: list[bytes], 74 | lower_bounds=random.sample(range(1000), 12), 75 | ): 76 | potential_blockers = sorted(set(potential_blockers), key=lambda s: (len(s), s)) 77 | 78 | for lower_bound in lower_bounds: 79 | 80 | async def acondition(x: bytes) -> bool: 81 | await trio.lowlevel.checkpoint() 82 | if len(x) < lower_bound: 83 | return False 84 | return is_interesting(x) 85 | 86 | current_best = None 87 | 88 | max_blockers = 1000 89 | candidates = [ 90 | candidate 91 | for candidate in potential_blockers 92 | if len(candidate) >= lower_bound 93 | ] 94 | 95 | if len(candidates) > max_blockers: 96 | candidates = random.sample(candidates, max_blockers) 97 | candidates.sort(key=lambda s: (len(s), s)) 98 | 99 | for initial in candidates: 100 | if len(initial) < lower_bound or not is_interesting(initial): 101 | continue 102 | problem: BasicReductionProblem[T] = BasicReductionProblem( 103 | initial=initial, 104 | is_interesting=acondition, 105 | work=WorkContext(parallelism=1), 106 | ) 107 | 108 | async def calc_result() -> bytes: 109 | reducer = ShrinkRay( 110 | target=problem, 111 | ) 112 | 113 | await reducer.run() 114 | 115 | return problem.current_test_case 116 | 117 | result = trio.run(calc_result) 118 | 119 | if current_best is None: 120 | current_best = result 121 | elif result != current_best: 122 | current_best, result = sorted( 123 | (result, current_best), key=problem.sort_key 124 | ) 125 | 126 | raise AssertionError( 127 | f"With lower bound {lower_bound}, {result} does not reduce to {current_best}" 128 | ) 129 | 130 | 131 | def direct_reductions(origin: bytes, *, parallelism=1) -> set[bytes]: 132 | children = set() 133 | 134 | def is_interesting(b: bytes) -> bool: 135 | if default_sort_key(b) < default_sort_key(origin): 136 | children.add(b) 137 | return b == origin 138 | 139 | reduce(origin, is_interesting, parallelism=parallelism) 140 | 141 | return children 142 | 143 | 144 | class Completed(Exception): 145 | pass 146 | 147 | 148 | def assert_reduces_to( 149 | *, 150 | origin: bytes, 151 | target: bytes, 152 | parallelism=1, 153 | language_restrictions=True, 154 | passes=None, 155 | ): 156 | if origin == target: 157 | raise AssertionError("A value cannot reduce to itself") 158 | if default_sort_key(origin) < default_sort_key(target): 159 | raise AssertionError( 160 | f"It is impossible for {origin} to reduce to {target} as it is more reduced." 161 | ) 162 | 163 | if language_restrictions and is_python(origin) and is_python(target): 164 | require_python = True 165 | else: 166 | require_python = False 167 | 168 | def is_interesting(value: bytes) -> bool: 169 | if require_python and not is_python(value): 170 | return False 171 | return default_sort_key(value) >= default_sort_key(target) 172 | 173 | if passes is None: 174 | best = reduce(origin, is_interesting, parallelism=parallelism) 175 | else: 176 | best = reduce_with(passes, origin, is_interesting, parallelism=parallelism) 177 | 178 | if best == target: 179 | return 180 | 181 | if best == origin: 182 | raise AssertionError(f"Unable to make any progress from {origin}") 183 | raise AssertionError( 184 | f"Unable to reduce {origin} to {target}. Best achieve was {best}" 185 | ) 186 | -------------------------------------------------------------------------------- /tests/test_byte_reduction_passes.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from collections import Counter 3 | 4 | import pytest 5 | from hypothesis import assume, example, given, strategies as st 6 | 7 | from shrinkray.passes.bytes import ( 8 | WHITESPACE, 9 | ByteReplacement, 10 | debracket, 11 | find_ngram_endpoints, 12 | lower_bytes, 13 | lower_individual_bytes, 14 | short_deletions, 15 | sort_whitespace, 16 | ) 17 | from shrinkray.passes.patching import apply_patches 18 | from shrinkray.passes.python import is_python 19 | from shrinkray.problem import BasicReductionProblem 20 | from shrinkray.work import WorkContext 21 | 22 | from tests.helpers import assert_reduces_to, reduce_with 23 | 24 | 25 | def is_hello(data: bytes) -> bool: 26 | try: 27 | tree = ast.parse(data) 28 | except SyntaxError: 29 | return False 30 | 31 | for node in ast.walk(tree): 32 | if isinstance(node, ast.Constant) and node.value == "Hello world!": 33 | return True 34 | 35 | return False 36 | 37 | 38 | def test_short_deletions_can_delete_brackets() -> None: 39 | assert ( 40 | reduce_with([short_deletions], b'"Hello world!"()', is_hello) 41 | == b'"Hello world!"' 42 | ) 43 | 44 | 45 | @example(b"") 46 | @example(b"\x00") 47 | @example(b"\x00\x00") 48 | @given(st.binary()) 49 | def test_ngram_endpoints(b): 50 | find_ngram_endpoints(b) 51 | 52 | 53 | def count_whitespace(b): 54 | return len([c for c in b if c in WHITESPACE]) 55 | 56 | 57 | def count_regions(b): 58 | n = 0 59 | is_whitespace_last = True 60 | for c in b: 61 | typ = c in WHITESPACE 62 | if not typ and is_whitespace_last: 63 | n += 1 64 | is_whitespace_last = typ 65 | return n 66 | 67 | 68 | @pytest.mark.skip 69 | @example(initial=b"\n0\r") 70 | @given(st.builds(bytes, st.lists(st.sampled_from(b"\t\n\r 0123.")))) 71 | def test_sorting_whitespace(initial): 72 | initial_count = count_whitespace(initial) 73 | 74 | assume(initial_count) > 0 75 | 76 | def is_interesting(tc): 77 | assert count_whitespace(tc) == initial_count 78 | assert Counter(tc) == Counter(initial) 79 | return True 80 | 81 | result = reduce_with([sort_whitespace], initial, is_interesting) 82 | 83 | for i, c in enumerate(result): 84 | assert (c in WHITESPACE) == (i < initial_count) 85 | 86 | 87 | @pytest.mark.skip 88 | @given(st.builds(bytes, st.lists(st.sampled_from(b"\t\n\r 0123.")))) 89 | def test_sorting_whitespace_preserving_regions(initial): 90 | initial_count = count_whitespace(initial) 91 | initial_regions = count_regions(initial) 92 | 93 | assume(initial_count > 0) 94 | assume(initial_regions > 1) 95 | 96 | def is_interesting(tc): 97 | assert count_whitespace(tc) == initial_count 98 | assert Counter(tc) == Counter(initial) 99 | return count_regions(tc) == initial_regions 100 | 101 | result = reduce_with([sort_whitespace], initial, is_interesting) 102 | 103 | for run in runs_of_whitespace(result)[1:]: 104 | assert len(run) <= 1 105 | 106 | 107 | @pytest.mark.skip 108 | @pytest.mark.parametrize( 109 | "initial", 110 | [ 111 | b"\t\t\t\t\nfrom\t\t\t\t\t\t\t\t\t\t.\timport\tA\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\nfrom\t\t\t.\t\t\t\t\t\t\t\t\t\timport\t\to\t\t\t\t\t\t\t\t\nfrom\t\t.\t\t\t\t\t\t\t\timport\t\tr\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", 112 | b"from\t\t.\t\t\t\t\t\t\t\t\timport\ta\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\nclass\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\ta\t\t\t\t\t\t\t\t\t\t\t(\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t)\t\t\t\t\t\t\t\t:\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t()\ndef\tr\t\t\t\t\t\t\t\t\t\t\t\t\t\t(\t\t\t\t\t\t\t\t\t\t\t\t\t\t)\t\t\t\t\t\t\t\t:\t\t\t\t\t\t\t\t\t\t\t\t\t\t...\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", 113 | ], 114 | ) 115 | def test_sorting_whitespace_preserving_python(initial): 116 | initial_count = count_whitespace(initial) 117 | 118 | def is_interesting(tc): 119 | assert count_whitespace(tc) == initial_count 120 | assert Counter(tc) == Counter(initial) 121 | return is_python(tc) 122 | 123 | result = reduce_with([sort_whitespace], initial, is_interesting) 124 | 125 | for run in runs_of_whitespace(result)[1:]: 126 | assert len(run) <= 1 127 | 128 | 129 | def runs_of_whitespace(result): 130 | whitespace_runs = [] 131 | i = 0 132 | while i < len(result): 133 | c = result[i] 134 | if c not in WHITESPACE: 135 | i += 1 136 | continue 137 | j = i + 1 138 | while j < len(result) and result[j] in WHITESPACE: 139 | j += 1 140 | whitespace_runs.append(result[i:j]) 141 | i = j 142 | return whitespace_runs 143 | 144 | 145 | def test_debracket(): 146 | assert ( 147 | reduce_with([debracket], b"(1 + 2) + (3 + 4)", lambda x: b"(3 + 4)" in x) 148 | == b"1 + 2 + (3 + 4)" 149 | ) 150 | 151 | 152 | @pytest.mark.parametrize("parallelism", [1, 2]) 153 | def test_byte_reduction_example_1(parallelism): 154 | assert ( 155 | reduce_with( 156 | [lower_bytes], 157 | b"\x00\x03", 158 | lambda x: (len(x) == 2 and x[1] >= 2), 159 | parallelism=parallelism, 160 | ) 161 | == b"\x00\x02" 162 | ) 163 | 164 | 165 | @pytest.mark.parametrize("parallelism", [1, 2]) 166 | def test_byte_reduction_example_2(parallelism): 167 | assert ( 168 | reduce_with( 169 | [lower_bytes, lower_individual_bytes], 170 | b"\x03\x00", 171 | lambda x: (len(x) == 2 and x[0] >= 2), 172 | parallelism=parallelism, 173 | ) 174 | == b"\x02\x00" 175 | ) 176 | 177 | 178 | @pytest.mark.parametrize("parallelism", [1, 2]) 179 | def test_byte_reduction_example_3(parallelism): 180 | assert_reduces_to( 181 | origin=b"1200\x00\x01\x02", 182 | target=b"120", 183 | parallelism=parallelism, 184 | ) 185 | 186 | 187 | @st.composite 188 | def lowering_problem(draw): 189 | initial = bytes(draw(st.lists(st.integers(0, 255), unique=True, min_size=1))) 190 | target = bytes([draw(st.integers(0, c)) for c in initial]) 191 | patches = draw(st.permutations([{c: d} for c, d in zip(initial, target)])) 192 | 193 | if len(initial) > 1: 194 | n_pair_patches = draw(st.integers(0, len(initial) * (len(initial) - 1))) 195 | pair_patch_indexes = draw( 196 | st.lists( 197 | st.lists( 198 | st.integers(0, len(initial) - 1), 199 | min_size=2, 200 | max_size=2, 201 | unique=True, 202 | ), 203 | min_size=n_pair_patches, 204 | max_size=n_pair_patches, 205 | ) 206 | ) 207 | patches += [{initial[i]: target[i] for i in ls} for ls in pair_patch_indexes] 208 | 209 | return (initial, target, patches) 210 | 211 | 212 | async def always_true(x): 213 | return True 214 | 215 | 216 | @given(lowering_problem(), st.integers(1, 5)) 217 | @example(lowering=(b"\x01\x02", b"\x00\x00", [{1: 0}, {2: 0}]), parallelism=2).via( 218 | "discovered failure" 219 | ) 220 | @example( 221 | lowering=( 222 | b"\x00\x01\x02\x03\x04\x05\x06\x07", 223 | b"\x00\x00\x00\x00\x00\x00\x00\x00", 224 | [{0: 0}, {1: 0}, {2: 0}, {3: 0}, {4: 0}, {5: 0}, {6: 0}, {7: 0}], 225 | ), 226 | parallelism=3, 227 | ).via("discovered failure") 228 | async def test_apply_byte_replacement_patches(lowering, parallelism): 229 | initial, target, patches = lowering 230 | 231 | trivial_problem = BasicReductionProblem( 232 | initial, always_true, work=WorkContext(parallelism=parallelism) 233 | ) 234 | 235 | await apply_patches(trivial_problem, ByteReplacement(), patches) 236 | 237 | assert trivial_problem.current_test_case == target 238 | -------------------------------------------------------------------------------- /tests/test_clang_delta.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from shrinkray.passes.clangdelta import ( 6 | TRANSFORMATIONS, 7 | ClangDelta, 8 | clang_delta_pump, 9 | find_clang_delta, 10 | ) 11 | from shrinkray.problem import BasicReductionProblem, WorkContext 12 | 13 | BAD_HELLO = b'namespace\x00std{inline\x00namespace\x00__1{templatestruct\x00char_traits;}}typedef\x00long\x00D;namespace\x00std{namespace\x00__1{template>class\x00basic_streambuf;template>class\x00C;typedef\x00CE;}D\x00__constexpr_strlen(const\x00char*__str){return\x00__builtin_strlen(__str);}namespace\x00__1{template<>struct\x00char_traits{using\x00C=char;static\x00C\x00$(const\x00C*__s){return\x00__constexpr_strlen(__s);}};templateclass\x00${public:typedef\x00basic_streambuf<_Bhar$>D;typedef\x00C<_Bhar$>C;D*__sbuf_;$(C&__s):__sbuf_(__s.E()){}$\x00operator=(_Bhar$\x00__c){__sbuf_->sputc(__c);}};class\x00A{public:typedef\x00D\x00$;typedef\x00D\x00C;virtual~A();void*E()const{return\x00__rdbuf_;}$\x00A;C\x00__;C\x00D;C\x00_;void*__rdbuf_;};templateclass\x00B:public\x00A{public:typedef\x00_Bhar$\x00$;typedef\x00_$1\x00C;basic_streambuf<$,C>*E()const;$\x00D()const;};templatebasic_streambuf<_Bhar$,_$1>*B<_Bhar$,_$1>::E()const{return\x00static_cast*>(A::E());}template_Bhar$\x00B<_Bhar$,_$1>::D()const{}templateclass\x00basic_streambuf{public:int\x00sputc(char);};}template_$1$1\x00__pad_and_output(_$1$1\x00__s,const\x00_Bhar$*__ob,const\x00_Bhar$*__oe){for(;__ob<__oe;++__ob)__s=*__ob;}namespace\x00__1{templateclass\x00C:virtual\x00public\x00B{};templateC<_Bhar$>&__put_character_se$1(C<_$1>&__os,const\x00_Bhar$*__str,D\x00__len){try{typedef\x00$<_Bhar$>_$A;__pad_and_output(_$A(__os),__str,__str+__len);}catch(...){}return\x00__os;}templateC<_Bhar$>&operator<<(C<_Bhar$,_$1>&__os,const\x00_Bhar$*__str){return\x00__put_character_se$1(__os,__str,_$1::$(__str));}extern\x00E\x00cout;}}int\x00main(){std::cout<<"Hello";}' 14 | CRASHER = b'namespace{\n inline namespace __1{\n templatestruct __attribute(())char_traits;\n template>class __attribute(())C;\n }\n}\nnamespace{\n namespace __1{\n class __attribute(())ios_base{\n public:~ios_base();\n }\n ;\n templateclass __attribute(())D:ios_base{}\n ;\n templateclass __attribute(())C:D<_C,_s>{\n public:void operator<<(C&(C&));\n }\n ;\n template__attribute(())Coperator<<(C<_s>,_C){}\n templateC<_C>&endl(C<_s>&);\n }\n}\nnamespace{\n extern __attribute(())Ccout;\n}\nint main(){\n cout<<""< None: 11 | assert ( 12 | reduce_with([reduce_integer_literals], b"bobcats99999hello", lambda x: True) 13 | == b"bobcats0hello" 14 | ) 15 | 16 | 17 | def test_can_reduce_integers_to_boundaries() -> None: 18 | assert ( 19 | reduce_with([reduce_integer_literals], b"100", lambda x: eval(x) >= 73) == b"73" 20 | ) 21 | 22 | 23 | def test_can_combine_expressions() -> None: 24 | assert reduce_with([combine_expressions], b"10 + 10", lambda x: True) == b"20" 25 | 26 | 27 | def test_does_not_error_on_bad_expression() -> None: 28 | assert reduce_with([combine_expressions], b"1 / 0", lambda x: True) == b"1 / 0" 29 | 30 | 31 | def test_can_combine_expressions_with_no_expressions() -> None: 32 | assert ( 33 | reduce_with([combine_expressions], b"hello world", lambda x: True) 34 | == b"hello world" 35 | ) 36 | 37 | 38 | LOTS_OF_COMMENTS = b""" 39 | hello # this is a single line comment 40 | /* this 41 | is 42 | a 43 | multiline 44 | comment */ world // some extraneous garbage 45 | """ 46 | 47 | 48 | def test_comment_removal(): 49 | x = reduce_with([cut_comment_like_things], LOTS_OF_COMMENTS, lambda x: True) 50 | lines = [line.strip() for line in x.splitlines() if line.strip()] 51 | assert lines == [b"hello", b"world"] 52 | -------------------------------------------------------------------------------- /tests/test_generic_shrinking_properties.py: -------------------------------------------------------------------------------- 1 | from random import Random 2 | 3 | import hypothesmith 4 | import trio 5 | from hypothesis import Phase, assume, example, given, note, settings, strategies as st 6 | from hypothesis.errors import Frozen, StopTest 7 | 8 | from shrinkray.passes.python import is_python 9 | from shrinkray.problem import BasicReductionProblem, default_sort_key 10 | from shrinkray.reducer import ShrinkRay 11 | from shrinkray.work import Volume, WorkContext 12 | 13 | from tests.helpers import assert_no_blockers, assert_reduces_to, direct_reductions 14 | 15 | 16 | def tidy_python_example(s): 17 | results = [] 18 | for line in s.splitlines(): 19 | line, *_ = line.split("#") 20 | line = line.strip() 21 | if line: 22 | results.append(line) 23 | output = "\n".join(results) 24 | if output.startswith('"""'): 25 | output = output[3:] 26 | i = output.index('"""') 27 | output = output[i + 3 :] 28 | return output.strip() + "\n" 29 | 30 | 31 | POTENTIAL_BLOCKERS = [ 32 | b"\n", 33 | b"s", 34 | b"0", 35 | b"()", 36 | b"[]", 37 | b"\t\t", 38 | b"#\x00", 39 | b"#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 40 | b"\t\t\t\t\t\t\t\t\t\t\t\t", 41 | # b"from\t\t\t.\t\t\t\t\t\timport\ta\t\t\t\t\t\t\t\t\t\t\nclass\t\to\t\t\t\t\t\t\t:\n\tdef\t\t\t\t\ta\t(\t\t\t\t\t\t\t\t\t)\t\t\t\t\t\t\t:...\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", 42 | # "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", 43 | ] 44 | 45 | python_files = st.builds( 46 | lambda s, e: s.encode(e), 47 | hypothesmith.from_grammar(), 48 | st.sampled_from(("utf-8",)), 49 | ) 50 | test_cases = (python_files).filter(lambda b: 1 < len(b) <= 1000) 51 | 52 | 53 | common_settings = settings(deadline=None, max_examples=10, report_multiple_bugs=False) 54 | 55 | 56 | @common_settings 57 | @given( 58 | initial=test_cases, 59 | rnd=st.randoms(use_true_random=True), 60 | is_interesting_sync=st.functions( 61 | returns=st.booleans(), pure=True, like=lambda test_case: False 62 | ), 63 | data=st.data(), 64 | parallelism=st.integers(1, 1), 65 | ) 66 | async def test_can_shrink_arbitrary_problems( 67 | initial, rnd, data, parallelism, is_interesting_sync 68 | ): 69 | is_interesting_cache = {} 70 | 71 | initial_is_python = is_python(initial) 72 | 73 | async def is_interesting(test_case: bytes) -> bool: 74 | await trio.lowlevel.checkpoint() 75 | if test_case == initial: 76 | return True 77 | elif not test_case: 78 | return False 79 | try: 80 | return is_interesting_cache[test_case] 81 | except KeyError: 82 | pass 83 | if initial_is_python and not is_python(test_case): 84 | result = False 85 | else: 86 | try: 87 | result = is_interesting_sync(test_case) 88 | except (StopTest, Frozen): # pragma: no cover 89 | result = False 90 | is_interesting_cache[test_case] = result 91 | if result: 92 | note(f"{test_case} is interesting") 93 | return result 94 | 95 | work = WorkContext( 96 | random=rnd, 97 | volume=Volume.quiet, 98 | parallelism=parallelism, 99 | ) 100 | problem = BasicReductionProblem( 101 | initial=initial, is_interesting=is_interesting, work=work 102 | ) 103 | 104 | reducer = ShrinkRay(problem) 105 | 106 | with trio.move_on_after(10) as cancel_scope: 107 | await reducer.run() 108 | assert not cancel_scope.cancelled_caught 109 | 110 | assert len(problem.current_test_case) <= len(initial) 111 | 112 | 113 | @settings(common_settings, phases=[Phase.explicit]) 114 | @example(b":\x80", 1) 115 | @example(b"#\x80", 1) 116 | @example( 117 | initial=b"..........................................................................................................................................................................................................................................................................................................................................................................................", 118 | parallelism=2, 119 | ) 120 | @given( 121 | initial=test_cases, 122 | parallelism=st.integers(1, 10), 123 | ) 124 | async def test_can_fail_to_shrink_arbitrary_problems(initial, parallelism): 125 | async def is_interesting(test_case: bytes) -> bool: 126 | await trio.lowlevel.checkpoint() 127 | return test_case == initial 128 | 129 | work = WorkContext( 130 | random=Random(0), 131 | volume=Volume.quiet, 132 | parallelism=parallelism, 133 | ) 134 | problem = BasicReductionProblem( 135 | initial=initial, is_interesting=is_interesting, work=work 136 | ) 137 | 138 | reducer = ShrinkRay(problem) 139 | 140 | with trio.move_on_after(10) as cancel_scope: 141 | await reducer.run() 142 | assert not cancel_scope.cancelled_caught 143 | 144 | assert problem.current_test_case == initial 145 | 146 | 147 | @example(b"from p import*", 1) 148 | @example( 149 | b'from types import MethodType\ndef is_hypothesis_test(test):\nif isinstance(test, MethodType):\nreturn is_hypothesis_test(test.__func__)\nreturn getattr(test, "is_hypothesis_test", False)', 150 | 1, 151 | ) 152 | @example(b"''", 1) 153 | @example(b"# Hello world", 1) 154 | @example(b"\x00\x01", 1) 155 | @common_settings 156 | @given( 157 | initial=test_cases, 158 | parallelism=st.integers(1, 10), 159 | ) 160 | async def test_can_succeed_at_shrinking_arbitrary_problems(initial, parallelism): 161 | initial_is_python = is_python(initial) 162 | 163 | async def is_interesting(test_case: bytes) -> bool: 164 | if initial_is_python and not is_python(test_case): 165 | return False 166 | await trio.lowlevel.checkpoint() 167 | return len(test_case) > 0 168 | 169 | work = WorkContext( 170 | random=Random(0), 171 | volume=Volume.quiet, 172 | parallelism=parallelism, 173 | ) 174 | problem = BasicReductionProblem( 175 | initial=initial, is_interesting=is_interesting, work=work 176 | ) 177 | 178 | reducer = ShrinkRay(problem) 179 | 180 | with trio.move_on_after(10) as cancel_scope: 181 | await reducer.run() 182 | assert not cancel_scope.cancelled_caught 183 | 184 | assert len(problem.current_test_case) == 1 185 | 186 | 187 | def test_no_blockers(): 188 | assert_no_blockers( 189 | potential_blockers=POTENTIAL_BLOCKERS, 190 | is_interesting=is_python, 191 | lower_bounds=[1, 2, 5, 10], 192 | ) 193 | 194 | 195 | @common_settings 196 | @given(st.binary(), st.data()) 197 | def test_always_reduces_to_each_direct_reduction(origin, data): 198 | reductions = sorted(direct_reductions(origin), key=default_sort_key, reverse=True) 199 | 200 | assume(reductions) 201 | 202 | target = data.draw(st.sampled_from(reductions)) 203 | 204 | assert_reduces_to(origin=origin, target=target, language_restrictions=False) 205 | 206 | 207 | @common_settings 208 | @given(st.binary(), st.integers(2, 8), st.data()) 209 | def test_parallelism_never_prevents_reduction(origin, parallelism, data): 210 | reductions = sorted(direct_reductions(origin), key=default_sort_key, reverse=True) 211 | 212 | assume(reductions) 213 | 214 | parallel_reductions = sorted( 215 | direct_reductions(origin, parallelism=parallelism), 216 | key=default_sort_key, 217 | reverse=True, 218 | ) 219 | 220 | assert set(reductions).issubset(parallel_reductions) 221 | 222 | target = data.draw(st.sampled_from(reductions)) 223 | 224 | assert_reduces_to( 225 | origin=origin, 226 | target=target, 227 | parallelism=parallelism, 228 | language_restrictions=False, 229 | ) 230 | -------------------------------------------------------------------------------- /tests/test_json_passes.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from shrinkray.passes.json import delete_identifiers 4 | from shrinkray.problem import BasicReductionProblem 5 | from shrinkray.work import WorkContext 6 | 7 | 8 | def json_problem(initial, is_interesting): 9 | return BasicReductionProblem( 10 | initial, 11 | is_interesting, 12 | work=WorkContext(parallelism=1), 13 | sort_key=lambda s: len(json.dumps(s)), 14 | ) 15 | 16 | 17 | async def test_can_remove_some_identifiers(): 18 | initial = [{"a": 0, "b": 0, "c": i} for i in range(10)] 19 | 20 | async def is_interesting(ls): 21 | return len(ls) == 10 and all(x.get("c", -1) == i for i, x in enumerate(ls)) 22 | 23 | assert await is_interesting(initial) 24 | 25 | problem = json_problem(initial, is_interesting) 26 | 27 | await delete_identifiers(problem) 28 | 29 | assert problem.current_test_case == [{"c": i} for i in range(10)] 30 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | import pathlib 5 | import pytest 6 | 7 | import trio 8 | import black 9 | from shrinkray.__main__ import interrupt_wait_and_kill 10 | 11 | 12 | def format(s): 13 | return black.format_str(s, mode=black.Mode()).strip() 14 | 15 | 16 | async def test_kill_process(): 17 | async with trio.open_nursery() as nursery: 18 | kwargs = dict( 19 | universal_newlines=False, 20 | preexec_fn=os.setsid, 21 | check=False, 22 | stdout=subprocess.PIPE, 23 | ) 24 | 25 | def call_with_kwargs(task_status=trio.TASK_STATUS_IGNORED): # type: ignore 26 | # start a subprocess that will just ignore SIGINT signals 27 | return trio.run_process( 28 | [ 29 | sys.executable, 30 | "-c", 31 | "import signal, sys, time; signal.signal(signal.SIGINT, lambda *a: 1); print(1); sys.stdout.flush(); time.sleep(1000)", 32 | ], 33 | **kwargs, 34 | task_status=task_status, 35 | ) 36 | 37 | sp = await nursery.start(call_with_kwargs) 38 | line = await sp.stdout.receive_some(2) 39 | assert line == b"1\n" 40 | # must not raise ValueError but succeed at killing the process 41 | await interrupt_wait_and_kill(sp) 42 | assert sp.returncode is not None 43 | assert sp.returncode != 0 44 | 45 | 46 | def test_can_reduce_a_directory(tmp_path: pathlib.Path): 47 | target = tmp_path / "foo" 48 | target.mkdir() 49 | a = target / "a.py" 50 | a.write_text("x = 1\ny=2\nz=3\n") 51 | b = target / "b.py" 52 | b.write_text("y = 'hello world'") 53 | c = target / "c.py" 54 | c.write_text("from a import x\n\n...\nassert x == 2") 55 | 56 | script = tmp_path / "test.py" 57 | script.write_text( 58 | f""" 59 | #!/usr/bin/env {sys.executable} 60 | import sys 61 | sys.path.append(sys.argv[1]) 62 | 63 | try: 64 | import c 65 | sys.exit(1) 66 | except AssertionError: 67 | sys.exit(0) 68 | """.strip() 69 | ) 70 | script.chmod(0o777) 71 | 72 | subprocess.check_call( 73 | [ 74 | str(script), 75 | str(target), 76 | ] 77 | ) 78 | 79 | subprocess.check_call( 80 | [sys.executable, "-m", "shrinkray", str(script), str(target), "--ui=basic"], 81 | ) 82 | 83 | assert a.exists() 84 | assert not b.exists() 85 | assert c.exists() 86 | 87 | # TODO: Remove calls to format when formatting is implemented properly for 88 | # directories. 89 | assert format(a.read_text()) == "x = 0" 90 | assert format(c.read_text()) == "from a import x\n\nassert x" 91 | 92 | 93 | def test_gives_informative_error_when_script_does_not_work_outside_current_directory(tmpdir): 94 | target = (tmpdir / "hello.txt") 95 | target.write_text("hello world", encoding='utf-8') 96 | script = tmpdir / "test.py" 97 | script.write_text( 98 | f""" 99 | #!/usr/bin/env {sys.executable} 100 | import sys 101 | 102 | if sys.argv[1] != {repr(str(target))}: 103 | sys.exit(1) 104 | """.strip(), encoding='utf-8' 105 | ) 106 | script.chmod(0o777) 107 | 108 | subprocess.check_call([script, target]) 109 | 110 | with pytest.raises(subprocess.CalledProcessError) as excinfo: 111 | subprocess.run( 112 | [sys.executable, "-m", "shrinkray", str(script), str(target), "--ui=basic"], 113 | check=True, 114 | stderr=subprocess.PIPE, 115 | stdout=subprocess.PIPE, 116 | universal_newlines=True, 117 | ) 118 | 119 | assert 'your script depends' in excinfo.value.stderr 120 | 121 | 122 | def test_prints_the_output_on_an_initially_uninteresting_test_case(tmpdir): 123 | target = (tmpdir / "hello.txt") 124 | target.write_text("hello world", encoding='utf-8') 125 | script = tmpdir / "test.py" 126 | script.write_text( 127 | f""" 128 | #!/usr/bin/env {sys.executable} 129 | import sys 130 | 131 | print("Hello world") 132 | 133 | sys.exit(1) 134 | """.strip(), encoding='utf-8' 135 | ) 136 | script.chmod(0o777) 137 | 138 | with pytest.raises(subprocess.CalledProcessError) as excinfo: 139 | subprocess.run( 140 | [sys.executable, "-m", "shrinkray", str(script), str(target), "--ui=basic"], 141 | check=True, 142 | stderr=subprocess.PIPE, 143 | stdout=subprocess.PIPE, 144 | universal_newlines=True, 145 | ) 146 | 147 | assert 'Hello world' in excinfo.value.stdout 148 | -------------------------------------------------------------------------------- /tests/test_misc_reduction_performance.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | from warnings import catch_warnings, filterwarnings 3 | 4 | import pytest 5 | 6 | from tests.helpers import reduce 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "initial", 11 | [ 12 | b"." * 100, 13 | ], 14 | ) 15 | @pytest.mark.parametrize("parallelism", [1, 2]) 16 | def test_failure_performance(initial, parallelism): 17 | start = time() 18 | final = reduce(initial, lambda s: s == initial) 19 | assert final == initial 20 | runtime = time() - start 21 | assert runtime <= 2 22 | 23 | 24 | ASSIGNMENT_CHAIN = b""" 25 | agmas = 42 26 | benighted = agmas 27 | squabashing = benighted 28 | paradisaically = squabashing 29 | simar = paradisaically 30 | output.append(simar) 31 | """ 32 | 33 | 34 | def test_can_normalize_identifiers(): 35 | def is_interesting(test_case): 36 | output = [] 37 | data = {"output": output} 38 | try: 39 | with catch_warnings(): 40 | filterwarnings("ignore", category=SyntaxWarning) 41 | exec(test_case, data, data) 42 | except BaseException: 43 | return False 44 | return output == [42] 45 | 46 | # Ideally we would reduce further than this, but it's tricky for now. 47 | assert reduce(ASSIGNMENT_CHAIN, is_interesting) == b"A=42\noutput.append(A)" 48 | -------------------------------------------------------------------------------- /tests/test_python_reducers.py: -------------------------------------------------------------------------------- 1 | from shrinkray.passes.python import ( 2 | lift_indented_constructs, 3 | replace_bodies_with_ellipsis, 4 | replace_statements_with_pass, 5 | strip_annotations, 6 | PYTHON_PASSES, 7 | ) 8 | import os 9 | from tests.helpers import reduce, reduce_with 10 | from glob import glob 11 | import pytest 12 | 13 | 14 | def test_can_replace_blocks_with_body() -> None: 15 | body = reduce_with( 16 | [lift_indented_constructs], b"if True:\n x = 1", lambda t: b"x" in t 17 | ) 18 | assert body == b"x = 1" 19 | 20 | 21 | ELIF_BLOCK = b""" 22 | if True: 23 | x = 1 24 | elif True: 25 | x = 2 26 | """ 27 | 28 | 29 | def test_lifts_bodies_of_elif(): 30 | assert ( 31 | reduce_with([lift_indented_constructs], ELIF_BLOCK, lambda x: True).strip() 32 | == b"x = 1" 33 | ) 34 | 35 | 36 | def test_does_not_error_on_elif(): 37 | assert ( 38 | reduce_with([lift_indented_constructs], ELIF_BLOCK, lambda x: b"elif" in x) 39 | == ELIF_BLOCK 40 | ) 41 | 42 | 43 | def test_lifts_bodies_of_with(): 44 | assert ( 45 | reduce_with( 46 | [lift_indented_constructs], "with ...:\n x = 1", lambda x: True 47 | ).strip() 48 | == b"x = 1" 49 | ) 50 | 51 | 52 | def test_can_replace_statements_with_pass() -> None: 53 | body = reduce_with( 54 | [replace_statements_with_pass], b"from x import *", lambda t: len(t) > 1 55 | ) 56 | assert body == b"pass" 57 | 58 | 59 | def test_can_reduce_an_example_that_crashes_lib_cst(): 60 | reduce(b"() if 0 else(lambda:())", lambda x: len(x) >= 5) 61 | 62 | 63 | ANNOTATED = b""" 64 | def has_an_annotation(x: list[int]) -> list[int]: 65 | y: list[int] = list(reversed(x)) 66 | return x + y 67 | """ 68 | 69 | DEANNOTATED = b""" 70 | def has_an_annotation(x): 71 | y = list(reversed(x)) 72 | return x + y 73 | """ 74 | 75 | 76 | def test_strip_annotations(): 77 | assert reduce_with([strip_annotations], ANNOTATED, lambda x: True) == DEANNOTATED 78 | 79 | 80 | def test_single_annotation(): 81 | x = b"x:A\n" 82 | assert reduce_with(PYTHON_PASSES, x, lambda y: True).strip() == b"" 83 | 84 | 85 | IF_BLOCK = """ 86 | if True: 87 | x = 1 88 | y = 2 89 | assert x + y 90 | """ 91 | 92 | 93 | def test_body_replacement_of_if(): 94 | assert ( 95 | reduce_with([replace_bodies_with_ellipsis], IF_BLOCK, lambda x: True).strip() 96 | == b"if True:\n ..." 97 | ) 98 | 99 | 100 | ROOT = os.path.dirname(os.path.dirname(__file__)) 101 | 102 | 103 | PYTHON_FILES = glob(os.path.join(ROOT, "src", "**", "*.py"), recursive=True) + glob( 104 | os.path.join(ROOT, "tests", "**", "*.py"), recursive=True 105 | ) 106 | 107 | 108 | @pytest.mark.parametrize("pyfile", PYTHON_FILES) 109 | def test_reduce_all(pyfile): 110 | with open(pyfile, "rb") as i: 111 | code = i.read() 112 | 113 | def is_interesting(x): 114 | return True 115 | 116 | reduce_with(PYTHON_PASSES, code, is_interesting) 117 | 118 | 119 | ISSUE_12_INPUT = b""" 120 | import asyncio 121 | import _lsprof 122 | 123 | if True: 124 | a = 1 125 | b = 2 126 | c = 3 127 | 128 | if True: 129 | obj = _lsprof.Profiler() 130 | obj.enable() 131 | obj._pystart_callback(lambda: 0, 0) 132 | obj = None 133 | loop = asyncio.get_event_loop() 134 | """ 135 | 136 | ISSUE_12_OUTPUT = b""" 137 | import asyncio 138 | import _lsprof 139 | 140 | if True: 141 | ... 142 | 143 | if True: 144 | obj = _lsprof.Profiler() 145 | obj.enable() 146 | obj._pystart_callback(lambda: 0, 0) 147 | obj = None 148 | loop = asyncio.get_event_loop() 149 | """ 150 | 151 | 152 | def test_reduce_with_ellipsis_can_reduce_single_block(): 153 | reduced = reduce_with([replace_bodies_with_ellipsis], ISSUE_12_INPUT, lambda x: b"Profiler" in x, parallelism=1) 154 | assert reduced == ISSUE_12_OUTPUT 155 | -------------------------------------------------------------------------------- /tests/test_sat.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import os 3 | import subprocess 4 | import tempfile 5 | from shutil import which 6 | from typing import Callable 7 | 8 | import pytest 9 | from hypothesis import assume, example, given, settings, strategies as st 10 | 11 | from shrinkray.passes.sat import SAT, SAT_PASSES, DimacsCNF 12 | 13 | from .helpers import reduce_with 14 | 15 | HAS_MINISAT = which("minisat") is not None 16 | 17 | sat_settings = settings(deadline=None) 18 | 19 | 20 | class MinisatNotInstalled(Exception): ... 21 | 22 | 23 | def check_minisat(): 24 | if not HAS_MINISAT: 25 | raise MinisatNotInstalled() 26 | 27 | 28 | def is_satisfiable(clauses): 29 | check_minisat() 30 | if not clauses: 31 | return True 32 | if not all(clauses): 33 | return False 34 | 35 | f, sat_file = tempfile.mkstemp() 36 | os.close(f) 37 | 38 | with open(sat_file, "wb") as o: 39 | o.write(DimacsCNF.dumps(clauses)) 40 | try: 41 | result = subprocess.run( 42 | ["minisat", sat_file], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL 43 | ).returncode 44 | assert result in (10, 20) 45 | return result == 10 46 | finally: 47 | os.unlink(sat_file) 48 | 49 | 50 | def find_solution(clauses): 51 | if not clauses: 52 | return [] 53 | if not all(clauses): 54 | return None 55 | 56 | f, sat_file = tempfile.mkstemp() 57 | os.close(f) 58 | 59 | f, out_file = tempfile.mkstemp() 60 | os.close(f) 61 | 62 | with open(sat_file, "wb") as o: 63 | o.write(DimacsCNF.dumps(clauses)) 64 | try: 65 | result = subprocess.run( 66 | ["minisat", sat_file, out_file], 67 | stdout=subprocess.DEVNULL, 68 | stderr=subprocess.DEVNULL, 69 | ).returncode 70 | assert result in (10, 20) 71 | if result == 20: 72 | return None 73 | with open(out_file) as i: 74 | satline, resultline = i 75 | assert satline == "SAT\n" 76 | result = list(map(int, resultline.strip().split())) 77 | assert result[-1] == 0 78 | result.pop() 79 | return result 80 | finally: 81 | os.unlink(sat_file) 82 | os.unlink(out_file) 83 | 84 | 85 | @st.composite 86 | def sat_clauses(draw, min_clause_size=1): 87 | n_variables = draw(st.integers(min_clause_size, min(10, min_clause_size * 2))) 88 | variables = range(1, n_variables + 1) 89 | 90 | literal = st.builds( 91 | operator.mul, st.sampled_from(variables), st.sampled_from((-1, 1)) 92 | ) 93 | 94 | return draw( 95 | st.lists(st.lists(literal, unique_by=abs, min_size=min_clause_size), min_size=1) 96 | ) 97 | 98 | 99 | @st.composite 100 | def unsatisfiable_clauses(draw, min_clause_size=1): 101 | clauses = draw(sat_clauses(min_clause_size=min_clause_size)) 102 | assume(clauses) 103 | 104 | while True: 105 | sol = find_solution(clauses) 106 | if sol is None: 107 | return clauses 108 | assert len(sol) >= min_clause_size, (sol, clauses) 109 | subset = draw( 110 | st.lists(st.sampled_from(sol), min_size=min_clause_size, unique=True) 111 | ) 112 | clauses.append([-n for n in subset]) 113 | 114 | 115 | @st.composite 116 | def has_unique_solution(draw): 117 | clauses = draw(sat_clauses(min_clause_size=2)) 118 | sol = find_solution(clauses) 119 | assume(sol is not None) 120 | assert sol is not None 121 | 122 | while True: 123 | other_sol = find_solution(clauses + [[-literal for literal in sol]]) 124 | if other_sol is None: 125 | assert is_satisfiable(clauses) 126 | return clauses 127 | 128 | to_rule_out = sorted(set(other_sol) - set(sol)) 129 | assert to_rule_out 130 | subset = draw( 131 | st.lists( 132 | st.sampled_from(to_rule_out), 133 | min_size=min(2, len(to_rule_out)), 134 | unique=True, 135 | ) 136 | ) 137 | clauses.append([-n for n in subset]) 138 | 139 | 140 | def shrink_sat(clauses: SAT, test_function: Callable[[SAT], bool]) -> SAT: 141 | return reduce_with(SAT_PASSES, clauses, test_function) 142 | 143 | 144 | @sat_settings 145 | @example([[1]]) 146 | @given(sat_clauses()) 147 | def test_shrink_to_one_single_literal_clause(clauses): 148 | result = shrink_sat(clauses, any) 149 | assert result == [[1]] 150 | 151 | 152 | @pytest.mark.parametrize("n", range(2, 11)) 153 | def test_can_shrink_chain_to_two(n): 154 | chain = [[-i, i + 1] for i in range(1, n + 1)] 155 | 156 | def test(clauses): 157 | clauses = list(clauses) 158 | return ( 159 | is_satisfiable(clauses) 160 | and is_satisfiable(clauses + [[1], [n]]) 161 | and is_satisfiable(clauses + [[-1], [-n]]) 162 | and not is_satisfiable(clauses + [[1], [-n]]) 163 | ) 164 | 165 | assert test(chain) 166 | 167 | shrunk = shrink_sat(chain, test) 168 | 169 | assert shrunk == [[-1, n]] 170 | 171 | 172 | @sat_settings 173 | @given(unsatisfiable_clauses()) 174 | def test_reduces_unsatisfiable_to_trivial(unsat): 175 | def test(clauses): 176 | return clauses and all(clauses) and not is_satisfiable(clauses) 177 | 178 | shrunk = shrink_sat(unsat, test) 179 | 180 | assert shrunk == [ 181 | [-1], 182 | [ 183 | 1, 184 | ], 185 | ] 186 | 187 | 188 | @sat_settings 189 | @example([[-1], [-2], [-3], [-4, -5], [4, 5], [-6], [4, -5]]) 190 | @given(has_unique_solution()) 191 | def test_reduces_unique_satisfiable_to_trivial(unique_sat): 192 | def test(clauses): 193 | if not clauses: 194 | return False 195 | sol = find_solution(clauses) 196 | if sol is None: 197 | return False 198 | return not is_satisfiable(list(clauses) + [[-literal for literal in sol]]) 199 | 200 | shrunk = shrink_sat(unique_sat, test) 201 | assert test(shrunk) 202 | 203 | assert shrunk == [[1]] 204 | -------------------------------------------------------------------------------- /tests/test_work.py: -------------------------------------------------------------------------------- 1 | from contextlib import aclosing 2 | 3 | import pytest 4 | 5 | from shrinkray.work import WorkContext, parallel_map 6 | 7 | 8 | async def identity(x: int) -> int: 9 | return x 10 | 11 | 12 | @pytest.mark.parametrize("p", [1, 2, 3, 4]) 13 | async def test_parallel_map(p: int) -> None: 14 | input = [1, 2, 3] 15 | async with parallel_map(input, identity, parallelism=p) as mapped: 16 | values = [x async for x in mapped] 17 | print(values) 18 | assert values == input 19 | 20 | 21 | @pytest.mark.parametrize("p", [1, 2, 3, 4]) 22 | async def test_worker_map(p: int) -> None: 23 | work = WorkContext(parallelism=p) 24 | 25 | input = range(1000) 26 | 27 | i = 0 28 | async with work.map(input, identity) as mapped: 29 | async for x in mapped: 30 | assert input[i] == x 31 | i += 1 32 | --------------------------------------------------------------------------------