├── .coveragerc ├── .github └── workflows │ ├── docs.yml │ ├── lints.yml │ └── tests.yml ├── .gitignore ├── .readthedocs.yml ├── LICENSE.md ├── README.md ├── TODO.rst ├── coverage.sh ├── docs ├── Makefile ├── conf.py ├── index.rst └── make.bat ├── mypy.ini ├── pdm.lock ├── peval.sublime-project ├── peval ├── __init__.py ├── components │ ├── __init__.py │ ├── fold.py │ ├── inline.py │ ├── peval_function_header.py │ ├── prune_assignments.py │ └── prune_cfg.py ├── core │ ├── __init__.py │ ├── cfg.py │ ├── expression.py │ ├── function.py │ ├── gensym.py │ ├── mangler.py │ ├── reify.py │ └── scope.py ├── highlevelapi.py ├── tags.py ├── tools │ ├── __init__.py │ ├── dispatcher.py │ ├── immutable.py │ ├── utils.py │ └── walker.py ├── typing.py └── wisdom.py ├── pylintrc ├── pyproject.toml └── tests ├── test_components ├── __init__.py ├── test_fold.py ├── test_inline.py ├── test_peval_function_header.py └── test_prune_cfg.py ├── test_core ├── __init__.py ├── test_cfg.py ├── test_expression.py ├── test_function.py ├── test_mangler.py └── test_reify.py ├── test_highlevelapi.py ├── test_tags.py ├── test_tools ├── __init__.py ├── test_immutable.py ├── test_utils.py └── test_walker.py ├── test_wisdom.py └── utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = peval 4 | 5 | [report] 6 | exclude_lines = 7 | # Have to re-enable the standard pragma 8 | pragma: no cover 9 | 10 | # Exclude abstract methods 11 | @abstractmethod 12 | 13 | # Exclude overload annotations 14 | @overload 15 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Documentation 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | sphinx-build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | # synchronized with `.readthedocs.yml` 15 | python-version: ["3.8"] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install PDM 24 | run: curl -sSL https://raw.githubusercontent.com/pdm-project/pdm/main/install-pdm.py | python3 25 | - name: Install dependencies 26 | run: | 27 | pdm sync -G docs 28 | - name: Run sphinx-build 29 | run: | 30 | pdm run sphinx-build -n docs build/sphinx 31 | -------------------------------------------------------------------------------- /.github/workflows/lints.yml: -------------------------------------------------------------------------------- 1 | name: Lints 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | black: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: psf/black@stable 15 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Tests 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | test-astunparse: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | python-version: ["3.8"] 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install PDM 26 | run: curl -sSL https://raw.githubusercontent.com/pdm-project/pdm/main/install-pdm.py | python3 27 | - name: Install dependencies 28 | run: | 29 | pdm sync -G tests -G astunparse 30 | - name: Test with pytest 31 | run: | 32 | pdm run py.test --cov=peval --cov-report=xml tests 33 | - name: Upload coverage 34 | run: | 35 | curl -Os https://uploader.codecov.io/latest/linux/codecov 36 | chmod +x codecov 37 | ./codecov 38 | 39 | test-astor: 40 | runs-on: ubuntu-latest 41 | strategy: 42 | fail-fast: false 43 | matrix: 44 | python-version: ["3.8"] 45 | steps: 46 | - uses: actions/checkout@v2 47 | - name: Set up Python ${{ matrix.python-version }} 48 | uses: actions/setup-python@v2 49 | with: 50 | python-version: ${{ matrix.python-version }} 51 | - name: Install PDM 52 | run: curl -sSL https://raw.githubusercontent.com/pdm-project/pdm/main/install-pdm.py | python3 53 | - name: Install dependencies 54 | run: | 55 | pdm sync -G tests -G astor 56 | - name: Test with pytest 57 | run: | 58 | pdm run py.test --cov=peval --cov-report=xml tests 59 | - name: Upload coverage 60 | run: | 61 | curl -Os https://uploader.codecov.io/latest/linux/codecov 62 | chmod +x codecov 63 | ./codecov 64 | 65 | test: 66 | runs-on: ubuntu-latest 67 | strategy: 68 | fail-fast: false 69 | matrix: 70 | python-version: ["3.9", "3.10", "3.11"] 71 | steps: 72 | - uses: actions/checkout@v2 73 | - name: Set up Python ${{ matrix.python-version }} 74 | uses: actions/setup-python@v2 75 | with: 76 | python-version: ${{ matrix.python-version }} 77 | - name: Install PDM 78 | run: curl -sSL https://raw.githubusercontent.com/pdm-project/pdm/main/install-pdm.py | python3 79 | - name: Install dependencies 80 | run: | 81 | pdm sync -G tests 82 | - name: Test with pytest 83 | run: | 84 | pdm run py.test --cov=peval --cov-report=xml tests 85 | - name: Upload coverage 86 | run: | 87 | curl -Os https://uploader.codecov.io/latest/linux/codecov 88 | chmod +x codecov 89 | ./codecov 90 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | *.pyc 3 | peval.egg-info/ 4 | env/ 5 | build/ 6 | dist/ 7 | htmlcov/ 8 | *.sublime-workspace 9 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Required 2 | version: 2 3 | 4 | # Build documentation in the docs/ directory with Sphinx 5 | sphinx: 6 | configuration: docs/conf.py 7 | 8 | # Build documentation with MkDocs 9 | #mkdocs: 10 | # configuration: mkdocs.yml 11 | 12 | # Optionally build your docs in additional formats such as PDF and ePub 13 | formats: all 14 | 15 | # Optionally set the version of Python and requirements required to build your docs 16 | python: 17 | version: 3.8 18 | install: 19 | - method: pip 20 | path: . 21 | extra_requirements: 22 | - docs 23 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | peval is licensed under the MIT license: 2 | 3 | Copyright (c) Bogdan Opanchuk (2016-now) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | The Software is provided "as is", without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the Software or the use or other dealings in the Software. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Partial evaluation of Python code 2 | 3 | [![pypi package][pypi-image]][pypi-link] ![License][pypi-license-image] [![Docs][rtd-image]][rtd-link] [![Coverage][cov-image]][cov-link] [![Code style: black][black-image]][black-link] 4 | 5 | Read more at the [project documentation page](http://peval.readthedocs.org). 6 | 7 | 8 | [pypi-image]: https://img.shields.io/pypi/v/peval 9 | [pypi-link]: https://pypi.org/project/peval/ 10 | [pypi-license-image]: https://img.shields.io/pypi/l/peval 11 | [rtd-image]: https://readthedocs.org/projects/peval/badge/?version=latest 12 | [rtd-link]: https://peval.readthedocs.io/en/latest/ 13 | [cov-image]: https://codecov.io/gh/fjarri/peval/branch/master/graph/badge.svg?token=RZP1LK1HB2 14 | [cov-link]: https://codecov.io/gh/fjarri/peval 15 | [black-image]: https://img.shields.io/badge/code%20style-black-000000.svg 16 | [black-link]: https://github.com/psf/black 17 | -------------------------------------------------------------------------------- /TODO.rst: -------------------------------------------------------------------------------- 1 | First issues: 2 | 3 | - Support all operations from https://docs.python.org/3/reference/datamodel.html, by properly converting sugared calls into dunder methods before evaluating. 4 | 5 | - Convert TODO and License to Markdown, remove TODO from github? 6 | 7 | - why does `forward_transfer()` returns lists of one element? 8 | 9 | - why are we calling `map_reify()` on something that is always a single element and not a container in `peval_expression()`? 10 | 11 | - `map_accum()` should also take `**kwds` to pass to the predicate. 12 | 13 | - Need to mark types and add comments for internal functions. Not to mention just go through the library and deconvolute it a little. 14 | 15 | - update `function.py::FUTURE_NAMES` for Py3.7/3.8 16 | 17 | 18 | General 19 | ------- 20 | 21 | * STYLE: Add exports from ``core`` and ``components`` submodules to their ``__init__``'s. 22 | 23 | * IMPROVEMENT: It seems to be common to compare two ASTs before and after some function to check if there were any changes. If it takes much time, we can make walkers set some flag if they made a change in the AST and then just propagate it. Drawbacks: propagating an additional value; changes can take place outside of walkers. 24 | 25 | * IMPROVEMENT: It will be great to have some way to "flatten" nested functions. That will eliminate the need for complicated scope analyzer. 26 | 27 | * Use relative imports. 28 | 29 | 30 | Core 31 | ---- 32 | 33 | * FEATURE (core/scope): needs a thorough improvement, with the ability to detect various ways of symbol usages, ``global``/``nonlocal`` modifiers, nested scopes and so on. 34 | 35 | * FEATURE (core/gensym): use randomized symbols (SHA or something) instead of a counter 36 | 37 | pros: 38 | - no need to walk the AST before generating symbols 39 | - no need to carry around the generator 40 | 41 | cons: 42 | - a tiny probability of a collision 43 | - non-deterministic output (harder to test) 44 | 45 | We can use a random seed, but then we'll have to carry the generator. Alternatively, we can use a global seed just for tests. 46 | 47 | * IMPROVEMENT (core/expression): ``peval_call()`` is sometimes called with a known function (e.g. from evaluating an operator). In this case there is no need to try to evaluate it, or replace it with a binding when evaluation failed, and remove this binding in the parent call. We can add proper handling of such cases to ``peval_call()`` and save on some state modifications. 48 | 49 | * DECIDE (core/callable) is it still necessary if we are limited to Py3.5+? 50 | 51 | * FEATURE (core/reify): extend the class of literals, adding non-trivial immutable objects, e.g. tuples and slices. 52 | 53 | Things to note: 54 | * It may be undesirable to make long lists (or dicts etc) into literals; an length limit can be introduced, but it requires some option object to be propagated. 55 | * Probably, making complex objects literals only makes sense if all their elements are literals; e.g. ``[_peval_temp1, _peval_temp2]`` is not much better than ``_peval_temp1``. 56 | * We can search for a ``__peval_reify__`` magic method that will return AST for custom objects, or ``None`` if it is non-reifiable. 57 | * In fact, custom objects may be reified by ``ast.parse(repr(obj))`` (for classes that define ``repr()`` properly) 58 | 59 | * FEATURE (core/walker): make ``visit_after`` call do nothing if we are actually in the visiting-after stage (so that one does not have to write ``if not visiting_after: visit_after()``). Or even make it return ``visiting_after`` value, similarly to how ``fork`` works. 60 | 61 | * BUG (core/mangler): when encountering a nested function definition, run it through ``Function`` and check which closure variables it uses (inlcuding the function name itself). 62 | Then mangle only them, leaving the rest intact. 63 | 64 | * PROBLEM (core/function): it would be nice to have the new peval'ed function to stay connected to the same globals dictionary as the original one. This way we will preserve the globals-modifying behavior, and the less changes we make to a function, the better. The problem is that during partial evaluation we will need to introduce new bindings, and we cannot just insert them into the original globals dictionary --- this may interfere with other functions. I am not sure how to solve this at the moment. 65 | 66 | * FEATURE (core/expression): add partial evaluation of lambdas (and nested function/class definitions). 67 | Things to look out to: 68 | 69 | * Need to see which outer variables the lambda uses as closures. 70 | These we need to lock --- ``prune_assignments`` cannot remove them. 71 | * Mark it as impure and mutating by default, unless the user explicitly marks it otherwise. 72 | * Run its own partial evaluation for the nested function? 73 | 74 | * FEATURE (core/expression): in Py3 ``iter()`` of a ``zip`` object returns itself, so list comprehension evaluator considers it unsafe to iterate it. 75 | Perhaps comprehensions need the same kind of policies as loop unroller does, to force evaluation in such cases (also in cases of various generator functions that can reference global variables and so on). 76 | 77 | * FEATURE (core/expression): support partial evaluation of starred arguments in invocations (see commented part of ``test_function_call``). 78 | 79 | * In ``check_peval_expression()``, sometimes we force the expected node (e.g. "-5" is parsed as "UnaryOp(op=USub(), Num(n=5))", not as "Num(n=-5)", but we enforce the latter). Is that really necessary? If Python parses it as the former, shouldn't we generate the same? 80 | 81 | * If we're limited to Py>=3.5, ``check_peval_expression_bool()`` is not needed anymore. 82 | 83 | * Compress bindings, eliminating all duplicates that point to the same object. 84 | 85 | * There seems to be an inconsistency regarding on whether the argument or mutated context is passed first/returned first in functions. 86 | 87 | * If we have an option object, a useful object would be: "assume builtins are not redefined", which will make the resulting code much more readable. 88 | 89 | 90 | Components 91 | ---------- 92 | 93 | * BUG (components/inline): when inlining a function, we must mangle the globals too, in case it uses a different set from what the parent function uses. 94 | 95 | * FEATURE (components/fold): skip unreachable code (e.g. ``if`` branches) right away when doing propagation. See the ``xfail``-ed test in ``test_fold``. 96 | 97 | * BUG (components/fold): possible mutation when comparing values in ``meet_values()`` (in case of a weird ``__eq()__`` definition) 98 | 99 | * BUG (components/prune_assignments): need to keep the variables that are used as closures in nested functions. 100 | 101 | * DECIDE (components/prune_assignments): The single assignments removal seems rather shady, although it is certainly useful to clean up after inlining. Need to think about corner cases, and also avoiding to call ``analyze_scope`` for every statement in the processed block. 102 | 103 | * FEATURE (components/prune_cfg): we can detect unconditional jumps in ``for`` loops as well, but in order to remove the loop, we need the loop unrolling functionality. 104 | 105 | * BUG (components/prune_cfg): evaluating ``bool(node.test)`` is potentially (albeit unlikely) unsafe (if it is some weird object with a weird ``__bool__()`` implementation). 106 | Need to run it through the safe call function from the expression evaluator. 107 | 108 | * BUG (components/prune_cfg): see several FIXME's in the code related to the processing of try-except blocks 109 | 110 | * FEATURE (components/inline): add support for inlining functions with varargs/kwargs. 111 | Probably just run the function through ``partial_apply`` before inlining? 112 | 113 | * BUG (components/inline): how does marking methods as inlineable work? Need to check and probably raise an exception. 114 | 115 | * FEATURE: support complex inlining scenarios: 116 | 1a. Inlining self (currently supported) 117 | 1b. Inlining a nested function 118 | 1c. Inlining a nesting function 119 | 2a. Inlining a function from the same module (currently supported) 120 | 2b. Inlining a function from the other module 121 | 122 | 123 | (new) components/unroll 124 | ----------------------- 125 | 126 | Conditionally unroll loops. 127 | Possible policies: 128 | 129 | * based on a *keyword* ``unroll`` (that is, look for a ``ast.Name(id='unroll')``); 130 | * based on a *function* ``unroll`` (check if the iterator in a loop is the unrolling iterator); 131 | * based on heuristics (unroll range iterators, lists, tuples or dicts with less than N entries). 132 | 133 | 134 | (new) components/macro 135 | ---------------------- 136 | 137 | Macros are similar to inlines, but the expressions passed to the function are substituted in its body without any changes and the resulting body is used to replace the macro call. 138 | If the function was called in an expression context, check that the body contains only one ``ast.Expr`` and substitute its value. 139 | 140 | :: 141 | 142 | @macro 143 | def mad(x, y, z): 144 | x * y + z 145 | 146 | a = mad(b[1], c + 10, d.value) 147 | # ---> 148 | # a = b[1] * (c + 10) + d.value 149 | 150 | 151 | (new) better code pick up 152 | ------------------------- 153 | 154 | In theory, the code of functions unreachable by ``inspect.getsource()`` (either the ones defined dynamically in the interactive prompt, or constructed at runtime) can be obtained by decompiling the code object. In theory, it seems pretty straightforward, but will require a lot of coding (to handle all the numerous opcodes). There is a decompiler for Py2 (https://github.com/wibiti/uncompyle2), but it uses some weird parsing and does not even involve the ``dis`` module. 155 | 156 | This will, in turn, allow us to create doctests, but otherwise it is tangential to the main ``peval`` functionality. 157 | 158 | 159 | (change) tools/immutable 160 | ------------------------ 161 | 162 | There are immutable data structure libraries that may be faster, e.g.: 163 | 164 | * https://github.com/zhemao/funktown 165 | * https://pythonhosted.org/pysistence/ 166 | * https://github.com/tobgu/pyrsistent (currently active) 167 | 168 | Alternatively, the embedded implementation can be optimized to reuse data instead of just making copies every time. 169 | 170 | Also, we can change ``update()`` and ``del_()`` to ``with_()`` and ``without()`` which better reflect the immutability of data structures. 171 | 172 | This is especially important in the light of https://www.reddit.com/r/Python/comments/42t9yw/til_dictmy_subclassed_dict_doesnt_use_dict_methods/ : subclassing from dict() and others is error-prone. 173 | 174 | 175 | Known limitations 176 | ================= 177 | 178 | In the process of partial evaluation, the target function needs to be discovered in the source code, parsed, optimized and re-evaluated by the interpreter. 179 | Due to the way the discovery of function code and metadata is implemented in Python, in some scenarios ``peval`` may lack necessary information and therefore fail to restore the function correctly. 180 | Fortunately, these scenarios are not very common, but one still needs to be aware of them. 181 | 182 | And, of course, there is a whole group of problems arising due to the highly dynamical nature of Python. 183 | 184 | 185 | Decorators 186 | ---------- 187 | 188 | * **Problem:** If the target function is decorated, the decorators must preserve the function metadata, in particular, closure variables, globals, and reference to the source file where it was defined. 189 | 190 | **Workaround:** One must either take care of the metadata manually, or use a metadata-aware decorator builder library like `wrapt `_. 191 | 192 | * **Problem:** Consider a function decorated inside another function: 193 | 194 | :: 195 | 196 | def outer(): 197 | arg1 = 1 198 | arg2 = 2 199 | 200 | @decorator(arg1, arg2) 201 | def innner(): 202 | # code_here 203 | 204 | return inner 205 | 206 | The variables used in the decorator declaration (``arg1``, ``arg2``) are not included neither in globals nor in closure variables of ``inner``. 207 | When the returned ``inner`` function is partially evaluated, it is not possible to restore the values of ``arg1`` and ``arg2``, and the final evaluation will fail. 208 | 209 | **Workaround:** Make sure all the variables used in the decorator declarations for target functions (including the decorators themselves) belong to the global namespace. 210 | 211 | * **Problem:** When the target function is re-evaluated, the decorators associated with it are applied to the new function. 212 | This may lead to unexpected behavior if the decorators have side effects, or rely on some particular function arguments (which may disappear after partial application). 213 | 214 | **Workaround:** Make sure that the second application of the decorators does not lead to undesired consequences, and that they can handle changes in the function signature. 215 | 216 | * **Problem:** Consider a case when a decorator uses the same symbol as one of the function arguments: 217 | 218 | :: 219 | 220 | @foo 221 | def test(foo, bar): 222 | return foo, bar 223 | 224 | If we bind the ``foo`` argument to some value, this value will be added to the globals and, therefore, will replace the value used for the ``foo`` decorator. 225 | Consequently, the evaluation of such partially applied function will fail 226 | (in fact, an assertion within ``Function.bind_partial()`` will fire before that). 227 | 228 | **Workaround:** Avoid using the same symbols in function argument lists and in the decorator declarations applied to these functions (which is usually a good general coding practice). 229 | -------------------------------------------------------------------------------- /coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | py.test --cov peval --cov-report=html "$@" 3 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/peval.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/peval.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/peval" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/peval" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | import importlib.metadata 16 | 17 | import setuptools_scm 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = "peval" 22 | copyright = "2016–now, Bogdan Opanchuk" 23 | author = "Bogdan Opanchuk" 24 | 25 | # The full version, including alpha/beta/rc tags 26 | try: 27 | release = importlib.metadata.version(project) 28 | except importlib.metadata.PackageNotFoundError: 29 | release = setuptools_scm.get_version(relative_to=os.path.abspath("../pyproject.toml")) 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | "sphinx.ext.autodoc", 39 | "sphinx.ext.doctest", 40 | "sphinx.ext.intersphinx", 41 | "sphinx.ext.viewcode", 42 | ] 43 | 44 | autoclass_content = "both" 45 | autodoc_member_order = "groupwise" 46 | 47 | # Add any paths that contain templates here, relative to this directory. 48 | templates_path = ["_templates"] 49 | 50 | # List of patterns, relative to source directory, that match files and 51 | # directories to ignore when looking for source files. 52 | # This pattern also affects html_static_path and html_extra_path. 53 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 54 | 55 | # Note: set to the lower bound of `numpy` version in the dependencies; 56 | # must be kept synchronized. 57 | intersphinx_mapping = { 58 | "python": ("https://docs.python.org/3", None), 59 | } 60 | 61 | # -- Options for HTML output ------------------------------------------------- 62 | 63 | # The theme to use for HTML and HTML Help pages. See the documentation for 64 | # a list of builtin themes. 65 | # 66 | html_theme = "furo" 67 | 68 | # Add any paths that contain custom static files (such as style sheets) here, 69 | # relative to this directory. They are copied after the builtin static files, 70 | # so a file named "default.css" will overwrite the builtin "default.css". 71 | html_static_path = [] 72 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ********************************* 2 | Partial evaluation of Python code 3 | ********************************* 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | 9 | Introduction 10 | ============ 11 | 12 | This library allows one to perform code specialization at run-time, turning this:: 13 | 14 | @inline 15 | def power(x, n): 16 | if n == 0: 17 | return 1 18 | elif n % 2 == 0: 19 | v = power(x, n // 2) 20 | return v * v 21 | else: 22 | return x * power(x, n - 1) 23 | 24 | with ``n`` set to, say, 27, into this:: 25 | 26 | def power_27(x): 27 | __peval_return_7 = (x * 1) 28 | __peval_return_6 = (__peval_return_7 * __peval_return_7) 29 | __peval_return_5 = (x * __peval_return_6) 30 | __peval_return_4 = (__peval_return_5 * __peval_return_5) 31 | __peval_return_3 = (__peval_return_4 * __peval_return_4) 32 | __peval_return_1 = (x * __peval_return_3) 33 | __peval_return_2 = (__peval_return_1 * __peval_return_1) 34 | return (x * __peval_return_2) 35 | 36 | The resulting code runs 6 times faster under CPython 3.5.1. 37 | 38 | The API is identical to that of ``functools.partial()``:: 39 | 40 | import peval 41 | power_27 = peval.partial_apply(power, n=27) 42 | 43 | You must mark the functions that you want inlined (maybe recursively) with :func:`peval.inline`. 44 | If you want some function to be evaluated during partial evaluation, mark it with :func:`peval.pure` (if it is not, in fact, pure, the results are unpredictable). 45 | 46 | 47 | Implementation details 48 | ====================== 49 | 50 | The partial evaluation is performed on an AST using a range of optimizations: 51 | 52 | * constant propagation 53 | * constant folding 54 | * unreachable code elimination 55 | * function inlining 56 | 57 | 58 | Restrictions on functions 59 | ========================= 60 | 61 | Currently not all functions can be partially evaluated or inlined. 62 | Hopefully, these restrictions will be lifted in the future as the package improves. 63 | 64 | The inlined functions: 65 | 66 | * cannot be ``async`` functions; 67 | * cannot be generators; 68 | * cannot have nested definitions (lambdas, functions or classes); 69 | * cannot have closures; 70 | * cannot have decorators and annotations (they will be ignored). 71 | 72 | The partially evaluated functions: 73 | 74 | * cannot be ``async`` functions; 75 | * cannot have nested definitions (lambdas, functions or classes); 76 | * cannot have decorators and annotations (they will be ignored). 77 | 78 | Also note that the partial evaluated function loses the connection to the globals dictionary of the original function. 79 | Therefore, it cannot reassign global variables (the copy is shallow, so mutation of global objects is still possible). 80 | 81 | 82 | API reference 83 | ============= 84 | 85 | .. automodule:: peval 86 | :members: 87 | 88 | .. autoclass:: peval.tools.ImmutableADict 89 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\peval.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\peval.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | # strict = true 3 | # otherwise mypy fails to pick up signatures from eth_* packages 4 | implicit_reexport = True 5 | -------------------------------------------------------------------------------- /peval.sublime-project: -------------------------------------------------------------------------------- 1 | { 2 | "folders": 3 | [ 4 | { 5 | "folder_exclude_patterns": 6 | [ 7 | "*.egg-info", 8 | "__pycache__", 9 | "htmlcov" 10 | ], 11 | "follow_symlinks": true, 12 | "path": "./" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /peval/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | High-level API 3 | -------------- 4 | 5 | .. autofunction:: partial_eval 6 | .. autofunction:: partial_apply 7 | 8 | 9 | Tags 10 | ==== 11 | 12 | .. autofunction:: inline 13 | .. autofunction:: pure 14 | 15 | 16 | Helper functions 17 | ================ 18 | 19 | .. autofunction:: getsource 20 | .. autofunction:: specialize_on 21 | 22 | 23 | Low-level tools 24 | --------------- 25 | 26 | .. autofunction:: ast_walker 27 | .. autofunction:: ast_inspector 28 | .. autofunction:: ast_transformer 29 | .. autofunction:: try_peval_expression 30 | .. autoclass:: Dispatcher 31 | .. autoclass:: Function 32 | :members: 33 | """ 34 | 35 | from peval.tools import Dispatcher, ast_walker, ast_inspector, ast_transformer 36 | from peval.core.expression import try_peval_expression 37 | from peval.highlevelapi import partial_eval, partial_apply, specialize_on 38 | from peval.tags import pure, inline 39 | from peval.core.function import getsource, Function 40 | -------------------------------------------------------------------------------- /peval/components/__init__.py: -------------------------------------------------------------------------------- 1 | from peval.components.inline import inline_functions 2 | from peval.components.prune_cfg import prune_cfg 3 | from peval.components.prune_assignments import prune_assignments 4 | from peval.components.fold import fold 5 | from peval.components.peval_function_header import peval_function_header 6 | -------------------------------------------------------------------------------- /peval/components/fold.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from functools import reduce 3 | from typing import Optional, Any, Dict, Callable, Iterable, List, Mapping, Tuple, List, Union 4 | 5 | from peval.tools import replace_fields, ast_transformer 6 | from peval.core.gensym import GenSym 7 | from peval.core.cfg import Graph, build_cfg 8 | from peval.core.expression import peval_expression 9 | from peval.tools.immutable import ImmutableADict 10 | from peval.typing import ConstsDictT, PassOutputT 11 | 12 | 13 | class Value: 14 | def __init__(self, value: Optional[Any] = None, undefined: bool = False) -> None: 15 | if undefined: 16 | self.defined = False 17 | self.value = None 18 | else: 19 | self.defined = True 20 | self.value = value 21 | 22 | def __str__(self): 23 | if not self.defined: 24 | return "" 25 | else: 26 | return "<" + str(self.value) + ">" 27 | 28 | def __eq__(self, other: "Value") -> bool: 29 | return self.defined == other.defined and self.value == other.value 30 | 31 | def __ne__(self, other): 32 | return self.defined != other.defined or self.value != other.value 33 | 34 | def __repr__(self): 35 | if not self.defined: 36 | return "Value(undefined=True)" 37 | else: 38 | return "Value(value={value})".format(value=repr(self.value)) 39 | 40 | 41 | def meet_values(val1: Value, val2: Value) -> Value: 42 | if not val1.defined or not val2.defined: 43 | return Value(undefined=True) 44 | 45 | v1 = val1.value 46 | v2 = val2.value 47 | 48 | if v1 is v2: 49 | return Value(value=v1) 50 | 51 | eq = False 52 | try: 53 | eq = v1 == v2 54 | except Exception: 55 | pass 56 | 57 | if eq: 58 | return Value(value=v1) 59 | else: 60 | return Value(undefined=True) 61 | 62 | 63 | class Environment: 64 | def __init__(self, values: Dict[str, Value]) -> None: 65 | self.values = values if values is not None else {} 66 | 67 | @classmethod 68 | def from_dict(cls, values: ConstsDictT) -> "Environment": 69 | return cls(values=dict((name, Value(value=value)) for name, value in values.items())) 70 | 71 | def known_values(self) -> ConstsDictT: 72 | return dict((name, value.value) for name, value in self.values.items() if value.defined) 73 | 74 | def __eq__(self, other): 75 | return self.values == other.values 76 | 77 | def __ne__(self, other: "Environment") -> bool: 78 | return self.values != other.values 79 | 80 | def __repr__(self): 81 | return "Environment(values={values})".format(values=self.values) 82 | 83 | 84 | def meet_envs(env1: Environment, env2: Environment) -> Environment: 85 | lhs = env1.values 86 | rhs = env2.values 87 | lhs_keys = set(lhs.keys()) 88 | rhs_keys = set(rhs.keys()) 89 | result = {} 90 | 91 | for var in lhs_keys - rhs_keys: 92 | result[var] = lhs[var] 93 | 94 | for var in rhs_keys - lhs_keys: 95 | result[var] = rhs[var] 96 | 97 | for var in lhs_keys & rhs_keys: 98 | result[var] = meet_values(lhs[var], rhs[var]) 99 | 100 | return Environment(values=result) 101 | 102 | 103 | def my_reduce(func: Callable, seq: Iterable[Environment]) -> Environment: 104 | if len(seq) == 1: 105 | return seq[0] 106 | else: 107 | return reduce(func, seq[1:], seq[0]) 108 | 109 | 110 | class CachedExpression: 111 | def __init__(self, path: List[str], node: ast.expr) -> None: 112 | self.node = node 113 | self.path = path 114 | 115 | 116 | TempBindingsT = Mapping[str, Any] 117 | 118 | 119 | def forward_transfer( 120 | gen_sym: GenSym, in_env: Environment, statement: ast.stmt 121 | ) -> Tuple[GenSym, Environment, List[CachedExpression], TempBindingsT]: 122 | if isinstance(statement, (ast.Assign, ast.AnnAssign)): 123 | if isinstance(statement, ast.AnnAssign): 124 | target = statement.target 125 | else: 126 | assert len(statement.targets) == 1 127 | target = statement.targets[0] 128 | 129 | if isinstance(target, ast.Name): 130 | target = target.id 131 | elif isinstance(target, (ast.Name, ast.Tuple)): 132 | raise ValueError( 133 | "Destructuring assignment (should have been eliminated by other pass)", 134 | target, 135 | ) 136 | else: 137 | raise ValueError("Incorrect assignment target", target) 138 | 139 | result, gen_sym = peval_expression(statement.value, gen_sym, in_env.known_values()) 140 | 141 | new_values = dict(in_env.values) 142 | 143 | if result.known_value is not None: 144 | new_value = Value(value=result.known_value.value) 145 | else: 146 | new_value = Value(undefined=True) 147 | new_values[target] = new_value 148 | 149 | out_env = Environment(values=new_values) 150 | new_exprs = [CachedExpression(path=["value"], node=result.node)] 151 | 152 | return gen_sym, out_env, new_exprs, result.temp_bindings 153 | 154 | elif isinstance(statement, (ast.Expr, ast.Return)): 155 | result, gen_sym = peval_expression(statement.value, gen_sym, in_env.known_values()) 156 | 157 | new_values = dict(in_env.values) 158 | 159 | new_exprs = [CachedExpression(path=["value"], node=result.node)] 160 | out_env = Environment(values=new_values) 161 | 162 | return gen_sym, out_env, new_exprs, result.temp_bindings 163 | 164 | elif isinstance(statement, ast.If): 165 | result, gen_sym = peval_expression(statement.test, gen_sym, in_env.known_values()) 166 | 167 | new_values = dict(in_env.values) 168 | 169 | out_env = Environment(values=new_values) 170 | 171 | new_exprs = [CachedExpression(path=["test"], node=result.node)] 172 | 173 | return gen_sym, out_env, new_exprs, result.temp_bindings 174 | 175 | else: 176 | return gen_sym, in_env, [], {} 177 | 178 | 179 | class State: 180 | def __init__( 181 | self, 182 | in_env: Environment, 183 | out_env: Environment, 184 | exprs: List[CachedExpression], 185 | temp_bindings: ImmutableADict, 186 | ) -> None: 187 | self.in_env = in_env 188 | self.out_env = out_env 189 | self.exprs = exprs 190 | self.temp_bindings = temp_bindings 191 | 192 | 193 | def get_sorted_nodes(graph: Graph, enter: int) -> List[int]: 194 | sorted_nodes = [] 195 | todo_list = [enter] 196 | visited = set() 197 | 198 | while len(todo_list) > 0: 199 | src_id = todo_list.pop() 200 | if src_id in visited: 201 | continue 202 | sorted_nodes.append(src_id) 203 | visited.add(src_id) 204 | 205 | for dest_id in sorted(graph.children_of(src_id)): 206 | todo_list.append(dest_id) 207 | 208 | return sorted_nodes 209 | 210 | 211 | def maximal_fixed_point( 212 | gen_sym: GenSym, graph: Graph, enter: int, bindings: ConstsDictT 213 | ) -> Tuple[List[CachedExpression], TempBindingsT]: 214 | states = dict( 215 | ( 216 | node_id, 217 | State(Environment.from_dict(bindings), Environment.from_dict(bindings), [], {}), 218 | ) 219 | for node_id in graph.nodes 220 | ) 221 | enter_env = Environment.from_dict(bindings) 222 | 223 | # First make a pass over each basic block 224 | # The todo list is sorted to make the names of the final bindings deterministic 225 | todo_forward = get_sorted_nodes(graph, enter) 226 | todo_forward_set = set(todo_forward) 227 | 228 | while len(todo_forward) > 0: 229 | node_id = todo_forward.pop(0) 230 | todo_forward_set.remove(node_id) 231 | state = states[node_id] 232 | 233 | # compute the environment at the entry of this BB 234 | if node_id == enter: 235 | new_in_env = enter_env 236 | else: 237 | parent_envs = [states[parent_id].out_env for parent_id in graph.parents_of(node_id)] 238 | new_in_env = my_reduce(meet_envs, parent_envs) 239 | 240 | # propagate information for this basic block 241 | gen_sym, new_out_env, new_exprs, temp_bindings = forward_transfer( 242 | gen_sym, new_in_env, graph.nodes[node_id].ast_node 243 | ) 244 | 245 | # TODO: merge it with the code in the condition above to avoid repetition 246 | states[node_id].in_env = new_in_env 247 | states[node_id].exprs = new_exprs 248 | states[node_id].temp_bindings = temp_bindings 249 | 250 | if new_out_env != states[node_id].out_env: 251 | states[node_id] = State(new_in_env, new_out_env, new_exprs, temp_bindings) 252 | for dest_id in sorted(graph.children_of(node_id)): 253 | if dest_id not in todo_forward_set: 254 | todo_forward_set.add(dest_id) 255 | todo_forward.append(dest_id) 256 | 257 | # Converged 258 | new_exprs = {} 259 | temp_bindings = {} 260 | for node_id, state in states.items(): 261 | node = graph.nodes[node_id].ast_node 262 | exprs = list(state.exprs) 263 | exprs_temp_bindings = dict(state.temp_bindings) 264 | 265 | # Evaluating annotations only after the MFP has converged, 266 | # since they don't introduce new bindings 267 | if isinstance(node, ast.AnnAssign): 268 | in_env = state.in_env 269 | annotation_result, gen_sym = peval_expression( 270 | node.annotation, 271 | gen_sym, 272 | state.in_env.known_values(), 273 | create_binding=True, 274 | ) 275 | exprs.append(CachedExpression(path=["annotation"], node=annotation_result.node)) 276 | exprs_temp_bindings.update(annotation_result.temp_bindings) 277 | 278 | new_exprs[node_id] = exprs 279 | temp_bindings.update(exprs_temp_bindings) 280 | 281 | return new_exprs, temp_bindings 282 | 283 | 284 | def replace_exprs( 285 | tree: ast.FunctionDef, new_exprs: Dict[int, List[CachedExpression]] 286 | ) -> Union[ast.FunctionDef, ast.Module]: 287 | return _replace_exprs(tree, ctx=dict(new_exprs=new_exprs)) 288 | 289 | 290 | ReplaceByPathNodeT = Union[ast.If, ast.Assign, ast.Expr, ast.Return] 291 | 292 | 293 | def replace_by_path( 294 | obj: ReplaceByPathNodeT, path: Iterable[str], new_value: ast.expr 295 | ) -> ReplaceByPathNodeT: 296 | ptr = path[0] 297 | 298 | if len(path) > 1: 299 | if isinstance(ptr, str): 300 | sub_obj = getattr(obj, ptr) 301 | elif isinstance(ptr, int): 302 | sub_obj = obj[ptr] 303 | new_value = replace_by_path(sub_obj, path[1:], new_value) 304 | 305 | if isinstance(ptr, str): 306 | return replace_fields(obj, **{ptr: new_value}) 307 | elif isinstance(ptr, int): 308 | return obj[:ptr] + [new_value] + obj[ptr + 1 :] 309 | 310 | 311 | @ast_transformer 312 | def _replace_exprs(node, ctx, walk_field, **_): 313 | if id(node) in ctx.new_exprs: 314 | exprs = ctx.new_exprs[id(node)] 315 | visited_fields = set() 316 | for expr in exprs: 317 | visited_fields.add(expr.path[0]) 318 | node = replace_by_path(node, expr.path, expr.node) 319 | 320 | for attr, value in ast.iter_fields(node): 321 | if attr not in visited_fields: 322 | setattr(node, attr, walk_field(value)) 323 | return node 324 | else: 325 | return node 326 | 327 | 328 | def fold(tree: ast.AST, constants: ConstsDictT) -> PassOutputT: 329 | statements = tree.body 330 | cfg = build_cfg(statements) 331 | gen_sym = GenSym.for_tree(tree) 332 | new_nodes, temp_bindings = maximal_fixed_point(gen_sym, cfg.graph, cfg.enter, constants) 333 | constants = dict(constants) 334 | constants.update(temp_bindings) 335 | new_tree = replace_exprs(tree, new_nodes) 336 | return new_tree, constants 337 | -------------------------------------------------------------------------------- /peval/components/inline.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from typing import List, Tuple, Dict, Any, Union 3 | 4 | from peval.tags import get_inline_tag 5 | from peval.core.reify import NONE_NODE, FALSE_NODE, TRUE_NODE 6 | from peval.core.expression import try_peval_expression 7 | from peval.core.function import Function 8 | from peval.core.mangler import mangle 9 | from peval.core.gensym import GenSym 10 | from peval.tools import ast_walker, replace_fields 11 | from peval.typing import ConstsDictT, PassOutputT 12 | 13 | 14 | def inline_functions(tree: ast.AST, constants: ConstsDictT) -> PassOutputT: 15 | gen_sym = GenSym.for_tree(tree) 16 | constants = dict(constants) 17 | state, tree = _inline_functions_walker(dict(gen_sym=gen_sym, constants=constants), tree) 18 | return tree, state.constants 19 | 20 | 21 | @ast_walker 22 | class _inline_functions_walker: 23 | @staticmethod 24 | def handle_Call(state, node, prepend, **_): 25 | gen_sym = state.gen_sym 26 | constants = state.constants 27 | 28 | evaluated, fn = try_peval_expression(node.func, constants) 29 | 30 | if not evaluated or not get_inline_tag(fn): 31 | return state, node 32 | 33 | return_name, gen_sym = gen_sym("return") 34 | inlined_body, gen_sym, constants = _inline(node, gen_sym, return_name, constants) 35 | prepend(inlined_body) 36 | new_state = state.with_(gen_sym=gen_sym, constants=constants) 37 | 38 | return new_state, ast.Name(id=return_name, ctx=ast.Load()) 39 | 40 | 41 | def _inline(node, gen_sym, return_name, constants): 42 | """ 43 | Return a list of nodes, representing inlined function call. 44 | """ 45 | fn = constants[node.func.id] 46 | fn_ast = Function.from_object(fn).tree 47 | 48 | gen_sym, new_fn_ast = mangle(gen_sym, fn_ast) 49 | 50 | parameter_assignments = _build_parameter_assignments(node, new_fn_ast) 51 | 52 | body_nodes = new_fn_ast.body 53 | 54 | gen_sym, inlined_body, new_bindings = _wrap_in_loop(gen_sym, body_nodes, return_name) 55 | constants = dict(constants) 56 | constants.update(new_bindings) 57 | 58 | return parameter_assignments + inlined_body, gen_sym, constants 59 | 60 | 61 | def _wrap_in_loop( 62 | gen_sym: GenSym, body_nodes: List[ast.If], return_name: str 63 | ) -> Tuple[GenSym, List[ast.While], Dict[Any, Any]]: 64 | new_bindings = dict() 65 | 66 | return_flag, gen_sym = gen_sym("return_flag") 67 | 68 | # Adding an explicit return at the end of the function, if it's not present. 69 | if type(body_nodes[-1]) != ast.Return: 70 | body_nodes = body_nodes + [ast.Return(value=NONE_NODE)] 71 | 72 | inlined_code, returns_ctr, returns_in_loops = _replace_returns( 73 | body_nodes, return_name, return_flag 74 | ) 75 | 76 | if returns_ctr == 1: 77 | # A shortcut for a common case with a single return at the end of the function. 78 | # No loop is required. 79 | inlined_body = inlined_code[:-1] 80 | else: 81 | # Multiple returns - wrap in a `while` loop. 82 | 83 | if returns_in_loops: 84 | # `return_flag` value will be used to detect returns from nested loops 85 | inlined_body = [ 86 | ast.Assign(targets=[ast.Name(return_flag, ast.Store())], value=FALSE_NODE) 87 | ] 88 | else: 89 | inlined_body = [] 90 | 91 | inlined_body.append(ast.While(test=TRUE_NODE, body=inlined_code, orelse=[])) 92 | 93 | return gen_sym, inlined_body, new_bindings 94 | 95 | 96 | def _build_parameter_assignments( 97 | call_node: ast.Call, functiondef_node: ast.FunctionDef 98 | ) -> List[ast.Assign]: 99 | # currently variadic arguments are not supported 100 | assert all(type(arg) != ast.Starred for arg in call_node.args) 101 | assert all(kw.arg is not None for kw in call_node.keywords) 102 | parameter_assignments = [] 103 | for callee_arg, fn_arg in zip(call_node.args, functiondef_node.args.args): 104 | parameter_assignments.append( 105 | ast.Assign(targets=[ast.Name(fn_arg.arg, ast.Store())], value=callee_arg) 106 | ) 107 | return parameter_assignments 108 | 109 | 110 | def _handle_loop(node, state, ctx, visit_after, visiting_after, walk_field, **_): 111 | if not visiting_after: 112 | # Need to traverse fields explicitly since for the purposes of _replace_returns(), 113 | # the body of `orelse` field is not inside a loop. 114 | state = state.with_(loop_nesting_ctr=state.loop_nesting_ctr + 1) 115 | state, new_body = walk_field(state, node.body, block_context=True) 116 | state = state.with_(loop_nesting_ctr=state.loop_nesting_ctr - 1) 117 | state, new_orelse = walk_field(state, node.orelse, block_context=True) 118 | 119 | visit_after() 120 | return state, replace_fields(node, body=new_body, orelse=new_orelse) 121 | else: 122 | # If there was a return inside a loop, append a conditional break 123 | # to propagate the return otside all nested loops 124 | if state.return_inside_a_loop: 125 | new_nodes = [ 126 | node, 127 | ast.If(test=ast.Name(id=ctx.return_flag_var), body=[ast.Break()], orelse=[]), 128 | ] 129 | else: 130 | new_nodes = node 131 | 132 | # if we are at root level, reset the return-inside-a-loop flag 133 | if state.loop_nesting_ctr == 0: 134 | state = state.with_(return_inside_a_loop=False) 135 | 136 | return state, new_nodes 137 | 138 | 139 | @ast_walker 140 | class _replace_returns_walker: 141 | """Replace returns with variable assignment + break.""" 142 | 143 | @staticmethod 144 | def handle_For(state, node, ctx, visit_after, visiting_after, **kwds): 145 | return _handle_loop(node, state, ctx, visit_after, visiting_after, **kwds) 146 | 147 | @staticmethod 148 | def handle_While(state, node, ctx, visit_after, visiting_after, **kwds): 149 | return _handle_loop(node, state, ctx, visit_after, visiting_after, **kwds) 150 | 151 | @staticmethod 152 | def handle_Return(state, node, ctx, **_): 153 | state_update = dict(returns_ctr=state.returns_ctr + 1) 154 | 155 | new_nodes = [ 156 | ast.Assign(targets=[ast.Name(id=ctx.return_var, ctx=ast.Store())], value=node.value) 157 | ] 158 | 159 | if state.loop_nesting_ctr > 0: 160 | new_nodes.append( 161 | ast.Assign( 162 | targets=[ast.Name(id=ctx.return_flag_var, ctx=ast.Store())], 163 | value=TRUE_NODE, 164 | ) 165 | ) 166 | state_update.update(return_inside_a_loop=True, returns_in_loops=True) 167 | 168 | new_nodes.append(ast.Break()) 169 | 170 | return state | state_update, new_nodes 171 | 172 | 173 | def _replace_returns( 174 | nodes: List[ast.AST], return_var: str, return_flag_var: str 175 | ) -> Tuple[List[Union[ast.If, ast.Assign, ast.Break]], int, bool]: 176 | state, new_nodes = _replace_returns_walker( 177 | dict( 178 | returns_ctr=0, 179 | loop_nesting_ctr=0, 180 | returns_in_loops=False, 181 | return_inside_a_loop=False, 182 | ), 183 | nodes, 184 | ctx=dict(return_var=return_var, return_flag_var=return_flag_var), 185 | ) 186 | return new_nodes, state.returns_ctr, state.returns_in_loops 187 | -------------------------------------------------------------------------------- /peval/components/peval_function_header.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import typing 3 | 4 | from peval.core.gensym import GenSym 5 | from peval.tools import replace_fields, ast_walker, ImmutableDict 6 | from peval.core.expression import peval_expression 7 | from peval.typing import ConstsDictT, PassOutputT 8 | 9 | 10 | @ast_walker 11 | class _peval_function_header: 12 | @staticmethod 13 | def handle_arg(state, node, ctx, **_): 14 | result, gen_sym = peval_expression(node.annotation, state.gen_sym, ctx.constants) 15 | new_bindings = state.new_bindings | result.temp_bindings 16 | 17 | state = state.with_(gen_sym=gen_sym, new_bindings=new_bindings) 18 | node = replace_fields(node, annotation=result.node) 19 | 20 | return state, node 21 | 22 | @staticmethod 23 | def handle_FunctionDef(state, node, ctx, skip_fields, walk_field, **_): 24 | # Avoid walking the body of the function 25 | skip_fields() 26 | 27 | # Walk function arguments 28 | state, new_args = walk_field(state, node.args) 29 | node = replace_fields(node, args=new_args) 30 | 31 | # Evaluate the return annotation 32 | result, gen_sym = peval_expression(node.returns, state.gen_sym, ctx.constants) 33 | new_bindings = state.new_bindings | result.temp_bindings 34 | node = replace_fields(node, returns=result.node) 35 | state = state.with_(gen_sym=gen_sym, new_bindings=new_bindings) 36 | 37 | return state, node 38 | 39 | 40 | def peval_function_header(tree: ast.FunctionDef, constants: ConstsDictT) -> PassOutputT: 41 | """ 42 | Partially evaluate argument annotations and return annotation of a function. 43 | """ 44 | gen_sym = GenSym.for_tree(tree) 45 | state, new_tree = _peval_function_header( 46 | dict(new_bindings=ImmutableDict(), gen_sym=gen_sym), 47 | tree, 48 | ctx=dict(constants=constants), 49 | ) 50 | return new_tree, state.new_bindings 51 | -------------------------------------------------------------------------------- /peval/components/prune_assignments.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from typing import Union, List, Tuple 3 | 4 | from peval.tools import ast_transformer, replace_fields 5 | from peval.core.scope import analyze_scope 6 | from peval.typing import ConstsDictT, PassOutputT 7 | 8 | 9 | def prune_assignments(node: ast.AST, constants: ConstsDictT) -> PassOutputT: 10 | scope = analyze_scope(node.body) 11 | node = remove_unused_assignments(node, ctx=dict(locals_used=scope.locals_used)) 12 | node = remove_simple_assignments(node) 13 | return node, constants 14 | 15 | 16 | @ast_transformer 17 | class remove_unused_assignments: 18 | @staticmethod 19 | def handle_Assign(node, ctx, **_): 20 | if all(type(target) == ast.Name for target in node.targets): 21 | names = set(target.id for target in node.targets) 22 | if ctx.locals_used.isdisjoint(names): 23 | return None 24 | else: 25 | return node 26 | else: 27 | return node 28 | 29 | 30 | def remove_simple_assignments( 31 | node: Union[ast.FunctionDef, ast.Module] 32 | ) -> Union[ast.FunctionDef, ast.Module]: 33 | """ 34 | Remove one assigment of the form ` = ` at a time, 35 | touching only the top level statements of the block. 36 | """ 37 | 38 | remaining_nodes = list(node.body) 39 | new_nodes = [] 40 | 41 | while len(remaining_nodes) > 0: 42 | cur_node = remaining_nodes.pop(0) 43 | if type(cur_node) == ast.Assign: 44 | can_remove, dest_name, src_name = _can_remove_assignment(cur_node, remaining_nodes) 45 | if can_remove: 46 | remaining_nodes = replace_name( 47 | remaining_nodes, ctx=dict(dest_name=dest_name, src_name=src_name) 48 | ) 49 | else: 50 | new_nodes.append(cur_node) 51 | else: 52 | new_nodes.append(cur_node) 53 | 54 | if len(new_nodes) == len(node.body): 55 | return node 56 | 57 | return replace_fields(node, body=new_nodes) 58 | 59 | 60 | def _can_remove_assignment( 61 | assign_node: ast.Assign, node_list: List[ast.AST] 62 | ) -> Union[Tuple[bool, str, str], Tuple[bool, None, None]]: 63 | """ 64 | Can remove it if: 65 | * it is "simple" 66 | * result it not used in "Store" context elsewhere 67 | """ 68 | if ( 69 | len(assign_node.targets) == 1 70 | and type(assign_node.targets[0]) == ast.Name 71 | and type(assign_node.value) == ast.Name 72 | ): 73 | src_name = assign_node.value.id 74 | dest_name = assign_node.targets[0].id 75 | if dest_name not in analyze_scope(node_list).locals: 76 | return True, dest_name, src_name 77 | return False, None, None 78 | 79 | 80 | @ast_transformer 81 | class replace_name: 82 | @staticmethod 83 | def handle_Name(node, ctx, **_): 84 | if type(node.ctx) == ast.Load and node.id == ctx.dest_name: 85 | return replace_fields(node, id=ctx.src_name) 86 | else: 87 | return node 88 | -------------------------------------------------------------------------------- /peval/components/prune_cfg.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from typing import List 3 | 4 | from peval.tools import replace_fields, ast_transformer, ast_inspector 5 | from peval.core.expression import try_peval_expression 6 | from peval.tools import ast_equal 7 | from peval.typing import ConstsDictT, PassOutputT 8 | 9 | 10 | def prune_cfg(node: ast.AST, bindings: ConstsDictT) -> PassOutputT: 11 | while True: 12 | new_node = node 13 | 14 | for func in ( 15 | remove_unreachable_statements, 16 | simplify_loops, 17 | remove_unreachable_branches, 18 | ): 19 | new_node = func(new_node, ctx=dict(bindings=bindings)) 20 | 21 | if ast_equal(new_node, node): 22 | break 23 | 24 | node = new_node 25 | 26 | return new_node, bindings 27 | 28 | 29 | @ast_transformer 30 | def remove_unreachable_statements(node, walk_field, **kwds): 31 | for attr in ("body", "orelse"): 32 | if hasattr(node, attr): 33 | old_list = getattr(node, attr) 34 | not_list = isinstance(old_list, ast.AST) 35 | if not_list: 36 | old_list = [old_list] 37 | new_list = filter_block(old_list) 38 | if new_list is not old_list: 39 | new_list = walk_field(new_list, block_context=True) 40 | if not_list: 41 | new_list = new_list[0] 42 | kwds = {attr: new_list} 43 | node = replace_fields(node, **kwds) 44 | return node 45 | 46 | 47 | def filter_block(node_list: List[ast.AST]) -> List[ast.AST]: 48 | """ 49 | Remove no-op code (``pass``), or any code after 50 | an unconditional jump (``return``, ``break``, ``continue``, ``raise``). 51 | """ 52 | if len(node_list) == 1: 53 | return node_list 54 | 55 | new_list = [] 56 | for node in node_list: 57 | if type(node) == ast.Pass: 58 | continue 59 | new_list.append(node) 60 | if type(node) in (ast.Return, ast.Break, ast.Continue, ast.Raise): 61 | break 62 | if len(new_list) == len(node_list): 63 | return node_list 64 | else: 65 | return new_list 66 | 67 | 68 | @ast_inspector 69 | class _find_jumps: 70 | @staticmethod 71 | def handle_FunctionDef(skip_fields, **_): 72 | skip_fields() 73 | 74 | @staticmethod 75 | def handle_ClassDef(skip_fields, **_): 76 | skip_fields() 77 | 78 | @staticmethod 79 | def handle_Break(state, **_): 80 | return state.with_(jumps_counter=state.jumps_counter + 1) 81 | 82 | @staticmethod 83 | def handle_Raise(state, **_): 84 | return state.with_(jumps_counter=state.jumps_counter + 1) 85 | 86 | @staticmethod 87 | def handle_Return(state, **_): 88 | return state.with_(jumps_counter=state.jumps_counter + 1) 89 | 90 | 91 | def find_jumps(node: List[ast.AST]) -> int: 92 | return _find_jumps(dict(jumps_counter=0), node).jumps_counter 93 | 94 | 95 | @ast_transformer 96 | class simplify_loops: 97 | @staticmethod 98 | def handle_While(node, **_): 99 | last_node = node.body[-1] 100 | unconditional_jump = type(last_node) in (ast.Break, ast.Raise, ast.Return) 101 | if unconditional_jump and find_jumps(node.body) == 1: 102 | if type(last_node) == ast.Break: 103 | new_body = node.body[:-1] 104 | else: 105 | new_body = node.body 106 | return ast.If(test=node.test, body=new_body, orelse=node.orelse) 107 | else: 108 | return node 109 | 110 | 111 | @ast_transformer 112 | class remove_unreachable_branches: 113 | @staticmethod 114 | def handle_If(node, ctx, walk_field, **_): 115 | evaluated, test = try_peval_expression(node.test, ctx.bindings) 116 | if evaluated: 117 | taken_node = node.body if test else node.orelse 118 | new_node = walk_field(taken_node, block_context=True) 119 | return new_node 120 | else: 121 | return node 122 | -------------------------------------------------------------------------------- /peval/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjarri/peval/e6b5d8c1be9b2da167eb27975aa70aacdb4eb660/peval/core/__init__.py -------------------------------------------------------------------------------- /peval/core/cfg.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from typing import Set, List, Optional, Union 3 | 4 | 5 | class Node: 6 | def __init__(self, ast_node: ast.AST) -> None: 7 | self.ast_node = ast_node 8 | self.parents = set() 9 | self.children = set() 10 | 11 | 12 | class Graph: 13 | def __init__(self) -> None: 14 | self.nodes = {} 15 | 16 | def add_node(self, ast_node: ast.AST) -> int: 17 | node_id = id(ast_node) 18 | self.nodes[node_id] = Node(ast_node) 19 | return node_id 20 | 21 | def add_edge(self, src: int, dest: int) -> None: 22 | assert src in self.nodes 23 | assert dest in self.nodes 24 | 25 | # assert dest not in self.children_of(src) 26 | # assert src not in self.parents_of(dest) 27 | 28 | self.nodes[src].children.add(dest) 29 | self.nodes[dest].parents.add(src) 30 | 31 | def children_of(self, node: int) -> Set[int]: 32 | return self.nodes[node].children 33 | 34 | def parents_of(self, node: int) -> Set[int]: 35 | return self.nodes[node].parents 36 | 37 | def update(self, other: "Graph") -> None: 38 | for node in other.nodes: 39 | assert node not in self.nodes 40 | self.nodes.update(other.nodes) 41 | 42 | def get_nontrivial_nodes(self) -> List[int]: 43 | # returns ids of nodes that can possibly raise an exception 44 | nodes = [] 45 | for node_id, node_obj in self.nodes.items(): 46 | node = node_obj.ast_node 47 | if type(node) not in (ast.Break, ast.Continue, ast.Pass, ast.Try): 48 | nodes.append(node_id) 49 | return nodes 50 | 51 | 52 | class Jumps: 53 | def __init__( 54 | self, 55 | returns: Optional[List[int]] = None, 56 | breaks: Optional[List[int]] = None, 57 | continues=None, 58 | raises=None, 59 | ) -> None: 60 | self.returns = [] if returns is None else returns 61 | self.breaks = [] if breaks is None else breaks 62 | self.continues = [] if continues is None else continues 63 | self.raises = [] if raises is None else raises 64 | 65 | def join(self, other: "Jumps") -> "Jumps": 66 | return Jumps( 67 | returns=self.returns + other.returns, 68 | breaks=self.breaks + other.breaks, 69 | continues=self.continues + other.continues, 70 | raises=self.raises + other.raises, 71 | ) 72 | 73 | 74 | class ControlFlowSubgraph: 75 | def __init__( 76 | self, 77 | graph: Graph, 78 | enter: int, 79 | exits: Optional[List[int]] = None, 80 | jumps: Optional[Jumps] = None, 81 | ) -> None: 82 | self.graph = graph 83 | self.enter = enter 84 | self.exits = [] if exits is None else exits 85 | self.jumps = Jumps() if jumps is None else jumps 86 | 87 | 88 | class ControlFlowGraph: 89 | def __init__( 90 | self, 91 | graph: Graph, 92 | enter: int, 93 | exits: Optional[List[int]] = None, 94 | raises=None, 95 | ) -> None: 96 | self.graph = graph 97 | self.enter = enter 98 | self.exits = [] if exits is None else exits 99 | self.raises = [] if raises is None else raises 100 | 101 | 102 | def _build_if_cfg(node: ast.If) -> ControlFlowSubgraph: 103 | cfg_true = _build_cfg(node.body) 104 | exits = cfg_true.exits 105 | jumps = cfg_true.jumps 106 | graph = cfg_true.graph 107 | 108 | node_id = graph.add_node(node) 109 | 110 | graph.add_edge(node_id, cfg_true.enter) 111 | 112 | if len(node.orelse) > 0: 113 | cfg_false = _build_cfg(node.orelse) 114 | exits += cfg_false.exits 115 | jumps = jumps.join(cfg_false.jumps) 116 | graph.update(cfg_false.graph) 117 | graph.add_edge(node_id, cfg_false.enter) 118 | else: 119 | exits.append(node_id) 120 | 121 | return ControlFlowSubgraph(graph, node_id, exits=exits, jumps=jumps) 122 | 123 | 124 | def _build_loop_cfg(node: Union[ast.For, ast.While]) -> ControlFlowSubgraph: 125 | cfg = _build_cfg(node.body) 126 | graph = cfg.graph 127 | 128 | node_id = graph.add_node(node) 129 | 130 | graph.add_edge(node_id, cfg.enter) 131 | 132 | for c_id in cfg.jumps.continues: 133 | graph.add_edge(c_id, node_id) 134 | exits = cfg.jumps.breaks 135 | jumps = Jumps(raises=cfg.jumps.raises) 136 | 137 | for exit_ in cfg.exits: 138 | graph.add_edge(exit_, node_id) 139 | 140 | if len(node.orelse) == 0: 141 | exits += cfg.exits 142 | else: 143 | cfg_orelse = _build_cfg(node.orelse) 144 | 145 | graph.update(cfg_orelse.graph) 146 | exits += cfg_orelse.exits 147 | jumps = jumps.join(Jumps(raises=cfg_orelse.jumps.raises)) 148 | for exit_ in cfg.exits: 149 | graph.add_edge(exit_, cfg_orelse.enter) 150 | 151 | return ControlFlowSubgraph(graph, node_id, exits=exits, jumps=jumps) 152 | 153 | 154 | def _build_with_cfg(node: ast.With) -> ControlFlowSubgraph: 155 | cfg = _build_cfg(node.body) 156 | graph = cfg.graph 157 | 158 | node_id = graph.add_node(node) 159 | 160 | graph.add_edge(node_id, cfg.enter) 161 | return ControlFlowSubgraph(graph, node_id, exits=cfg.exits, jumps=cfg.jumps) 162 | 163 | 164 | def _build_break_cfg(node: ast.Break) -> ControlFlowSubgraph: 165 | graph = Graph() 166 | node_id = graph.add_node(node) 167 | return ControlFlowSubgraph(graph, node_id, jumps=Jumps(breaks=[node_id])) 168 | 169 | 170 | def _build_continue_cfg(node): 171 | graph = Graph() 172 | node_id = graph.add_node(node) 173 | return ControlFlowSubgraph(graph, node_id, jumps=Jumps(continues=[node_id])) 174 | 175 | 176 | def _build_return_cfg(node: ast.Return) -> ControlFlowSubgraph: 177 | graph = Graph() 178 | node_id = graph.add_node(node) 179 | return ControlFlowSubgraph(graph, node_id, jumps=Jumps(returns=[node_id])) 180 | 181 | 182 | def _build_statement_cfg(node: ast.stmt) -> ControlFlowSubgraph: 183 | graph = Graph() 184 | node_id = graph.add_node(node) 185 | return ControlFlowSubgraph(graph, node_id, exits=[node_id]) 186 | 187 | 188 | def _build_excepthandler_cfg(node: ast.ExceptHandler) -> ControlFlowSubgraph: 189 | graph = Graph() 190 | enter = graph.add_node(node) 191 | 192 | cfg = _build_cfg(node.body) 193 | graph.update(cfg.graph) 194 | graph.add_edge(enter, cfg.enter) 195 | 196 | return ControlFlowSubgraph(graph, enter, exits=cfg.exits, jumps=cfg.jumps) 197 | 198 | 199 | def _build_try_block_cfg( 200 | try_node: ast.Try, 201 | body: List[ast.AST], 202 | handlers: List[ast.ExceptHandler], 203 | orelse: List[ast.AST], 204 | ) -> ControlFlowSubgraph: 205 | graph = Graph() 206 | enter = graph.add_node(try_node) 207 | 208 | body_cfg = _build_cfg(body) 209 | 210 | jumps = body_cfg.jumps 211 | jumps.raises = [] # raises will be connected to all the handlers anyway 212 | 213 | graph.update(body_cfg.graph) 214 | graph.add_edge(enter, body_cfg.enter) 215 | 216 | handler_cfgs = [_build_excepthandler_cfg(handler) for handler in handlers] 217 | for handler_cfg in handler_cfgs: 218 | graph.update(handler_cfg.graph) 219 | jumps = jumps.join(handler_cfg.jumps) 220 | 221 | # FIXME: is it correct in case of nested `try`s? 222 | body_ids = body_cfg.graph.get_nontrivial_nodes() 223 | if len(handler_cfgs) > 0: 224 | # FIXME: if there are exception handlers, 225 | # assuming that all the exceptions are caught by them 226 | for body_id in body_ids: 227 | for handler_cfg in handler_cfgs: 228 | graph.add_edge(body_id, handler_cfg.enter) 229 | else: 230 | # If there are no handlers, every statement can potentially raise 231 | # (otherwise they wouldn't be in a try block) 232 | jumps = jumps.join(Jumps(raises=body_ids)) 233 | 234 | exits = body_cfg.exits 235 | 236 | if len(orelse) > 0 and len(body_cfg.exits) > 0: 237 | # FIXME: show warning about unreachable code if there's `orelse`, but no exits from body? 238 | orelse_cfg = _build_cfg(orelse) 239 | graph.update(orelse_cfg.graph) 240 | jumps = jumps.join(orelse_cfg.jumps) 241 | for exit_ in exits: 242 | graph.add_edge(exit_, orelse_cfg.enter) 243 | exits = orelse_cfg.exits 244 | 245 | for handler_cfg in handler_cfgs: 246 | exits += handler_cfg.exits 247 | 248 | return ControlFlowSubgraph(graph, enter, exits=exits, jumps=jumps) 249 | 250 | 251 | def _build_try_finally_block_cfg( 252 | try_node: ast.Try, 253 | body: List[ast.AST], 254 | handlers: List[ast.ExceptHandler], 255 | orelse: List[ast.AST], 256 | finalbody: List[ast.AST], 257 | ) -> ControlFlowSubgraph: 258 | try_cfg = _build_try_block_cfg(try_node, body, handlers, orelse) 259 | 260 | if len(finalbody) == 0: 261 | return try_cfg 262 | 263 | # everything has to pass through finally 264 | final_cfg = _build_cfg(finalbody) 265 | graph = try_cfg.graph 266 | jumps = try_cfg.jumps 267 | graph.update(final_cfg.graph) 268 | 269 | for exit_ in try_cfg.exits: 270 | graph.add_edge(exit_, final_cfg.enter) 271 | 272 | def pass_through(jump_list): 273 | if len(jump_list) > 0: 274 | for jump_id in jump_list: 275 | graph.add_edge(jump_id, final_cfg.enter) 276 | return final_cfg.exits 277 | else: 278 | return [] 279 | 280 | returns = pass_through(jumps.returns) 281 | raises = pass_through(jumps.raises) 282 | continues = pass_through(jumps.continues) 283 | breaks = pass_through(jumps.breaks) 284 | 285 | return ControlFlowSubgraph( 286 | graph, 287 | try_cfg.enter, 288 | exits=final_cfg.exits, 289 | jumps=Jumps(returns=returns, raises=raises, continues=continues, breaks=breaks), 290 | ) 291 | 292 | 293 | def _build_try_finally_cfg(node): 294 | # If there are no exception handlers, the body is just a sequence of statements 295 | return _build_try_finally_block_cfg(node, node.body, [], [], node.finalbody) 296 | 297 | 298 | def _build_try_cfg(node: ast.Try) -> ControlFlowSubgraph: 299 | return _build_try_finally_block_cfg(node, node.body, node.handlers, node.orelse, node.finalbody) 300 | 301 | 302 | def _build_node_cfg(node) -> ControlFlowSubgraph: 303 | handlers = { 304 | ast.If: _build_if_cfg, 305 | ast.For: _build_loop_cfg, 306 | ast.While: _build_loop_cfg, 307 | ast.With: _build_with_cfg, 308 | ast.Break: _build_break_cfg, 309 | ast.Continue: _build_continue_cfg, 310 | ast.Return: _build_return_cfg, 311 | ast.Try: _build_try_cfg, 312 | } 313 | 314 | if type(node) in handlers: 315 | handler = handlers[type(node)] 316 | else: 317 | handler = _build_statement_cfg 318 | 319 | return handler(node) 320 | 321 | 322 | def _build_cfg(statements) -> ControlFlowSubgraph: 323 | enter = id(statements[0]) 324 | 325 | exits = [enter] 326 | graph = Graph() 327 | 328 | jumps = Jumps() 329 | 330 | for i, node in enumerate(statements): 331 | cfg = _build_node_cfg(node) 332 | 333 | graph.update(cfg.graph) 334 | 335 | if i > 0: 336 | for exit_ in exits: 337 | graph.add_edge(exit_, cfg.enter) 338 | 339 | exits = cfg.exits 340 | jumps = jumps.join(cfg.jumps) 341 | 342 | if type(node) in (ast.Break, ast.Continue, ast.Return): 343 | # Issue a warning about unreachable code? 344 | break 345 | 346 | return ControlFlowSubgraph(graph, enter, exits=exits, jumps=jumps) 347 | 348 | 349 | def build_cfg(statements) -> ControlFlowGraph: 350 | cfg = _build_cfg(statements) 351 | assert len(cfg.jumps.breaks) == 0 352 | assert len(cfg.jumps.continues) == 0 353 | return ControlFlowGraph( 354 | cfg.graph, cfg.enter, cfg.exits + cfg.jumps.returns, raises=cfg.jumps.raises 355 | ) 356 | -------------------------------------------------------------------------------- /peval/core/gensym.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from typing import Optional, DefaultDict, Tuple, FrozenSet 3 | 4 | from peval.core.scope import analyze_scope 5 | from ast import FunctionDef 6 | 7 | 8 | class GenSym: 9 | def __init__( 10 | self, 11 | taken_names: Optional[FrozenSet[str]] = None, 12 | counters: Optional[DefaultDict[str, int]] = None, 13 | ) -> None: 14 | self._taken_names = taken_names if taken_names is not None else frozenset() 15 | 16 | # Keeping per-tag counters affects performance, 17 | # but the creation of new names happens quite rarely, 18 | # so it is not noticeable. 19 | # On the other hand, it makes it easier to compare resulting code with reference code, 20 | # since in Py3.4 and later we do not need to mangle True/False/None any more, 21 | # so the joint counter would produce different variable names. 22 | if counters is None: 23 | self._counters = defaultdict(lambda: 1) 24 | else: 25 | self._counters = counters.copy() 26 | 27 | @classmethod 28 | def for_tree(cls, tree: Optional[FunctionDef] = None) -> "GenSym": 29 | if tree is not None: 30 | scope = analyze_scope(tree) 31 | taken_names = scope.locals | scope.globals 32 | else: 33 | taken_names = None 34 | return cls(taken_names=taken_names) 35 | 36 | def __call__(self, tag: str = "sym") -> Tuple[str, "GenSym"]: 37 | counter = self._counters[tag] 38 | while True: 39 | name = "__peval_" + tag + "_" + str(counter) 40 | counter += 1 41 | if name not in self._taken_names: 42 | break 43 | self._counters[tag] = counter 44 | 45 | return name, GenSym(taken_names=self._taken_names, counters=self._counters) 46 | -------------------------------------------------------------------------------- /peval/core/mangler.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from typing import Tuple, FrozenSet 3 | 4 | from peval.tools import ImmutableDict, ast_walker 5 | from peval.core.scope import analyze_scope 6 | from peval.tools.immutable import ImmutableADict 7 | from peval.core.gensym import GenSym 8 | from peval.typing import NameNodeT 9 | from peval.tools.immutable import ImmutableDict 10 | 11 | 12 | def _visit_local( 13 | gen_sym: GenSym, node: NameNodeT, to_mangle: FrozenSet[str], mangled: ImmutableADict 14 | ) -> Tuple[GenSym, NameNodeT, ImmutableDict]: 15 | """ 16 | Replacing known variables with literal values 17 | """ 18 | is_name = type(node) == ast.Name 19 | 20 | node_id = node.id if is_name else node.arg 21 | 22 | if node_id in to_mangle: 23 | if node_id in mangled: 24 | mangled_id = mangled[node_id] 25 | else: 26 | mangled_id, gen_sym = gen_sym("mangled") 27 | mangled = mangled.with_item(node_id, mangled_id) 28 | 29 | if is_name: 30 | new_node = ast.Name(id=mangled_id, ctx=node.ctx) 31 | else: 32 | new_node = ast.arg(arg=mangled_id, annotation=node.annotation) 33 | 34 | else: 35 | new_node = node 36 | 37 | return gen_sym, new_node, mangled 38 | 39 | 40 | @ast_walker 41 | class _mangle: 42 | """ 43 | Mangle all variable names, returns. 44 | """ 45 | 46 | @staticmethod 47 | def handle_arg(state, node, ctx, **_): 48 | gen_sym, new_node, mangled = _visit_local(state.gen_sym, node, ctx.fn_locals, state.mangled) 49 | new_state = state.with_(gen_sym=gen_sym, mangled=mangled) 50 | return new_state, new_node 51 | 52 | @staticmethod 53 | def handle_Name(state, node, ctx, **_): 54 | gen_sym, new_node, mangled = _visit_local(state.gen_sym, node, ctx.fn_locals, state.mangled) 55 | new_state = state.with_(gen_sym=gen_sym, mangled=mangled) 56 | return new_state, new_node 57 | 58 | 59 | def mangle(gen_sym: GenSym, node: ast.FunctionDef) -> Tuple[GenSym, ast.FunctionDef]: 60 | fn_locals = analyze_scope(node).locals 61 | state, new_node = _mangle( 62 | dict(gen_sym=gen_sym, mangled=ImmutableDict()), 63 | node, 64 | ctx=dict(fn_locals=fn_locals), 65 | ) 66 | return state.gen_sym, new_node 67 | -------------------------------------------------------------------------------- /peval/core/reify.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import sys 3 | from typing import Any, Optional, Tuple, Dict 4 | 5 | from peval.core.gensym import GenSym 6 | from peval.typing import ConstantOrNameNodeT, ConsantOrASTNodeT 7 | 8 | 9 | NONE_NODE = ast.Constant(value=None, kind=None) 10 | FALSE_NODE = ast.Constant(value=False, kind=None) 11 | TRUE_NODE = ast.Constant(value=True, kind=None) 12 | 13 | 14 | class KnownValue: 15 | def __init__(self, value: Any, preferred_name: Optional[str] = None) -> None: 16 | self.value = value 17 | self.preferred_name = preferred_name 18 | 19 | def __str__(self): 20 | return ( 21 | "<" 22 | + str(self.value) 23 | + (" (" + self.preferred_name + ")" if self.preferred_name is not None else "") 24 | + ">" 25 | ) 26 | 27 | def __repr__(self): 28 | return "KnownValue({value}, preferred_name={name})".format( 29 | value=repr(self.value), name=repr(self.preferred_name) 30 | ) 31 | 32 | 33 | ReifyResT = Tuple[ConstantOrNameNodeT, GenSym, Dict[str, ConsantOrASTNodeT]] 34 | 35 | 36 | def reify(kvalue: KnownValue, gen_sym: GenSym, create_binding: bool = False) -> ReifyResT: 37 | value = kvalue.value 38 | 39 | # TODO: add a separate reify_constant() method that guarantees not to change the bindings 40 | if value is True or value is False or value is None: 41 | return ast.Constant(value=value, kind=None), gen_sym, {} 42 | elif type(value) == str: 43 | return ast.Constant(value=value, kind=None), gen_sym, {} 44 | elif type(value) == bytes: 45 | return ast.Constant(value=value, kind=None), gen_sym, {} 46 | elif type(value) in (int, float, complex): 47 | return ast.Constant(value=value, kind=None), gen_sym, {} 48 | else: 49 | if kvalue.preferred_name is None or create_binding: 50 | name, gen_sym = gen_sym("temp") 51 | else: 52 | name = kvalue.preferred_name 53 | return ast.Name(id=name, ctx=ast.Load()), gen_sym, {name: value} 54 | 55 | 56 | def reify_unwrapped(value: ConstantOrNameNodeT, gen_sym: GenSym) -> ReifyResT: 57 | return reify(KnownValue(value), gen_sym) 58 | -------------------------------------------------------------------------------- /peval/core/scope.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from collections import namedtuple 3 | 4 | from peval.tools import ast_inspector 5 | 6 | 7 | Scope = namedtuple("Scope", "locals locals_used globals") 8 | 9 | 10 | @ast_inspector 11 | class _analyze_scope: 12 | @staticmethod 13 | def handle_arg(state, node: ast.AST, **_): 14 | return state.with_(locals=state.locals | {node.arg}) 15 | 16 | @staticmethod 17 | def handle_Name(state, node: ast.AST, **_): 18 | name = node.id 19 | if type(node.ctx) == ast.Store: 20 | state = state.with_(locals=state.locals | {name}) 21 | if name in state.globals: 22 | state = state.with_(globals=state.globals - {name}) 23 | elif type(node.ctx) == ast.Load: 24 | if name in state.locals: 25 | state = state.with_(locals_used=state.locals_used | {name}) 26 | else: 27 | state = state.with_(globals=state.globals | {name}) 28 | 29 | return state 30 | 31 | @staticmethod 32 | def handle_alias(state, node: ast.AST, **_): 33 | name = node.asname if node.asname else node.name 34 | if "." in name: 35 | name = name.split(".", 1)[0] 36 | return state.with_(locals=state.locals | {name}) 37 | 38 | 39 | def analyze_scope(node: ast.AST) -> Scope: 40 | state = _analyze_scope( 41 | dict(locals=frozenset(), locals_used=frozenset(), globals=frozenset()), 42 | node, 43 | ) 44 | return Scope(locals=state.locals, locals_used=state.locals_used, globals=state.globals) 45 | -------------------------------------------------------------------------------- /peval/highlevelapi.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from functools import lru_cache 3 | from typing import Callable, Union, Tuple 4 | 5 | from peval.core.function import Function, has_nested_definitions, is_async 6 | from peval.typing import ConstsDictT, PassOutputT 7 | from peval.components import ( 8 | inline_functions, 9 | prune_cfg, 10 | prune_assignments, 11 | fold, 12 | peval_function_header, 13 | ) 14 | from peval.tools import ast_equal 15 | from ast import AST 16 | 17 | 18 | def _run_components(tree: AST, constants: ConstsDictT) -> PassOutputT: 19 | while True: 20 | new_tree = tree 21 | new_constants = constants 22 | 23 | for func in (fold, prune_cfg, prune_assignments, inline_functions): 24 | new_tree, new_constants = func(new_tree, new_constants) 25 | 26 | if ast_equal(new_tree, tree) and new_constants == constants: 27 | break 28 | 29 | tree = new_tree 30 | constants = new_constants 31 | 32 | return new_tree, new_constants 33 | 34 | 35 | def partial_apply(func: Callable, *args, **kwds) -> Callable: 36 | """ 37 | Same as :func:`partial_eval`, but in addition uses the provided values of 38 | positional and keyword arguments in the partial evaluation. 39 | """ 40 | function = Function.from_object(func, ignore_decorators=True) 41 | 42 | if has_nested_definitions(function): 43 | raise ValueError( 44 | "A partially evaluated function cannot have nested function or class definitions" 45 | ) 46 | 47 | if is_async(function): 48 | raise ValueError("A partially evaluated function cannot be an async coroutine") 49 | 50 | if len(args) > 0 or len(kwds) > 0: 51 | bound_function = function.bind_partial(*args, **kwds) 52 | else: 53 | bound_function = function 54 | 55 | ext_vars = bound_function.get_external_variables() 56 | 57 | # We don't need to run signature evaluation several times until convergence, 58 | # since there is no inlining/folding going on. 59 | new_tree, signature_bindings = peval_function_header(bound_function.tree, ext_vars) 60 | 61 | # The components do have to be run iteratively until convergence for the body of the function. 62 | new_tree, body_bindings = _run_components(new_tree, ext_vars) 63 | 64 | globals_ = dict(bound_function.globals) 65 | globals_.update(signature_bindings) 66 | globals_.update(body_bindings) 67 | 68 | new_function = bound_function.replace(tree=new_tree, globals_=globals_) 69 | 70 | return new_function.eval() 71 | 72 | 73 | def partial_eval(func: Callable) -> Callable: 74 | """ 75 | Returns a partially evaluated version of ``func``, using the values of 76 | associated global and closure variables. 77 | """ 78 | return partial_apply(func) 79 | 80 | 81 | def specialize_on(names: Union[str, Tuple[str, str]], maxsize=None) -> Callable: 82 | """ 83 | A decorator that wraps a function, partially evaluating it with the parameters 84 | defined by ``names`` (can be a string or an iterable of strings) being fixed. 85 | The partially evaluated versions are cached based on the values of these parameters 86 | using ``functools.lru_cache`` with the provided ``maxsize`` 87 | (consequently, these values should be hashable). 88 | """ 89 | if isinstance(names, str): 90 | names = [names] 91 | names_set = set(names) 92 | 93 | def _specializer(func): 94 | signature = inspect.signature(func) 95 | 96 | if not names_set.issubset(signature.parameters): 97 | missing_names = names_set.intersection(signature.parameters) 98 | raise ValueError( 99 | "The provided function does not have parameters: " + ", ".join(missing_names) 100 | ) 101 | 102 | @lru_cache(maxsize=maxsize) 103 | def get_pevaled_func(args): 104 | return partial_apply(func, **{name: val for name, val in args}) 105 | 106 | def _wrapper(*args, **kwds): 107 | bargs = signature.bind(*args, **kwds) 108 | call_arguments = bargs.arguments.copy() 109 | for name in list(bargs.arguments): 110 | if name not in names_set: 111 | del bargs.arguments[name] # automatically changes .args and .kwargs 112 | else: 113 | del call_arguments[name] 114 | 115 | cache_args = tuple((name, val) for name, val in bargs.arguments.items()) 116 | pevaled_func = get_pevaled_func(cache_args) 117 | 118 | bargs.arguments = call_arguments # automatically changes .args and .kwargs 119 | 120 | return pevaled_func(*bargs.args, **bargs.kwargs) 121 | 122 | return _wrapper 123 | 124 | return _specializer 125 | -------------------------------------------------------------------------------- /peval/tags.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional 2 | 3 | from peval.core.function import Function, has_nested_definitions, is_a_generator, is_async 4 | 5 | 6 | def pure(func: Callable) -> Callable: 7 | """ 8 | Marks the function as pure (not having any side effects, except maybe argument mutation). 9 | 10 | Note that for class- and staticmethods ``@pure`` must be declared 11 | within the corresponding decorator. 12 | """ 13 | func.__peval_pure__ = True 14 | return func 15 | 16 | 17 | def get_pure_tag(func: Callable) -> Optional[bool]: 18 | return getattr(func, "__peval_pure__", None) 19 | 20 | 21 | def inline(func: Callable) -> Callable: 22 | """ 23 | Marks the function for inlining. 24 | """ 25 | function = Function.from_object(func, ignore_decorators=True) 26 | 27 | if has_nested_definitions(function): 28 | raise ValueError("An inlined function cannot have nested function or class definitions") 29 | 30 | if is_a_generator(function): 31 | raise ValueError("An inlined function cannot be a generator") 32 | 33 | if is_async(function): 34 | raise ValueError("An inlined function cannot be an async coroutine") 35 | 36 | if len(function.closure_vals) > 0: 37 | raise ValueError("An inlined function cannot have a closure") 38 | 39 | func.__peval_inline__ = True 40 | return func 41 | 42 | 43 | def get_inline_tag(func: Callable) -> Optional[bool]: 44 | return getattr(func, "__peval_inline__", None) 45 | -------------------------------------------------------------------------------- /peval/tools/__init__.py: -------------------------------------------------------------------------------- 1 | from peval.tools.dispatcher import Dispatcher 2 | from peval.tools.immutable import ImmutableDict, ImmutableADict 3 | from peval.tools.utils import unparse, unindent, replace_fields, ast_equal, map_accum, fold_and 4 | from peval.tools.walker import ast_walker, ast_inspector, ast_transformer 5 | -------------------------------------------------------------------------------- /peval/tools/dispatcher.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import types 3 | from typing import Tuple, Callable, Optional, Generic, TypeVar, Any, Dict, Type, cast 4 | from typing_extensions import ParamSpec, Concatenate 5 | 6 | from .immutable import ImmutableDict 7 | 8 | _Params = ParamSpec("_Params") 9 | _Return = TypeVar("_Return") 10 | 11 | 12 | class Dispatcher(Generic[_Params, _Return]): 13 | """ 14 | A dispatcher that maps a call to a group of functions 15 | based on the type of the first argument 16 | (hardcoded to be an AST node at the moment). 17 | 18 | ``handler_obj`` can be either a function with the signature:: 19 | 20 | def handler(*args, **kwds) 21 | 22 | or a class with the static methods:: 23 | 24 | @staticmethod 25 | def handle_(*args, **kwds) 26 | 27 | where ```` is the name of the type that this function will handle 28 | (e.g., ``handle_FunctionDef`` for ``ast.FunctionDef``). 29 | The class can also define the default handler:: 30 | 31 | @staticmethod 32 | def handle(*args, **kwds) 33 | 34 | If it is not defined, the ``default_handler`` value will be used 35 | (which must be a function with the same signature as above). 36 | If neither ``handle`` exists or ``default_handler`` is provided, 37 | a ``ValueError`` is thrown. 38 | """ 39 | 40 | def __init__( 41 | self, handler_obj: Any, default_handler: Optional[Callable[_Params, _Return]] = None 42 | ): 43 | self._handlers: Dict[Type[ast.AST], Callable[_Params, _Return]] = {} 44 | if isinstance(handler_obj, types.FunctionType): 45 | self._default_handler = cast(Callable[_Params, _Return], handler_obj) 46 | else: 47 | handler_prefix = "handle" 48 | if hasattr(handler_obj, handler_prefix): 49 | self._default_handler = cast( 50 | Callable[_Params, _Return], getattr(handler_obj, handler_prefix) 51 | ) 52 | elif default_handler is not None: 53 | self._default_handler = default_handler 54 | else: 55 | raise ValueError("Default handler was not provided") 56 | 57 | attr_prefix = handler_prefix + "_" 58 | for attr in vars(handler_obj): 59 | if attr.startswith(attr_prefix): 60 | typename = attr[len(attr_prefix) :] 61 | if hasattr(ast, typename): 62 | self._handlers[getattr(ast, typename)] = getattr(handler_obj, attr) 63 | 64 | def __call__( 65 | self, dispatch_node: ast.AST, *args: _Params.args, **kwargs: _Params.kwargs 66 | ) -> _Return: 67 | handler = self._handlers.get(type(dispatch_node), self._default_handler) 68 | return handler(*args, **kwargs) 69 | -------------------------------------------------------------------------------- /peval/tools/immutable.py: -------------------------------------------------------------------------------- 1 | """ 2 | Immutable data structures. 3 | 4 | The classes in this module have the prefix 'immutable' to avoid confusion 5 | with the built-in ``frozenset``, which does not have any modification methods, 6 | even pure ones. 7 | """ 8 | from typing import Any, TypeVar, Mapping, Iterator 9 | 10 | 11 | _Key = TypeVar("_Key") 12 | _Val = TypeVar("_Val") 13 | 14 | 15 | class ImmutableDict(Mapping[_Key, _Val]): 16 | """ 17 | An immutable version of ``dict``. 18 | 19 | TODO: switch to `frozendict` when it fixes its typing problems 20 | (see https://github.com/Marco-Sulla/python-frozendict/issues/39) 21 | 22 | Mutating syntax (``del d[k]``, ``d[k] = v``) is prohibited, 23 | pure methods ``del_`` and ``set`` are available instead. 24 | Mutating methods are overridden to return the new dictionary 25 | (or a tuple ``(value, new_dict)`` where applicable) 26 | without mutating the source dictionary. 27 | If a mutating method does not change the dictionary, 28 | the source dictionary itself is returned as the new dictionary. 29 | """ 30 | 31 | def __init__(self, *args: Any, **kwargs: Any) -> None: 32 | self._dict = dict(*args, **kwargs) 33 | 34 | def __getitem__(self, key: object) -> _Val: 35 | return self._dict[key] 36 | 37 | def __contains__(self, key: object) -> bool: 38 | return key in self._dict 39 | 40 | def __iter__(self) -> Iterator[_Key]: 41 | return iter(self._dict) 42 | 43 | def __len__(self) -> int: 44 | return len(self._dict) 45 | 46 | def __or__(self, other: Mapping[_Key, _Val]) -> "ImmutableDict[_Key, _Val]": 47 | new = dict(self._dict) 48 | new.update(other) 49 | return self.__class__(new) 50 | 51 | def with_item(self, key: _Key, val: _Val) -> "ImmutableDict[_Key, _Val]": 52 | if key in self._dict and self._dict[key] is val: 53 | return self 54 | new = dict(self._dict) 55 | new[key] = val 56 | return self.__class__(new) 57 | 58 | def without(self, key: _Key) -> "ImmutableDict[_Key, _Val]": 59 | new = dict(self._dict) 60 | del new[key] 61 | return self.__class__(new) 62 | 63 | def __repr__(self): 64 | return f"ImmutableDict({repr(self._dict)})" 65 | 66 | 67 | class ImmutableADict(ImmutableDict[str, _Val]): 68 | """ 69 | A subclass of ``ImmutableDict`` with values being accessible as attributes 70 | (e.g. ``d['a']`` is equivalent to ``d.a``). 71 | """ 72 | 73 | def __getattr__(self, attr: str) -> _Val: 74 | return self._dict[attr] 75 | 76 | def with_(self, **kwds: Mapping[str, _Val]) -> "ImmutableADict[_Val]": 77 | # TODO: need to think this over again. 78 | # In some places we check if the dicts were updated or not with `is`, 79 | # to avoid a lengthy equality check. 80 | # But e.g. equal strings are not guaranteed to be the same object in Python. 81 | # Is this fine? 82 | if all(key in self._dict and self._dict[key] is val for key, val in kwds.items()): 83 | return self 84 | new = dict(self._dict) 85 | new.update(**kwds) 86 | return self.__class__(new) 87 | 88 | def __repr__(self): 89 | return f"ImmutableADict({repr(self._dict)})" 90 | -------------------------------------------------------------------------------- /peval/tools/utils.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import re 3 | from typing import ( 4 | Callable, 5 | Any, 6 | Iterable, 7 | Optional, 8 | Tuple, 9 | Sequence, 10 | Union, 11 | TypeVar, 12 | List, 13 | Dict, 14 | cast, 15 | ) 16 | from typing_extensions import ParamSpec, Concatenate 17 | 18 | 19 | # TODO: as long as we're supporting Python 3.8 we have to rely on third-party unparsers. 20 | # Clean it up when Py3.8 is dropped. 21 | 22 | _unparse = None 23 | 24 | try: 25 | from ast import unparse as _unparse # type: ignore 26 | except ImportError: 27 | pass 28 | 29 | if _unparse is None: 30 | # Enabled by the `astunparse` feature. 31 | try: 32 | from astunparse import unparse as _unparse # type: ignore 33 | except ImportError: 34 | pass 35 | 36 | if _unparse is None: 37 | # Enabled by the `astor` feature. 38 | try: 39 | from astor import to_source as _unparse # type: ignore 40 | except ImportError: 41 | pass 42 | 43 | if _unparse is None: 44 | raise ImportError( 45 | "Unparsing functionality is not available; switch to Python 3.9+, " 46 | "install with 'astunparse' feature, or install with 'astor' feature." 47 | ) 48 | 49 | unparse = cast(Callable[[ast.AST], str], _unparse) 50 | 51 | 52 | def unindent(source: str) -> str: 53 | """ 54 | Shift source to the left so that it starts with zero indentation. 55 | """ 56 | source = source.rstrip("\n ").lstrip("\n") 57 | # Casting to Match here because this particular regex always matches 58 | indent = cast(re.Match, re.match(r"([ \t])*", source)).group(0) 59 | lines = source.split("\n") 60 | shifted_lines = [] 61 | for line in lines: 62 | line = line.rstrip() 63 | if len(line) > 0: 64 | if not line.startswith(indent): 65 | raise ValueError("Inconsistent indent at line " + repr(line)) 66 | shifted_lines.append(line[len(indent) :]) 67 | else: 68 | shifted_lines.append(line) 69 | return "\n".join(shifted_lines) 70 | 71 | 72 | def replace_fields(node: ast.AST, **kwds) -> ast.AST: 73 | """ 74 | Return a node with several of its fields replaced by the given values. 75 | """ 76 | new_kwds = dict(ast.iter_fields(node)) 77 | for key, value in kwds.items(): 78 | if value is not new_kwds[key]: 79 | break 80 | else: 81 | return node 82 | new_kwds.update(kwds) 83 | return type(node)(**new_kwds) 84 | 85 | 86 | def _ast_equal(node1: Any, node2: Any): 87 | if node1 is node2: 88 | return True 89 | 90 | if type(node1) != type(node2): 91 | return False 92 | if isinstance(node1, list): 93 | if len(node1) != len(node2): 94 | return False 95 | for elem1, elem2 in zip(node1, node2): 96 | if not _ast_equal(elem1, elem2): 97 | return False 98 | elif isinstance(node1, ast.AST): 99 | for attr, value1 in ast.iter_fields(node1): 100 | value2 = getattr(node2, attr) 101 | if not _ast_equal(value1, value2): 102 | return False 103 | else: 104 | if node1 != node2: 105 | return False 106 | 107 | return True 108 | 109 | 110 | def ast_equal(node1: ast.AST, node2: ast.AST) -> bool: 111 | """ 112 | Test two AST nodes or two lists of AST nodes for equality. 113 | """ 114 | # Type-gating it to make sure it's applied to AST nodes only. 115 | return _ast_equal(node1, node2) 116 | 117 | 118 | _Accum = TypeVar("_Accum") 119 | _Elem = TypeVar("_Elem") 120 | _Container = TypeVar("_Container") 121 | _Params = ParamSpec("_Params") 122 | 123 | 124 | def map_accum( 125 | func: Callable[Concatenate[_Accum, _Elem, _Params], Tuple[_Accum, _Elem]], 126 | acc: _Accum, 127 | container: _Container, 128 | *args: _Params.args, 129 | **kwargs: _Params.kwargs, 130 | ) -> Tuple[_Accum, _Container]: 131 | # Unfortunately we have to do some casting, because mypy does not support higher-ranked types 132 | # (what we want here is to make the type of `func` something like 133 | # `forall[_Elem] Callable[Concatenate[_Accum, _Elem, _Params], Tuple[_Accum, _Elem]]`). 134 | if container is None: 135 | return acc, None 136 | elif isinstance(container, (tuple, list)): 137 | new_list = [] 138 | for elem in container: 139 | acc, new_elem = map_accum(func, acc, elem, *args, **kwargs) 140 | new_list.append(new_elem) 141 | return acc, cast(_Container, type(container)(new_list)) 142 | elif isinstance(container, dict): 143 | new_dict = {} 144 | for key, elem in container.items(): 145 | acc, new_dict[key] = map_accum(func, acc, elem, *args, **kwargs) 146 | return acc, cast(_Container, new_dict) 147 | else: 148 | acc, new_container = func(acc, cast(_Elem, container), *args, **kwargs) 149 | return acc, cast(_Container, new_container) 150 | 151 | 152 | def fold_and(func: Callable[[Any], bool], container: Union[List, Tuple, Dict, Any]) -> bool: 153 | if type(container) in (list, tuple): 154 | return all(fold_and(func, elem) for elem in container) 155 | elif type(container) == dict: 156 | return all(fold_and(func, elem) for elem in container.values()) 157 | else: 158 | return func(container) 159 | -------------------------------------------------------------------------------- /peval/typing.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import ast 3 | from typing import Dict, Any, Tuple, Union, Type 4 | 5 | ConstsDictT = Dict[str, Any] 6 | PassOutputT = Tuple[ast.AST, ConstsDictT] 7 | 8 | NodeTypeT = Type[ast.AST] 9 | NodeTypeIsInstanceCriteriaT = Union[Tuple[NodeTypeT], NodeTypeT] 10 | 11 | NameNodeT = Union[ast.arg, ast.Name] 12 | 13 | ConstantNodeT = ast.Constant 14 | 15 | ConstantOrNameNodeT = Union[ConstantNodeT, ast.Name] 16 | 17 | ConstantT = Union[int, float, complex, str, bytes, bool] 18 | ConsantOrASTNodeT = Union[ConstantT, ast.AST] 19 | -------------------------------------------------------------------------------- /peval/wisdom.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import builtins 3 | from typing import Callable 4 | import types 5 | 6 | from peval.tags import get_pure_tag 7 | 8 | 9 | _PURE_METHODS = { 10 | "__abs__", 11 | "__add__", 12 | "__and__", 13 | "__bool__", 14 | "__bool__", 15 | "__bytes__", 16 | "__ceil__", 17 | "__class_getitem__", 18 | "__complex__", 19 | "__contains__", 20 | "__dir__", 21 | "__eq__", 22 | "__float__", 23 | "__floor__", 24 | "__floordiv__", 25 | "__format__", 26 | "__ge__", 27 | "__get__", 28 | "__getattr__", 29 | "__getattribute__", 30 | "__getitem__", 31 | "__gt__", 32 | "__hash__", 33 | "__index__", 34 | "__init__", 35 | "__instancecheck__", 36 | "__int__", 37 | "__invert__", 38 | "__iter__", 39 | "__le__", 40 | "__len__", 41 | "__length_hint__", 42 | "__lshift__", 43 | "__lt__", 44 | "__matmul__", 45 | "__missing__", 46 | "__mod__", 47 | "__mul__", 48 | "__ne__", 49 | "__neg__", 50 | "__or__", 51 | "__pos__", 52 | "__pow__", 53 | "__radd__", 54 | "__rand__", 55 | "__repr__", 56 | "__reversed__", 57 | "__rfloordiv__", 58 | "__rlshift__", 59 | "__rmatmul__", 60 | "__rmod__", 61 | "__rmul__", 62 | "__ror__", 63 | "__round__", 64 | "__rpow__", 65 | "__rrshift__", 66 | "__rshift__", 67 | "__rsub__", 68 | "__rtruediv__", 69 | "__rxor__", 70 | "__str__", 71 | "__sub__", 72 | "__subclasscheck__", 73 | "__truediv__", 74 | "__trunc__", 75 | "__xor__", 76 | } 77 | 78 | _IMPURE_BUILTINS = {delattr, setattr, eval, exec, input, print, next, open} 79 | 80 | _BUILTIN_PURE_CALLABLES = set() 81 | for name in dir(builtins): 82 | builtin = getattr(builtins, name) 83 | if type(builtin) == type: 84 | for method in _PURE_METHODS: 85 | if hasattr(builtin, method): 86 | _BUILTIN_PURE_CALLABLES.add(getattr(builtin, method)) 87 | elif callable(builtin) and builtin not in _IMPURE_BUILTINS: 88 | _BUILTIN_PURE_CALLABLES.add(builtin) 89 | 90 | 91 | def is_pure_callable(callable_) -> bool: 92 | if type(callable_) == type: 93 | # A regular class or a builtin type 94 | unbound_callable = callable_.__init__ 95 | elif type(callable_) == types.FunctionType: 96 | unbound_callable = callable_ 97 | elif type(callable_) == types.BuiltinFunctionType: 98 | # A builtin function (e.g. `isinstance`) 99 | unbound_callable = callable_ 100 | elif type(callable_) == types.WrapperDescriptorType: 101 | # An unbound method of some builtin classes (e.g. `str.__getitem__`) 102 | unbound_callable = callable_ 103 | elif type(callable_) == types.MethodWrapperType: 104 | # An bound method of some builtin classes (e.g. `"a".__getitem__`) 105 | unbound_callable = getattr(callable_.__objclass__, callable_.__name__) 106 | elif type(callable_) == types.MethodType: 107 | unbound_callable = callable_.__func__ 108 | elif hasattr(callable_, "__call__") and callable(callable_.__call__): 109 | unbound_callable = callable_.__call__ 110 | else: 111 | return False 112 | 113 | if unbound_callable in _BUILTIN_PURE_CALLABLES: 114 | return True 115 | 116 | pure_tag = get_pure_tag(unbound_callable) 117 | if pure_tag is not None: 118 | return pure_tag 119 | 120 | return False 121 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Add files or directories to the blacklist. They should be base names, not 11 | # paths. 12 | ignore=CVS 13 | 14 | # Pickle collected data for later comparisons. 15 | persistent=yes 16 | 17 | # List of plugins (as comma separated values of python modules names) to load, 18 | # usually to register additional checkers. 19 | load-plugins= 20 | 21 | # Use multiple processes to speed up Pylint. 22 | jobs=1 23 | 24 | # Allow loading of arbitrary C extensions. Extensions are imported into the 25 | # active Python interpreter and may run arbitrary code. 26 | unsafe-load-any-extension=no 27 | 28 | # A comma-separated list of package or module names from where C extensions may 29 | # be loaded. Extensions are loading into the active Python interpreter and may 30 | # run arbitrary code 31 | extension-pkg-whitelist= 32 | 33 | # Allow optimization of some AST trees. This will activate a peephole AST 34 | # optimizer, which will apply various small optimizations. For instance, it can 35 | # be used to obtain the result of joining multiple strings with the addition 36 | # operator. Joining a lot of strings can lead to a maximum recursion error in 37 | # Pylint and this flag can prevent that. It has one side effect, the resulting 38 | # AST will be different than the one from reality. 39 | optimize-ast=no 40 | 41 | 42 | [MESSAGES CONTROL] 43 | 44 | # Only show warnings with the listed confidence levels. Leave empty to show 45 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 46 | confidence= 47 | 48 | # Enable the message, report, category or checker with the given id(s). You can 49 | # either give multiple identifier separated by comma (,) or put this option 50 | # multiple time. See also the "--disable" option for examples. 51 | #enable= 52 | 53 | # Disable the message, report, category or checker with the given id(s). You 54 | # can either give multiple identifiers separated by comma (,) or put this 55 | # option multiple times (only on the command line, not in the configuration 56 | # file where it should appear only once).You can also use "--disable=all" to 57 | # disable everything first and then reenable specific checks. For example, if 58 | # you want to run only the similarities checker, you can use "--disable=all 59 | # --enable=similarities". If you want to run only the classes checker, but have 60 | # no Warning level messages displayed, use"--disable=all --enable=classes 61 | # --disable=W" 62 | disable=print-statement,xrange-builtin,raw_input-builtin,reduce-builtin,oct-method,raising-string,round-builtin,apply-builtin,dict-iter-method,next-method-called,unichr-builtin,buffer-builtin,basestring-builtin,map-builtin-not-iterating,filter-builtin-not-iterating,long-builtin,input-builtin,setslice-method,range-builtin-not-iterating,import-star-module-level,cmp-builtin,getslice-method,coerce-method,standarderror-builtin,file-builtin,suppressed-message,hex-method,old-raise-syntax,parameter-unpacking,delslice-method,metaclass-assignment,unicode-builtin,no-absolute-import,old-ne-operator,old-division,nonzero-method,using-cmp-argument,dict-view-method,cmp-method,unpacking-in-except,coerce-builtin,old-octal-literal,zip-builtin-not-iterating,reload-builtin,backtick,intern-builtin,indexing-exception,long-suffix,useless-suppression,execfile-builtin,missing-docstring 63 | 64 | 65 | [REPORTS] 66 | 67 | # Set the output format. Available formats are text, parseable, colorized, msvs 68 | # (visual studio) and html. You can also give a reporter class, eg 69 | # mypackage.mymodule.MyReporterClass. 70 | output-format=text 71 | 72 | # Put messages in a separate file for each module / package specified on the 73 | # command line instead of printing them on stdout. Reports (if any) will be 74 | # written in a file name "pylint_global.[txt|html]". 75 | files-output=no 76 | 77 | # Tells whether to display a full report or only the messages 78 | reports=yes 79 | 80 | # Python expression which should return a note less than 10 (10 is the highest 81 | # note). You have access to the variables errors warning, statement which 82 | # respectively contain the number of errors / warnings messages and the total 83 | # number of statements analyzed. This is used by the global evaluation report 84 | # (RP0004). 85 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 86 | 87 | # Template used to display messages. This is a python new-style format string 88 | # used to format the message information. See doc for all details 89 | #msg-template= 90 | 91 | 92 | [BASIC] 93 | 94 | # List of builtins function names that should not be used, separated by a comma 95 | bad-functions=map,filter 96 | 97 | # Good variable names which should always be accepted, separated by a comma 98 | good-names=i,j,k,ex,Run,_ 99 | 100 | # Bad variable names which should always be refused, separated by a comma 101 | bad-names=foo,bar,baz,toto,tutu,tata 102 | 103 | # Colon-delimited sets of names that determine each other's naming style when 104 | # the name regexes allow several styles. 105 | name-group= 106 | 107 | # Include a hint for the correct naming format with invalid-name 108 | include-naming-hint=no 109 | 110 | # Regular expression matching correct module names 111 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 112 | 113 | # Naming hint for module names 114 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 115 | 116 | # Regular expression matching correct class names 117 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 118 | 119 | # Naming hint for class names 120 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 121 | 122 | # Regular expression matching correct attribute names 123 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 124 | 125 | # Naming hint for attribute names 126 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 127 | 128 | # Regular expression matching correct inline iteration names 129 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 130 | 131 | # Naming hint for inline iteration names 132 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 133 | 134 | # Regular expression matching correct method names 135 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 136 | 137 | # Naming hint for method names 138 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 139 | 140 | # Regular expression matching correct argument names 141 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 142 | 143 | # Naming hint for argument names 144 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 145 | 146 | # Regular expression matching correct variable names 147 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 148 | 149 | # Naming hint for variable names 150 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 151 | 152 | # Regular expression matching correct function names 153 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 154 | 155 | # Naming hint for function names 156 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 157 | 158 | # Regular expression matching correct constant names 159 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 160 | 161 | # Naming hint for constant names 162 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 163 | 164 | # Regular expression matching correct class attribute names 165 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 166 | 167 | # Naming hint for class attribute names 168 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 169 | 170 | # Regular expression which should only match function or class names that do 171 | # not require a docstring. 172 | no-docstring-rgx=^_ 173 | 174 | # Minimum line length for functions/classes that require docstrings, shorter 175 | # ones are exempt. 176 | docstring-min-length=-1 177 | 178 | 179 | [ELIF] 180 | 181 | # Maximum number of nested blocks for function / method body 182 | max-nested-blocks=5 183 | 184 | 185 | [FORMAT] 186 | 187 | # Maximum number of characters on a single line. 188 | max-line-length=100 189 | 190 | # Regexp for a line that is allowed to be longer than the limit. 191 | ignore-long-lines=^\s*(# )??$ 192 | 193 | # Allow the body of an if to be on the same line as the test if there is no 194 | # else. 195 | single-line-if-stmt=no 196 | 197 | # List of optional constructs for which whitespace checking is disabled. `dict- 198 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 199 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 200 | # `empty-line` allows space-only lines. 201 | no-space-check=trailing-comma,dict-separator 202 | 203 | # Maximum number of lines in a module 204 | max-module-lines=1000 205 | 206 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 207 | # tab). 208 | indent-string=' ' 209 | 210 | # Number of spaces of indent required inside a hanging or continued line. 211 | indent-after-paren=4 212 | 213 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 214 | expected-line-ending-format= 215 | 216 | 217 | [LOGGING] 218 | 219 | # Logging modules to check that the string format arguments are in logging 220 | # function parameter format 221 | logging-modules=logging 222 | 223 | 224 | [MISCELLANEOUS] 225 | 226 | # List of note tags to take in consideration, separated by a comma. 227 | notes=FIXME,XXX,TODO 228 | 229 | 230 | [SIMILARITIES] 231 | 232 | # Minimum lines number of a similarity. 233 | min-similarity-lines=4 234 | 235 | # Ignore comments when computing similarities. 236 | ignore-comments=yes 237 | 238 | # Ignore docstrings when computing similarities. 239 | ignore-docstrings=yes 240 | 241 | # Ignore imports when computing similarities. 242 | ignore-imports=no 243 | 244 | 245 | [SPELLING] 246 | 247 | # Spelling dictionary name. Available dictionaries: none. To make it working 248 | # install python-enchant package. 249 | spelling-dict= 250 | 251 | # List of comma separated words that should not be checked. 252 | spelling-ignore-words= 253 | 254 | # A path to a file that contains private dictionary; one word per line. 255 | spelling-private-dict-file= 256 | 257 | # Tells whether to store unknown words to indicated private dictionary in 258 | # --spelling-private-dict-file option instead of raising a message. 259 | spelling-store-unknown-words=no 260 | 261 | 262 | [TYPECHECK] 263 | 264 | # Tells whether missing members accessed in mixin class should be ignored. A 265 | # mixin class is detected if its name ends with "mixin" (case insensitive). 266 | ignore-mixin-members=yes 267 | 268 | # List of module names for which member attributes should not be checked 269 | # (useful for modules/projects where namespaces are manipulated during runtime 270 | # and thus existing member attributes cannot be deduced by static analysis. It 271 | # supports qualified module names, as well as Unix pattern matching. 272 | ignored-modules= 273 | 274 | # List of classes names for which member attributes should not be checked 275 | # (useful for classes with attributes dynamically set). This supports can work 276 | # with qualified names. 277 | ignored-classes= 278 | 279 | # List of members which are set dynamically and missed by pylint inference 280 | # system, and so shouldn't trigger E1101 when accessed. Python regular 281 | # expressions are accepted. 282 | generated-members= 283 | 284 | 285 | [VARIABLES] 286 | 287 | # Tells whether we should check for unused import in __init__ files. 288 | init-import=no 289 | 290 | # A regular expression matching the name of dummy variables (i.e. expectedly 291 | # not used). 292 | dummy-variables-rgx=_$|dummy 293 | 294 | # List of additional names supposed to be defined in builtins. Remember that 295 | # you should avoid to define new builtins when possible. 296 | additional-builtins= 297 | 298 | # List of strings which can identify a callback function by name. A callback 299 | # name must start or end with one of those strings. 300 | callbacks=cb_,_cb 301 | 302 | 303 | [CLASSES] 304 | 305 | # List of method names used to declare (i.e. assign) instance attributes. 306 | defining-attr-methods=__init__,__new__,setUp 307 | 308 | # List of valid names for the first argument in a class method. 309 | valid-classmethod-first-arg=cls 310 | 311 | # List of valid names for the first argument in a metaclass class method. 312 | valid-metaclass-classmethod-first-arg=mcs 313 | 314 | # List of member names, which should be excluded from the protected access 315 | # warning. 316 | exclude-protected=_asdict,_fields,_replace,_source,_make 317 | 318 | 319 | [DESIGN] 320 | 321 | # Maximum number of arguments for function / method 322 | max-args=5 323 | 324 | # Argument names that match this expression will be ignored. Default to name 325 | # with leading underscore 326 | ignored-argument-names=_.* 327 | 328 | # Maximum number of locals for function / method body 329 | max-locals=15 330 | 331 | # Maximum number of return / yield for function / method body 332 | max-returns=6 333 | 334 | # Maximum number of branch for function / method body 335 | max-branches=12 336 | 337 | # Maximum number of statements in function / method body 338 | max-statements=50 339 | 340 | # Maximum number of parents for a class (see R0901). 341 | max-parents=7 342 | 343 | # Maximum number of attributes for a class (see R0902). 344 | max-attributes=7 345 | 346 | # Minimum number of public methods for a class (see R0903). 347 | min-public-methods=2 348 | 349 | # Maximum number of public methods for a class (see R0904). 350 | max-public-methods=20 351 | 352 | # Maximum number of boolean expressions in a if statement 353 | max-bool-expr=5 354 | 355 | 356 | [IMPORTS] 357 | 358 | # Deprecated modules which should not be used, separated by a comma 359 | deprecated-modules=optparse 360 | 361 | # Create a graph of every (i.e. internal and external) dependencies in the 362 | # given file (report RP0402 must not be disabled) 363 | import-graph= 364 | 365 | # Create a graph of external dependencies in the given file (report RP0402 must 366 | # not be disabled) 367 | ext-import-graph= 368 | 369 | # Create a graph of internal dependencies in the given file (report RP0402 must 370 | # not be disabled) 371 | int-import-graph= 372 | 373 | 374 | [EXCEPTIONS] 375 | 376 | # Exceptions that will emit a warning when being caught. Defaults to 377 | # "Exception" 378 | overgeneral-exceptions=Exception 379 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "peval" 3 | version = "0.2.0.dev" 4 | license = {text = "MIT"} 5 | authors = [{name = "Bogdan Opanchuk", email = "bogdan@opanchuk.net"}] 6 | description = "Partial evaluation on AST level" 7 | keywords = ["AST", "partial", "optimization"] 8 | readme = "README.md" 9 | classifiers = [ 10 | "Development Status :: 3 - Alpha", 11 | "Operating System :: OS Independent", 12 | "Intended Audience :: Science/Research", 13 | "Intended Audience :: Developers", 14 | "License :: OSI Approved :: MIT License", 15 | "Programming Language :: Python", 16 | "Programming Language :: Python :: 3", 17 | "Programming Language :: Python :: 3.8", 18 | "Programming Language :: Python :: 3.9", 19 | "Programming Language :: Python :: 3.10", 20 | "Programming Language :: Python :: 3.11", 21 | "Topic :: Software Development :: Libraries :: Python Modules", 22 | "Topic :: Software Development :: Code Generators", 23 | ] 24 | urls = {Homepage = "https://github.com/fjarri/peval"} 25 | requires-python = ">=3.8.0" 26 | dependencies = [ 27 | "typing-extensions>=4.2", 28 | ] 29 | 30 | [project.optional-dependencies] 31 | astunparse = [ 32 | "astunparse>=1.3", 33 | ] 34 | astor = [ 35 | "astor>=0.8", 36 | ] 37 | tests = [ 38 | "pytest>=7", 39 | "pytest-cov", 40 | ] 41 | docs = [ 42 | "sphinx>=4", 43 | "furo", 44 | "setuptools-scm>=7", 45 | ] 46 | lint = [ 47 | "mypy>=0.941", 48 | "black>=23", 49 | ] 50 | 51 | [build-system] 52 | requires = ["pdm-backend"] 53 | build-backend = "pdm.backend" 54 | 55 | [tool.pdm.build] 56 | source-includes = [ 57 | "tests/**/*.py", 58 | "docs/*.rst", 59 | "docs/*.py", 60 | "docs/*.bat", 61 | "docs/Makefile", 62 | ".coveragerc", 63 | "mypy.ini", 64 | ] 65 | 66 | [tool.setuptools_scm] 67 | 68 | [tool.black] 69 | line-length = 100 70 | target-version = ['py38'] 71 | -------------------------------------------------------------------------------- /tests/test_components/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjarri/peval/e6b5d8c1be9b2da167eb27975aa70aacdb4eb660/tests/test_components/__init__.py -------------------------------------------------------------------------------- /tests/test_components/test_fold.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from peval.components import fold 6 | from peval import pure 7 | 8 | from utils import check_component, function_from_source 9 | 10 | 11 | def dummy(x): 12 | a = 1 13 | if a > 2: 14 | b = 3 15 | c = 4 + 6 16 | else: 17 | b = 2 18 | c = 3 + a 19 | return a + b + c + x 20 | 21 | 22 | def test_fold(): 23 | check_component( 24 | fold, 25 | dummy, 26 | expected_source=""" 27 | def dummy(x): 28 | a = 1 29 | if False: 30 | b = 3 31 | c = 10 32 | else: 33 | b = 2 34 | c = 4 35 | return 1 + b + c + x 36 | """, 37 | ) 38 | 39 | 40 | def test_if_visit_only_true_branch(): 41 | # This optimization can potentially save some time during constant propagation 42 | # (by not evaluating the functions that will be eliminated anyway). 43 | # Not implemented at the moment. 44 | 45 | pytest.xfail() 46 | 47 | global_state = dict(cnt=0) 48 | 49 | @pure 50 | def inc(): 51 | global_state["cnt"] += 1 52 | return True 53 | 54 | def if_body(): 55 | if a: 56 | inc() 57 | 58 | def if_else(): 59 | if a: 60 | dec() 61 | else: 62 | inc() 63 | 64 | check_component( 65 | fold, 66 | if_body, 67 | additional_bindings=dict(a=False, inc=inc), 68 | expected_source=""" 69 | def if_body(): 70 | if False: 71 | inc() 72 | """, 73 | ) 74 | assert global_state["cnt"] == 0 75 | 76 | check_component( 77 | fold, 78 | if_else, 79 | additional_bindings=dict(a=False, inc=inc), 80 | expected_source=""" 81 | def if_else(): 82 | if False: 83 | dec() 84 | else: 85 | inc() 86 | """, 87 | ) 88 | assert global_state["cnt"] == 1 89 | 90 | 91 | @pure 92 | def int32(): 93 | return int 94 | 95 | 96 | def test_variable_annotation(): 97 | if sys.version_info < (3, 6): 98 | pytest.skip() 99 | 100 | func_annotations = function_from_source( 101 | """ 102 | def func_annotations(): 103 | x = int 104 | a: x 105 | x = float 106 | b: x 107 | c: int32() 108 | """, 109 | globals_=dict(int32=int32), 110 | ).eval() 111 | 112 | check_component( 113 | fold, 114 | func_annotations, 115 | expected_source=""" 116 | def func_annotations(): 117 | x = int 118 | a: __peval_temp_1 119 | x = float 120 | b: __peval_temp_2 121 | c: __peval_temp_3 122 | """, 123 | expected_new_bindings=dict(__peval_temp_1=int, __peval_temp_2=float, __peval_temp_3=int), 124 | ) 125 | -------------------------------------------------------------------------------- /tests/test_components/test_inline.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | 3 | import ast 4 | import sys 5 | 6 | import pytest 7 | 8 | from peval.core.gensym import GenSym 9 | from peval.tags import inline 10 | from peval.components.inline import ( 11 | inline_functions, 12 | _replace_returns, 13 | _wrap_in_loop, 14 | _build_parameter_assignments, 15 | ) 16 | 17 | from utils import check_component, unindent, assert_ast_equal 18 | 19 | 20 | def _test_replace_returns(source, expected_source, expected_returns_ctr, expected_returns_in_loops): 21 | nodes = ast.parse(unindent(source)).body 22 | 23 | return_var = "return_var" 24 | return_flag_var = "return_flag" 25 | 26 | expected_source = expected_source.format(return_var=return_var, return_flag=return_flag_var) 27 | expected_nodes = ast.parse(unindent(expected_source)).body 28 | 29 | new_nodes, returns_ctr, returns_in_loops = _replace_returns(nodes, return_var, return_flag_var) 30 | 31 | assert_ast_equal(new_nodes, expected_nodes) 32 | assert returns_ctr == expected_returns_ctr 33 | assert returns_in_loops == expected_returns_in_loops 34 | 35 | 36 | class TestReplaceReturns: 37 | def test_single_return(self): 38 | _test_replace_returns( 39 | source=""" 40 | b = y + list(x) 41 | return b 42 | """, 43 | expected_source=""" 44 | b = y + list(x) 45 | {return_var} = b 46 | break 47 | """, 48 | expected_returns_ctr=1, 49 | expected_returns_in_loops=False, 50 | ) 51 | 52 | def test_several_returns(self): 53 | _test_replace_returns( 54 | source=""" 55 | if a: 56 | return y + list(x) 57 | elif b: 58 | return b 59 | return c 60 | """, 61 | expected_source=""" 62 | if a: 63 | {return_var} = y + list(x) 64 | break 65 | elif b: 66 | {return_var} = b 67 | break 68 | {return_var} = c 69 | break 70 | """, 71 | expected_returns_ctr=3, 72 | expected_returns_in_loops=False, 73 | ) 74 | 75 | def test_returns_in_loops(self): 76 | _test_replace_returns( 77 | source=""" 78 | for x in range(10): 79 | for y in range(10): 80 | if x + y > 10: 81 | return 2 82 | else: 83 | return 3 84 | 85 | if x: 86 | return 1 87 | 88 | while z: 89 | if z: 90 | return 3 91 | 92 | return 0 93 | """, 94 | expected_source=""" 95 | for x in range(10): 96 | for y in range(10): 97 | if ((x + y) > 10): 98 | {return_var} = 2 99 | {return_flag} = True 100 | break 101 | else: 102 | {return_var} = 3 103 | {return_flag} = True 104 | break 105 | if {return_flag}: 106 | break 107 | if {return_flag}: 108 | break 109 | if x: 110 | {return_var} = 1 111 | break 112 | while z: 113 | if z: 114 | {return_var} = 3 115 | {return_flag} = True 116 | break 117 | if {return_flag}: 118 | break 119 | {return_var} = 0 120 | break 121 | """, 122 | expected_returns_ctr=5, 123 | expected_returns_in_loops=True, 124 | ) 125 | 126 | def test_returns_in_loop_else(self): 127 | _test_replace_returns( 128 | source=""" 129 | for y in range(10): 130 | x += y 131 | else: 132 | return 1 133 | 134 | return 0 135 | """, 136 | expected_source=""" 137 | for y in range(10): 138 | x += y 139 | else: 140 | {return_var} = 1 141 | break 142 | 143 | {return_var} = 0 144 | break 145 | """, 146 | expected_returns_ctr=2, 147 | expected_returns_in_loops=False, 148 | ) 149 | 150 | 151 | def _test_build_parameter_assignments(call_str, signature_str, expected_assignments): 152 | call_node = ast.parse("func(" + call_str + ")").body[0].value 153 | signature_node = ast.parse("def func(" + signature_str + "):\n\tpass").body[0] 154 | 155 | assignments = _build_parameter_assignments(call_node, signature_node) 156 | 157 | expected_assignments = ast.parse(unindent(expected_assignments)).body 158 | 159 | assert_ast_equal(assignments, expected_assignments) 160 | 161 | 162 | class TestBuildParameterAssignments: 163 | def test_positional_args(self): 164 | _test_build_parameter_assignments( 165 | "a, b, 1, 3", 166 | "c, d, e, f", 167 | """ 168 | c = a 169 | d = b 170 | e = 1 171 | f = 3 172 | """, 173 | ) 174 | 175 | 176 | def _test_wrap_in_loop(body_src, expected_src, format_kwds={}, expected_bindings={}): 177 | gen_sym = GenSym.for_tree() 178 | 179 | body_nodes = ast.parse(unindent(body_src)).body 180 | 181 | return_name = "_return_val" 182 | 183 | gen_sym, inlined_body, new_bindings = _wrap_in_loop(gen_sym, body_nodes, return_name) 184 | 185 | expected_body = ast.parse( 186 | unindent(expected_src.format(return_val=return_name, **format_kwds)) 187 | ).body 188 | 189 | assert_ast_equal(inlined_body, expected_body) 190 | 191 | assert new_bindings == expected_bindings 192 | 193 | 194 | class TestWrapInLoop: 195 | def test_no_return(self): 196 | _test_wrap_in_loop( 197 | """ 198 | do_something() 199 | do_something_else() 200 | """, 201 | """ 202 | do_something() 203 | do_something_else() 204 | {return_val} = None 205 | """, 206 | ) 207 | 208 | def test_single_return(self): 209 | _test_wrap_in_loop( 210 | """ 211 | do_something() 212 | do_something_else() 213 | return 1 214 | """, 215 | """ 216 | do_something() 217 | do_something_else() 218 | {return_val} = 1 219 | """, 220 | ) 221 | 222 | def test_several_returns(self): 223 | _test_wrap_in_loop( 224 | """ 225 | if a > 4: 226 | do_something() 227 | return 2 228 | do_something_else() 229 | return 1 230 | """, 231 | """ 232 | while True: 233 | if a > 4: 234 | do_something() 235 | {return_val} = 2 236 | break 237 | do_something_else() 238 | {return_val} = 1 239 | break 240 | """, 241 | ) 242 | 243 | def test_returns_in_loops(self): 244 | _test_wrap_in_loop( 245 | """ 246 | for x in range(10): 247 | do_something() 248 | if b: 249 | return 2 250 | do_something_else() 251 | return 1 252 | """, 253 | """ 254 | {return_flag} = False 255 | while True: 256 | for x in range(10): 257 | do_something() 258 | if b: 259 | {return_val} = 2 260 | {return_flag} = True 261 | break 262 | if {return_flag}: 263 | break 264 | do_something_else() 265 | {return_val} = 1 266 | break 267 | """, 268 | dict(return_flag="__peval_return_flag_1"), 269 | ) 270 | 271 | 272 | def test_component(): 273 | @inline 274 | def inlined(y): 275 | l = [] 276 | for _ in range(y): 277 | l.append(y.do_stuff()) 278 | return l 279 | 280 | def outer(x): 281 | a = x.foo() 282 | if a: 283 | b = a * 10 284 | a = b + inlined(x) 285 | return a 286 | 287 | check_component( 288 | inline_functions, 289 | outer, 290 | expected_source=""" 291 | def outer(x): 292 | a = x.foo() 293 | if a: 294 | b = (a * 10) 295 | __peval_mangled_1 = x 296 | __peval_mangled_2 = [] 297 | for __peval_mangled_3 in range(__peval_mangled_1): 298 | __peval_mangled_2.append(__peval_mangled_1.do_stuff()) 299 | __peval_return_1 = __peval_mangled_2 300 | a = (b + __peval_return_1) 301 | return a 302 | """, 303 | ) 304 | -------------------------------------------------------------------------------- /tests/test_components/test_peval_function_header.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from peval.tags import pure 4 | from peval.components import peval_function_header 5 | 6 | from utils import check_component 7 | 8 | 9 | class Dummy: 10 | tp1 = int 11 | 12 | 13 | @pure 14 | def get_type(): 15 | return str 16 | 17 | 18 | def dummy(x: int, y: Dummy.tp1 = "aaa") -> get_type(): 19 | pass 20 | 21 | 22 | def test_peval_function_header(): 23 | check_component( 24 | peval_function_header, 25 | dummy, 26 | expected_source=""" 27 | def dummy(x: int, y:__peval_temp_1='aaa') -> __peval_temp_2: 28 | pass 29 | """, 30 | expected_new_bindings=dict(__peval_temp_1=Dummy.tp1, __peval_temp_2=get_type()), 31 | ) 32 | 33 | 34 | @pure 35 | def make_annotation(p): 36 | return str(p + 1) 37 | 38 | 39 | @pytest.mark.parametrize("str_annotation", [False, True], ids=["ast_annotation", "str_annotation"]) 40 | def test_peval_annotations(str_annotation): 41 | if str_annotation: 42 | 43 | def dummy_annotations(x: "make_annotation(1)"): 44 | pass 45 | 46 | else: 47 | 48 | def dummy_annotations(x: make_annotation(1)): 49 | pass 50 | 51 | check_component( 52 | peval_function_header, 53 | dummy_annotations, 54 | expected_source=""" 55 | def dummy_annotations(x: "2"): 56 | pass 57 | """, 58 | ) 59 | -------------------------------------------------------------------------------- /tests/test_components/test_prune_cfg.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import pytest 4 | 5 | from peval.components.prune_cfg import prune_cfg 6 | 7 | from utils import check_component 8 | 9 | 10 | def test_if_true(): 11 | """ 12 | Eliminate if test, if the value is known at compile time 13 | """ 14 | 15 | true_values = [True, 1, 2.0, object(), "foo", int] 16 | assert all(true_values) 17 | 18 | def f_if(): 19 | if x: 20 | print("x is True") 21 | 22 | for x in true_values: 23 | check_component( 24 | prune_cfg, 25 | f_if, 26 | additional_bindings=dict(x=x), 27 | expected_source=""" 28 | def f_if(): 29 | print('x is True') 30 | """, 31 | ) 32 | 33 | def f_if_else(): 34 | if x: 35 | print("x is True") 36 | else: 37 | print("x is False") 38 | 39 | check_component( 40 | prune_cfg, 41 | f_if_else, 42 | additional_bindings=dict(x=2), 43 | expected_source=""" 44 | def f_if_else(): 45 | print("x is True") 46 | """, 47 | ) 48 | 49 | 50 | def test_if_false_elimination(): 51 | """ 52 | Eliminate if test, when test is false 53 | """ 54 | 55 | class Falsy: 56 | def __bool__(self): 57 | # For Python 3 58 | return False 59 | 60 | false_values = [0, "", [], {}, set(), False, None, Falsy()] 61 | assert not any(false_values) 62 | 63 | def f_if(): 64 | if x: 65 | print("x is True") 66 | 67 | for x in false_values: 68 | check_component( 69 | prune_cfg, 70 | f_if, 71 | additional_bindings=dict(x=x), 72 | expected_source=""" 73 | def f_if(): 74 | pass 75 | """, 76 | ) 77 | 78 | def f_if_else(): 79 | if x: 80 | print("x is True") 81 | else: 82 | print("x is False") 83 | 84 | check_component( 85 | prune_cfg, 86 | f_if_else, 87 | additional_bindings=dict(x=False), 88 | expected_source=""" 89 | def f_if_else(): 90 | print("x is False") 91 | """, 92 | ) 93 | 94 | 95 | def test_if_no_elimination(): 96 | """ 97 | Test that there is no unneeded elimination of if test 98 | """ 99 | 100 | def f(x): 101 | if x: 102 | a = 1 103 | else: 104 | a = 2 105 | 106 | check_component(prune_cfg, f, dict(y=2)) 107 | 108 | 109 | def test_visit_all_branches(): 110 | def f(): 111 | if x > 0: 112 | if True: 113 | x += 1 114 | else: 115 | if False: 116 | return 0 117 | 118 | check_component( 119 | prune_cfg, 120 | f, 121 | {}, 122 | expected_source=""" 123 | def f(): 124 | if x > 0: 125 | x += 1 126 | else: 127 | pass 128 | """, 129 | ) 130 | 131 | 132 | def test_remove_pass(): 133 | def f(x): 134 | x += 1 135 | pass 136 | x += 1 137 | 138 | check_component( 139 | prune_cfg, 140 | f, 141 | {}, 142 | expected_source=""" 143 | def f(x): 144 | x += 1 145 | x += 1 146 | """, 147 | ) 148 | 149 | 150 | def test_not_remove_pass(): 151 | def f(x): 152 | pass 153 | 154 | check_component(prune_cfg, f, {}) 155 | 156 | 157 | def test_remove_code_after_jump(): 158 | def f(x): 159 | x += 1 160 | return x 161 | x += 1 162 | 163 | check_component( 164 | prune_cfg, 165 | f, 166 | {}, 167 | expected_source=""" 168 | def f(x): 169 | x += 1 170 | return x 171 | """, 172 | ) 173 | 174 | 175 | def test_not_simplify_while(): 176 | def f(x): 177 | while x > 1: 178 | x += 1 179 | else: 180 | x = 10 181 | 182 | check_component(prune_cfg, f, {}) 183 | 184 | 185 | def test_simplify_while(): 186 | def f(x): 187 | while x > 1: 188 | x += 1 189 | raise Exception 190 | else: 191 | x = 10 192 | 193 | check_component( 194 | prune_cfg, 195 | f, 196 | {}, 197 | expected_source=""" 198 | def f(x): 199 | if x > 1: 200 | x += 1 201 | raise Exception 202 | else: 203 | x = 10 204 | """, 205 | ) 206 | 207 | 208 | def test_simplify_while_with_break(): 209 | def f(x): 210 | while x > 1: 211 | x += 1 212 | break 213 | else: 214 | x = 10 215 | 216 | check_component( 217 | prune_cfg, 218 | f, 219 | {}, 220 | expected_source=""" 221 | def f(x): 222 | if x > 1: 223 | x += 1 224 | else: 225 | x = 10 226 | """, 227 | ) 228 | -------------------------------------------------------------------------------- /tests/test_core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjarri/peval/e6b5d8c1be9b2da167eb27975aa70aacdb4eb660/tests/test_core/__init__.py -------------------------------------------------------------------------------- /tests/test_core/test_cfg.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import inspect 3 | import os, os.path 4 | import subprocess 5 | import sys 6 | 7 | from peval.tools import unparse 8 | from peval.core.cfg import build_cfg 9 | 10 | from utils import print_diff, unparser 11 | 12 | RENDER_GRAPHS = False 13 | 14 | 15 | def _if_expr(a, b): 16 | if unparser() == "astunparse": 17 | return f"if ({a} > {b}):" 18 | else: 19 | return f"if {a} > {b}:" 20 | 21 | 22 | def make_label(node): 23 | return unparse(node.ast_node).strip().split("\n")[0] 24 | 25 | 26 | def get_edges(cfg): 27 | edges = [] 28 | for node_id in cfg.graph.nodes: 29 | for child_id in cfg.graph.children_of(node_id): 30 | edges.append((node_id, child_id)) 31 | return edges 32 | 33 | 34 | def get_labeled_edges(cfg): 35 | edges = [] 36 | todo_list = [cfg.enter] 37 | visited = set() 38 | 39 | while len(todo_list) > 0: 40 | src_id = todo_list.pop() 41 | if src_id in visited: 42 | continue 43 | visited.add(src_id) 44 | 45 | src_label = make_label(cfg.graph.nodes[src_id]) 46 | 47 | dests = [ 48 | (dest_id, make_label(cfg.graph.nodes[dest_id])) 49 | for dest_id in cfg.graph.children_of(src_id) 50 | ] 51 | dests = sorted(dests, key=lambda pair: pair[1]) 52 | 53 | for dest_id, dest_label in dests: 54 | edges.append((src_label, dest_label)) 55 | todo_list.append(dest_id) 56 | 57 | return edges 58 | 59 | 60 | def render_cfg(cfg, fname): 61 | node_str = lambda node_id, label: ( 62 | ' {node_id} [label="{label}"]'.format(node_id=node_id, label=label) 63 | ) 64 | edge_str = lambda node1, node2: " {node1} -> {node2}".format(node1=node1, node2=node2) 65 | 66 | directives = [] 67 | 68 | directives.append(node_str("enter", "enter")) 69 | directives.append(node_str("exit", "exit")) 70 | 71 | for node_id, node in cfg.graph.nodes.items(): 72 | directives.append(node_str(node_id, make_label(node))) 73 | 74 | directives.append(edge_str("enter", cfg.enter)) 75 | for exit in cfg.exits: 76 | directives.append(edge_str(exit, "exit")) 77 | 78 | for node_id in cfg.graph.nodes: 79 | for child_id in cfg.graph.children_of(node_id): 80 | directives.append(edge_str(node_id, child_id)) 81 | 82 | base, ext = os.path.splitext(os.path.abspath(fname)) 83 | dotfile = base + ".dot" 84 | 85 | with open(dotfile, "w") as f: 86 | f.write("\n".join(["strict digraph {", ' node [label="\\N"];'] + directives + ["}"])) 87 | 88 | picfile = base + ext 89 | subprocess.check_call(["dot", "-T" + ext[1:], dotfile, "-o", picfile]) 90 | os.remove(dotfile) 91 | 92 | 93 | def get_body(function): 94 | src = inspect.getsource(function) 95 | return ast.parse(src).body[0].body 96 | 97 | 98 | def assert_labels_equal(cfg, expected_edges, expected_exits, expected_raises): 99 | test_edges = get_labeled_edges(cfg) 100 | 101 | expected_exits = list(sorted(expected_exits)) 102 | test_exits = list(sorted([make_label(cfg.graph.nodes[exit_id]) for exit_id in cfg.exits])) 103 | 104 | assert expected_exits == test_exits 105 | 106 | expected_raises = list(sorted(expected_raises)) 107 | test_raises = list(sorted([make_label(cfg.graph.nodes[exit_id]) for exit_id in cfg.raises])) 108 | 109 | assert expected_raises == test_raises 110 | 111 | equal = test_edges == expected_edges 112 | if not equal: 113 | make_str = lambda edges: "\n".join(src + " --> " + dest for src, dest in edges) 114 | test_str = make_str(test_edges) 115 | expected_str = make_str(expected_edges) 116 | print_diff(test_str, expected_str) 117 | assert equal 118 | 119 | 120 | def check_cfg(function, expected_edges, expected_exits, expected_raises): 121 | statements = get_body(function) 122 | cfg = build_cfg(statements) 123 | 124 | assert_labels_equal(cfg, expected_edges, expected_exits, expected_raises) 125 | 126 | if RENDER_GRAPHS: 127 | render_cfg(cfg, "test_" + function.__name__ + ".pdf") 128 | 129 | 130 | def func_if(): 131 | a = 1 132 | b = 2 133 | if a > 2: 134 | do_stuff() 135 | do_smth_else() 136 | return 3 137 | elif a > 4: 138 | foo() 139 | else: 140 | bar() 141 | 142 | return b 143 | 144 | 145 | def test_func_if(): 146 | check_cfg( 147 | func_if, 148 | expected_edges=[ 149 | ("a = 1", "b = 2"), 150 | ("b = 2", _if_expr("a", 2)), 151 | (_if_expr("a", 2), "do_stuff()"), 152 | (_if_expr("a", 2), _if_expr("a", 4)), 153 | (_if_expr("a", 4), "bar()"), 154 | (_if_expr("a", 4), "foo()"), 155 | ("foo()", "return b"), 156 | ("bar()", "return b"), 157 | ("do_stuff()", "do_smth_else()"), 158 | ("do_smth_else()", "return 3"), 159 | ], 160 | expected_exits=["return 3", "return b"], 161 | expected_raises=[], 162 | ) 163 | 164 | 165 | def func_for(): 166 | a = 1 167 | 168 | for i in range(5): 169 | b = 2 170 | if i > 4: 171 | break 172 | elif i > 2: 173 | continue 174 | else: 175 | foo() 176 | else: 177 | c = 3 178 | 179 | return b 180 | 181 | 182 | def test_func_for(): 183 | check_cfg( 184 | func_for, 185 | expected_edges=[ 186 | ("a = 1", "for i in range(5):"), 187 | ("for i in range(5):", "b = 2"), 188 | ("b = 2", _if_expr("i", 4)), 189 | (_if_expr("i", 4), "break"), 190 | (_if_expr("i", 4), _if_expr("i", 2)), 191 | (_if_expr("i", 2), "continue"), 192 | (_if_expr("i", 2), "foo()"), 193 | ("foo()", "c = 3"), 194 | ("foo()", "for i in range(5):"), 195 | ("c = 3", "return b"), 196 | ("continue", "for i in range(5):"), 197 | ("break", "return b"), 198 | ], 199 | expected_exits=["return b"], 200 | expected_raises=[], 201 | ) 202 | 203 | 204 | def func_try_except(): 205 | a = 1 206 | 207 | for i in range(5): 208 | try: 209 | do() 210 | if i > 3: 211 | break 212 | stuff() 213 | except Exception: 214 | foo() 215 | except ValueError: 216 | bar() 217 | else: 218 | do_else() 219 | 220 | return b 221 | 222 | 223 | def test_func_try_except(): 224 | check_cfg( 225 | func_try_except, 226 | expected_edges=[ 227 | ("a = 1", "for i in range(5):"), 228 | ("for i in range(5):", "try:"), 229 | ("try:", "do()"), 230 | ("do()", "except Exception:"), 231 | ("do()", "except ValueError:"), 232 | ("do()", _if_expr("i", 3)), 233 | (_if_expr("i", 3), "break"), 234 | (_if_expr("i", 3), "except Exception:"), 235 | (_if_expr("i", 3), "except ValueError:"), 236 | (_if_expr("i", 3), "stuff()"), 237 | ("stuff()", "do_else()"), 238 | ("stuff()", "except Exception:"), 239 | ("stuff()", "except ValueError:"), 240 | ("except ValueError:", "bar()"), 241 | ("bar()", "for i in range(5):"), 242 | ("bar()", "return b"), 243 | ("except Exception:", "foo()"), 244 | ("foo()", "for i in range(5):"), 245 | ("foo()", "return b"), 246 | ("do_else()", "for i in range(5):"), 247 | ("do_else()", "return b"), 248 | ("break", "return b"), 249 | ], 250 | expected_exits=["return b"], 251 | expected_raises=[], 252 | ) 253 | 254 | 255 | def func_try_finally(): 256 | a = 1 257 | 258 | try: 259 | do() 260 | stuff() 261 | return c 262 | finally: 263 | do_finally() 264 | 265 | return b 266 | 267 | 268 | def test_func_try_finally(): 269 | check_cfg( 270 | func_try_finally, 271 | expected_edges=[ 272 | ("a = 1", "try:"), 273 | ("try:", "do()"), 274 | ("do()", "do_finally()"), 275 | ("do()", "stuff()"), 276 | ("stuff()", "do_finally()"), 277 | ("stuff()", "return c"), 278 | ("return c", "do_finally()"), 279 | ("do_finally()", "return b"), 280 | ], 281 | expected_exits=["do_finally()", "return b"], 282 | expected_raises=["do_finally()"], 283 | ) 284 | 285 | 286 | def func_try_except_finally(): 287 | a = 1 288 | 289 | try: 290 | do() 291 | stuff() 292 | return c 293 | except Exception: 294 | foo() 295 | finally: 296 | do_finally() 297 | 298 | return b 299 | 300 | 301 | def test_func_try_except_finally(): 302 | check_cfg( 303 | func_try_except_finally, 304 | expected_edges=[ 305 | ("a = 1", "try:"), 306 | ("try:", "do()"), 307 | ("do()", "except Exception:"), 308 | ("do()", "stuff()"), 309 | ("stuff()", "except Exception:"), 310 | ("stuff()", "return c"), 311 | ("return c", "do_finally()"), 312 | ("return c", "except Exception:"), 313 | ("except Exception:", "foo()"), 314 | ("foo()", "do_finally()"), 315 | ("do_finally()", "return b"), 316 | ], 317 | expected_exits=["do_finally()", "return b"], 318 | expected_raises=[], 319 | ) 320 | 321 | 322 | def func_try_except_else_finally(): 323 | a = 1 324 | 325 | try: 326 | do() 327 | stuff() 328 | except Exception: 329 | foo() 330 | else: 331 | do_else() 332 | finally: 333 | do_finally() 334 | 335 | return b 336 | 337 | 338 | def test_func_try_except_else_finally(): 339 | check_cfg( 340 | func_try_except_else_finally, 341 | expected_edges=[ 342 | ("a = 1", "try:"), 343 | ("try:", "do()"), 344 | ("do()", "except Exception:"), 345 | ("do()", "stuff()"), 346 | ("stuff()", "do_else()"), 347 | ("stuff()", "except Exception:"), 348 | ("except Exception:", "foo()"), 349 | ("foo()", "do_finally()"), 350 | ("do_finally()", "return b"), 351 | ("do_else()", "do_finally()"), 352 | ], 353 | expected_exits=["return b"], 354 | expected_raises=[], 355 | ) 356 | -------------------------------------------------------------------------------- /tests/test_core/test_function.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import copy 3 | import sys 4 | import inspect 5 | 6 | from peval.core.function import Function 7 | from peval.tools import unindent 8 | 9 | from utils import normalize_source, function_from_source, unparser 10 | 11 | 12 | global_var = 1 13 | 14 | 15 | def global_var_writer(x): 16 | global global_var 17 | global_var = x 18 | 19 | 20 | def global_var_reader(): 21 | return global_var 22 | 23 | 24 | def make_one_var_closure(): 25 | closure_var = [1] 26 | 27 | def closure(): 28 | a = global_var 29 | some_local_var = 3 30 | closure_var[0] += 1 31 | return closure_var[0] 32 | 33 | return closure 34 | 35 | 36 | def make_two_var_closure(): 37 | closure_var1 = [1] 38 | closure_var2 = [2] 39 | 40 | def closure(): 41 | some_local_var = closure_var1[0] 42 | closure_var2[0] += 1 43 | return closure_var2[0] 44 | 45 | return closure 46 | 47 | 48 | def dummy_func(a, b, c=4, d=5): 49 | return a, b, c, d 50 | 51 | 52 | def dummy_func_arg_groups(a, b, *args, **kwds): 53 | return a, b, args, kwds 54 | 55 | 56 | def test_bind_partial_args(): 57 | func = Function.from_object(dummy_func) 58 | 59 | new_func = func.bind_partial(1).eval() 60 | sig = inspect.signature(new_func) 61 | 62 | assert new_func(2, 3, 4) == (1, 2, 3, 4) 63 | assert "a" not in sig.parameters 64 | assert "b" in sig.parameters 65 | assert "c" in sig.parameters 66 | assert "d" in sig.parameters 67 | 68 | 69 | def test_bind_partial_kwds(): 70 | func = Function.from_object(dummy_func) 71 | 72 | new_func = func.bind_partial(1, d=10).eval() 73 | sig = inspect.signature(new_func) 74 | 75 | assert new_func(2, 3) == (1, 2, 3, 10) 76 | assert "a" not in sig.parameters 77 | assert "b" in sig.parameters 78 | assert "c" in sig.parameters 79 | assert "d" not in sig.parameters 80 | 81 | 82 | def test_bind_partial_varargs(): 83 | func = Function.from_object(dummy_func_arg_groups) 84 | 85 | new_func = func.bind_partial(1, 2, 3).eval() 86 | sig = inspect.signature(new_func) 87 | 88 | assert new_func(d=10) == (1, 2, (3,), {"d": 10}) 89 | assert "a" not in sig.parameters 90 | assert "b" not in sig.parameters 91 | assert "args" not in sig.parameters 92 | assert "kwds" in sig.parameters 93 | 94 | 95 | def test_bind_partial_varkwds(): 96 | func = Function.from_object(dummy_func_arg_groups) 97 | 98 | new_func = func.bind_partial(1, 2, d=10).eval() 99 | sig = inspect.signature(new_func) 100 | 101 | assert new_func(3, 4) == (1, 2, (3, 4), {"d": 10}) 102 | assert "a" not in sig.parameters 103 | assert "b" not in sig.parameters 104 | assert "args" in sig.parameters 105 | assert "kwds" not in sig.parameters 106 | 107 | 108 | def test_globals_contents(): 109 | func = Function.from_object(make_one_var_closure()) 110 | 111 | assert "global_var" in func.globals 112 | assert "closure_var" not in func.globals 113 | 114 | 115 | def test_closure_contents(): 116 | func = Function.from_object(make_one_var_closure()) 117 | 118 | assert "global_var" not in func.closure_vals 119 | assert "closure_var" in func.closure_vals 120 | 121 | 122 | def test_copy_globals(): 123 | """ 124 | Checks that a restored function does not refer to the same globals dictionary, 125 | but a copy of it. Therefore it cannot reassign global values. 126 | """ 127 | 128 | global_var_writer(10) 129 | assert global_var_reader() == 10 130 | assert global_var == 10 131 | 132 | reader = Function.from_object(global_var_reader).eval() 133 | writer = Function.from_object(global_var_writer).eval() 134 | 135 | writer(20) 136 | assert reader() == 10 137 | assert global_var == 10 138 | 139 | 140 | def test_restore_simple_closure(): 141 | closure_ref = make_one_var_closure() 142 | assert closure_ref() == 2 143 | 144 | closure = Function.from_object(closure_ref).eval() 145 | assert closure() == 3 146 | assert closure_ref() == 4 147 | 148 | 149 | def test_restore_modified_closure(): 150 | def remove_first_line(node): 151 | assert isinstance(node, ast.FunctionDef) 152 | node = copy.deepcopy(node) 153 | node.body = node.body[1:] 154 | return node 155 | 156 | closure_ref = make_two_var_closure() 157 | assert closure_ref() == 3 158 | 159 | closure = Function.from_object(closure_ref) 160 | closure = closure.replace(tree=remove_first_line(closure.tree)).eval() 161 | 162 | assert closure() == 4 163 | assert closure_ref() == 5 164 | 165 | 166 | def recursive_outer(x): 167 | if x > 1: 168 | return recursive_outer(x - 1) 169 | else: 170 | return x 171 | 172 | 173 | def make_recursive(): 174 | def recursive_inner(x): 175 | if x > 2: 176 | return recursive_inner(x - 1) 177 | else: 178 | return x 179 | 180 | return recursive_inner 181 | 182 | 183 | def test_recursive_call(): 184 | # When evaluated inside a fake closure (to detect closure variables), 185 | # the function name will be included in the list of closure variables 186 | # (if it is used in the body of the function). 187 | # So if the function was not a closure to begin with, 188 | # the corresponding cell will be missing. 189 | # This tests checks that Function evaluates non-closure functions 190 | # without a fake closure to prevent that. 191 | 192 | func = Function.from_object(recursive_outer) 193 | func = func.replace() 194 | func = func.eval() 195 | assert func(10) == 1 196 | 197 | func = Function.from_object(make_recursive()) 198 | func = func.replace() 199 | func = func.eval() 200 | assert func(10) == 2 201 | 202 | 203 | def test_construct_from_eval(): 204 | # Test that the function returned from Function.eval() 205 | # can be used to construct a new Function object. 206 | func = Function.from_object(dummy_func).eval() 207 | func2 = Function.from_object(func).eval() 208 | assert func2(1, 2, c=10) == (1, 2, 10, 5) 209 | 210 | 211 | # The decorator must be in the global namespace 212 | # otherwise the current implementation of ``from_object()`` will fail 213 | def tag(f): 214 | vars(f)["_tag"] = True 215 | return f 216 | 217 | 218 | def test_reapply_decorators(): 219 | @tag 220 | def tagged(x): 221 | return x 222 | 223 | func = Function.from_object(tagged).eval() 224 | 225 | assert "_tag" in vars(func) and vars(func)["_tag"] 226 | 227 | 228 | def test_detect_future_features(): 229 | # Test that the presence of a future feature is detected 230 | 231 | src = """ 232 | from __future__ import generator_stop 233 | def f(): 234 | pass 235 | """ 236 | 237 | func = function_from_source(src) 238 | assert func.future_features.generator_stop 239 | 240 | # Test that the absence of a future feature is detected 241 | 242 | src = """ 243 | def f(): 244 | pass 245 | """ 246 | 247 | func = function_from_source(src) 248 | if sys.version_info >= (3, 7): 249 | assert func.future_features.generator_stop 250 | else: 251 | assert not func.future_features.generator_stop 252 | 253 | 254 | def test_preserve_future_feature_presence(): 255 | src = """ 256 | from __future__ import generator_stop 257 | def f(): 258 | error = lambda: next(i for i in range(3) if i==10) 259 | try: 260 | return all(error() for i in range(2)) 261 | except RuntimeError: 262 | return False 263 | """ 264 | 265 | func = function_from_source(src) 266 | new_func_obj = func.eval() 267 | new_func = Function.from_object(new_func_obj) 268 | 269 | assert new_func.future_features.generator_stop 270 | assert new_func_obj() == False 271 | 272 | 273 | def test_preserve_future_feature_absence(): 274 | src = """ 275 | def f(): 276 | error = lambda: next(i for i in range(3) if i==10) 277 | try: 278 | return all(error() for i in range(2)) 279 | except RuntimeError: 280 | return False 281 | """ 282 | 283 | func = function_from_source(src) 284 | new_func_obj = func.eval() 285 | new_func = Function.from_object(new_func_obj) 286 | 287 | if sys.version_info >= (3, 7): 288 | assert new_func.future_features.generator_stop 289 | assert new_func_obj() == False 290 | else: 291 | assert not new_func.future_features.generator_stop 292 | assert new_func_obj() == True 293 | 294 | 295 | def test_compile_ast(): 296 | function = Function.from_object(sample_fn) 297 | compiled_fn = function.eval() 298 | assert compiled_fn(3, -9) == sample_fn(3, -9) 299 | assert compiled_fn(3, -9, "z", zzz=map) == sample_fn(3, -9, "z", zzz=map) 300 | 301 | 302 | def test_get_source(): 303 | function = Function.from_object(sample_fn) 304 | source = normalize_source(function.get_source()) 305 | 306 | if unparser() == "astunparse": 307 | cond = "(foo == 'bar')" 308 | ret = "(x + y)" 309 | else: 310 | cond = "foo == 'bar'" 311 | ret = "x + y" 312 | 313 | expected_source = unindent( 314 | f""" 315 | def sample_fn(x, y, foo='bar', **kw): 316 | if {cond}: 317 | return {ret} 318 | else: 319 | return kw['zzz'] 320 | """ 321 | ) 322 | 323 | assert source == expected_source 324 | 325 | 326 | def sample_fn(x, y, foo="bar", **kw): 327 | if foo == "bar": 328 | return x + y 329 | else: 330 | return kw["zzz"] 331 | -------------------------------------------------------------------------------- /tests/test_core/test_mangler.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from peval.tools import unindent 4 | from peval.core.gensym import GenSym 5 | from peval.core.mangler import mangle 6 | 7 | from utils import assert_ast_equal 8 | 9 | 10 | def test_mutiple_returns(): 11 | source = unindent( 12 | """ 13 | def f(x, y, z='foo'): 14 | if x: 15 | b = y + list(x) 16 | return b 17 | else: 18 | return z 19 | """ 20 | ) 21 | tree = ast.parse(source) 22 | 23 | expected_source = unindent( 24 | """ 25 | def f(__peval_mangled_1, __peval_mangled_2, __peval_mangled_3='foo'): 26 | if __peval_mangled_1: 27 | __peval_mangled_4 = __peval_mangled_2 + list(__peval_mangled_1) 28 | return __peval_mangled_4 29 | else: 30 | return __peval_mangled_3 31 | """ 32 | ) 33 | expected_tree = ast.parse(expected_source) 34 | 35 | gen_sym = GenSym.for_tree(tree) 36 | gen_sym, new_tree = mangle(gen_sym, tree) 37 | 38 | assert_ast_equal(new_tree, expected_tree) 39 | -------------------------------------------------------------------------------- /tests/test_core/test_reify.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import sys 3 | 4 | import pytest 5 | 6 | from peval.core.reify import KnownValue, reify, reify_unwrapped 7 | from peval.core.gensym import GenSym 8 | 9 | from utils import assert_ast_equal 10 | 11 | 12 | def check_reify(value, expected_ast, preferred_name=None, expected_binding=None): 13 | kvalue = KnownValue(value, preferred_name=preferred_name) 14 | gen_sym = GenSym() 15 | node, gen_sym, binding = reify(kvalue, gen_sym) 16 | 17 | assert_ast_equal(node, expected_ast) 18 | if expected_binding is not None: 19 | assert binding == expected_binding 20 | 21 | 22 | def check_node_to_maybe_kvalue(node, bindings, expected_result, expected_preferred_name=None): 23 | node_or_kvalue = node_to_maybe_kvalue(node, bindings) 24 | 25 | if isinstance(node_or_kvalue, KnownValue): 26 | assert node_or_kvalue.value == expected_result 27 | assert node_or_kvalue.preferred_name == expected_preferred_name 28 | else: 29 | assert_ast_equal(node_or_kvalue, expected_result) 30 | 31 | 32 | def test_simple_reify(): 33 | check_reify(True, ast.Constant(value=True, kind=None)) 34 | check_reify(False, ast.Constant(value=False, kind=None)) 35 | check_reify(None, ast.Constant(value=None, kind=None)) 36 | 37 | class Dummy: 38 | pass 39 | 40 | x = Dummy() 41 | check_reify( 42 | x, 43 | ast.Name(id="__peval_temp_1", ctx=ast.Load()), 44 | expected_binding=dict(__peval_temp_1=x), 45 | ) 46 | check_reify( 47 | x, 48 | ast.Name(id="y", ctx=ast.Load()), 49 | preferred_name="y", 50 | expected_binding=dict(y=x), 51 | ) 52 | 53 | check_reify(1, ast.Constant(value=1, kind=None)) 54 | check_reify(2.3, ast.Constant(value=2.3, kind=None)) 55 | check_reify(3 + 4j, ast.Constant(value=3 + 4j, kind=None)) 56 | check_reify("abc", ast.Constant(value="abc", kind=None)) 57 | 58 | s = bytes("abc", encoding="ascii") 59 | check_reify(s, ast.Constant(value=s, kind=None)) 60 | 61 | 62 | def test_reify_unwrapped(): 63 | class Dummy: 64 | pass 65 | 66 | x = Dummy() 67 | gen_sym = GenSym() 68 | node, gen_sym, binding = reify_unwrapped(x, gen_sym) 69 | assert_ast_equal(node, ast.Name(id="__peval_temp_1", ctx=ast.Load())) 70 | assert binding == dict(__peval_temp_1=x) 71 | 72 | 73 | def test_str_repr(): 74 | kv = KnownValue(1, preferred_name="x") 75 | s = str(kv) 76 | nkv = eval(repr(kv)) 77 | assert nkv.value == kv.value 78 | assert nkv.preferred_name == kv.preferred_name 79 | -------------------------------------------------------------------------------- /tests/test_highlevelapi.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import sys 4 | import ast 5 | import functools 6 | 7 | import pytest 8 | 9 | from peval.core.function import Function 10 | from peval import partial_eval, partial_apply, specialize_on, getsource, inline 11 | from peval.tools import unindent 12 | 13 | from utils import assert_ast_equal, function_from_source 14 | 15 | 16 | def assert_func_equal_on(fn1, fn2, *args, **kwargs): 17 | """ 18 | Check that functions are the same, or raise the same exception 19 | """ 20 | v1 = v2 = e1 = e2 = None 21 | try: 22 | v1 = fn1(*args, **kwargs) 23 | except Exception as _e1: 24 | e1 = _e1 25 | try: 26 | v2 = fn2(*args, **kwargs) 27 | except Exception as _e2: 28 | e2 = _e2 29 | if e1 or e2: 30 | # reraise exception, if there is only one 31 | if e1 is None: 32 | fn2(*args, **kwargs) 33 | if e2 is None: 34 | fn1(*args, **kwargs) 35 | if type(e1) != type(e2): 36 | # assume that fn1 is more correct, so raise exception from fn2 37 | fn2(*args, **kwargs) 38 | assert type(e1) == type(e2) 39 | assert e1.args == e2.args 40 | else: 41 | assert e1 is None 42 | assert e2 is None 43 | assert v1 == v2 44 | 45 | 46 | def check_partial_apply( 47 | func, args=None, kwds=None, expected_source=None, expected_new_bindings=None 48 | ): 49 | """ 50 | Test that with given constants, optimized_ast transforms 51 | source to expected_source. 52 | It :expected_new_bindings: is given, we check that they 53 | are among new bindings returned by optimizer. 54 | """ 55 | 56 | if args is None: 57 | args = tuple() 58 | if kwds is None: 59 | kwds = {} 60 | 61 | new_func = partial_apply(func, *args, **kwds) 62 | function = Function.from_object(new_func) 63 | 64 | if expected_source is not None: 65 | assert_ast_equal(function.tree, ast.parse(unindent(expected_source)).body[0]) 66 | 67 | if expected_new_bindings is not None: 68 | for k in expected_new_bindings: 69 | if k not in function.globals: 70 | print("Expected binding missing:", k) 71 | 72 | binding = function.globals[k] 73 | expected_binding = expected_new_bindings[k] 74 | 75 | assert binding == expected_binding 76 | 77 | 78 | def check_partial_fn(base_fn, get_partial_kwargs, get_kwargs): 79 | """ 80 | Check that partial evaluation of base_fn with partial_args 81 | gives the same result on args_list 82 | as functools.partial(base_fn, partial_args) 83 | """ 84 | fn = partial_apply(base_fn, **get_partial_kwargs()) 85 | partial_fn = functools.partial(base_fn, **get_partial_kwargs()) 86 | # call two times to check for possible side-effects 87 | assert_func_equal_on(partial_fn, fn, **get_kwargs()) # first 88 | assert_func_equal_on(partial_fn, fn, **get_kwargs()) # second 89 | 90 | 91 | def test_args_handling(): 92 | def args_kwargs(a, b, c=None): 93 | return 1.0 * a / b * (c or 3) 94 | 95 | assert partial_apply(args_kwargs, 1)(2) == 1.0 / 2 * 3 96 | assert partial_apply(args_kwargs, 1, 2, 1)() == 1.0 / 2 * 1 97 | 98 | 99 | def test_kwargs_handling(): 100 | def args_kwargs(a, b, c=None): 101 | return 1.0 * a / b * (c or 3) 102 | 103 | assert partial_apply(args_kwargs, c=4)(1, 2) == 1.0 / 2 * 4 104 | assert partial_apply(args_kwargs, 2, c=4)(6) == 2.0 / 6 * 4 105 | 106 | 107 | @inline 108 | def smart_power(n, x): 109 | if not isinstance(n, int) or n < 0: 110 | raise ValueError("Base should be a positive integer") 111 | elif n == 0: 112 | return 1 113 | elif n % 2 == 0: 114 | v = smart_power(n // 2, x) 115 | return v * v 116 | else: 117 | return x * smart_power(n - 1, x) 118 | 119 | 120 | @inline 121 | def stupid_power(n, x): 122 | if not isinstance(n, int) or n < 0: 123 | raise ValueError("Base should be a positive integer") 124 | else: 125 | if n == 0: 126 | return 1 127 | if n == 1: 128 | return x 129 | v = 1 130 | for _ in range(n): 131 | v = v * x 132 | return v 133 | 134 | 135 | def test_if_on_stupid_power(): 136 | for n in ("foo", 0, 1, 2, 3): 137 | for x in [0, 1, 0.01, 5e10]: 138 | check_partial_fn(stupid_power, lambda: dict(n=n), lambda: {"x": x}) 139 | 140 | 141 | def test_if_on_recursive_power(): 142 | for n in ("foo", 0, 1, 2, 3): 143 | for x in [0, 1, 0.01, 5e10]: 144 | check_partial_fn(smart_power, lambda: dict(n=n), lambda: {"x": x}) 145 | 146 | 147 | def for_specialize(a, b, c=4, d=5): 148 | return a + b + c + d 149 | 150 | 151 | @pytest.mark.parametrize("names", ["b", "d", ("a", "b"), ("b", "c"), ("c", "d")]) 152 | def test_specialize_on(names): 153 | f = specialize_on(names)(for_specialize) 154 | assert f(1, 2) == for_specialize(1, 2) 155 | assert f(1, 2, d=10) == for_specialize(1, 2, d=10) 156 | assert f(a=3, b=4, d=2) == for_specialize(a=3, b=4, d=2) 157 | 158 | 159 | def test_specialize_on_missing_names(): 160 | with pytest.raises(ValueError): 161 | f = specialize_on("k")(for_specialize) 162 | 163 | 164 | def test_peval_closure(): 165 | a = 1 166 | b = 2 167 | 168 | def f(x): 169 | return x + (a + b) 170 | 171 | ff = partial_eval(f) 172 | 173 | assert ff(1) == 1 + a + b 174 | 175 | 176 | def test_peval_prohibit_nested_definitions(): 177 | def f(x): 178 | g = lambda y: x + y 179 | return g(x) 180 | 181 | with pytest.raises(ValueError): 182 | ff = partial_eval(f) 183 | 184 | 185 | def test_peval_prohibit_async(): 186 | f = function_from_source( 187 | """ 188 | async def f(x): 189 | return x 190 | """ 191 | ).eval() 192 | 193 | with pytest.raises(ValueError): 194 | ff = partial_eval(f) 195 | -------------------------------------------------------------------------------- /tests/test_tags.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from peval.tags import pure, get_pure_tag, inline, get_inline_tag 4 | 5 | from utils import function_from_source 6 | 7 | 8 | def test_pure_tag(): 9 | @pure 10 | def func(x): 11 | return x 12 | 13 | assert get_pure_tag(func) 14 | 15 | 16 | def test_inline_tag(): 17 | @inline 18 | def func(x): 19 | return x 20 | 21 | assert get_inline_tag(func) 22 | 23 | 24 | def test_inline_prohibit_nested_definitions(): 25 | def func(x): 26 | return lambda y: x + y 27 | 28 | with pytest.raises(ValueError): 29 | func = inline(func) 30 | 31 | 32 | def test_inline_prohibit_generator(): 33 | def func(x): 34 | for i in range(x): 35 | yield i 36 | 37 | with pytest.raises(ValueError): 38 | func = inline(func) 39 | 40 | 41 | def test_inline_prohibit_async(): 42 | func = function_from_source( 43 | """ 44 | async def func(x): 45 | return x 46 | """ 47 | ).eval() 48 | 49 | with pytest.raises(ValueError): 50 | func = inline(func) 51 | 52 | 53 | def test_inline_prohibit_closure(): 54 | @inline 55 | def no_closure(x): 56 | return x 57 | 58 | assert get_inline_tag(no_closure) 59 | 60 | a = 1 61 | 62 | def with_closure(x): 63 | return x + a 64 | 65 | with pytest.raises(ValueError): 66 | with_closure = inline(with_closure) 67 | -------------------------------------------------------------------------------- /tests/test_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fjarri/peval/e6b5d8c1be9b2da167eb27975aa70aacdb4eb660/tests/test_tools/__init__.py -------------------------------------------------------------------------------- /tests/test_tools/test_immutable.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from peval.tools import ImmutableDict, ImmutableADict 4 | 5 | 6 | # Immutable dictionary 7 | 8 | 9 | def test_del(): 10 | d = ImmutableDict(a=1) 11 | with pytest.raises(KeyError): 12 | nd = d.without("b") 13 | 14 | d = ImmutableDict(a=1) 15 | nd = d.without("a") 16 | assert nd == {} 17 | assert d == dict(a=1) 18 | 19 | 20 | def test_with_item(): 21 | d = ImmutableDict(a=1) 22 | nd = d.with_item("a", 1) 23 | assert nd is d 24 | 25 | d = ImmutableDict(a=1) 26 | nd = d.with_item("b", 2) 27 | assert nd == dict(a=1, b=2) 28 | assert d == dict(a=1) 29 | 30 | 31 | def test_with(): 32 | d = ImmutableADict(a=1) 33 | nd = d.with_(b=2) 34 | assert nd == dict(a=1, b=2) 35 | assert d == dict(a=1) 36 | 37 | d = ImmutableADict(a=1) 38 | nd = d.with_(a=1) 39 | assert nd is d 40 | 41 | 42 | def test_dict_repr(): 43 | d = ImmutableDict(a=1) 44 | nd = eval(repr(d)) 45 | assert type(nd) == type(d) 46 | assert nd == d 47 | 48 | 49 | # Immutable attribute dictionary 50 | 51 | 52 | def test_adict_getattr(): 53 | d = ImmutableADict(a=1) 54 | assert d.a == 1 55 | 56 | 57 | def test_adict_repr(): 58 | d = ImmutableADict(a=1) 59 | nd = eval(repr(d)) 60 | assert type(nd) == type(d) 61 | assert nd == d 62 | -------------------------------------------------------------------------------- /tests/test_tools/test_utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import ast 3 | 4 | import pytest 5 | 6 | from peval.tools import unindent, ast_equal, replace_fields 7 | from peval.core.function import Function 8 | 9 | 10 | def test_unindent(): 11 | src = """ 12 | def sample_fn(x, y, foo='bar', **kw): 13 | if (foo == 'bar'): 14 | return (x + y) 15 | else: 16 | return kw['zzz'] 17 | """ 18 | expected_src = """def sample_fn(x, y, foo='bar', **kw): 19 | if (foo == 'bar'): 20 | return (x + y) 21 | else: 22 | return kw['zzz']""" 23 | 24 | assert unindent(src) == expected_src 25 | 26 | 27 | def test_unindent_unexpected_indentation(): 28 | src = """ 29 | def sample_fn(x, y, foo='bar', **kw): 30 | if (foo == 'bar'): 31 | return (x + y) 32 | else: 33 | return kw['zzz'] 34 | some_code() # indentation here does not start from the same position as the first line! 35 | """ 36 | 37 | with pytest.raises(ValueError): 38 | result = unindent(src) 39 | 40 | 41 | def test_unindent_empty_line(): 42 | src = ( 43 | """ 44 | def sample_fn(x, y, foo='bar', **kw):\n""" 45 | # Technically, this line would be an unexpected indentation, 46 | # because it does not start with 8 spaces. 47 | # But `unindent` will see that it's just an empty line 48 | # and just replace it with a single `\n`. 49 | " \n" 50 | """ if (foo == 'bar'): 51 | return (x + y) 52 | else: 53 | return kw['zzz'] 54 | """ 55 | ) 56 | 57 | expected_src = ( 58 | "def sample_fn(x, y, foo='bar', **kw):\n" 59 | "\n" 60 | """ if (foo == 'bar'): 61 | return (x + y) 62 | else: 63 | return kw['zzz']""" 64 | ) 65 | 66 | assert unindent(src) == expected_src 67 | 68 | 69 | def test_ast_equal(): 70 | src = """ 71 | def sample_fn(x, y, foo='bar', **kw): 72 | if (foo == 'bar'): 73 | return (x + y) 74 | else: 75 | return kw['zzz'] 76 | """ 77 | 78 | # Different node type (`-` instead of `+`) 79 | different_node = """ 80 | def sample_fn(x, y, foo='bar', **kw): 81 | if (foo == 'bar'): 82 | return (x - y) 83 | else: 84 | return kw['zzz'] 85 | """ 86 | 87 | # Different value in a node ('zzy' instead of 'zzz') 88 | different_value = """ 89 | def sample_fn(x, y, foo='bar', **kw): 90 | if (foo == 'bar'): 91 | return (x + y) 92 | else: 93 | return kw['zzy'] 94 | """ 95 | 96 | # Additional element in a body 97 | different_length = """ 98 | def sample_fn(x, y, foo='bar', **kw): 99 | if (foo == 'bar'): 100 | return (x + y) 101 | return 1 102 | else: 103 | return kw['zzz'] 104 | """ 105 | 106 | tree = ast.parse(unindent(src)) 107 | different_node = ast.parse(unindent(different_node)) 108 | different_value = ast.parse(unindent(different_value)) 109 | different_length = ast.parse(unindent(different_length)) 110 | 111 | assert ast_equal(tree, tree) 112 | assert not ast_equal(tree, different_node) 113 | assert not ast_equal(tree, different_value) 114 | assert not ast_equal(tree, different_length) 115 | 116 | 117 | def test_replace_fields(): 118 | node = ast.Name(id="x", ctx=ast.Load()) 119 | 120 | new_node = replace_fields(node, id="y") 121 | assert new_node is not node 122 | assert new_node.id == "y" and type(new_node.ctx) == ast.Load 123 | 124 | new_node = replace_fields(node, id="x") 125 | # no new object is created if the new value is the same as the old value 126 | assert new_node is node 127 | assert new_node.id == "x" and type(new_node.ctx) == ast.Load 128 | -------------------------------------------------------------------------------- /tests/test_tools/test_walker.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import ast 3 | import inspect 4 | import sys 5 | 6 | import pytest 7 | 8 | from peval.tools import ( 9 | unindent, 10 | replace_fields, 11 | ast_inspector, 12 | ast_transformer, 13 | ast_walker, 14 | ) 15 | from peval.tools.walker import _Walker 16 | 17 | from utils import assert_ast_equal 18 | 19 | 20 | def get_ast(function): 21 | if isinstance(function, str): 22 | return ast.parse(unindent(function)) 23 | else: 24 | return ast.parse(inspect.getsource(function)) 25 | 26 | 27 | def check_mutation(node, walker): 28 | node_ref = copy.deepcopy(node) 29 | new_node = walker(node) 30 | assert ast.dump(node) != ast.dump(new_node) 31 | assert_ast_equal(node, node_ref) 32 | return new_node 33 | 34 | 35 | def dummy(x, y): 36 | c = 4 37 | a = 1 38 | 39 | 40 | def dummy_blocks(x, y): 41 | a = 1 42 | if x: 43 | b = 2 44 | c = 3 45 | 46 | 47 | def dummy_nested(x, y): 48 | def inner_function(z): 49 | return z 50 | 51 | return inner_function 52 | 53 | 54 | def dummy_if(): 55 | if a: 56 | if b: 57 | pass 58 | 59 | 60 | global_var = 1 61 | 62 | 63 | def dummy_globals(): 64 | global global_var 65 | return global_var 66 | 67 | 68 | def test_inspector(): 69 | @ast_inspector 70 | def collect_numbers(state, node, **kwds): 71 | if isinstance(node, ast.Num): 72 | return state.with_(numbers=state.numbers | {node.n}) 73 | else: 74 | return state 75 | 76 | node = get_ast(dummy) 77 | state = collect_numbers(dict(numbers=frozenset()), node) 78 | assert state.numbers == set([1, 4]) 79 | 80 | 81 | def test_walk_list(): 82 | @ast_inspector 83 | def collect_numbers(state, node, **kwds): 84 | if isinstance(node, ast.Num): 85 | return state.with_(numbers=state.numbers | {node.n}) 86 | else: 87 | return state 88 | 89 | node = get_ast(dummy) 90 | state = collect_numbers(dict(numbers=frozenset()), node.body) 91 | assert state.numbers == set([1, 4]) 92 | 93 | 94 | def test_walker(): 95 | @ast_walker 96 | def process_numbers(state, node, **kwds): 97 | if isinstance(node, ast.Num): 98 | return state.with_(numbers=state.numbers | {node.n}), ast.Num(n=node.n + 1) 99 | else: 100 | return state, node 101 | 102 | node = get_ast(dummy) 103 | state, new_node = process_numbers(dict(numbers=frozenset()), node) 104 | 105 | assert state.numbers == set([1, 4]) 106 | assert_ast_equal( 107 | new_node, 108 | get_ast( 109 | """ 110 | def dummy(x, y): 111 | c = 5 112 | a = 2 113 | """ 114 | ), 115 | ) 116 | 117 | 118 | # Transformations 119 | 120 | 121 | def test_change_node(): 122 | @ast_transformer 123 | def change_name(node, **kwds): 124 | if isinstance(node, ast.Name) and node.id == "a": 125 | return ast.Name(id="b", ctx=node.ctx) 126 | else: 127 | return node 128 | 129 | node = get_ast(dummy) 130 | new_node = check_mutation(node, change_name) 131 | assert_ast_equal( 132 | new_node, 133 | get_ast( 134 | """ 135 | def dummy(x, y): 136 | c = 4 137 | b = 1 138 | """ 139 | ), 140 | ) 141 | 142 | 143 | def test_add_statement(): 144 | @ast_transformer 145 | def add_statement(node, **kwds): 146 | if isinstance(node, ast.Assign): 147 | return [node, ast.parse("b = 2").body[0]] 148 | else: 149 | return node 150 | 151 | node = get_ast(dummy) 152 | new_node = check_mutation(node, add_statement) 153 | assert_ast_equal( 154 | new_node, 155 | get_ast( 156 | """ 157 | def dummy(x, y): 158 | c = 4 159 | b = 2 160 | a = 1 161 | b = 2 162 | """ 163 | ), 164 | ) 165 | 166 | 167 | def test_list_element(): 168 | """ 169 | Tests the removal of an AST node that is an element of a list 170 | referenced by a field of the parent node. 171 | """ 172 | 173 | @ast_transformer 174 | def remove_list_element(node, **kwds): 175 | if isinstance(node, ast.Assign) and node.targets[0].id == "a": 176 | return None 177 | else: 178 | return node 179 | 180 | node = get_ast(dummy) 181 | new_node = check_mutation(node, remove_list_element) 182 | assert_ast_equal( 183 | new_node, 184 | get_ast( 185 | """ 186 | def dummy(x, y): 187 | c = 4 188 | """ 189 | ), 190 | ) 191 | 192 | 193 | def test_remove_field(): 194 | """ 195 | Tests the removal of an AST node that is referenced by a field of the parent node. 196 | """ 197 | 198 | @ast_transformer 199 | def remove_field(node, **kwds): 200 | if isinstance(node, ast.arg) and node.arg == "x": 201 | return None 202 | else: 203 | return node 204 | 205 | node = get_ast(dummy) 206 | new_node = check_mutation(node, remove_field) 207 | assert_ast_equal( 208 | new_node, 209 | get_ast( 210 | """ 211 | def dummy(y): 212 | c = 4 213 | a = 1 214 | """ 215 | ), 216 | ) 217 | 218 | 219 | # Error checks 220 | 221 | 222 | def test_walker_contract(): 223 | """ 224 | Test that the backend _Walker cannot be created 225 | with both ``transform`` and ``inspect`` set to ``False``. 226 | """ 227 | 228 | def pass_through(node, **kwds): 229 | pass 230 | 231 | with pytest.raises(ValueError): 232 | w = _Walker(pass_through) 233 | 234 | 235 | def test_wrong_root_type(): 236 | @ast_inspector 237 | def pass_through(node, **kwds): 238 | pass 239 | 240 | with pytest.raises(TypeError): 241 | pass_through(None, {}) 242 | 243 | 244 | def test_wrong_root_return_value(): 245 | @ast_transformer 246 | def wrong_root_return_value(node, **kwds): 247 | return 1 248 | 249 | node = get_ast(dummy) 250 | with pytest.raises(TypeError): 251 | wrong_root_return_value(node) 252 | 253 | 254 | def test_wrong_field_return_value(): 255 | @ast_transformer 256 | def wrong_field_return_value(node, **kwds): 257 | if isinstance(node, ast.Num): 258 | return 1 259 | else: 260 | return node 261 | 262 | node = get_ast(dummy) 263 | with pytest.raises(TypeError): 264 | wrong_field_return_value(node) 265 | 266 | 267 | def test_wrong_list_return_value(): 268 | @ast_transformer 269 | def wrong_list_return_value(node, **kwds): 270 | if isinstance(node, ast.Assign): 271 | return 1 272 | else: 273 | return node 274 | 275 | node = get_ast(dummy) 276 | with pytest.raises(TypeError): 277 | wrong_list_return_value(node) 278 | 279 | 280 | def test_walker_call_signature(): 281 | @ast_walker 282 | def pass_through(state, node, **kwds): 283 | return state, node 284 | 285 | node = get_ast(dummy) 286 | with pytest.raises(TypeError): 287 | pass_through(node) 288 | 289 | 290 | def test_inspector_call_signature(): 291 | @ast_inspector 292 | def pass_through(state, node, **kwds): 293 | return state, node 294 | 295 | node = get_ast(dummy) 296 | with pytest.raises(TypeError): 297 | pass_through(node) 298 | 299 | 300 | def test_transformer_call_signature(): 301 | @ast_transformer 302 | def pass_through(node, **kwds): 303 | return node 304 | 305 | node = get_ast(dummy) 306 | with pytest.raises(TypeError): 307 | pass_through({}, node) 308 | 309 | 310 | # Handler dispatchers 311 | 312 | 313 | def test_dispatched_walker(): 314 | @ast_inspector 315 | class collect_numbers_with_default: 316 | @staticmethod 317 | def handle_Num(state, node, **kwds): 318 | return state.with_(numbers=state.numbers | {node.n}) 319 | 320 | @staticmethod 321 | def handle_Constant(state, node, **kwds): 322 | return state.with_(numbers=state.numbers | {node.n}) 323 | 324 | @staticmethod 325 | def handle(state, node, **kwds): 326 | return state 327 | 328 | @ast_inspector 329 | class collect_numbers: 330 | @staticmethod 331 | def handle_Num(state, node, **kwds): 332 | return state.with_(numbers=state.numbers | {node.n}) 333 | 334 | @staticmethod 335 | def handle_Constant(state, node, **kwds): 336 | return state.with_(numbers=state.numbers | {node.n}) 337 | 338 | node = get_ast(dummy) 339 | 340 | state = collect_numbers(dict(numbers=frozenset()), node) 341 | assert state.numbers == set([1, 4]) 342 | 343 | state = collect_numbers_with_default(dict(numbers=frozenset()), node) 344 | assert state.numbers == set([1, 4]) 345 | 346 | 347 | # Advanced functionality 348 | 349 | 350 | def test_walk_children(): 351 | @ast_transformer 352 | def mangle_outer_functions(node, **kwds): 353 | if isinstance(node, ast.FunctionDef): 354 | return replace_fields(node, name="__" + node.name) 355 | else: 356 | return node 357 | 358 | @ast_transformer 359 | def mangle_all_functions(node, walk_field, **kwds): 360 | if isinstance(node, ast.FunctionDef): 361 | return replace_fields( 362 | node, name="__" + node.name, body=walk_field(node.body, block_context=True) 363 | ) 364 | else: 365 | return node 366 | 367 | node = get_ast(dummy_nested) 368 | 369 | new_node = mangle_outer_functions(node) 370 | assert_ast_equal( 371 | new_node, 372 | get_ast( 373 | """ 374 | def __dummy_nested(x, y): 375 | def inner_function(z): 376 | return z 377 | return inner_function 378 | """ 379 | ), 380 | ) 381 | 382 | new_node = mangle_all_functions(node) 383 | assert_ast_equal( 384 | new_node, 385 | get_ast( 386 | """ 387 | def __dummy_nested(x, y): 388 | def __inner_function(z): 389 | return z 390 | return inner_function 391 | """ 392 | ), 393 | ) 394 | 395 | 396 | def test_global_context(): 397 | @ast_transformer 398 | def rename(node, ctx, **kwds): 399 | if isinstance(node, ast.Name) and node.id == ctx.old_name: 400 | return ast.Name(id=ctx.new_name, ctx=node.ctx) 401 | else: 402 | return node 403 | 404 | node = get_ast(dummy) 405 | new_node = rename(node, ctx=dict(old_name="c", new_name="d")) 406 | 407 | assert_ast_equal( 408 | new_node, 409 | get_ast( 410 | """ 411 | def dummy(x, y): 412 | d = 4 413 | a = 1 414 | """ 415 | ), 416 | ) 417 | 418 | 419 | def test_prepend(): 420 | @ast_transformer 421 | def prepender(node, prepend, **kwds): 422 | if isinstance(node, ast.Name): 423 | if node.id == "a": 424 | prepend( 425 | [ast.Assign(targets=[ast.Name(id="k", ctx=ast.Store())], value=ast.Num(n=10))] 426 | ) 427 | return node 428 | elif node.id == "b": 429 | prepend( 430 | [ast.Assign(targets=[ast.Name(id="l", ctx=ast.Store())], value=ast.Num(n=20))] 431 | ) 432 | return ast.Name(id="d", ctx=node.ctx) 433 | elif node.id == "c": 434 | prepend( 435 | [ast.Assign(targets=[ast.Name(id="m", ctx=ast.Store())], value=ast.Num(n=30))] 436 | ) 437 | return node 438 | else: 439 | return node 440 | else: 441 | return node 442 | 443 | node = get_ast(dummy_blocks) 444 | new_node = prepender(node) 445 | 446 | assert_ast_equal( 447 | new_node, 448 | get_ast( 449 | """ 450 | def dummy_blocks(x, y): 451 | k = 10 452 | a = 1 453 | if x: 454 | l = 20 455 | d = 2 456 | m = 30 457 | c = 3 458 | """ 459 | ), 460 | ) 461 | 462 | 463 | def test_visit_after(): 464 | @ast_transformer 465 | def simplify(node, visit_after, visiting_after, **kwds): 466 | if isinstance(node, ast.If): 467 | if not visiting_after: 468 | visit_after() 469 | return node 470 | 471 | # This wouldn't work if we didn't simplify the child nodes first 472 | if len(node.orelse) == 0 and len(node.body) == 1 and isinstance(node.body[0], ast.Pass): 473 | return ast.Pass() 474 | else: 475 | return node 476 | else: 477 | return node 478 | 479 | node = get_ast(dummy_if) 480 | new_node = simplify(node) 481 | 482 | assert_ast_equal( 483 | new_node, 484 | get_ast( 485 | """ 486 | def dummy_if(): 487 | pass 488 | """ 489 | ), 490 | ) 491 | 492 | 493 | def test_block_autofix(): 494 | # This transformer removes If nodes from statement blocks, 495 | # but it has no way to check whether the resulting body still has some nodes or not. 496 | # That's why the walker adds a Pass node automatically if after all the transformations 497 | # a statement block turns out to be empty. 498 | @ast_transformer 499 | def delete_ifs(node, **kwds): 500 | if isinstance(node, ast.If): 501 | return None 502 | else: 503 | return node 504 | 505 | node = get_ast(dummy_if) 506 | new_node = delete_ifs(node) 507 | 508 | assert_ast_equal( 509 | new_node, 510 | get_ast( 511 | """ 512 | def dummy_if(): 513 | pass 514 | """ 515 | ), 516 | ) 517 | 518 | 519 | def test_walk_field_transform(): 520 | @ast_transformer 521 | def increment(node, walk_field, **kwds): 522 | if isinstance(node, ast.Assign): 523 | return replace_fields(node, targets=node.targets, value=walk_field(node.value)) 524 | elif isinstance(node, ast.Num): 525 | return ast.Num(n=node.n + 1) 526 | else: 527 | return node 528 | 529 | node = get_ast(dummy) 530 | new_node = increment(node) 531 | 532 | assert_ast_equal( 533 | new_node, 534 | get_ast( 535 | """ 536 | def dummy(x, y): 537 | c = 5 538 | a = 2 539 | """ 540 | ), 541 | ) 542 | 543 | 544 | def test_walk_field_inspect(): 545 | @ast_inspector 546 | def names_and_nums(state, node, walk_field, **kwds): 547 | if isinstance(node, ast.Assign): 548 | state = walk_field(state, node.value) 549 | return state.with_(objs=state.objs | {node.targets[0].id}) 550 | elif isinstance(node, ast.Num): 551 | return state.with_(objs=state.objs | {node.n}) 552 | else: 553 | return state 554 | 555 | node = get_ast(dummy) 556 | state = names_and_nums(dict(objs=frozenset()), node) 557 | assert state.objs == set(["a", "c", 1, 4]) 558 | 559 | 560 | def test_walk_field_transform_inspect(): 561 | @ast_walker 562 | def names_and_incremented_nums(state, node, walk_field, **kwds): 563 | if isinstance(node, ast.Assign): 564 | state, value_node = walk_field(state, node.value) 565 | new_node = replace_fields(node, targets=node.targets, value=value_node) 566 | new_state = state.with_(objs=state.objs | {node.targets[0].id}) 567 | return new_state, new_node 568 | elif isinstance(node, ast.Num): 569 | return state.with_(objs=state.objs | {node.n}), ast.Num(n=node.n + 1) 570 | else: 571 | return state, node 572 | 573 | node = get_ast(dummy) 574 | state, new_node = names_and_incremented_nums(dict(objs=frozenset()), node) 575 | assert state.objs == set(["a", "c", 1, 4]) 576 | assert_ast_equal( 577 | new_node, 578 | get_ast( 579 | """ 580 | def dummy(x, y): 581 | c = 5 582 | a = 2 583 | """ 584 | ), 585 | ) 586 | 587 | 588 | def test_skip_fields(): 589 | @ast_transformer 590 | def increment(node, skip_fields, **kwds): 591 | if isinstance(node, ast.Assign) and node.targets[0].id == "c": 592 | skip_fields() 593 | 594 | if isinstance(node, ast.Num): 595 | return ast.Num(n=node.n + 1) 596 | else: 597 | return node 598 | 599 | node = get_ast(dummy) 600 | new_node = increment(node) 601 | 602 | assert_ast_equal( 603 | new_node, 604 | get_ast( 605 | """ 606 | def dummy(x, y): 607 | c = 4 608 | a = 2 609 | """ 610 | ), 611 | ) 612 | 613 | 614 | def test_globals(): 615 | """ 616 | A regression test. The nodes for ``globals`` and ``nonlocals`` statements 617 | contain lists of strings, which confused the walker, which was expecting nodes there 618 | and tried to walk them. 619 | """ 620 | 621 | @ast_inspector 622 | def pass_through(node, state, **kwds): 623 | return state 624 | 625 | node = get_ast(dummy_globals) 626 | state = pass_through({}, node) 627 | -------------------------------------------------------------------------------- /tests/test_wisdom.py: -------------------------------------------------------------------------------- 1 | import types 2 | import sys 3 | 4 | from peval.tags import pure 5 | from peval.wisdom import is_pure_callable 6 | 7 | 8 | class StrPure(str): 9 | pass 10 | 11 | 12 | class StrImpureMethod(str): 13 | def __getitem__(self, idx): 14 | return str.__getitem__(self, idx) 15 | 16 | 17 | class StrPureMethod(str): 18 | @pure 19 | def __getitem__(self, idx): 20 | return str.__getitem__(self, idx) 21 | 22 | 23 | @pure 24 | def dummy_pure(): 25 | pass 26 | 27 | 28 | def dummy_impure(): 29 | pass 30 | 31 | 32 | class DummyPureInit: 33 | @pure 34 | def __init__(self): 35 | pass 36 | 37 | 38 | class DummyPureCall: 39 | @pure 40 | def __call__(self): 41 | pass 42 | 43 | 44 | class DummyImpureInit: 45 | def __init__(self): 46 | pass 47 | 48 | 49 | class DummyImpureCall: 50 | def __call__(self): 51 | pass 52 | 53 | 54 | class Dummy: 55 | @pure 56 | def pure_method(self): 57 | pass 58 | 59 | def impure_method(self): 60 | pass 61 | 62 | @classmethod 63 | @pure 64 | def pure_classmethod(cls): 65 | pass 66 | 67 | @classmethod 68 | def impure_classmethod(cls): 69 | pass 70 | 71 | @staticmethod 72 | @pure 73 | def pure_staticmethod(): 74 | pass 75 | 76 | @staticmethod 77 | def impure_staticmethod(): 78 | pass 79 | 80 | 81 | def test_is_pure(): 82 | # a builtin function 83 | assert is_pure_callable(isinstance) 84 | 85 | # a builtin type 86 | assert is_pure_callable(str) 87 | 88 | # an unbound method of a built-in type 89 | assert is_pure_callable(str.__getitem__) 90 | 91 | # a bound method of a built-in type 92 | assert is_pure_callable("a".__getitem__) 93 | 94 | # a class derived from a builtin type 95 | assert is_pure_callable(StrPure) 96 | assert is_pure_callable(StrPure("a").__getitem__) 97 | 98 | # Overridden methods need to be explicitly marked as pure 99 | assert is_pure_callable(StrPureMethod("a").__getitem__) 100 | assert not is_pure_callable(StrImpureMethod("a").__getitem__) 101 | 102 | # A function 103 | assert is_pure_callable(dummy_pure) 104 | assert not is_pure_callable(dummy_impure) 105 | 106 | # A class 107 | assert is_pure_callable(DummyPureInit) 108 | assert not is_pure_callable(DummyImpureInit) 109 | 110 | # A callable object 111 | assert is_pure_callable(DummyPureCall()) 112 | assert not is_pure_callable(DummyImpureCall()) 113 | 114 | # Various methods 115 | assert is_pure_callable(Dummy().pure_method) 116 | assert is_pure_callable(Dummy.pure_method) 117 | assert is_pure_callable(Dummy().pure_classmethod) 118 | assert is_pure_callable(Dummy.pure_classmethod) 119 | assert is_pure_callable(Dummy().pure_staticmethod) 120 | assert is_pure_callable(Dummy.pure_staticmethod) 121 | assert not is_pure_callable(Dummy().impure_method) 122 | assert not is_pure_callable(Dummy.impure_method) 123 | assert not is_pure_callable(Dummy().impure_classmethod) 124 | assert not is_pure_callable(Dummy.impure_classmethod) 125 | assert not is_pure_callable(Dummy().impure_staticmethod) 126 | assert not is_pure_callable(Dummy.impure_staticmethod) 127 | 128 | # a non-callable 129 | assert not is_pure_callable("a") 130 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import ast 4 | import difflib 5 | import sys 6 | 7 | from peval.tools import ast_equal, unindent, unparse 8 | from peval.core.function import Function 9 | 10 | 11 | def unparser() -> str: 12 | # Different unparsers we use render some nodes differently. 13 | # For example, `astunparse` encloses logical expressions in parentheses when unparsing, 14 | # while `ast` and `astor` don't. 15 | # So while we can have either of the three backends available, 16 | # we need this compatibility stub, because some tests check the unparsed source. 17 | 18 | # follows the logic in `tools.utils.unparse()` 19 | try: 20 | from ast import unparse 21 | 22 | return "ast" 23 | except ImportError: 24 | pass 25 | 26 | try: 27 | from astunparse import unparse 28 | 29 | return "astunparse" 30 | except ImportError: 31 | pass 32 | 33 | try: 34 | from astor import to_source 35 | 36 | return "astor" 37 | except ImportError: 38 | pass 39 | 40 | raise ImportError( 41 | "Unparsing functionality is not available; switch to Python 3.9+, " 42 | "install with 'astunparse' feature, or install with 'astor' feature." 43 | ) 44 | 45 | 46 | def normalize_source(source): 47 | # trim newlines and trailing spaces --- some pretty printers add it 48 | 49 | # Note: this may change multiline string literals, 50 | # but we are assuming we won't have the ones susceptible to this in tests. 51 | source = "\n".join(line.rstrip() for line in source.split("\n")) 52 | 53 | source = source.strip("\n") 54 | 55 | return source 56 | 57 | 58 | def print_diff(test, expected): 59 | print("\n" + "=" * 40 + " expected:\n\n" + expected) 60 | print("\n" + "=" * 40 + " result:\n\n" + test) 61 | print("\n") 62 | 63 | expected_lines = expected.split("\n") 64 | test_lines = test.split("\n") 65 | 66 | for line in difflib.unified_diff( 67 | expected_lines, test_lines, fromfile="expected", tofile="test" 68 | ): 69 | print(line) 70 | 71 | 72 | def assert_ast_equal(test_ast, expected_ast, print_ast=True): 73 | """ 74 | Check that test_ast is equal to expected_ast, 75 | printing helpful error message if they are not equal 76 | """ 77 | 78 | equal = ast_equal(test_ast, expected_ast) 79 | if not equal: 80 | if print_ast: 81 | expected_ast_str = ast.dump(expected_ast) 82 | test_ast_str = ast.dump(test_ast) 83 | print_diff(test_ast_str, expected_ast_str) 84 | 85 | expected_source = normalize_source(unparse(expected_ast)) 86 | test_source = normalize_source(unparse(test_ast)) 87 | print_diff(test_source, expected_source) 88 | 89 | assert equal 90 | 91 | 92 | def check_component( 93 | component, 94 | func, 95 | additional_bindings=None, 96 | expected_source=None, 97 | expected_new_bindings=None, 98 | ): 99 | function = Function.from_object(func) 100 | bindings = function.get_external_variables() 101 | if additional_bindings is not None: 102 | bindings.update(additional_bindings) 103 | 104 | new_tree, new_bindings = component(function.tree, bindings) 105 | 106 | if expected_source is None: 107 | expected_ast = function.tree 108 | else: 109 | expected_ast = ast.parse(unindent(expected_source)).body[0] 110 | 111 | assert_ast_equal(new_tree, expected_ast) 112 | 113 | if expected_new_bindings is not None: 114 | for k in expected_new_bindings: 115 | if k not in new_bindings: 116 | print("Expected binding missing:", k) 117 | 118 | binding = new_bindings[k] 119 | expected_binding = expected_new_bindings[k] 120 | assert binding == expected_binding 121 | 122 | 123 | def function_from_source(source, globals_=None): 124 | """ 125 | A helper function to construct a Function object from source. 126 | Helpful if you need to create a test function with syntax 127 | that's not supported by some of the Py versions 128 | that are used to run tests or build docs, 129 | or if you need a function with custom __future__ imports. 130 | """ 131 | 132 | module = ast.parse(unindent(source)) 133 | ast.fix_missing_locations(module) 134 | 135 | for stmt in module.body: 136 | if type(stmt) in (ast.FunctionDef, ast.AsyncFunctionDef): 137 | tree = stmt 138 | name = stmt.name 139 | break 140 | else: 141 | raise ValueError("No function definitions found in the provided source") 142 | 143 | code_object = compile(module, "", "exec", dont_inherit=True) 144 | locals_ = {} 145 | eval(code_object, globals_, locals_) 146 | 147 | function_obj = locals_[name] 148 | function_obj._peval_source = unparse(tree) 149 | 150 | return Function.from_object(function_obj) 151 | --------------------------------------------------------------------------------