├── tests ├── __init__.py ├── test_rtrip.py ├── check_astunparse.py ├── test_optional.py ├── test_code_to_ast.py ├── support.py ├── check_expressions.py ├── test_misc.py ├── build_expressions.py └── test_code_gen.py ├── setup.py ├── requirements-tox.txt ├── MANIFEST.in ├── .github └── FUNDING.yml ├── requirements-dev.txt ├── .coveragerc ├── astor ├── codegen.py ├── __init__.py ├── string_repr.py ├── op_util.py ├── file_util.py ├── tree_walk.py ├── node_util.py ├── rtrip.py ├── source_repr.py └── code_gen.py ├── .travis.yml ├── Makefile ├── tox.ini ├── docs ├── conf.py ├── Makefile ├── index.rst └── changelog.rst ├── AUTHORS ├── .gitignore ├── LICENSE ├── setup.cfg └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /requirements-tox.txt: -------------------------------------------------------------------------------- 1 | nose>=1.3.0 2 | flake8>=3.7.0 3 | coverage>=4.5.0 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst AUTHORS LICENSE 2 | recursive-include tests *.py 3 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: berkerpeksag 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements-tox.txt 2 | 3 | wheel>=0.23.0 4 | sphinx-rtd-theme>=0.1.6 5 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = 4 | # omit the codegen because of deprecation 5 | astor/codegen.py 6 | .tox/* 7 | [report] 8 | show_missing = True 9 | -------------------------------------------------------------------------------- /astor/codegen.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from .code_gen import * # NOQA 4 | 5 | 6 | warnings.warn( 7 | 'astor.codegen module is deprecated. Please import ' 8 | 'astor.code_gen module instead.', 9 | DeprecationWarning, 10 | stacklevel=2 11 | ) 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false # run travis jobs in containers 2 | language: python 3 | python: 4 | - 2.7 5 | - 3.4 6 | - 3.5 7 | - 3.6 8 | - 3.7 9 | - 3.8 10 | - 3.9-dev 11 | - pypy 12 | - pypy3.5 13 | matrix: 14 | fast_finish: true 15 | allow_failures: 16 | - python: 3.9-dev 17 | cache: pip 18 | install: 19 | - pip install tox-travis 20 | script: tox 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | testenv: 2 | pip install -e . 3 | 4 | create-sdist: 5 | python setup.py sdist bdist_wheel 6 | 7 | release: create-sdist 8 | twine upload dist/* 9 | 10 | # Test it via `pip install -i https://test.pypi.org/simple/ ` 11 | test-release: create-sdist 12 | twine upload -r test dist/* 13 | 14 | clean: 15 | find . -name "*.pyc" -exec rm {} \; 16 | rm -rf *.egg-info 17 | rm -rf build/ dist/ __pycache__/ 18 | 19 | .PHONY: clean release test-release create-sdist 20 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{27, 34, 35, 36, 37, 38, py, py3.5} 4 | lint 5 | skipsdist = True 6 | skip_missing_interpreters = true 7 | 8 | [testenv] 9 | usedevelop = True 10 | commands = 11 | coverage run {envbindir}/nosetests -v --nocapture {posargs} 12 | coverage report 13 | deps = 14 | -rrequirements-tox.txt 15 | py27,pypy: unittest2 16 | 17 | [testenv:lint] 18 | deps = flake8 19 | commands = flake8 astor/ 20 | 21 | [flake8] 22 | ignore = E114, E116, E501, W504 23 | 24 | [travis] 25 | python = 26 | 3.7: py37, lint 27 | -------------------------------------------------------------------------------- /tests/test_rtrip.py: -------------------------------------------------------------------------------- 1 | """ 2 | Part of the astor library for Python AST manipulation 3 | 4 | License: 3-clause BSD 5 | 6 | Copyright (c) 2017 Patrick Maupin 7 | """ 8 | 9 | import os 10 | 11 | try: 12 | import unittest2 as unittest 13 | except ImportError: 14 | import unittest 15 | 16 | import astor.rtrip 17 | 18 | 19 | class RtripTestCase(unittest.TestCase): 20 | 21 | def test_convert_stdlib(self): 22 | srcdir = os.path.dirname(os.__file__) 23 | result = astor.rtrip.convert(srcdir) 24 | self.assertEqual(result, []) 25 | 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import os.path 4 | import sys 5 | import time 6 | 7 | extensions = [] 8 | 9 | templates_path = ['_templates'] 10 | 11 | source_suffix = '.rst' 12 | 13 | master_doc = 'index' 14 | 15 | project = u'astor' 16 | copyright = u'2013-%s, Berker Peksag' % time.strftime('%Y') 17 | 18 | version = release = '0.8.1' 19 | 20 | exclude_patterns = ['_build'] 21 | 22 | pygments_style = 'sphinx' 23 | 24 | try: 25 | import sphinx_rtd_theme 26 | except ImportError: 27 | html_theme = 'default' 28 | else: 29 | html_theme = 'sphinx_rtd_theme' 30 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 31 | 32 | htmlhelp_basename = 'astordoc' 33 | -------------------------------------------------------------------------------- /tests/check_astunparse.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | try: 4 | import unittest2 as unittest 5 | except ImportError: 6 | import unittest 7 | 8 | from . import test_code_gen 9 | 10 | import astunparse 11 | 12 | 13 | class MyTests(test_code_gen.CodegenTestCase): 14 | to_source = staticmethod(astunparse.unparse) 15 | 16 | # Just see if it'll do anything good at all 17 | assertSrcRoundtrips = test_code_gen.CodegenTestCase.assertAstRoundtrips 18 | 19 | # Don't look for exact comparison; see if ASTs match 20 | def assertSrcEqual(self, src1, src2): 21 | self.assertAstEqual(ast.parse(src1), ast.parse(src2)) 22 | 23 | 24 | if __name__ == '__main__': 25 | unittest.main() 26 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Original author of astor/codegen.py: 2 | * Armin Ronacher 3 | 4 | And with some modifications based on Armin's code: 5 | * Paul Dubs 6 | 7 | * Berker Peksag 8 | * Patrick Maupin 9 | * Abhishek L 10 | * Bob Tolbert 11 | * Whyzgeek 12 | * Zack M. Davis 13 | * Ryan Gonzalez 14 | * Lenny Truong 15 | * Radomír Bosák 16 | * Kodi Arfer 17 | * Felix Yan 18 | * Chris Rink 19 | * Batuhan Taskaya 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bad.txt 2 | test_codegen_dump_1.txt 3 | test_codegen_dump_2.txt 4 | 5 | # scratch directory for astor.rtrip 6 | tmp_rtrip/ 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | bin/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | eggs/ 20 | lib/ 21 | lib64/ 22 | lib64 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | pyvenv.cfg 30 | pip-selfcheck.json 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .cache 41 | nosetests.xml 42 | coverage.xml 43 | 44 | # Sphinx documentation 45 | docs/_build/ 46 | 47 | all_expr_*.py 48 | -------------------------------------------------------------------------------- /tests/test_optional.py: -------------------------------------------------------------------------------- 1 | """ 2 | Part of the astor library for Python AST manipulation 3 | 4 | License: 3-clause BSD 5 | 6 | Copyright (c) 2014 Berker Peksag 7 | Copyright (c) 2015, 2017 Patrick Maupin 8 | 9 | Use this by putting a link to astunparse's common.py test file. 10 | 11 | """ 12 | 13 | try: 14 | import unittest2 as unittest 15 | except ImportError: 16 | import unittest 17 | 18 | try: 19 | from test_code_gen import Comparisons 20 | except ImportError: 21 | from .test_code_gen import Comparisons 22 | 23 | try: 24 | from astunparse_common import AstunparseCommonTestCase 25 | except ImportError: 26 | AstunparseCommonTestCase = None 27 | 28 | if AstunparseCommonTestCase is not None: 29 | 30 | class UnparseTestCase(AstunparseCommonTestCase, unittest.TestCase, 31 | Comparisons): 32 | 33 | def check_roundtrip(self, code1, mode=None): 34 | self.assertAstRoundtrips(code1) 35 | 36 | def test_files(self): 37 | """ Don't bother -- we do this manually and more thoroughly """ 38 | 39 | if __name__ == '__main__': 40 | unittest.main() 41 | -------------------------------------------------------------------------------- /tests/test_code_to_ast.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import unittest 3 | 4 | from astor import code_to_ast 5 | 6 | 7 | def decorator(f): 8 | @functools.wraps(f) 9 | def wrapper(*args, **kwargs): 10 | return f(*args, **kwargs) 11 | return wrapper 12 | 13 | 14 | def simple_decorator(f): 15 | f.__decorated__ = True 16 | return f 17 | 18 | 19 | def undecorated_func(): 20 | pass 21 | 22 | 23 | @decorator 24 | def decorated_func(): 25 | pass 26 | 27 | 28 | @decorator 29 | @decorator 30 | def twice_decorated_func(): 31 | pass 32 | 33 | 34 | @simple_decorator 35 | def plain_decorated_func(): 36 | pass 37 | 38 | 39 | @simple_decorator 40 | def simple_decorated_func(): 41 | pass 42 | 43 | 44 | @simple_decorator 45 | @decorator 46 | def twice_decorated_func_2(): 47 | pass 48 | 49 | 50 | class CodeToASTTestCase(unittest.TestCase): 51 | 52 | def test_decorated(self): 53 | self.assertIsNotNone(code_to_ast(decorated_func)) 54 | 55 | def test_undecorated(self): 56 | self.assertIsNotNone(code_to_ast(undecorated_func)) 57 | 58 | def test_twice_decorated(self): 59 | self.assertIsNotNone(code_to_ast(twice_decorated_func)) 60 | 61 | def test_twice_decorated_2(self): 62 | self.assertIsNotNone(code_to_ast(twice_decorated_func_2)) 63 | 64 | def test_plain_decorator(self): 65 | self.assertIsNotNone(code_to_ast(simple_decorated_func)) 66 | 67 | def test_module(self): 68 | self.assertIsNotNone(code_to_ast(unittest)) 69 | 70 | 71 | if __name__ == '__main__': 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Patrick Maupin 2 | Copyright (c) 2013, Berker Peksag 3 | Copyright (c) 2008, Armin Ronacher 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, 7 | are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation and/or 14 | other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its contributors 17 | may be used to endorse or promote products derived from this software without 18 | specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 24 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = astor 3 | description = Read/rewrite/write Python ASTs 4 | long_description = file:README.rst 5 | version = attr:astor.__version__ 6 | author = Patrick Maupin 7 | author_email = pmaupin@gmail.com 8 | platforms = Independent 9 | url = https://github.com/berkerpeksag/astor 10 | license = BSD-3-Clause 11 | keywords = ast, codegen, PEP 8 12 | classifiers = 13 | Development Status :: 5 - Production/Stable 14 | Environment :: Console 15 | Intended Audience :: Developers 16 | License :: OSI Approved :: BSD License 17 | Operating System :: OS Independent 18 | Programming Language :: Python 19 | Programming Language :: Python :: 2 20 | Programming Language :: Python :: 2.7 21 | Programming Language :: Python :: 3 22 | Programming Language :: Python :: 3.4 23 | Programming Language :: Python :: 3.5 24 | Programming Language :: Python :: 3.6 25 | Programming Language :: Python :: 3.7 26 | Programming Language :: Python :: 3.8 27 | Programming Language :: Python :: Implementation 28 | Programming Language :: Python :: Implementation :: CPython 29 | Programming Language :: Python :: Implementation :: PyPy 30 | Topic :: Software Development :: Code Generators 31 | Topic :: Software Development :: Compilers 32 | 33 | [options] 34 | zip_safe = True 35 | include_package_data = True 36 | packages = find: 37 | python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* 38 | tests_requires = ["nose", "astunparse"] 39 | test_suite = nose.collector 40 | 41 | [options.packages.find] 42 | exclude = tests 43 | 44 | [bdist_wheel] 45 | universal = 1 46 | 47 | [build-system] 48 | requires = ['setuptools', 'wheel'] 49 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " linkcheck to check all external links for integrity" 28 | clean: 29 | rm -rf $(BUILDDIR)/* 30 | 31 | html: 32 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 33 | @echo 34 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 35 | 36 | linkcheck: 37 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 38 | @echo 39 | @echo "Link check complete; look for any errors in the above output " \ 40 | "or in $(BUILDDIR)/linkcheck/output.txt." 41 | -------------------------------------------------------------------------------- /tests/support.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import sys 3 | 4 | 5 | def _save_and_remove_module(name, orig_modules): 6 | """Helper function to save and remove a module from sys.modules 7 | Raise ImportError if the module can't be imported. 8 | """ 9 | # try to import the module and raise an error if it can't be imported 10 | if name not in sys.modules: 11 | __import__(name) 12 | del sys.modules[name] 13 | for modname in list(sys.modules): 14 | if modname == name or modname.startswith(name + '.'): 15 | orig_modules[modname] = sys.modules[modname] 16 | del sys.modules[modname] 17 | 18 | 19 | def import_fresh_module(name, fresh=(), blocked=()): 20 | """Import and return a module, deliberately bypassing sys.modules. 21 | 22 | This function imports and returns a fresh copy of the named Python module 23 | by removing the named module from sys.modules before doing the import. 24 | Note that unlike reload, the original module is not affected by 25 | this operation. 26 | """ 27 | orig_modules = {} 28 | names_to_remove = [] 29 | _save_and_remove_module(name, orig_modules) 30 | try: 31 | for fresh_name in fresh: 32 | _save_and_remove_module(fresh_name, orig_modules) 33 | for blocked_name in blocked: 34 | if not _save_and_block_module(blocked_name, orig_modules): 35 | names_to_remove.append(blocked_name) 36 | fresh_module = importlib.import_module(name) 37 | except ImportError: 38 | fresh_module = None 39 | finally: 40 | for orig_name, module in orig_modules.items(): 41 | sys.modules[orig_name] = module 42 | for name_to_remove in names_to_remove: 43 | del sys.modules[name_to_remove] 44 | return fresh_module 45 | -------------------------------------------------------------------------------- /astor/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright 2012 (c) Patrick Maupin 8 | Copyright 2013 (c) Berker Peksag 9 | 10 | """ 11 | 12 | import warnings 13 | 14 | from .code_gen import SourceGenerator, to_source # NOQA 15 | from .node_util import iter_node, strip_tree, dump_tree # NOQA 16 | from .node_util import ExplicitNodeVisitor # NOQA 17 | from .file_util import CodeToAst, code_to_ast # NOQA 18 | from .op_util import get_op_symbol, get_op_precedence # NOQA 19 | from .op_util import symbol_data # NOQA 20 | from .tree_walk import TreeWalk # NOQA 21 | 22 | __version__ = '0.8.1' 23 | 24 | parse_file = code_to_ast.parse_file 25 | 26 | # DEPRECATED!!! 27 | # These aliases support old programs. Please do not use in future. 28 | 29 | deprecated = """ 30 | get_boolop = get_binop = get_cmpop = get_unaryop = get_op_symbol 31 | get_anyop = get_op_symbol 32 | parsefile = code_to_ast.parse_file 33 | codetoast = code_to_ast 34 | dump = dump_tree 35 | all_symbols = symbol_data 36 | treewalk = tree_walk 37 | codegen = code_gen 38 | """ 39 | 40 | exec(deprecated) 41 | 42 | 43 | def deprecate(): 44 | def wrap(deprecated_name, target_name): 45 | if '.' in target_name: 46 | target_mod, target_fname = target_name.split('.') 47 | target_func = getattr(globals()[target_mod], target_fname) 48 | else: 49 | target_func = globals()[target_name] 50 | msg = "astor.%s is deprecated. Please use astor.%s." % ( 51 | deprecated_name, target_name) 52 | if callable(target_func): 53 | def newfunc(*args, **kwarg): 54 | warnings.warn(msg, DeprecationWarning, stacklevel=2) 55 | return target_func(*args, **kwarg) 56 | else: 57 | class ModProxy: 58 | def __getattr__(self, name): 59 | warnings.warn(msg, DeprecationWarning, stacklevel=2) 60 | return getattr(target_func, name) 61 | newfunc = ModProxy() 62 | 63 | globals()[deprecated_name] = newfunc 64 | 65 | for line in deprecated.splitlines(): # NOQA 66 | line = line.split('#')[0].replace('=', '').split() 67 | if line: 68 | target_name = line.pop() 69 | for deprecated_name in line: 70 | wrap(deprecated_name, target_name) 71 | 72 | 73 | deprecate() 74 | 75 | del deprecate, deprecated 76 | -------------------------------------------------------------------------------- /tests/check_expressions.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Part of the astor library for Python AST manipulation. 5 | 6 | License: 3-clause BSD 7 | 8 | Copyright (c) 2015 Patrick Maupin 9 | 10 | This module reads the strings generated by build_expressions, 11 | and runs them through the Python interpreter. 12 | 13 | For strings that are suboptimal (too many spaces, etc.), 14 | it simply dumps them to a miscompare file. 15 | 16 | For strings that seem broken (do not parse after roundtrip) 17 | or are maybe too compressed, it dumps information to the console. 18 | 19 | This module does not take too long to execute; however, the 20 | underlying build_expressions module takes forever, so this 21 | should not be part of the automated regressions. 22 | 23 | """ 24 | 25 | import sys 26 | import ast 27 | import astor 28 | 29 | try: 30 | import importlib 31 | except ImportError: 32 | try: 33 | import all_expr_2_6 as mymod 34 | except ImportError: 35 | print("Expression list does not exist -- building") 36 | from . import build_expressions 37 | build_expressions.makelib() 38 | print("Expression list built") 39 | import all_expr_2_6 as mymod 40 | else: 41 | mymodname = 'all_expr_%s_%s' % sys.version_info[:2] 42 | 43 | try: 44 | mymod = importlib.import_module(mymodname) 45 | except ImportError: 46 | print("Expression list does not exist -- building") 47 | from . import build_expressions 48 | build_expressions.makelib() 49 | print("Expression list built") 50 | mymod = importlib.import_module(mymodname) 51 | 52 | 53 | def checklib(): 54 | print("Checking expressions") 55 | parse = ast.parse 56 | dump_tree = astor.dump_tree 57 | to_source = astor.to_source 58 | with open('mismatch_%s_%s.txt' % sys.version_info[:2], 'wb') as f: 59 | for srctxt in mymod.all_expr.strip().splitlines(): 60 | srcast = parse(srctxt) 61 | dsttxt = to_source(srcast) 62 | if dsttxt != srctxt: 63 | srcdmp = dump_tree(srcast) 64 | try: 65 | dstast = parse(dsttxt) 66 | except SyntaxError: 67 | bad = True 68 | dstdmp = 'aborted' 69 | else: 70 | dstdmp = dump_tree(dstast) 71 | bad = srcdmp != dstdmp 72 | if bad or len(dsttxt) < len(srctxt): 73 | print(srctxt, dsttxt) 74 | if bad: 75 | print('****************** Original') 76 | print(srcdmp) 77 | print('****************** Extra Crispy') 78 | print(dstdmp) 79 | print('******************') 80 | print() 81 | print() 82 | f.write(('%s %s\n' % (repr(srctxt), 83 | repr(dsttxt))).encode('utf-8')) 84 | 85 | 86 | if __name__ == '__main__': 87 | checklib() 88 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | astor -- AST observe/rewrite 3 | ============================= 4 | 5 | :PyPI: https://pypi.org/project/astor/ 6 | :Documentation: https://astor.readthedocs.io 7 | :Source: https://github.com/berkerpeksag/astor 8 | :License: 3-clause BSD 9 | :Build status: 10 | .. image:: https://secure.travis-ci.org/berkerpeksag/astor.svg 11 | :alt: Travis CI 12 | :target: https://travis-ci.org/berkerpeksag/astor/ 13 | 14 | astor is designed to allow easy manipulation of Python source via the AST. 15 | 16 | There are some other similar libraries, but astor focuses on the following areas: 17 | 18 | - Round-trip an AST back to Python [1]_: 19 | 20 | - Modified AST doesn't need linenumbers, ctx, etc. or otherwise 21 | be directly compileable for the round-trip to work. 22 | - Easy to read generated code as, well, code 23 | - Can round-trip two different source trees to compare for functional 24 | differences, using the astor.rtrip tool (for example, after PEP8 edits). 25 | 26 | - Dump pretty-printing of AST 27 | 28 | - Harder to read than round-tripped code, but more accurate to figure out what 29 | is going on. 30 | 31 | - Easier to read than dump from built-in AST module 32 | 33 | - Non-recursive treewalk 34 | 35 | - Sometimes you want a recursive treewalk (and astor supports that, starting 36 | at any node on the tree), but sometimes you don't need to do that. astor 37 | doesn't require you to explicitly visit sub-nodes unless you want to: 38 | 39 | - You can add code that executes before a node's children are visited, and/or 40 | - You can add code that executes after a node's children are visited, and/or 41 | - You can add code that executes and keeps the node's children from being 42 | visited (and optionally visit them yourself via a recursive call) 43 | 44 | - Write functions to access the tree based on object names and/or attribute names 45 | - Enjoy easy access to parent node(s) for tree rewriting 46 | 47 | .. [1] 48 | The decompilation back to Python is based on code originally written 49 | by Armin Ronacher. Armin's code was well-structured, but failed on 50 | some obscure corner cases of the Python language (and even more corner 51 | cases when the AST changed on different versions of Python), and its 52 | output arguably had cosmetic issues -- for example, it produced 53 | parentheses even in some cases where they were not needed, to 54 | avoid having to reason about precedence. 55 | 56 | Other derivatives of Armin's code are floating around, and typically 57 | have fixes for a few corner cases that happened to be noticed by the 58 | maintainers, but most of them have not been tested as thoroughly as 59 | astor. One exception may be the version of codegen 60 | `maintained at github by CensoredUsername`__. This has been tested 61 | to work properly on Python 2.7 using astor's test suite, and, as it 62 | is a single source file, it may be easier to drop into some applications 63 | that do not require astor's other features or Python 3.x compatibility. 64 | 65 | __ https://github.com/CensoredUsername/codegen 66 | -------------------------------------------------------------------------------- /astor/string_repr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright (c) 2015 Patrick Maupin 8 | 9 | Pretty-print strings for the decompiler 10 | 11 | We either return the repr() of the string, 12 | or try to format it as a triple-quoted string. 13 | 14 | This is a lot harder than you would think. 15 | 16 | This has lots of Python 2 / Python 3 ugliness. 17 | 18 | """ 19 | 20 | import re 21 | 22 | try: 23 | special_unicode = unicode 24 | except NameError: 25 | class special_unicode(object): 26 | pass 27 | 28 | try: 29 | basestring = basestring 30 | except NameError: 31 | basestring = str 32 | 33 | 34 | def _properly_indented(s, line_indent): 35 | mylist = s.split('\n')[1:] 36 | mylist = [x.rstrip() for x in mylist] 37 | mylist = [x for x in mylist if x] 38 | if not s: 39 | return False 40 | counts = [(len(x) - len(x.lstrip())) for x in mylist] 41 | return counts and min(counts) >= line_indent 42 | 43 | 44 | mysplit = re.compile(r'(\\|\"\"\"|\"$)').split 45 | replacements = {'\\': '\\\\', '"""': '""\\"', '"': '\\"'} 46 | 47 | 48 | def _prep_triple_quotes(s, mysplit=mysplit, replacements=replacements): 49 | """ Split the string up and force-feed some replacements 50 | to make sure it will round-trip OK 51 | """ 52 | 53 | s = mysplit(s) 54 | s[1::2] = (replacements[x] for x in s[1::2]) 55 | return ''.join(s) 56 | 57 | 58 | def string_triplequote_repr(s): 59 | """Return string's python representation in triple quotes. 60 | """ 61 | return '"""%s"""' % _prep_triple_quotes(s) 62 | 63 | 64 | def pretty_string(s, embedded, current_line, uni_lit=False, 65 | min_trip_str=20, max_line=100): 66 | """There are a lot of reasons why we might not want to or 67 | be able to return a triple-quoted string. We can always 68 | punt back to the default normal string. 69 | """ 70 | 71 | default = repr(s) 72 | 73 | # Punt on abnormal strings 74 | if (isinstance(s, special_unicode) or not isinstance(s, basestring)): 75 | return default 76 | if uni_lit and isinstance(s, bytes): 77 | return 'b' + default 78 | 79 | len_s = len(default) 80 | 81 | if current_line.strip(): 82 | len_current = len(current_line) 83 | second_line_start = s.find('\n') + 1 84 | if embedded > 1 and not second_line_start: 85 | return default 86 | 87 | if len_s < min_trip_str: 88 | return default 89 | 90 | line_indent = len_current - len(current_line.lstrip()) 91 | 92 | # Could be on a line by itself... 93 | if embedded and not second_line_start: 94 | return default 95 | 96 | total_len = len_current + len_s 97 | if total_len < max_line and not _properly_indented(s, line_indent): 98 | return default 99 | 100 | fancy = string_triplequote_repr(s) 101 | 102 | # Sometimes this doesn't work. One reason is that 103 | # the AST has no understanding of whether \r\n was 104 | # entered that way in the string or was a cr/lf in the 105 | # file. So we punt just so we can round-trip properly. 106 | 107 | try: 108 | if eval(fancy) == s and '\r' not in fancy: 109 | return fancy 110 | except Exception: 111 | pass 112 | return default 113 | -------------------------------------------------------------------------------- /astor/op_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright (c) 2015 Patrick Maupin 8 | 9 | This module provides data and functions for mapping 10 | AST nodes to symbols and precedences. 11 | 12 | """ 13 | 14 | import ast 15 | 16 | op_data = """ 17 | GeneratorExp 1 18 | 19 | Assign 1 20 | AnnAssign 1 21 | AugAssign 0 22 | Expr 0 23 | Yield 1 24 | YieldFrom 0 25 | If 1 26 | For 0 27 | AsyncFor 0 28 | While 0 29 | Return 1 30 | 31 | Slice 1 32 | Subscript 0 33 | Index 1 34 | ExtSlice 1 35 | comprehension_target 1 36 | Tuple 0 37 | FormattedValue 0 38 | 39 | Comma 1 40 | NamedExpr 1 41 | Assert 0 42 | Raise 0 43 | call_one_arg 1 44 | 45 | Lambda 1 46 | IfExp 0 47 | 48 | comprehension 1 49 | Or or 1 50 | And and 1 51 | Not not 1 52 | 53 | Eq == 1 54 | Gt > 0 55 | GtE >= 0 56 | In in 0 57 | Is is 0 58 | NotEq != 0 59 | Lt < 0 60 | LtE <= 0 61 | NotIn not in 0 62 | IsNot is not 0 63 | 64 | BitOr | 1 65 | BitXor ^ 1 66 | BitAnd & 1 67 | LShift << 1 68 | RShift >> 0 69 | Add + 1 70 | Sub - 0 71 | Mult * 1 72 | Div / 0 73 | Mod % 0 74 | FloorDiv // 0 75 | MatMult @ 0 76 | PowRHS 1 77 | Invert ~ 1 78 | UAdd + 0 79 | USub - 0 80 | Pow ** 1 81 | Await 1 82 | Num 1 83 | Constant 1 84 | """ 85 | 86 | op_data = [x.split() for x in op_data.splitlines()] 87 | op_data = [[x[0], ' '.join(x[1:-1]), int(x[-1])] for x in op_data if x] 88 | for index in range(1, len(op_data)): 89 | op_data[index][2] *= 2 90 | op_data[index][2] += op_data[index - 1][2] 91 | 92 | precedence_data = dict((getattr(ast, x, None), z) for x, y, z in op_data) 93 | symbol_data = dict((getattr(ast, x, None), y) for x, y, z in op_data) 94 | 95 | 96 | def get_op_symbol(obj, fmt='%s', symbol_data=symbol_data, type=type): 97 | """Given an AST node object, returns a string containing the symbol. 98 | """ 99 | return fmt % symbol_data[type(obj)] 100 | 101 | 102 | def get_op_precedence(obj, precedence_data=precedence_data, type=type): 103 | """Given an AST node object, returns the precedence. 104 | """ 105 | return precedence_data[type(obj)] 106 | 107 | 108 | class Precedence(object): 109 | vars().update((x, z) for x, y, z in op_data) 110 | highest = max(z for x, y, z in op_data) + 2 111 | -------------------------------------------------------------------------------- /astor/file_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright (c) 2012-2015 Patrick Maupin 8 | Copyright (c) 2013-2015 Berker Peksag 9 | 10 | Functions that interact with the filesystem go here. 11 | 12 | """ 13 | 14 | import ast 15 | import sys 16 | import os 17 | 18 | try: 19 | from tokenize import open as fopen 20 | except ImportError: 21 | fopen = open 22 | 23 | 24 | class CodeToAst(object): 25 | """Given a module, or a function that was compiled as part 26 | of a module, re-compile the module into an AST and extract 27 | the sub-AST for the function. Allow caching to reduce 28 | number of compiles. 29 | 30 | Also contains static helper utility functions to 31 | look for python files, to parse python files, and to extract 32 | the file/line information from a code object. 33 | """ 34 | 35 | @staticmethod 36 | def find_py_files(srctree, ignore=None): 37 | """Return all the python files in a source tree 38 | 39 | Ignores any path that contains the ignore string 40 | 41 | This is not used by other class methods, but is 42 | designed to be used in code that uses this class. 43 | """ 44 | 45 | if not os.path.isdir(srctree): 46 | yield os.path.split(srctree) 47 | for srcpath, _, fnames in os.walk(srctree): 48 | # Avoid infinite recursion for silly users 49 | if ignore is not None and ignore in srcpath: 50 | continue 51 | for fname in (x for x in fnames if x.endswith('.py')): 52 | yield srcpath, fname 53 | 54 | @staticmethod 55 | def parse_file(fname): 56 | """Parse a python file into an AST. 57 | 58 | This is a very thin wrapper around ast.parse 59 | 60 | TODO: Handle encodings other than the default for Python 2 61 | (issue #26) 62 | """ 63 | try: 64 | with fopen(fname) as f: 65 | fstr = f.read() 66 | except IOError: 67 | if fname != 'stdin': 68 | raise 69 | sys.stdout.write('\nReading from stdin:\n\n') 70 | fstr = sys.stdin.read() 71 | fstr = fstr.replace('\r\n', '\n').replace('\r', '\n') 72 | if not fstr.endswith('\n'): 73 | fstr += '\n' 74 | return ast.parse(fstr, filename=fname) 75 | 76 | @staticmethod 77 | def get_file_info(codeobj): 78 | """Returns the file and line number of a code object. 79 | 80 | If the code object has a __file__ attribute (e.g. if 81 | it is a module), then the returned line number will 82 | be 0 83 | """ 84 | fname = getattr(codeobj, '__file__', None) 85 | linenum = 0 86 | if fname is None: 87 | func_code = codeobj.__code__ 88 | fname = func_code.co_filename 89 | linenum = func_code.co_firstlineno 90 | fname = fname.replace('.pyc', '.py') 91 | return fname, linenum 92 | 93 | def __init__(self, cache=None): 94 | self.cache = cache or {} 95 | 96 | def __call__(self, codeobj): 97 | cache = self.cache 98 | fname = self.get_file_info(codeobj)[0] 99 | key = (fname, codeobj.__name__) 100 | result = cache.get(key) 101 | if result is not None: 102 | return result 103 | cache[key] = mod_ast = self.parse_file(fname) 104 | for obj in mod_ast.body: 105 | if not isinstance(obj, ast.FunctionDef): 106 | continue 107 | cache[(fname, obj.name)] = obj 108 | return cache[key] 109 | 110 | 111 | code_to_ast = CodeToAst() 112 | -------------------------------------------------------------------------------- /tests/test_misc.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import sys 3 | import warnings 4 | try: 5 | import unittest2 as unittest 6 | except ImportError: 7 | import unittest 8 | 9 | import astor 10 | 11 | from astor.source_repr import split_lines 12 | 13 | from .support import import_fresh_module 14 | 15 | 16 | class GetSymbolTestCase(unittest.TestCase): 17 | 18 | @unittest.skipUnless(sys.version_info >= (3, 5), 19 | "ast.MatMult introduced in Python 3.5") 20 | def test_get_mat_mult(self): 21 | self.assertEqual('@', astor.get_op_symbol(ast.MatMult())) 22 | 23 | 24 | class PublicAPITestCase(unittest.TestCase): 25 | 26 | def test_aliases(self): 27 | self.assertIs(astor.parse_file, astor.code_to_ast.parse_file) 28 | 29 | def test_codegen_from_root(self): 30 | with self.assertWarns(DeprecationWarning) as cm: 31 | astor = import_fresh_module('astor') 32 | astor.codegen.SourceGenerator 33 | self.assertEqual(len(cm.warnings), 1) 34 | # This message comes from 'astor/__init__.py'. 35 | self.assertEqual( 36 | str(cm.warning), 37 | 'astor.codegen is deprecated. Please use astor.code_gen.' 38 | ) 39 | 40 | def test_codegen_as_submodule(self): 41 | with self.assertWarns(DeprecationWarning) as cm: 42 | import astor.codegen 43 | self.assertEqual(len(cm.warnings), 1) 44 | # This message comes from 'astor/codegen.py'. 45 | self.assertEqual( 46 | str(cm.warning), 47 | 'astor.codegen module is deprecated. Please import ' 48 | 'astor.code_gen module instead.' 49 | ) 50 | 51 | def test_to_source_invalid_customize_generator(self): 52 | class InvalidGenerator: 53 | pass 54 | 55 | node = ast.parse('spam = 42') 56 | 57 | with self.assertRaises(TypeError) as cm: 58 | astor.to_source(node, source_generator_class=InvalidGenerator) 59 | self.assertEqual( 60 | str(cm.exception), 61 | 'source_generator_class should be a subclass of SourceGenerator', 62 | ) 63 | 64 | with self.assertRaises(TypeError) as cm: 65 | astor.to_source( 66 | node, 67 | source_generator_class=astor.SourceGenerator(indent_with=' ' * 4), 68 | ) 69 | self.assertEqual( 70 | str(cm.exception), 71 | 'source_generator_class should be a class', 72 | ) 73 | 74 | 75 | class FastCompareTestCase(unittest.TestCase): 76 | 77 | def test_fast_compare(self): 78 | fast_compare = astor.node_util.fast_compare 79 | 80 | def check(a, b): 81 | ast_a = ast.parse(a) 82 | ast_b = ast.parse(b) 83 | dump_a = astor.dump_tree(ast_a) 84 | dump_b = astor.dump_tree(ast_b) 85 | self.assertEqual(dump_a == dump_b, fast_compare(ast_a, ast_b)) 86 | check('a = 3', 'a = 3') 87 | check('a = 3', 'a = 5') 88 | check('a = 3 - (3, 4, 5)', 'a = 3 - (3, 4, 5)') 89 | check('a = 3 - (3, 4, 5)', 'a = 3 - (3, 4, 6)') 90 | 91 | 92 | class TreeWalkTestCase(unittest.TestCase): 93 | 94 | def test_auto_generated_attributes(self): 95 | # See #136 for more details. 96 | treewalk = astor.TreeWalk() 97 | self.assertIsInstance(treewalk.__dict__, dict) 98 | # Check that the initial state of the instance is empty. 99 | self.assertEqual(treewalk.__dict__['nodestack'], []) 100 | self.assertEqual(treewalk.__dict__['pre_handlers'], {}) 101 | self.assertEqual(treewalk.__dict__['post_handlers'], {}) 102 | 103 | 104 | class SourceReprTestCase(unittest.TestCase): 105 | """ 106 | Tests for helpers in astor.source_repr module. 107 | 108 | Note that these APIs are not public. 109 | """ 110 | 111 | @unittest.skipUnless(sys.version_info[0] == 2, 'only applies to Python 2') 112 | def test_split_lines_unicode_support(self): 113 | source = [u'copy', '\n'] 114 | self.assertEqual(split_lines(source), source) 115 | 116 | 117 | if __name__ == '__main__': 118 | unittest.main() 119 | -------------------------------------------------------------------------------- /astor/tree_walk.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright 2012 (c) Patrick Maupin 8 | Copyright 2013 (c) Berker Peksag 9 | 10 | This file contains a TreeWalk class that views a node tree 11 | as a unified whole and allows several modes of traversal. 12 | 13 | """ 14 | 15 | from .node_util import iter_node 16 | 17 | 18 | class MetaFlatten(type): 19 | """This metaclass is used to flatten classes to remove 20 | class hierarchy. 21 | 22 | This makes it easier to manipulate classes (find 23 | attributes in a single dict, etc.) 24 | 25 | """ 26 | def __new__(clstype, name, bases, clsdict): 27 | newbases = (object,) 28 | newdict = {} 29 | for base in reversed(bases): 30 | if base not in newbases: 31 | newdict.update(vars(base)) 32 | newdict.update(clsdict) 33 | # These are class-bound, we should let Python recreate them. 34 | newdict.pop('__dict__', None) 35 | newdict.pop('__weakref__', None) 36 | # Delegate the real work to type 37 | return type.__new__(clstype, name, newbases, newdict) 38 | 39 | 40 | MetaFlatten = MetaFlatten('MetaFlatten', (object,), {}) 41 | 42 | 43 | class TreeWalk(MetaFlatten): 44 | """The TreeWalk class can be used as a superclass in order 45 | to walk an AST or similar tree. 46 | 47 | Unlike other treewalkers, this class can walk a tree either 48 | recursively or non-recursively. Subclasses can define 49 | methods with the following signatures:: 50 | 51 | def pre_xxx(self): 52 | pass 53 | 54 | def post_xxx(self): 55 | pass 56 | 57 | def init_xxx(self): 58 | pass 59 | 60 | Where 'xxx' is one of: 61 | 62 | - A class name 63 | - An attribute member name concatenated with '_name' 64 | For example, 'pre_targets_name' will process nodes 65 | that are referenced by the name 'targets' in their 66 | parent's node. 67 | - An attribute member name concatenated with '_item' 68 | For example, 'pre_targets_item' will process nodes 69 | that are in a list that is the targets attribute 70 | of some node. 71 | 72 | pre_xxx will process a node before processing any of its subnodes. 73 | if the return value from pre_xxx evalates to true, then walk 74 | will not process any of the subnodes. Those can be manually 75 | processed, if desired, by calling self.walk(node) on the subnodes 76 | before returning True. 77 | 78 | post_xxx will process a node after processing all its subnodes. 79 | 80 | init_xxx methods can decorate the class instance with subclass-specific 81 | information. A single init_whatever method could be written, but to 82 | make it easy to keep initialization with use, any number of init_xxx 83 | methods can be written. They will be called in alphabetical order. 84 | 85 | """ 86 | 87 | def __init__(self, node=None): 88 | self.nodestack = [] 89 | self.setup() 90 | if node is not None: 91 | self.walk(node) 92 | 93 | def setup(self): 94 | """All the node-specific handlers are setup at 95 | object initialization time. 96 | 97 | """ 98 | self.pre_handlers = pre_handlers = {} 99 | self.post_handlers = post_handlers = {} 100 | for name in sorted(vars(type(self))): 101 | if name.startswith('init_'): 102 | getattr(self, name)() 103 | elif name.startswith('pre_'): 104 | pre_handlers[name[4:]] = getattr(self, name) 105 | elif name.startswith('post_'): 106 | post_handlers[name[5:]] = getattr(self, name) 107 | 108 | def walk(self, node, name='', list=list, len=len, type=type): 109 | """Walk the tree starting at a given node. 110 | 111 | Maintain a stack of nodes. 112 | 113 | """ 114 | pre_handlers = self.pre_handlers.get 115 | post_handlers = self.post_handlers.get 116 | nodestack = self.nodestack 117 | emptystack = len(nodestack) 118 | append, pop = nodestack.append, nodestack.pop 119 | append([node, name, list(iter_node(node, name + '_item')), -1]) 120 | while len(nodestack) > emptystack: 121 | node, name, subnodes, index = nodestack[-1] 122 | if index >= len(subnodes): 123 | handler = (post_handlers(type(node).__name__) or 124 | post_handlers(name + '_name')) 125 | if handler is None: 126 | pop() 127 | continue 128 | self.cur_node = node 129 | self.cur_name = name 130 | handler() 131 | current = nodestack and nodestack[-1] 132 | popstack = current and current[0] is node 133 | if popstack and current[-1] >= len(current[-2]): 134 | pop() 135 | continue 136 | nodestack[-1][-1] = index + 1 137 | if index < 0: 138 | handler = (pre_handlers(type(node).__name__) or 139 | pre_handlers(name + '_name')) 140 | if handler is not None: 141 | self.cur_node = node 142 | self.cur_name = name 143 | if handler(): 144 | pop() 145 | else: 146 | node, name = subnodes[index] 147 | append([node, name, list(iter_node(node, name + '_item')), -1]) 148 | 149 | @property 150 | def parent(self): 151 | """Return the parent node of the current node.""" 152 | nodestack = self.nodestack 153 | if len(nodestack) < 2: 154 | return None 155 | return nodestack[-2][0] 156 | 157 | @property 158 | def parent_name(self): 159 | """Return the parent node and name.""" 160 | nodestack = self.nodestack 161 | if len(nodestack) < 2: 162 | return None 163 | return nodestack[-2][:2] 164 | 165 | def replace(self, new_node): 166 | """Replace a node after first checking integrity of node stack.""" 167 | cur_node = self.cur_node 168 | nodestack = self.nodestack 169 | cur = nodestack.pop() 170 | prev = nodestack[-1] 171 | index = prev[-1] - 1 172 | oldnode, name = prev[-2][index] 173 | assert cur[0] is cur_node is oldnode, (cur[0], cur_node, prev[-2], 174 | index) 175 | parent = prev[0] 176 | if isinstance(parent, list): 177 | parent[index] = new_node 178 | else: 179 | setattr(parent, name, new_node) 180 | -------------------------------------------------------------------------------- /astor/node_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright 2012-2015 (c) Patrick Maupin 8 | Copyright 2013-2015 (c) Berker Peksag 9 | 10 | Utilities for node (and, by extension, tree) manipulation. 11 | For a whole-tree approach, see the treewalk submodule. 12 | 13 | """ 14 | 15 | import ast 16 | import itertools 17 | 18 | try: 19 | zip_longest = itertools.zip_longest 20 | except AttributeError: 21 | zip_longest = itertools.izip_longest 22 | 23 | 24 | class NonExistent(object): 25 | """This is not the class you are looking for. 26 | """ 27 | pass 28 | 29 | 30 | def iter_node(node, name='', unknown=None, 31 | # Runtime optimization 32 | list=list, getattr=getattr, isinstance=isinstance, 33 | enumerate=enumerate, missing=NonExistent): 34 | """Iterates over an object: 35 | 36 | - If the object has a _fields attribute, 37 | it gets attributes in the order of this 38 | and returns name, value pairs. 39 | 40 | - Otherwise, if the object is a list instance, 41 | it returns name, value pairs for each item 42 | in the list, where the name is passed into 43 | this function (defaults to blank). 44 | 45 | - Can update an unknown set with information about 46 | attributes that do not exist in fields. 47 | """ 48 | fields = getattr(node, '_fields', None) 49 | if fields is not None: 50 | for name in fields: 51 | value = getattr(node, name, missing) 52 | if value is not missing: 53 | yield value, name 54 | if unknown is not None: 55 | unknown.update(set(vars(node)) - set(fields)) 56 | elif isinstance(node, list): 57 | for value in node: 58 | yield value, name 59 | 60 | 61 | def dump_tree(node, name=None, initial_indent='', indentation=' ', 62 | maxline=120, maxmerged=80, 63 | # Runtime optimization 64 | iter_node=iter_node, special=ast.AST, 65 | list=list, isinstance=isinstance, type=type, len=len): 66 | """Dumps an AST or similar structure: 67 | 68 | - Pretty-prints with indentation 69 | - Doesn't print line/column/ctx info 70 | 71 | """ 72 | def dump(node, name=None, indent=''): 73 | level = indent + indentation 74 | name = name and name + '=' or '' 75 | values = list(iter_node(node)) 76 | if isinstance(node, list): 77 | prefix, suffix = '%s[' % name, ']' 78 | elif values: 79 | prefix, suffix = '%s%s(' % (name, type(node).__name__), ')' 80 | elif isinstance(node, special): 81 | prefix, suffix = name + type(node).__name__, '' 82 | else: 83 | return '%s%s' % (name, repr(node)) 84 | node = [dump(a, b, level) for a, b in values if b != 'ctx'] 85 | oneline = '%s%s%s' % (prefix, ', '.join(node), suffix) 86 | if len(oneline) + len(indent) < maxline: 87 | return '%s' % oneline 88 | if node and len(prefix) + len(node[0]) < maxmerged: 89 | prefix = '%s%s,' % (prefix, node.pop(0)) 90 | node = (',\n%s' % level).join(node).lstrip() 91 | return '%s\n%s%s%s' % (prefix, level, node, suffix) 92 | return dump(node, name, initial_indent) 93 | 94 | 95 | def strip_tree(node, 96 | # Runtime optimization 97 | iter_node=iter_node, special=ast.AST, 98 | list=list, isinstance=isinstance, type=type, len=len): 99 | """Strips an AST by removing all attributes not in _fields. 100 | 101 | Returns a set of the names of all attributes stripped. 102 | 103 | This canonicalizes two trees for comparison purposes. 104 | """ 105 | stripped = set() 106 | 107 | def strip(node, indent): 108 | unknown = set() 109 | leaf = True 110 | for subnode, _ in iter_node(node, unknown=unknown): 111 | leaf = False 112 | strip(subnode, indent + ' ') 113 | if leaf: 114 | if isinstance(node, special): 115 | unknown = set(vars(node)) 116 | stripped.update(unknown) 117 | for name in unknown: 118 | delattr(node, name) 119 | if hasattr(node, 'ctx'): 120 | delattr(node, 'ctx') 121 | if 'ctx' in node._fields: 122 | mylist = list(node._fields) 123 | mylist.remove('ctx') 124 | node._fields = mylist 125 | strip(node, '') 126 | return stripped 127 | 128 | 129 | class ExplicitNodeVisitor(ast.NodeVisitor): 130 | """This expands on the ast module's NodeVisitor class 131 | to remove any implicit visits. 132 | 133 | """ 134 | 135 | def abort_visit(node): # XXX: self? 136 | msg = 'No defined handler for node of type %s' 137 | raise AttributeError(msg % node.__class__.__name__) 138 | 139 | def visit(self, node, abort=abort_visit): 140 | """Visit a node.""" 141 | method = 'visit_' + node.__class__.__name__ 142 | visitor = getattr(self, method, abort) 143 | return visitor(node) 144 | 145 | 146 | def allow_ast_comparison(): 147 | """This ugly little monkey-patcher adds in a helper class 148 | to all the AST node types. This helper class allows 149 | eq/ne comparisons to work, so that entire trees can 150 | be easily compared by Python's comparison machinery. 151 | Used by the anti8 functions to compare old and new ASTs. 152 | Could also be used by the test library. 153 | 154 | 155 | """ 156 | 157 | class CompareHelper(object): 158 | def __eq__(self, other): 159 | return type(self) == type(other) and vars(self) == vars(other) 160 | 161 | def __ne__(self, other): 162 | return type(self) != type(other) or vars(self) != vars(other) 163 | 164 | for item in vars(ast).values(): 165 | if type(item) != type: 166 | continue 167 | if issubclass(item, ast.AST): 168 | try: 169 | item.__bases__ = tuple(list(item.__bases__) + [CompareHelper]) 170 | except TypeError: 171 | pass 172 | 173 | 174 | def fast_compare(tree1, tree2): 175 | """ This is optimized to compare two AST trees for equality. 176 | It makes several assumptions that are currently true for 177 | AST trees used by rtrip, and it doesn't examine the _attributes. 178 | """ 179 | 180 | geta = ast.AST.__getattribute__ 181 | 182 | work = [(tree1, tree2)] 183 | pop = work.pop 184 | extend = work.extend 185 | # TypeError in cPython, AttributeError in PyPy 186 | exception = TypeError, AttributeError 187 | zipl = zip_longest 188 | type_ = type 189 | list_ = list 190 | while work: 191 | n1, n2 = pop() 192 | try: 193 | f1 = geta(n1, '_fields') 194 | f2 = geta(n2, '_fields') 195 | except exception: 196 | if type_(n1) is list_: 197 | extend(zipl(n1, n2)) 198 | continue 199 | if n1 == n2: 200 | continue 201 | return False 202 | else: 203 | f1 = [x for x in f1 if x != 'ctx'] 204 | if f1 != [x for x in f2 if x != 'ctx']: 205 | return False 206 | extend((geta(n1, fname), geta(n2, fname)) for fname in f1) 207 | 208 | return True 209 | -------------------------------------------------------------------------------- /astor/rtrip.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Part of the astor library for Python AST manipulation. 5 | 6 | License: 3-clause BSD 7 | 8 | Copyright (c) 2015 Patrick Maupin 9 | """ 10 | 11 | import sys 12 | import os 13 | import ast 14 | import shutil 15 | import logging 16 | 17 | from astor.code_gen import to_source 18 | from astor.file_util import code_to_ast 19 | from astor.node_util import (allow_ast_comparison, dump_tree, 20 | strip_tree, fast_compare) 21 | 22 | 23 | dsttree = 'tmp_rtrip' 24 | 25 | # TODO: Remove this workaround once we remove version 2 support 26 | 27 | 28 | def out_prep(s, pre_encoded=(sys.version_info[0] == 2)): 29 | return s if pre_encoded else s.encode('utf-8') 30 | 31 | 32 | def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False, 33 | ignore_exceptions=False, fullcomp=False): 34 | """Walk the srctree, and convert/copy all python files 35 | into the dsttree 36 | 37 | """ 38 | 39 | if fullcomp: 40 | allow_ast_comparison() 41 | 42 | parse_file = code_to_ast.parse_file 43 | find_py_files = code_to_ast.find_py_files 44 | srctree = os.path.normpath(srctree) 45 | 46 | if not readonly: 47 | dsttree = os.path.normpath(dsttree) 48 | logging.info('') 49 | logging.info('Trashing ' + dsttree) 50 | shutil.rmtree(dsttree, True) 51 | 52 | unknown_src_nodes = set() 53 | unknown_dst_nodes = set() 54 | badfiles = set() 55 | broken = [] 56 | 57 | oldpath = None 58 | 59 | allfiles = find_py_files(srctree, None if readonly else dsttree) 60 | for srcpath, fname in allfiles: 61 | # Create destination directory 62 | if not readonly and srcpath != oldpath: 63 | oldpath = srcpath 64 | if srcpath >= srctree: 65 | dstpath = srcpath.replace(srctree, dsttree, 1) 66 | if not dstpath.startswith(dsttree): 67 | raise ValueError("%s not a subdirectory of %s" % 68 | (dstpath, dsttree)) 69 | else: 70 | assert srctree.startswith(srcpath) 71 | dstpath = dsttree 72 | os.makedirs(dstpath) 73 | 74 | srcfname = os.path.join(srcpath, fname) 75 | logging.info('Converting %s' % srcfname) 76 | try: 77 | srcast = parse_file(srcfname) 78 | except SyntaxError: 79 | badfiles.add(srcfname) 80 | continue 81 | 82 | try: 83 | dsttxt = to_source(srcast) 84 | except Exception: 85 | if not ignore_exceptions: 86 | raise 87 | dsttxt = '' 88 | 89 | if not readonly: 90 | dstfname = os.path.join(dstpath, fname) 91 | try: 92 | with open(dstfname, 'wb') as f: 93 | f.write(out_prep(dsttxt)) 94 | except UnicodeEncodeError: 95 | badfiles.add(dstfname) 96 | 97 | # As a sanity check, make sure that ASTs themselves 98 | # round-trip OK 99 | try: 100 | dstast = ast.parse(dsttxt) if readonly else parse_file(dstfname) 101 | except SyntaxError: 102 | dstast = [] 103 | if fullcomp: 104 | unknown_src_nodes.update(strip_tree(srcast)) 105 | unknown_dst_nodes.update(strip_tree(dstast)) 106 | bad = srcast != dstast 107 | else: 108 | bad = not fast_compare(srcast, dstast) 109 | if dumpall or bad: 110 | srcdump = dump_tree(srcast) 111 | dstdump = dump_tree(dstast) 112 | logging.warning(' calculating dump -- %s' % 113 | ('bad' if bad else 'OK')) 114 | if bad: 115 | broken.append(srcfname) 116 | if dumpall or bad: 117 | if not readonly: 118 | try: 119 | with open(dstfname[:-3] + '.srcdmp', 'wb') as f: 120 | f.write(out_prep(srcdump)) 121 | except UnicodeEncodeError: 122 | badfiles.add(dstfname[:-3] + '.srcdmp') 123 | try: 124 | with open(dstfname[:-3] + '.dstdmp', 'wb') as f: 125 | f.write(out_prep(dstdump)) 126 | except UnicodeEncodeError: 127 | badfiles.add(dstfname[:-3] + '.dstdmp') 128 | elif dumpall: 129 | sys.stdout.write('\n\nAST:\n\n ') 130 | sys.stdout.write(srcdump.replace('\n', '\n ')) 131 | sys.stdout.write('\n\nDecompile:\n\n ') 132 | sys.stdout.write(dsttxt.replace('\n', '\n ')) 133 | sys.stdout.write('\n\nNew AST:\n\n ') 134 | sys.stdout.write('(same as old)' if dstdump == srcdump 135 | else dstdump.replace('\n', '\n ')) 136 | sys.stdout.write('\n') 137 | 138 | if badfiles: 139 | logging.warning('\nFiles not processed due to syntax errors:') 140 | for fname in sorted(badfiles): 141 | logging.warning(' %s' % fname) 142 | if broken: 143 | logging.warning('\nFiles failed to round-trip to AST:') 144 | for srcfname in broken: 145 | logging.warning(' %s' % srcfname) 146 | 147 | ok_to_strip = 'col_offset _precedence _use_parens lineno _p_op _pp' 148 | ok_to_strip = set(ok_to_strip.split()) 149 | bad_nodes = (unknown_dst_nodes | unknown_src_nodes) - ok_to_strip 150 | if bad_nodes: 151 | logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes) 152 | logging.info('\n') 153 | return broken 154 | 155 | 156 | def usage(msg): 157 | raise SystemExit(textwrap.dedent(""" 158 | 159 | Error: %s 160 | 161 | Usage: 162 | 163 | python -m astor.rtrip [readonly] [] 164 | 165 | 166 | This utility tests round-tripping of Python source to AST 167 | and back to source. 168 | 169 | If readonly is specified, then the source will be tested, 170 | but no files will be written. 171 | 172 | if the source is specified to be "stdin" (without quotes) 173 | then any source entered at the command line will be compiled 174 | into an AST, converted back to text, and then compiled to 175 | an AST again, and the results will be displayed to stdout. 176 | 177 | If neither readonly nor stdin is specified, then rtrip 178 | will create a mirror directory named tmp_rtrip and will 179 | recursively round-trip all the Python source from the source 180 | into the tmp_rtrip dir, after compiling it and then reconstituting 181 | it through code_gen.to_source. 182 | 183 | If the source is not specified, the entire Python library will be used. 184 | 185 | """) % msg) 186 | 187 | 188 | if __name__ == '__main__': 189 | import textwrap 190 | 191 | args = sys.argv[1:] 192 | 193 | readonly = 'readonly' in args 194 | if readonly: 195 | args.remove('readonly') 196 | 197 | if not args: 198 | args = [os.path.dirname(textwrap.__file__)] 199 | 200 | if len(args) > 1: 201 | usage("Too many arguments") 202 | 203 | fname, = args 204 | dumpall = False 205 | if not os.path.exists(fname): 206 | dumpall = fname == 'stdin' or usage("Cannot find directory %s" % fname) 207 | 208 | logging.basicConfig(format='%(msg)s', level=logging.INFO) 209 | convert(fname, readonly=readonly or dumpall, dumpall=dumpall) 210 | -------------------------------------------------------------------------------- /astor/source_repr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright (c) 2015 Patrick Maupin 8 | 9 | Pretty-print source -- post-process for the decompiler 10 | 11 | The goals of the initial cut of this engine are: 12 | 13 | 1) Do a passable, if not PEP8, job of line-wrapping. 14 | 15 | 2) Serve as an example of an interface to the decompiler 16 | for anybody who wants to do a better job. :) 17 | """ 18 | 19 | 20 | def pretty_source(source): 21 | """ Prettify the source. 22 | """ 23 | 24 | return ''.join(split_lines(source)) 25 | 26 | 27 | def split_lines(source, maxline=79): 28 | """Split inputs according to lines. 29 | If a line is short enough, just yield it. 30 | Otherwise, fix it. 31 | """ 32 | result = [] 33 | extend = result.extend 34 | append = result.append 35 | line = [] 36 | multiline = False 37 | count = 0 38 | for item in source: 39 | newline = type(item)('\n') 40 | index = item.find(newline) 41 | if index: 42 | line.append(item) 43 | multiline = index > 0 44 | count += len(item) 45 | else: 46 | if line: 47 | if count <= maxline or multiline: 48 | extend(line) 49 | else: 50 | wrap_line(line, maxline, result) 51 | count = 0 52 | multiline = False 53 | line = [] 54 | append(item) 55 | return result 56 | 57 | 58 | def count(group, slen=str.__len__): 59 | return sum([slen(x) for x in group]) 60 | 61 | 62 | def wrap_line(line, maxline=79, result=[], count=count): 63 | """ We have a line that is too long, 64 | so we're going to try to wrap it. 65 | """ 66 | 67 | # Extract the indentation 68 | 69 | append = result.append 70 | extend = result.extend 71 | 72 | indentation = line[0] 73 | lenfirst = len(indentation) 74 | indent = lenfirst - len(indentation.lstrip()) 75 | assert indent in (0, lenfirst) 76 | indentation = line.pop(0) if indent else '' 77 | 78 | # Get splittable/non-splittable groups 79 | 80 | dgroups = list(delimiter_groups(line)) 81 | unsplittable = dgroups[::2] 82 | splittable = dgroups[1::2] 83 | 84 | # If the largest non-splittable group won't fit 85 | # on a line, try to add parentheses to the line. 86 | 87 | if max(count(x) for x in unsplittable) > maxline - indent: 88 | line = add_parens(line, maxline, indent) 89 | dgroups = list(delimiter_groups(line)) 90 | unsplittable = dgroups[::2] 91 | splittable = dgroups[1::2] 92 | 93 | # Deal with the first (always unsplittable) group, and 94 | # then set up to deal with the remainder in pairs. 95 | 96 | first = unsplittable[0] 97 | append(indentation) 98 | extend(first) 99 | if not splittable: 100 | return result 101 | pos = indent + count(first) 102 | indentation += ' ' 103 | indent += 4 104 | if indent >= maxline / 2: 105 | maxline = maxline / 2 + indent 106 | 107 | for sg, nsg in zip(splittable, unsplittable[1:]): 108 | 109 | if sg: 110 | # If we already have stuff on the line and even 111 | # the very first item won't fit, start a new line 112 | if pos > indent and pos + len(sg[0]) > maxline: 113 | append('\n') 114 | append(indentation) 115 | pos = indent 116 | 117 | # Dump lines out of the splittable group 118 | # until the entire thing fits 119 | csg = count(sg) 120 | while pos + csg > maxline: 121 | ready, sg = split_group(sg, pos, maxline) 122 | if ready[-1].endswith(' '): 123 | ready[-1] = ready[-1][:-1] 124 | extend(ready) 125 | append('\n') 126 | append(indentation) 127 | pos = indent 128 | csg = count(sg) 129 | 130 | # Dump the remainder of the splittable group 131 | if sg: 132 | extend(sg) 133 | pos += csg 134 | 135 | # Dump the unsplittable group, optionally 136 | # preceded by a linefeed. 137 | cnsg = count(nsg) 138 | if pos > indent and pos + cnsg > maxline: 139 | append('\n') 140 | append(indentation) 141 | pos = indent 142 | extend(nsg) 143 | pos += cnsg 144 | 145 | 146 | def split_group(source, pos, maxline): 147 | """ Split a group into two subgroups. The 148 | first will be appended to the current 149 | line, the second will start the new line. 150 | 151 | Note that the first group must always 152 | contain at least one item. 153 | 154 | The original group may be destroyed. 155 | """ 156 | first = [] 157 | source.reverse() 158 | while source: 159 | tok = source.pop() 160 | first.append(tok) 161 | pos += len(tok) 162 | if source: 163 | tok = source[-1] 164 | allowed = (maxline + 1) if tok.endswith(' ') else (maxline - 4) 165 | if pos + len(tok) > allowed: 166 | break 167 | 168 | source.reverse() 169 | return first, source 170 | 171 | 172 | begin_delim = set('([{') 173 | end_delim = set(')]}') 174 | end_delim.add('):') 175 | 176 | 177 | def delimiter_groups(line, begin_delim=begin_delim, 178 | end_delim=end_delim): 179 | """Split a line into alternating groups. 180 | The first group cannot have a line feed inserted, 181 | the next one can, etc. 182 | """ 183 | text = [] 184 | line = iter(line) 185 | while True: 186 | # First build and yield an unsplittable group 187 | for item in line: 188 | text.append(item) 189 | if item in begin_delim: 190 | break 191 | if not text: 192 | break 193 | yield text 194 | 195 | # Now build and yield a splittable group 196 | level = 0 197 | text = [] 198 | for item in line: 199 | if item in begin_delim: 200 | level += 1 201 | elif item in end_delim: 202 | level -= 1 203 | if level < 0: 204 | yield text 205 | text = [item] 206 | break 207 | text.append(item) 208 | else: 209 | assert not text, text 210 | break 211 | 212 | 213 | statements = set(['del ', 'return', 'yield ', 'if ', 'while ']) 214 | 215 | 216 | def add_parens(line, maxline, indent, statements=statements, count=count): 217 | """Attempt to add parentheses around the line 218 | in order to make it splittable. 219 | """ 220 | 221 | if line[0] in statements: 222 | index = 1 223 | if not line[0].endswith(' '): 224 | index = 2 225 | assert line[1] == ' ' 226 | line.insert(index, '(') 227 | if line[-1] == ':': 228 | line.insert(-1, ')') 229 | else: 230 | line.append(')') 231 | 232 | # That was the easy stuff. Now for assignments. 233 | groups = list(get_assign_groups(line)) 234 | if len(groups) == 1: 235 | # So sad, too bad 236 | return line 237 | 238 | counts = list(count(x) for x in groups) 239 | didwrap = False 240 | 241 | # If the LHS is large, wrap it first 242 | if sum(counts[:-1]) >= maxline - indent - 4: 243 | for group in groups[:-1]: 244 | didwrap = False # Only want to know about last group 245 | if len(group) > 1: 246 | group.insert(0, '(') 247 | group.insert(-1, ')') 248 | didwrap = True 249 | 250 | # Might not need to wrap the RHS if wrapped the LHS 251 | if not didwrap or counts[-1] > maxline - indent - 10: 252 | groups[-1].insert(0, '(') 253 | groups[-1].append(')') 254 | 255 | return [item for group in groups for item in group] 256 | 257 | 258 | # Assignment operators 259 | ops = list('|^&+-*/%@~') + '<< >> // **'.split() + [''] 260 | ops = set(' %s= ' % x for x in ops) 261 | 262 | 263 | def get_assign_groups(line, ops=ops): 264 | """ Split a line into groups by assignment (including 265 | augmented assignment) 266 | """ 267 | group = [] 268 | for item in line: 269 | group.append(item) 270 | if item in ops: 271 | yield group 272 | group = [] 273 | yield group 274 | -------------------------------------------------------------------------------- /tests/build_expressions.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Part of the astor library for Python AST manipulation. 5 | 6 | License: 3-clause BSD 7 | 8 | Copyright (c) 2015 Patrick Maupin 9 | 10 | This module generates a lot of permutations of Python 11 | expressions, and dumps them into a python module 12 | all_expr_x_y.py (where x and y are the python version tuple) 13 | as a string. 14 | 15 | This string is later used by check_expressions. 16 | 17 | This module takes a loooooooooong time to execute. 18 | 19 | """ 20 | 21 | import sys 22 | import collections 23 | import itertools 24 | import textwrap 25 | import ast 26 | import astor 27 | 28 | all_operators = ( 29 | # Selected special operands 30 | '3 -3 () yield', 31 | # operators with one parameter 32 | 'yield lambda_: not + - ~ $, yield_from', 33 | # operators with two parameters 34 | 'or and == != > >= < <= in not_in is is_not ' 35 | '| ^ & << >> + - * / % // @ ** for$in$ $($) $[$] . ' 36 | '$,$ ', 37 | # operators with 3 parameters 38 | '$if$else$ $for$in$' 39 | ) 40 | 41 | 42 | select_operators = ( 43 | # Selected special operands -- remove 44 | # some at redundant precedence levels 45 | '-3', 46 | # operators with one parameter 47 | 'yield lambda_: not - ~ $,', 48 | # operators with two parameters 49 | 'or and == in is ' 50 | '| ^ & >> - % ** for$in$ $($) . ', 51 | # operators with 3 parameters 52 | '$if$else$ $for$in$' 53 | ) 54 | 55 | 56 | def get_primitives(base): 57 | """Attempt to return formatting strings for all operators, 58 | and selected operands. 59 | Here, I use the term operator loosely to describe anything 60 | that accepts an expression and can be used in an additional 61 | expression. 62 | """ 63 | 64 | operands = [] 65 | operators = [] 66 | for nparams, s in enumerate(base): 67 | s = s.replace('%', '%%').split() 68 | for s in (x.replace('_', ' ') for x in s): 69 | if nparams and '$' not in s: 70 | assert nparams in (1, 2) 71 | s = '%s%s$' % ('$' if nparams == 2 else '', s) 72 | assert nparams == s.count('$'), (nparams, s) 73 | s = s.replace('$', ' %s ').strip() 74 | 75 | # Normalize the spacing 76 | s = s.replace(' ,', ',') 77 | s = s.replace(' . ', '.') 78 | s = s.replace(' [ ', '[').replace(' ]', ']') 79 | s = s.replace(' ( ', '(').replace(' )', ')') 80 | if nparams == 1: 81 | s = s.replace('+ ', '+') 82 | s = s.replace('- ', '-') 83 | s = s.replace('~ ', '~') 84 | 85 | if nparams: 86 | operators.append((s, nparams)) 87 | else: 88 | operands.append(s) 89 | return operators, operands 90 | 91 | 92 | def get_sub_combinations(maxop): 93 | """Return a dictionary of lists of combinations suitable 94 | for recursively building expressions. 95 | 96 | Each dictionary key is a tuple of (numops, numoperands), 97 | where: 98 | 99 | numops is the number of operators we 100 | should build an expression for 101 | 102 | numterms is the number of operands required 103 | by the current operator. 104 | 105 | Each list contains all permutations of the number 106 | of operators that the recursively called function 107 | should use for each operand. 108 | """ 109 | combo = collections.defaultdict(list) 110 | for numops in range(maxop+1): 111 | if numops: 112 | combo[numops, 1].append((numops-1,)) 113 | for op1 in range(numops): 114 | combo[numops, 2].append((op1, numops - op1 - 1)) 115 | for op2 in range(numops - op1): 116 | combo[numops, 3].append((op1, op2, numops - op1 - op2 - 1)) 117 | return combo 118 | 119 | 120 | def get_paren_combos(): 121 | """This function returns a list of lists. 122 | The first list is indexed by the number of operands 123 | the current operator has. 124 | 125 | Each sublist contains all permutations of wrapping 126 | the operands in parentheses or not. 127 | """ 128 | results = [None] * 4 129 | options = [('%s', '(%s)')] 130 | for i in range(1, 4): 131 | results[i] = list(itertools.product(*(i * options))) 132 | return results 133 | 134 | 135 | def operand_combo(expressions, operands, max_operand=13): 136 | op_combos = [] 137 | operands = list(operands) 138 | operands.append('%s') 139 | for n in range(max_operand): 140 | this_combo = [] 141 | op_combos.append(this_combo) 142 | for i in range(n): 143 | for op in operands: 144 | mylist = ['%s'] * n 145 | mylist[i] = op 146 | this_combo.append(tuple(mylist)) 147 | for expr in expressions: 148 | expr = expr.replace('%%', '%%%%') 149 | for op in op_combos[expr.count('%s')]: 150 | yield expr % op 151 | 152 | 153 | def build(numops=2, all_operators=all_operators, use_operands=False, 154 | # Runtime optimization 155 | tuple=tuple): 156 | operators, operands = get_primitives(all_operators) 157 | combo = get_sub_combinations(numops) 158 | paren_combos = get_paren_combos() 159 | product = itertools.product 160 | try: 161 | izip = itertools.izip 162 | except AttributeError: 163 | izip = zip 164 | 165 | def recurse_build(numops): 166 | if not numops: 167 | yield '%s' 168 | for myop, nparams in operators: 169 | myop = myop.replace('%%', '%%%%') 170 | myparens = paren_combos[nparams] 171 | # print combo[numops, nparams] 172 | for mycombo in combo[numops, nparams]: 173 | # print mycombo 174 | call_again = (recurse_build(x) for x in mycombo) 175 | for subexpr in product(*call_again): 176 | for parens in myparens: 177 | wrapped = tuple(x % y for (x, y) 178 | in izip(parens, subexpr)) 179 | yield myop % wrapped 180 | result = recurse_build(numops) 181 | return operand_combo(result, operands) if use_operands else result 182 | 183 | 184 | def makelib(): 185 | parse = ast.parse 186 | dump_tree = astor.dump_tree 187 | 188 | def default_value(): return 1000000, '' 189 | mydict = collections.defaultdict(default_value) 190 | 191 | allparams = [tuple('abcdefghijklmnop'[:x]) for x in range(13)] 192 | alltxt = itertools.chain(build(1, use_operands=True), 193 | build(2, use_operands=True), 194 | build(3, select_operators)) 195 | 196 | yieldrepl = list(('yield %s %s' % (operator, operand), 197 | 'yield %s%s' % (operator, operand)) 198 | for operator in '+-' for operand in '(ab') 199 | yieldrepl.append(('yield[', 'yield [')) 200 | # alltxt = itertools.chain(build(1), build(2)) 201 | badexpr = 0 202 | goodexpr = 0 203 | silly = '3( 3.( 3[ 3.['.split() 204 | for expr in alltxt: 205 | params = allparams[expr.count('%s')] 206 | expr %= params 207 | try: 208 | myast = parse(expr) 209 | except: 210 | badexpr += 1 211 | continue 212 | goodexpr += 1 213 | key = dump_tree(myast) 214 | expr = expr.replace(', - ', ', -') 215 | ignore = [x for x in silly if x in expr] 216 | if ignore: 217 | continue 218 | if 'yield' in expr: 219 | for x in yieldrepl: 220 | expr = expr.replace(*x) 221 | mydict[key] = min(mydict[key], (len(expr), expr)) 222 | print(badexpr, goodexpr) 223 | 224 | stuff = [x[1] for x in mydict.values()] 225 | stuff.sort() 226 | 227 | lineend = '\n'.encode('utf-8') 228 | with open('all_expr_%s_%s.py' % sys.version_info[:2], 'wb') as f: 229 | f.write(textwrap.dedent(''' 230 | # AUTOMAGICALLY GENERATED!!! DO NOT MODIFY!! 231 | # 232 | all_expr = """ 233 | ''').encode('utf-8')) 234 | for item in stuff: 235 | f.write(item.encode('utf-8')) 236 | f.write(lineend) 237 | f.write('"""\n'.encode('utf-8')) 238 | 239 | 240 | if __name__ == '__main__': 241 | makelib() 242 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. 2 | ************************************************************** 3 | Note that his file was designed to be viewed at Read the Docs. 4 | Some content will not display properly when viewing using the 5 | GitHub browser. 6 | ************************************************************** 7 | 8 | .. currentmodule:: astor 9 | 10 | ############################ 11 | astor -- AST observe/rewrite 12 | ############################ 13 | 14 | :PyPI: https://pypi.org/project/astor/ 15 | :Source: https://github.com/berkerpeksag/astor 16 | :Issues: https://github.com/berkerpeksag/astor/issues/ 17 | :License: 3-clause BSD 18 | :Build status: 19 | .. image:: https://secure.travis-ci.org/berkerpeksag/astor.svg 20 | :alt: Travis CI 21 | :target: https://travis-ci.org/berkerpeksag/astor/ 22 | 23 | 24 | .. toctree:: 25 | :hidden: 26 | 27 | self 28 | changelog 29 | 30 | 31 | astor is designed to allow easy manipulation of Python source via the AST. 32 | 33 | *************** 34 | Getting Started 35 | *************** 36 | 37 | Install with **pip**: 38 | 39 | .. code-block:: bash 40 | 41 | $ pip install astor 42 | 43 | or clone the latest version from GitHub_. 44 | 45 | 46 | ******** 47 | Features 48 | ******** 49 | 50 | There are some other similar libraries, but astor focuses on the following 51 | areas: 52 | 53 | - Round-trip back to Python via Armin Ronacher's codegen.py module: 54 | 55 | - Modified AST doesn't need linenumbers, ctx, etc. or otherwise be directly 56 | compileable 57 | - Easy to read generated code as, well, code 58 | 59 | - Dump pretty-printing of AST 60 | 61 | - Harder to read than round-tripped code, but more accurate to figure out what 62 | is going on. 63 | 64 | - Easier to read than dump from built-in AST module 65 | 66 | - Non-recursive treewalk 67 | 68 | - Sometimes you want a recursive treewalk (and astor supports that, starting 69 | at any node on the tree), but sometimes you don't need to do that. astor 70 | doesn't require you to explicitly visit sub-nodes unless you want to: 71 | 72 | - You can add code that executes before a node's children are visited, and/or 73 | - You can add code that executes after a node's children are visited, and/or 74 | - You can add code that executes and keeps the node's children from being 75 | visited (and optionally visit them yourself via a recursive call) 76 | 77 | - Write functions to access the tree based on object names and/or attribute 78 | names 79 | - Enjoy easy access to parent node(s) for tree rewriting 80 | 81 | .. _deprecations: 82 | 83 | ************ 84 | Deprecations 85 | ************ 86 | 87 | .. versionadded:: 0.6 88 | 89 | Modules 90 | ~~~~~~~ 91 | 92 | =================== ==================== 93 | astor 0.5 astor 0.6+ 94 | =================== ==================== 95 | ``astor.codegen`` ``astor.code_gen`` 96 | ``astor.misc`` ``astor.file_util`` 97 | ``astor.treewalk`` ``astor.tree_walk`` 98 | =================== ==================== 99 | 100 | Functions 101 | ~~~~~~~~~ 102 | 103 | ======================== ==================== 104 | astor 0.5 astor 0.6+ 105 | ======================== ==================== 106 | ``astor.codetoast()`` ``astor.code_to_ast()`` 107 | ``astor.parsefile()`` ``astor.parse_file()`` 108 | ``astor.dump()`` ``astor.dump_tree()`` 109 | ``astor.get_anyop()`` ``astor.get_op_symbol()`` 110 | ``astor.get_boolop()`` ``astor.get_op_symbol()`` 111 | ``astor.get_binop()`` ``astor.get_op_symbol()`` 112 | ``astor.get_cmpop()`` ``astor.get_op_symbol()`` 113 | ``astor.get_unaryop()`` ``astor.get_op_symbol()`` 114 | ======================== ==================== 115 | 116 | Attributes 117 | ~~~~~~~~~~ 118 | 119 | ======================== ==================== 120 | astor 0.5 astor 0.6+ 121 | ======================== ==================== 122 | ``astor.codetoast`` ``astor.code_to_ast`` 123 | ``astor.all_symbols`` ``astor.symbol_data`` 124 | ======================== ==================== 125 | 126 | 127 | ********* 128 | Functions 129 | ********* 130 | 131 | .. function:: to_source(source, indent_with=' ' * 4, \ 132 | add_line_information=False, 133 | source_generator_class=astor.SourceGenerator) 134 | 135 | Convert a node tree back into Python source code. 136 | 137 | Each level of indentation is replaced with *indent_with*. Per default this 138 | parameter is equal to four spaces as suggested by :pep:`8`. 139 | 140 | If *add_line_information* is set to ``True`` comments for the line numbers 141 | of the nodes are added to the output. This can be used to spot wrong line 142 | number information of statement nodes. 143 | 144 | *source_generator_class* defaults to :class:`astor.SourceGenerator`, and 145 | specifies the class that will be instantiated and used to generate the 146 | source code. 147 | 148 | .. versionchanged:: 0.8 149 | *source_generator_class* was added. 150 | 151 | .. function:: codetoast 152 | .. function:: code_to_ast(codeobj) 153 | 154 | Given a module, or a function that was compiled as part 155 | of a module, re-compile the module into an AST and extract 156 | the sub-AST for the function. Allow caching to reduce 157 | number of compiles. 158 | 159 | .. deprecated:: 0.6 160 | ``codetoast()`` is deprecated. 161 | 162 | 163 | .. function:: astor.parsefile 164 | .. function:: astor.parse_file 165 | .. function:: astor.code_to_ast.parse_file(fname) 166 | 167 | Parse a Python file into an AST. 168 | 169 | This is a very thin wrapper around :func:`ast.parse`. 170 | 171 | .. deprecated:: 0.6 172 | ``astor.parsefile()`` is deprecated. 173 | 174 | .. versionadded:: 0.6.1 175 | Added the ``astor.parse_file()`` function as an alias. 176 | 177 | .. function:: astor.code_to_ast.get_file_info(codeobj) 178 | 179 | Returns the file and line number of *codeobj*. 180 | 181 | If *codeobj* has a ``__file__`` attribute (e.g. if 182 | it is a module), then the returned line number will be 0. 183 | 184 | .. versionadded:: 0.6 185 | 186 | 187 | .. function:: astor.code_to_ast.find_py_files(srctree, ignore=None) 188 | 189 | Recursively returns the path and filename for all 190 | Python files under the *srctree* directory. 191 | 192 | If *ignore* is not ``None``, it will ignore any path 193 | that contains the ignore string. 194 | 195 | .. versionadded:: 0.6 196 | 197 | 198 | .. function:: iter_node(node, unknown=None) 199 | 200 | This function iterates over an AST node object: 201 | 202 | - If the object has a _fields attribute, 203 | it gets attributes in the order of this 204 | and returns name, value pairs. 205 | 206 | - Otherwise, if the object is a list instance, 207 | it returns name, value pairs for each item 208 | in the list, where the name is passed into 209 | this function (defaults to blank). 210 | 211 | - Can update an unknown set with information about 212 | attributes that do not exist in fields. 213 | 214 | 215 | .. function:: dump 216 | .. function:: dump_tree(node, name=None, initial_indent='', \ 217 | indentation=' ', maxline=120, maxmerged=80) 218 | 219 | This function pretty prints an AST or similar structure 220 | with indentation. 221 | 222 | .. deprecated:: 0.6 223 | ``astor.dump()`` is deprecated. 224 | 225 | 226 | .. function:: strip_tree(node) 227 | 228 | This function recursively removes all attributes from 229 | an AST tree that are not referenced by the _fields member. 230 | 231 | Returns a set of the names of all attributes stripped. 232 | By default, this should just be the line number and column. 233 | 234 | This canonicalizes two trees for comparison purposes. 235 | 236 | .. versionadded:: 0.6 237 | 238 | 239 | .. function:: get_boolop 240 | .. function:: get_binop 241 | .. function:: get_cmpop 242 | .. function:: get_unaryop 243 | .. function:: get_anyop 244 | .. function:: get_op_symbol(node, fmt='%s') 245 | 246 | Given an ast node, returns the string representing the 247 | corresponding symbol. 248 | 249 | .. deprecated:: 0.6 250 | ``get_boolop()``, ``get_binop()``, ``get_cmpop()``, ``get_unaryop()`` 251 | and ``get_anyop()`` functions are deprecated. 252 | 253 | 254 | ******* 255 | Classes 256 | ******* 257 | 258 | .. class:: file_util.CodeToAst 259 | 260 | This is the base class for the helper function :func:`code_to_ast`. 261 | It may be subclassed, but probably will not need to be. 262 | 263 | 264 | .. class:: tree_walk.TreeWalk(node=None) 265 | 266 | The ``TreeWalk`` class is designed to be subclassed in order 267 | to walk a tree in arbitrary fashion. 268 | 269 | 270 | .. class:: node_util.ExplicitNodeVisitor 271 | 272 | The ``ExplicitNodeVisitor`` class subclasses the :class:`ast.NodeVisitor` 273 | class, and removes the ability to perform implicit visits. 274 | This allows for rapid failure when your code encounters a 275 | tree with a node type it was not expecting. 276 | 277 | 278 | ********************** 279 | Command-line utilities 280 | ********************** 281 | 282 | There is currently one command-line utility: 283 | 284 | rtrip 285 | ~~~~~ 286 | 287 | .. versionadded:: 0.6 288 | 289 | :: 290 | 291 | python -m astor.rtrip [readonly] [] 292 | 293 | This utility tests round-tripping of Python source to AST 294 | and back to source. 295 | 296 | .. warning:: 297 | This tool **will trash** the *tmp_rtrip* directory unless 298 | the *readonly* option is specified. 299 | 300 | If readonly is specified, then the source will be tested, 301 | but no files will be written. 302 | 303 | if the source is specified to be "stdin" (without quotes) 304 | then any source entered at the command line will be compiled 305 | into an AST, converted back to text, and then compiled to 306 | an AST again, and the results will be displayed to stdout. 307 | 308 | If neither readonly nor stdin is specified, then rtrip 309 | will create a mirror directory named tmp_rtrip and will 310 | recursively round-trip all the Python source from the source 311 | into the tmp_rtrip dir, after compiling it and then reconstituting 312 | it through code_gen.to_source. 313 | 314 | If the source is not specified, the entire Python library will be used. 315 | 316 | The purpose of rtrip is to place Python code into a canonical form. 317 | 318 | This is useful both for functional testing of astor, and for 319 | validating code edits. 320 | 321 | For example, if you make manual edits for PEP8 compliance, 322 | you can diff the rtrip output of the original code against 323 | the rtrip output of the edited code, to insure that you 324 | didn't make any functional changes. 325 | 326 | For testing astor itself, it is useful to point to a big codebase, 327 | e.g:: 328 | 329 | python -m astor.rtrip 330 | 331 | to round-trip the standard library. 332 | 333 | If any round-tripped files fail to be built or to match, the 334 | tmp_rtrip directory will also contain fname.srcdmp and fname.dstdmp, 335 | which are textual representations of the ASTs. 336 | 337 | 338 | .. note:: 339 | The canonical form is only canonical for a given version of 340 | this module and the astor toolbox. It is not guaranteed to 341 | be stable. The only desired guarantee is that two source modules 342 | that parse to the same AST will be converted back into the same 343 | canonical form. 344 | 345 | .. _GitHub: https://github.com/berkerpeksag/astor/ 346 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Release Notes 3 | ============= 4 | 5 | 0.9.0 - in development 6 | ---------------------- 7 | 8 | New features 9 | ~~~~~~~~~~~~ 10 | * Add support for Structural Pattern Matching, see :pep:`634` for more 11 | details. 12 | (Reported by avelican in `Issue 215`_ and contributed by Skurikhin Alexandr in `PR 219`_.) 13 | 14 | .. _`Issue 215`: https://github.com/berkerpeksag/astor/issues/215 15 | .. _`PR 219`: https://github.com/berkerpeksag/astor/pull/219 16 | 17 | * Add support for Type Parameter Syntax, see :pep:`695` for more details. 18 | (Contributed by am230 in `PR 222`_.) 19 | 20 | .. _`PR 222`: https://github.com/berkerpeksag/astor/pull/222 21 | 22 | Bug fixes 23 | ~~~~~~~~~ 24 | 25 | * Use ``codeobj.__name__`` in the key for the internal cache of 26 | :class:`astor.file_util.CodeToAst` rather than the line number to 27 | prevent :exc:`KeyError`. 28 | (Reported and fixed by David Charboneau in `Issue 174`_ and `PR 175`_.) 29 | 30 | .. _`Issue 174`: https://github.com/berkerpeksag/astor/pull/174 31 | .. _`PR 175`: https://github.com/berkerpeksag/astor/pull/175 32 | 33 | * Change formatting of function and assignment type annotations to be more 34 | :pep:`8` friendly. 35 | (Contributed by Venkatesh-Prasad Ranganath in `PR 170`_.) 36 | 37 | .. _`PR 170`: https://github.com/berkerpeksag/astor/pull/170 38 | 39 | * Include parentheses for keyword unpacking when syntactically required. 40 | (Contributed by Kodi Arfer in `PR 202`_.) 41 | 42 | .. _`PR 202`: https://github.com/berkerpeksag/astor/pull/202 43 | 44 | 0.8.1 - 2019-12-10 45 | ------------------ 46 | 47 | Bug fixes 48 | ~~~~~~~~~ 49 | 50 | * Fixed precedence issue for f-string expressions that caused 51 | redundant parenthesis around expression. 52 | (Reported by Ilya Kamenshchikov in `Issue 153`_ and fixed by Batuhan Taskaya in `PR 155`_.) 53 | 54 | .. _`Issue 153`: https://github.com/berkerpeksag/astor/issues/153 55 | .. _`PR 155`: https://github.com/berkerpeksag/astor/pull/155 56 | 57 | * Fixed :func:`astor.to_source` incorrectly checking whether 58 | *source_generator_class* is a subclass of :class:`astor.code_gen.SourceGenerator`. 59 | (Reported by Yu-Chia "Hank" Liu in `Issue 158`_ and fixed by Will Crichton in `PR 164`_.) 60 | 61 | .. _`Issue 158`: https://github.com/berkerpeksag/astor/issues/158 62 | .. _`PR 164`: https://github.com/berkerpeksag/astor/pull/164 63 | 64 | * Fixed :exc:`TypeError` when AST nodes with unicode strings are passed to 65 | :func:`astor.to_source`. 66 | (Reported and fixed by Dominik Moritz in `PR 154`_.) 67 | 68 | .. _`PR 154`: https://github.com/berkerpeksag/astor/pull/154 69 | 70 | * Fixed installation issue with setuptools 41.4.0 or later due to the use of 71 | an undocumented feature. 72 | (Reported and fixed by Jonathan Ringer in `Issue 162`_ and `PR 163`_.) 73 | 74 | .. _`Issue 162`: https://github.com/berkerpeksag/astor/issues/162 75 | .. _`PR 163`: https://github.com/berkerpeksag/astor/pull/163 76 | 77 | 0.8.0 - 2019-05-19 78 | ------------------ 79 | 80 | New features 81 | ~~~~~~~~~~~~ 82 | 83 | * Support ``ast.Constant`` nodes being emitted by Python 3.8 (and initially 84 | created in Python 3.6). 85 | (Reported and fixed by Chris Rink in `Issue 120`_ and `PR 121`_.) 86 | 87 | .. _`Issue 120`: https://github.com/berkerpeksag/astor/issues/120 88 | .. _`PR 121`: https://github.com/berkerpeksag/astor/pull/121 89 | 90 | * Support Python 3.8's assignment expressions. 91 | (Reported and fixed by Kodi Arfer in `Issue 126`_ and `PR 134`_.) 92 | 93 | .. _`Issue 126`: https://github.com/berkerpeksag/astor/issues/126 94 | .. _`PR 134`: https://github.com/berkerpeksag/astor/pull/134 95 | 96 | * Support Python 3.8's f-string debugging syntax. 97 | (Reported and fixed by Batuhan Taskaya in `Issue 138`_ and `PR 139`_.) 98 | 99 | .. _`Issue 138`: https://github.com/berkerpeksag/astor/issues/138 100 | .. _`PR 139`: https://github.com/berkerpeksag/astor/pull/139 101 | 102 | * :func:`astor.to_source` now has a *source_generator_class* parameter to 103 | customize source code generation. 104 | (Reported and fixed by matham in `Issue 113`_ and `PR 114`_.) 105 | 106 | .. _`Issue 113`: https://github.com/berkerpeksag/astor/issues/113 107 | .. _`PR 114`: https://github.com/berkerpeksag/astor/pull/114 108 | 109 | * The :class:`~SourceGenerator` class can now be imported from the 110 | :mod:`astor` package directly. Previously, the ``astor.code_gen`` 111 | submodule was needed to be imported. 112 | 113 | * Support Python 3.8's positional only arguments. See :pep:`570` for 114 | more details. 115 | (Reported and fixed by Batuhan Taskaya in `Issue 142`_ and `PR 143`_.) 116 | 117 | .. _`Issue 142`: https://github.com/berkerpeksag/astor/issues/142 118 | .. _`PR 143`: https://github.com/berkerpeksag/astor/pull/143 119 | 120 | Bug fixes 121 | ~~~~~~~~~ 122 | 123 | * Fix string parsing when there is a newline inside an f-string. (Reported by 124 | Adam Cécile in `Issue 119`_ and fixed by Felix Yan in `PR 123`_.) 125 | 126 | * Fixed code generation with escaped braces in f-strings. 127 | (Reported by Felix Yan in `Issue 124`_ and fixed by Kodi Arfer in `PR 125`_.) 128 | 129 | .. _`Issue 119`: https://github.com/berkerpeksag/astor/issues/119 130 | .. _`PR 123`: https://github.com/berkerpeksag/astor/pull/123 131 | .. _`Issue 124`: https://github.com/berkerpeksag/astor/issues/124 132 | .. _`PR 125`: https://github.com/berkerpeksag/astor/pull/125 133 | 134 | * Fixed code generation with attributes of integer literals, and 135 | with ``u``-prefixed string literals. 136 | (Fixed by Kodi Arfer in `PR 133`_.) 137 | 138 | .. _`PR 133`: https://github.com/berkerpeksag/astor/pull/133 139 | 140 | * Fixed code generation with very large integers. 141 | (Reported by Adam Kucz in `Issue 127`_ and fixed by Kodi Arfer in `PR 130`_.) 142 | 143 | .. _`Issue 127`: https://github.com/berkerpeksag/astor/issues/127 144 | .. _`PR 130`: https://github.com/berkerpeksag/astor/pull/130 145 | 146 | * Fixed :class:`astor.tree_walk.TreeWalk` when attempting to access attributes 147 | created by Python's type system (such as ``__dict__`` and ``__weakref__``) 148 | (Reported and fixed by esupoff in `Issue 136`_ and `PR 137`_.) 149 | 150 | .. _`Issue 136`: https://github.com/berkerpeksag/astor/issues/136 151 | .. _`PR 137`: https://github.com/berkerpeksag/astor/pull/137 152 | 153 | 0.7.1 - 2018-07-06 154 | ------------------ 155 | 156 | Bug fixes 157 | ~~~~~~~~~ 158 | 159 | * Fixed installation error by adding the ``setuputils.py`` helper to the sdist. 160 | (Reported by Adam and fixed by Berker Peksag in `Issue 116`_.) 161 | 162 | .. _`Issue 116`: https://github.com/berkerpeksag/astor/issues/116 163 | 164 | 0.7.0 - 2018-07-05 165 | ------------------ 166 | 167 | New features 168 | ~~~~~~~~~~~~ 169 | 170 | * Added initial support for Python 3.7.0. 171 | 172 | Note that if you have a subclass of ``astor.code_gen.SourceGenerator``, you 173 | may need to rename the keyword argument ``async`` of the following methods 174 | to ``is_async``: 175 | 176 | - ``visit_FunctionDef(..., is_async=False)`` 177 | - ``visit_For(..., is_async=False)`` 178 | - ``visit_With(..., is_async=False)`` 179 | 180 | (Reported and fixed by Berker Peksag in `Issue 86`_.) 181 | 182 | .. _`Issue 86`: https://github.com/berkerpeksag/astor/issues/86 183 | 184 | * Dropped support for Python 2.6 and Python 3.3. 185 | 186 | Bug fixes 187 | ~~~~~~~~~ 188 | 189 | * Fixed a bug where newlines would be inserted to a wrong place during 190 | printing f-strings with trailing newlines. 191 | (Reported by Felix Yan and contributed by Radomír Bosák in 192 | `Issue 89`_.) 193 | 194 | .. _`Issue 89`: https://github.com/berkerpeksag/astor/issues/89 195 | 196 | * Improved code generation to support ``ast.Num`` nodes containing infinities 197 | or NaNs. 198 | (Reported and fixed by Kodi Arfer in `Issue 85`_ and `Issue 100`_.) 199 | 200 | .. _`Issue 85`: https://github.com/berkerpeksag/astor/issues/85 201 | .. _`Issue 100`: https://github.com/berkerpeksag/astor/issues/100 202 | 203 | * Improved code generation to support empty sets. 204 | (Reported and fixed by Kodi Arfer in `Issue 108`_.) 205 | 206 | .. _`Issue 108`: https://github.com/berkerpeksag/astor/issues/108 207 | 208 | 0.6.2 - 2017-11-11 209 | ------------------ 210 | 211 | Bug fixes 212 | ~~~~~~~~~ 213 | 214 | * Restore backwards compatibility that was broken after 0.6.1. 215 | You can now continue to use the following pattern:: 216 | 217 | import astor 218 | 219 | class SpamCodeGenerator(astor.codegen.SourceGenerator): 220 | ... 221 | 222 | (Reported by Dan Moldovan and fixed by Berker Peksag in `Issue 87`_.) 223 | 224 | .. _`Issue 87`: https://github.com/berkerpeksag/astor/issues/87 225 | 226 | 227 | 0.6.1 - 2017-11-11 228 | ------------------ 229 | 230 | New features 231 | ~~~~~~~~~~~~ 232 | 233 | * Added ``astor.parse_file()`` as an alias to 234 | ``astor.code_to_ast.parsefile()``. 235 | (Contributed by Berker Peksag.) 236 | 237 | Bug fixes 238 | ~~~~~~~~~ 239 | 240 | * Fix compatibility layer for the ``astor.codegen`` submodule. Importing 241 | ``astor.codegen`` now succeeds and raises a :exc:`DeprecationWarning` 242 | instead of :exc:`ImportError`. 243 | (Contributed by Berker Peksag.) 244 | 245 | 246 | 0.6 - 2017-10-31 247 | ---------------- 248 | 249 | New features 250 | ~~~~~~~~~~~~ 251 | 252 | * New ``astor.rtrip`` command-line tool to test round-tripping 253 | of Python source to AST and back to source. 254 | (Contributed by Patrick Maupin.) 255 | 256 | * New pretty printer outputs much better looking code: 257 | 258 | - Remove parentheses where not necessary 259 | 260 | - Use triple-quoted strings where it makes sense 261 | 262 | - Add placeholder for function to do nice line wrapping on output 263 | 264 | (Contributed by Patrick Maupin.) 265 | 266 | * Additional Python 3.5 support: 267 | 268 | - Additional unpacking generalizations (:pep:`448`) 269 | - Async and await (:pep:`492`) 270 | 271 | (Contributed by Zack M. Davis.) 272 | 273 | * Added Python 3.6 feature support: 274 | 275 | - f-strings (:pep:`498`) 276 | - async comprehensions (:pep:`530`) 277 | - variable annotations (:pep:`526`) 278 | 279 | (Contributed by Ryan Gonzalez.) 280 | 281 | * Code cleanup, including renaming for PEP8 and deprecation of old names. 282 | See :ref:`deprecations` for more information. 283 | (Contributed by Leonard Truong in `Issue 36`_.) 284 | 285 | .. _`Issue 36`: https://github.com/berkerpeksag/astor/issues/36 286 | 287 | Bug fixes 288 | ~~~~~~~~~ 289 | 290 | * Don't put trailing comma-spaces in dictionaries. astor will now create 291 | ``{'three': 3}`` instead of ``{'three': 3, }``. 292 | (Contributed by Zack M. Davis.) 293 | 294 | * Fixed several bugs in code generation: 295 | 296 | #. Keyword-only arguments should come before ``**`` 297 | #. ``from .. import `` with no trailing module name did not work 298 | #. Support ``from .. import foo as bar`` syntax 299 | #. Support ``with foo: ...``, ``with foo as bar: ...`` and 300 | ``with foo, bar as baz: ...`` syntax 301 | #. Support ``1eNNNN`` syntax 302 | #. Support ``return (yield foo)`` syntax 303 | #. Support unary operations such as ``-(1) + ~(2) + +(3)`` 304 | #. Support ``if (yield): pass`` 305 | #. Support ``if (yield from foo): pass`` 306 | #. ``try...finally`` block needs to come after the ``try...else`` clause 307 | #. Wrap integers with parentheses where applicable (e.g. ``(0).real`` 308 | should generated) 309 | #. When the ``yield`` keyword is an expression rather than a statement, 310 | it can be a syntax error if it is not enclosed in parentheses 311 | #. Remove extraneous parentheses around ``yield from`` 312 | 313 | (Contributed by Patrick Maupin in `Issue 27`_.) 314 | 315 | .. _`Issue 27`: https://github.com/berkerpeksag/astor/issues/27 316 | 317 | 318 | 0.5 - 2015-04-18 319 | ---------------- 320 | 321 | New features 322 | ~~~~~~~~~~~~ 323 | 324 | * Added support for Python 3.5 infix matrix multiplication (:pep:`465`) 325 | (Contributed by Zack M. Davis.) 326 | 327 | 0.4.1 - 2015-03-15 328 | ------------------ 329 | 330 | Bug fixes 331 | ~~~~~~~~~ 332 | 333 | * Added missing ``SourceGenerator.visit_arguments()`` 334 | 335 | 0.4 - 2014-06-29 336 | ---------------- 337 | 338 | New features 339 | ~~~~~~~~~~~~ 340 | 341 | * Added initial test suite and documentation 342 | 343 | Bug fixes 344 | ~~~~~~~~~ 345 | 346 | * Added a visitor for ``NameConstant`` 347 | 348 | 0.3 - 2013-12-10 349 | ---------------- 350 | 351 | New features 352 | ~~~~~~~~~~~~ 353 | 354 | * Added support for Python 3.3. 355 | 356 | - Added ``YieldFrom`` 357 | - Updated ``Try`` and ``With``. 358 | 359 | Bug fixes 360 | ~~~~~~~~~ 361 | 362 | * Fixed a packaging bug on Python 3 -- see pull requests #1 and #2 for more information. 363 | 364 | 0.2.1 -- 2012-09-20 365 | ------------------- 366 | 367 | Enhancements 368 | ~~~~~~~~~~~~ 369 | 370 | * Modified TreeWalk to add ``_name`` suffix for functions that work on attribute names 371 | 372 | 373 | 0.2 -- 2012-09-19 374 | ----------------- 375 | 376 | Enhancements 377 | ~~~~~~~~~~~~ 378 | 379 | * Initial Python 3 support 380 | * Test of treewalk 381 | 382 | 0.1 -- 2012-09-19 383 | ----------------- 384 | 385 | * Initial release 386 | * Based on Armin Ronacher's codegen 387 | * Several bug fixes to that and new tree walker 388 | -------------------------------------------------------------------------------- /astor/code_gen.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Part of the astor library for Python AST manipulation. 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright (c) 2008 Armin Ronacher 8 | Copyright (c) 2012-2017 Patrick Maupin 9 | Copyright (c) 2013-2017 Berker Peksag 10 | 11 | This module converts an AST into Python source code. 12 | 13 | Before being version-controlled as part of astor, 14 | this code came from here (in 2012): 15 | 16 | https://gist.github.com/1250562 17 | 18 | """ 19 | 20 | import ast 21 | import inspect 22 | import math 23 | import sys 24 | 25 | from .op_util import get_op_symbol, get_op_precedence, Precedence 26 | from .node_util import ExplicitNodeVisitor 27 | from .string_repr import pretty_string 28 | from .source_repr import pretty_source 29 | 30 | 31 | def to_source(node, indent_with=' ' * 4, add_line_information=False, 32 | pretty_string=pretty_string, pretty_source=pretty_source, 33 | source_generator_class=None): 34 | """This function can convert a node tree back into python sourcecode. 35 | This is useful for debugging purposes, especially if you're dealing with 36 | custom asts not generated by python itself. 37 | 38 | It could be that the sourcecode is evaluable when the AST itself is not 39 | compilable / evaluable. The reason for this is that the AST contains some 40 | more data than regular sourcecode does, which is dropped during 41 | conversion. 42 | 43 | Each level of indentation is replaced with `indent_with`. Per default this 44 | parameter is equal to four spaces as suggested by PEP 8, but it might be 45 | adjusted to match the application's styleguide. 46 | 47 | If `add_line_information` is set to `True` comments for the line numbers 48 | of the nodes are added to the output. This can be used to spot wrong line 49 | number information of statement nodes. 50 | 51 | `source_generator_class` defaults to `SourceGenerator`, and specifies the 52 | class that will be instantiated and used to generate the source code. 53 | 54 | """ 55 | if source_generator_class is None: 56 | source_generator_class = SourceGenerator 57 | elif not inspect.isclass(source_generator_class): 58 | raise TypeError('source_generator_class should be a class') 59 | elif not issubclass(source_generator_class, SourceGenerator): 60 | raise TypeError('source_generator_class should be a subclass of SourceGenerator') 61 | generator = source_generator_class( 62 | indent_with, add_line_information, pretty_string) 63 | generator.visit(node) 64 | generator.result.append('\n') 65 | if set(generator.result[0]) == set('\n'): 66 | generator.result[0] = '' 67 | return pretty_source(generator.result) 68 | 69 | 70 | def precedence_setter(AST=ast.AST, get_op_precedence=get_op_precedence, 71 | isinstance=isinstance, list=list): 72 | """ This only uses a closure for performance reasons, 73 | to reduce the number of attribute lookups. (set_precedence 74 | is called a lot of times.) 75 | """ 76 | 77 | def set_precedence(value, *nodes): 78 | """Set the precedence (of the parent) into the children. 79 | """ 80 | if isinstance(value, AST): 81 | value = get_op_precedence(value) 82 | for node in nodes: 83 | if isinstance(node, AST): 84 | node._pp = value 85 | elif isinstance(node, list): 86 | set_precedence(value, *node) 87 | else: 88 | assert node is None, node 89 | 90 | return set_precedence 91 | 92 | 93 | set_precedence = precedence_setter() 94 | 95 | 96 | class Delimit(object): 97 | """A context manager that can add enclosing 98 | delimiters around the output of a 99 | SourceGenerator method. By default, the 100 | parentheses are added, but the enclosed code 101 | may set discard=True to get rid of them. 102 | """ 103 | 104 | discard = False 105 | 106 | def __init__(self, tree, *args): 107 | """ use write instead of using result directly 108 | for initial data, because it may flush 109 | preceding data into result. 110 | """ 111 | delimiters = '()' 112 | node = None 113 | op = None 114 | for arg in args: 115 | if isinstance(arg, ast.AST): 116 | if node is None: 117 | node = arg 118 | else: 119 | op = arg 120 | else: 121 | delimiters = arg 122 | tree.write(delimiters[0]) 123 | result = self.result = tree.result 124 | self.index = len(result) 125 | self.closing = delimiters[1] 126 | if node is not None: 127 | self.p = p = get_op_precedence(op or node) 128 | self.pp = pp = tree.get__pp(node) 129 | self.discard = p >= pp 130 | 131 | def __enter__(self): 132 | return self 133 | 134 | def __exit__(self, *exc_info): 135 | result = self.result 136 | start = self.index - 1 137 | if self.discard: 138 | result[start] = '' 139 | else: 140 | result.append(self.closing) 141 | 142 | 143 | class SourceGenerator(ExplicitNodeVisitor): 144 | """This visitor is able to transform a well formed syntax tree into Python 145 | sourcecode. 146 | 147 | For more details have a look at the docstring of the `to_source` 148 | function. 149 | 150 | """ 151 | 152 | using_unicode_literals = False 153 | 154 | def __init__(self, indent_with, add_line_information=False, 155 | pretty_string=pretty_string, 156 | # constants 157 | len=len, isinstance=isinstance, callable=callable): 158 | self.result = [] 159 | self.indent_with = indent_with 160 | self.add_line_information = add_line_information 161 | self.indentation = 0 # Current indentation level 162 | self.new_lines = 0 # Number of lines to insert before next code 163 | self.colinfo = 0, 0 # index in result of string containing linefeed, and 164 | # position of last linefeed in that string 165 | self.pretty_string = pretty_string 166 | AST = ast.AST 167 | 168 | visit = self.visit 169 | result = self.result 170 | append = result.append 171 | 172 | self.discard_numeric_delim_for_const = False 173 | 174 | def write(*params): 175 | """ self.write is a closure for performance (to reduce the number 176 | of attribute lookups). 177 | """ 178 | for item in params: 179 | if isinstance(item, AST): 180 | visit(item) 181 | elif callable(item): 182 | item() 183 | else: 184 | if self.new_lines: 185 | append('\n' * self.new_lines) 186 | self.colinfo = len(result), 0 187 | append(self.indent_with * self.indentation) 188 | self.new_lines = 0 189 | if item: 190 | append(item) 191 | 192 | self.write = write 193 | 194 | def __getattr__(self, name, defaults=dict(keywords=(), 195 | _pp=Precedence.highest).get): 196 | """ Get an attribute of the node. 197 | like dict.get (returns None if doesn't exist) 198 | """ 199 | if not name.startswith('get_'): 200 | raise AttributeError 201 | geta = getattr 202 | shortname = name[4:] 203 | default = defaults(shortname) 204 | 205 | def getter(node): 206 | return geta(node, shortname, default) 207 | 208 | setattr(self, name, getter) 209 | return getter 210 | 211 | def delimit(self, *args): 212 | return Delimit(self, *args) 213 | 214 | def conditional_write(self, *stuff): 215 | if stuff[-1] is not None: 216 | self.write(*stuff) 217 | # Inform the caller that we wrote 218 | return True 219 | 220 | def newline(self, node=None, extra=0): 221 | self.new_lines = max(self.new_lines, 1 + extra) 222 | if node is not None and self.add_line_information: 223 | self.write('# line: %s' % node.lineno) 224 | self.new_lines = 1 225 | 226 | def body(self, statements): 227 | self.indentation += 1 228 | self.write(*statements) 229 | self.indentation -= 1 230 | 231 | def else_body(self, elsewhat): 232 | if elsewhat: 233 | self.write(self.newline, 'else:') 234 | self.body(elsewhat) 235 | 236 | def body_or_else(self, node): 237 | self.body(node.body) 238 | self.else_body(node.orelse) 239 | 240 | def visit_arguments(self, node): 241 | want_comma = [] 242 | 243 | def write_comma(): 244 | if want_comma: 245 | self.write(', ') 246 | else: 247 | want_comma.append(True) 248 | 249 | def loop_args(args, defaults): 250 | set_precedence(Precedence.Comma, defaults) 251 | padding = [None] * (len(args) - len(defaults)) 252 | for arg, default in zip(args, padding + defaults): 253 | self.write(write_comma, arg) 254 | self.conditional_write('=', default) 255 | 256 | posonlyargs = getattr(node, 'posonlyargs', []) 257 | offset = 0 258 | if posonlyargs: 259 | offset += len(node.defaults) - len(node.args) 260 | loop_args(posonlyargs, node.defaults[:offset]) 261 | self.write(write_comma, '/') 262 | 263 | loop_args(node.args, node.defaults[offset:]) 264 | self.conditional_write(write_comma, '*', node.vararg) 265 | 266 | kwonlyargs = self.get_kwonlyargs(node) 267 | if kwonlyargs: 268 | if node.vararg is None: 269 | self.write(write_comma, '*') 270 | loop_args(kwonlyargs, node.kw_defaults) 271 | self.conditional_write(write_comma, '**', node.kwarg) 272 | 273 | def statement(self, node, *params, **kw): 274 | self.newline(node) 275 | self.write(*params) 276 | 277 | def decorators(self, node, extra): 278 | self.newline(extra=extra) 279 | for decorator in node.decorator_list: 280 | self.statement(decorator, '@', decorator) 281 | 282 | def comma_list(self, items, trailing=False): 283 | set_precedence(Precedence.Comma, *items) 284 | for idx, item in enumerate(items): 285 | self.write(', ' if idx else '', item) 286 | self.write(',' if trailing else '') 287 | 288 | def type_params(self, node): 289 | if getattr(node, 'type_params', []): # Python >= 3.12 290 | self.write('[') 291 | self.comma_list(node.type_params) 292 | self.write(']') 293 | 294 | # Statements 295 | 296 | def visit_Assign(self, node): 297 | set_precedence(node, node.value, *node.targets) 298 | self.newline(node) 299 | for target in node.targets: 300 | self.write(target, ' = ') 301 | self.visit(node.value) 302 | 303 | def visit_AugAssign(self, node): 304 | set_precedence(node, node.value, node.target) 305 | self.statement(node, node.target, get_op_symbol(node.op, ' %s= '), 306 | node.value) 307 | 308 | def visit_AnnAssign(self, node): 309 | set_precedence(node, node.target, node.annotation) 310 | set_precedence(Precedence.Comma, node.value) 311 | need_parens = isinstance(node.target, ast.Name) and not node.simple 312 | begin = '(' if need_parens else '' 313 | end = ')' if need_parens else '' 314 | self.statement(node, begin, node.target, end, ': ', node.annotation) 315 | self.conditional_write(' = ', node.value) 316 | 317 | def visit_ImportFrom(self, node): 318 | self.statement(node, 'from ', node.level * '.', 319 | node.module or '', ' import ') 320 | self.comma_list(node.names) 321 | # Goofy stuff for Python 2.7 _pyio module 322 | if node.module == '__future__' and 'unicode_literals' in ( 323 | x.name for x in node.names): 324 | self.using_unicode_literals = True 325 | 326 | def visit_Import(self, node): 327 | self.statement(node, 'import ') 328 | self.comma_list(node.names) 329 | 330 | def visit_Expr(self, node): 331 | set_precedence(node, node.value) 332 | self.statement(node) 333 | self.generic_visit(node) 334 | 335 | def visit_TypeAlias(self, node): 336 | self.statement(node, 'type ', node.name) 337 | self.type_params(node) 338 | self.write(' = ') 339 | self.visit(node.value) 340 | 341 | def visit_TypeVar(self, node): 342 | self.write(node.name) 343 | if node.bound: 344 | self.write(': ', node.bound) 345 | 346 | def visit_TypeVarTuple(self, node): 347 | self.write('*') 348 | self.write(node.name) 349 | 350 | def visit_ParamSpec(self, node): 351 | self.write('**') 352 | self.write(node.name) 353 | 354 | def visit_FunctionDef(self, node, is_async=False): 355 | prefix = 'async ' if is_async else '' 356 | self.decorators(node, 1 if self.indentation else 2) 357 | self.statement(node, '%sdef %s' % (prefix, node.name)) 358 | self.type_params(node) 359 | self.write('(') 360 | self.visit_arguments(node.args) 361 | self.write(')') 362 | self.conditional_write(' -> ', self.get_returns(node)) 363 | self.write(':') 364 | self.body(node.body) 365 | if not self.indentation: 366 | self.newline(extra=2) 367 | 368 | # introduced in Python 3.5 369 | def visit_AsyncFunctionDef(self, node): 370 | self.visit_FunctionDef(node, is_async=True) 371 | 372 | def visit_ClassDef(self, node): 373 | have_args = [] 374 | 375 | def paren_or_comma(): 376 | if have_args: 377 | self.write(', ') 378 | else: 379 | have_args.append(True) 380 | self.write('(') 381 | 382 | self.decorators(node, 2) 383 | self.statement(node, 'class %s' % node.name) 384 | self.type_params(node) 385 | for base in node.bases: 386 | self.write(paren_or_comma, base) 387 | # keywords not available in early version 388 | for keyword in self.get_keywords(node): 389 | self.write(paren_or_comma, keyword.arg or '', 390 | '=' if keyword.arg else '**', keyword.value) 391 | self.conditional_write(paren_or_comma, '*', self.get_starargs(node)) 392 | self.conditional_write(paren_or_comma, '**', self.get_kwargs(node)) 393 | self.write(have_args and '):' or ':') 394 | self.body(node.body) 395 | if not self.indentation: 396 | self.newline(extra=2) 397 | 398 | def visit_If(self, node): 399 | set_precedence(node, node.test) 400 | self.statement(node, 'if ', node.test, ':') 401 | self.body(node.body) 402 | while True: 403 | else_ = node.orelse 404 | if len(else_) == 1 and isinstance(else_[0], ast.If): 405 | node = else_[0] 406 | set_precedence(node, node.test) 407 | self.write(self.newline, 'elif ', node.test, ':') 408 | self.body(node.body) 409 | else: 410 | self.else_body(else_) 411 | break 412 | 413 | def visit_For(self, node, is_async=False): 414 | set_precedence(node, node.target) 415 | prefix = 'async ' if is_async else '' 416 | self.statement(node, '%sfor ' % prefix, 417 | node.target, ' in ', node.iter, ':') 418 | self.body_or_else(node) 419 | 420 | # introduced in Python 3.5 421 | def visit_AsyncFor(self, node): 422 | self.visit_For(node, is_async=True) 423 | 424 | def visit_While(self, node): 425 | set_precedence(node, node.test) 426 | self.statement(node, 'while ', node.test, ':') 427 | self.body_or_else(node) 428 | 429 | def visit_With(self, node, is_async=False): 430 | prefix = 'async ' if is_async else '' 431 | self.statement(node, '%swith ' % prefix) 432 | if hasattr(node, "context_expr"): # Python < 3.3 433 | self.visit_withitem(node) 434 | else: # Python >= 3.3 435 | self.comma_list(node.items) 436 | self.write(':') 437 | self.body(node.body) 438 | 439 | # new for Python 3.5 440 | def visit_AsyncWith(self, node): 441 | self.visit_With(node, is_async=True) 442 | 443 | # new for Python 3.3 444 | def visit_withitem(self, node): 445 | self.write(node.context_expr) 446 | self.conditional_write(' as ', node.optional_vars) 447 | 448 | # deprecated in Python 3.8 449 | def visit_NameConstant(self, node): 450 | self.write(repr(node.value)) 451 | 452 | def visit_Pass(self, node): 453 | self.statement(node, 'pass') 454 | 455 | def visit_Print(self, node): 456 | # XXX: python 2.6 only 457 | self.statement(node, 'print ') 458 | values = node.values 459 | if node.dest is not None: 460 | self.write(' >> ') 461 | values = [node.dest] + node.values 462 | self.comma_list(values, not node.nl) 463 | 464 | def visit_Delete(self, node): 465 | self.statement(node, 'del ') 466 | self.comma_list(node.targets) 467 | 468 | def visit_TryExcept(self, node): 469 | self.statement(node, 'try:') 470 | self.body(node.body) 471 | self.write(*node.handlers) 472 | self.else_body(node.orelse) 473 | 474 | # new for Python 3.3 475 | def visit_Try(self, node): 476 | self.statement(node, 'try:') 477 | self.body(node.body) 478 | self.write(*node.handlers) 479 | self.else_body(node.orelse) 480 | if node.finalbody: 481 | self.statement(node, 'finally:') 482 | self.body(node.finalbody) 483 | 484 | def visit_ExceptHandler(self, node): 485 | self.statement(node, 'except') 486 | if self.conditional_write(' ', node.type): 487 | self.conditional_write(' as ', node.name) 488 | self.write(':') 489 | self.body(node.body) 490 | 491 | def visit_TryFinally(self, node): 492 | self.statement(node, 'try:') 493 | self.body(node.body) 494 | self.statement(node, 'finally:') 495 | self.body(node.finalbody) 496 | 497 | def visit_Exec(self, node): 498 | dicts = node.globals, node.locals 499 | dicts = dicts[::-1] if dicts[0] is None else dicts 500 | self.statement(node, 'exec ', node.body) 501 | self.conditional_write(' in ', dicts[0]) 502 | self.conditional_write(', ', dicts[1]) 503 | 504 | def visit_Assert(self, node): 505 | set_precedence(node, node.test, node.msg) 506 | self.statement(node, 'assert ', node.test) 507 | self.conditional_write(', ', node.msg) 508 | 509 | def visit_Global(self, node): 510 | self.statement(node, 'global ', ', '.join(node.names)) 511 | 512 | def visit_Nonlocal(self, node): 513 | self.statement(node, 'nonlocal ', ', '.join(node.names)) 514 | 515 | def visit_Return(self, node): 516 | set_precedence(node, node.value) 517 | self.statement(node, 'return') 518 | self.conditional_write(' ', node.value) 519 | 520 | def visit_Break(self, node): 521 | self.statement(node, 'break') 522 | 523 | def visit_Continue(self, node): 524 | self.statement(node, 'continue') 525 | 526 | def visit_Raise(self, node): 527 | # XXX: Python 2.6 / 3.0 compatibility 528 | self.statement(node, 'raise') 529 | if self.conditional_write(' ', self.get_exc(node)): 530 | self.conditional_write(' from ', node.cause) 531 | elif self.conditional_write(' ', self.get_type(node)): 532 | set_precedence(node, node.inst) 533 | self.conditional_write(', ', node.inst) 534 | self.conditional_write(', ', node.tback) 535 | 536 | # Match statement (introduced in Python 3.10) 537 | def visit_Match(self, node): 538 | self.discard_numeric_delim_for_const = True 539 | self.statement(node, 'match ', node.subject, ':') 540 | self.body(node.cases) 541 | self.discard_numeric_delim_for_const = False 542 | 543 | def visit_match_case(self, node): 544 | self.statement(node, 'case ', node.pattern) 545 | self.conditional_write(' if ', node.guard) 546 | self.write(':') 547 | self.body(node.body) 548 | 549 | def visit_MatchSequence(self, node): 550 | with self.delimit('[]'): 551 | self.comma_list(node.patterns) 552 | 553 | def visit_MatchValue(self, node): 554 | self.write(node.value) 555 | 556 | def visit_MatchSingleton(self, node): 557 | self.write(str(node.value)) 558 | 559 | def visit_MatchStar(self, node): 560 | self.write('*', node.name or '_') 561 | 562 | def visit_MatchMapping(self, node): 563 | with self.delimit('{}'): 564 | for idx, (key, value) in enumerate(zip(node.keys, node.patterns)): 565 | if key: 566 | set_precedence(Precedence.Comma, value) 567 | self.write(', ' if idx else '', 568 | key if key else '', 569 | ': ' if key else '**', value) 570 | if node.rest: 571 | if node.keys: 572 | self.write(', ') 573 | self.write('**', node.rest) 574 | 575 | def visit_MatchAs(self, node): 576 | if not node.pattern: 577 | self.write(node.name or '_') 578 | else: 579 | self.write(node.pattern, ' as ', node.name) 580 | 581 | def visit_MatchOr(self, node): 582 | for idx, pattern in enumerate(node.patterns): 583 | self.write(' | ' if idx else '', pattern) 584 | 585 | def visit_MatchClass(self, node): 586 | write = self.write 587 | want_comma = [] 588 | 589 | def write_comma(): 590 | if want_comma: 591 | write(', ') 592 | else: 593 | want_comma.append(True) 594 | 595 | self.visit(node.cls) 596 | with self.delimit('()'): 597 | args = node.patterns 598 | for arg in args: 599 | write(write_comma, arg) 600 | 601 | kwd_attrs = node.kwd_attrs 602 | kwd_patterns = node.kwd_patterns 603 | 604 | for key, value in zip(kwd_attrs, kwd_patterns): 605 | write(write_comma, key, '=', value) 606 | 607 | # Expressions 608 | 609 | def visit_Attribute(self, node): 610 | self.write(node.value, '.', node.attr) 611 | 612 | def visit_Call(self, node, len=len): 613 | write = self.write 614 | want_comma = [] 615 | 616 | def write_comma(): 617 | if want_comma: 618 | write(', ') 619 | else: 620 | want_comma.append(True) 621 | 622 | args = node.args 623 | keywords = node.keywords 624 | starargs = self.get_starargs(node) 625 | kwargs = self.get_kwargs(node) 626 | numargs = len(args) + len(keywords) 627 | numargs += starargs is not None 628 | numargs += kwargs is not None 629 | p = Precedence.Comma if numargs > 1 else Precedence.call_one_arg 630 | set_precedence(p, *args) 631 | self.visit(node.func) 632 | write('(') 633 | for arg in args: 634 | write(write_comma, arg) 635 | 636 | set_precedence(Precedence.Comma, 637 | *(x.value for x in keywords if x.arg)) 638 | for keyword in keywords: 639 | # a keyword.arg of None indicates dictionary unpacking 640 | # (Python >= 3.5) 641 | arg = keyword.arg or '' 642 | write(write_comma, arg, '=' if arg else '**', keyword.value) 643 | # 3.5 no longer has these 644 | self.conditional_write(write_comma, '*', starargs) 645 | self.conditional_write(write_comma, '**', kwargs) 646 | write(')') 647 | 648 | def visit_Name(self, node): 649 | self.write(node.id) 650 | 651 | # ast.Constant is new in Python 3.6 and it replaces ast.Bytes, 652 | # ast.Ellipsis, ast.NameConstant, ast.Num, ast.Str in Python 3.8 653 | def visit_Constant(self, node): 654 | value = node.value 655 | 656 | if isinstance(value, (int, float, complex)): 657 | with self.delimit(node) as delimiters: 658 | if self.discard_numeric_delim_for_const: 659 | delimiters.discard = True 660 | self._handle_numeric_constant(value) 661 | elif isinstance(value, str): 662 | self._handle_string_constant(node, node.value) 663 | elif value is Ellipsis: 664 | self.write('...') 665 | else: 666 | self.write(repr(value)) 667 | 668 | def visit_JoinedStr(self, node): 669 | self._handle_string_constant(node, None, is_joined=True) 670 | 671 | def _handle_string_constant(self, node, value, is_joined=False): 672 | # embedded is used to control when we might want 673 | # to use a triple-quoted string. We determine 674 | # if we are in an assignment and/or in an expression 675 | precedence = self.get__pp(node) 676 | embedded = ((precedence > Precedence.Expr) + 677 | (precedence >= Precedence.Assign)) 678 | 679 | # Flush any pending newlines, because we're about 680 | # to severely abuse the result list. 681 | self.write('') 682 | result = self.result 683 | 684 | # Calculate the string representing the line 685 | # we are working on, up to but not including 686 | # the string we are adding. 687 | 688 | res_index, str_index = self.colinfo 689 | current_line = self.result[res_index:] 690 | if str_index: 691 | current_line[0] = current_line[0][str_index:] 692 | current_line = ''.join(current_line) 693 | 694 | has_ast_constant = sys.version_info >= (3, 6) 695 | 696 | if is_joined: 697 | # Handle new f-strings. This is a bit complicated, because 698 | # the tree can contain subnodes that recurse back to JoinedStr 699 | # subnodes... 700 | 701 | def recurse(node): 702 | for value in node.values: 703 | if isinstance(value, ast.Str): 704 | # Double up braces to escape them. 705 | self.write(value.s.replace('{', '{{').replace('}', '}}')) 706 | elif isinstance(value, ast.FormattedValue): 707 | with self.delimit('{}'): 708 | set_precedence(value, value.value) 709 | self.visit(value.value) 710 | if value.conversion != -1: 711 | self.write('!%s' % chr(value.conversion)) 712 | if value.format_spec is not None: 713 | self.write(':') 714 | recurse(value.format_spec) 715 | elif has_ast_constant and isinstance(value, ast.Constant): 716 | self.write(value.value) 717 | else: 718 | kind = type(value).__name__ 719 | assert False, 'Invalid node %s inside JoinedStr' % kind 720 | 721 | index = len(result) 722 | recurse(node) 723 | 724 | # Flush trailing newlines (so that they are part of mystr) 725 | self.write('') 726 | mystr = ''.join(result[index:]) 727 | del result[index:] 728 | self.colinfo = res_index, str_index # Put it back like we found it 729 | uni_lit = False # No formatted byte strings 730 | 731 | else: 732 | assert value is not None, "Node value cannot be None" 733 | mystr = value 734 | uni_lit = self.using_unicode_literals 735 | 736 | mystr = self.pretty_string(mystr, embedded, current_line, uni_lit) 737 | 738 | if is_joined: 739 | mystr = 'f' + mystr 740 | elif getattr(node, 'kind', False): 741 | # Constant.kind is a Python 3.8 addition. 742 | mystr = node.kind + mystr 743 | 744 | self.write(mystr) 745 | 746 | lf = mystr.rfind('\n') + 1 747 | if lf: 748 | self.colinfo = len(result) - 1, lf 749 | 750 | # deprecated in Python 3.8 751 | def visit_Str(self, node): 752 | self._handle_string_constant(node, node.s) 753 | 754 | # deprecated in Python 3.8 755 | def visit_Bytes(self, node): 756 | self.write(repr(node.s)) 757 | 758 | def _handle_numeric_constant(self, value): 759 | x = value 760 | 761 | def part(p, imaginary): 762 | # Represent infinity as 1e1000 and NaN as 1e1000-1e1000. 763 | s = 'j' if imaginary else '' 764 | try: 765 | if math.isinf(p): 766 | if p < 0: 767 | return '-1e1000' + s 768 | return '1e1000' + s 769 | if math.isnan(p): 770 | return '(1e1000%s-1e1000%s)' % (s, s) 771 | except OverflowError: 772 | # math.isinf will raise this when given an integer 773 | # that's too large to convert to a float. 774 | pass 775 | return repr(p) + s 776 | 777 | real = part(x.real if isinstance(x, complex) else x, imaginary=False) 778 | if isinstance(x, complex): 779 | imag = part(x.imag, imaginary=True) 780 | if x.real == 0: 781 | s = imag 782 | elif x.imag == 0: 783 | s = '(%s+0j)' % real 784 | else: 785 | # x has nonzero real and imaginary parts. 786 | s = '(%s%s%s)' % (real, ['+', ''][imag.startswith('-')], imag) 787 | else: 788 | s = real 789 | self.write(s) 790 | 791 | def visit_Num(self, node, 792 | # constants 793 | new=sys.version_info >= (3, 0)): 794 | with self.delimit(node) as delimiters: 795 | self._handle_numeric_constant(node.n) 796 | 797 | # We can leave the delimiters handling in visit_Num 798 | # since this is meant to handle a Python 2.x specific 799 | # issue and ast.Constant exists only in 3.6+ 800 | 801 | # The Python 2.x compiler merges a unary minus 802 | # with a number. This is a premature optimization 803 | # that we deal with here... 804 | if not new and delimiters.discard: 805 | if not isinstance(node.n, complex) and node.n < 0: 806 | pow_lhs = Precedence.Pow + 1 807 | delimiters.discard = delimiters.pp != pow_lhs 808 | else: 809 | op = self.get__p_op(node) 810 | delimiters.discard = not isinstance(op, ast.USub) 811 | 812 | def visit_Tuple(self, node): 813 | with self.delimit(node) as delimiters: 814 | # Two things are special about tuples: 815 | # 1) We cannot discard the enclosing parentheses if empty 816 | # 2) We need the trailing comma if only one item 817 | elts = node.elts 818 | delimiters.discard = delimiters.discard and elts 819 | self.comma_list(elts, len(elts) == 1) 820 | 821 | def visit_List(self, node): 822 | with self.delimit('[]'): 823 | self.comma_list(node.elts) 824 | 825 | def visit_Set(self, node): 826 | if node.elts: 827 | with self.delimit('{}'): 828 | self.comma_list(node.elts) 829 | else: 830 | # If we tried to use "{}" to represent an empty set, it would be 831 | # interpreted as an empty dictionary. We can't use "set()" either 832 | # because the name "set" might be rebound. 833 | self.write('{1}.__class__()') 834 | 835 | def visit_Dict(self, node): 836 | with self.delimit('{}'): 837 | for idx, (key, value) in enumerate(zip(node.keys, node.values)): 838 | if key: 839 | set_precedence(Precedence.Comma, value) 840 | self.write(', ' if idx else '', 841 | key if key else '', 842 | ': ' if key else '**', value) 843 | 844 | def visit_BinOp(self, node): 845 | op, left, right = node.op, node.left, node.right 846 | with self.delimit(node, op) as delimiters: 847 | ispow = isinstance(op, ast.Pow) 848 | p = delimiters.p 849 | set_precedence((Precedence.Pow + 1) if ispow else p, left) 850 | set_precedence(Precedence.PowRHS if ispow else (p + 1), right) 851 | self.write(left, get_op_symbol(op, ' %s '), right) 852 | 853 | def visit_BoolOp(self, node): 854 | with self.delimit(node, node.op) as delimiters: 855 | op = get_op_symbol(node.op, ' %s ') 856 | set_precedence(delimiters.p + 1, *node.values) 857 | for idx, value in enumerate(node.values): 858 | self.write(idx and op or '', value) 859 | 860 | def visit_Compare(self, node): 861 | with self.delimit(node, node.ops[0]) as delimiters: 862 | if self.discard_numeric_delim_for_const: 863 | delimiters.discard = True 864 | set_precedence(delimiters.p + 1, node.left, *node.comparators) 865 | self.visit(node.left) 866 | for op, right in zip(node.ops, node.comparators): 867 | self.write(get_op_symbol(op, ' %s '), right) 868 | 869 | # assignment expressions; new for Python 3.8 870 | def visit_NamedExpr(self, node): 871 | with self.delimit(node) as delimiters: 872 | p = delimiters.p 873 | set_precedence(p, node.target) 874 | set_precedence(p + 1, node.value) 875 | # Python is picky about delimiters for assignment 876 | # expressions: it requires at least one pair in any 877 | # statement that uses an assignment expression, even 878 | # when not necessary according to the precedence 879 | # rules. We address this with the kludge of forcing a 880 | # pair of parentheses around every assignment 881 | # expression. 882 | delimiters.discard = False 883 | self.write(node.target, ' := ', node.value) 884 | 885 | def visit_UnaryOp(self, node): 886 | with self.delimit(node, node.op) as delimiters: 887 | set_precedence(delimiters.p, node.operand) 888 | # In Python 2.x, a unary negative of a literal 889 | # number is merged into the number itself. This 890 | # bit of ugliness means it is useful to know 891 | # what the parent operation was... 892 | node.operand._p_op = node.op 893 | sym = get_op_symbol(node.op) 894 | self.write(sym, ' ' if sym.isalpha() else '', node.operand) 895 | 896 | def visit_Subscript(self, node): 897 | set_precedence(node, node.slice) 898 | self.write(node.value, '[', node.slice, ']') 899 | 900 | def visit_Slice(self, node): 901 | set_precedence(node, node.lower, node.upper, node.step) 902 | self.conditional_write(node.lower) 903 | self.write(':') 904 | self.conditional_write(node.upper) 905 | if node.step is not None: 906 | self.write(':') 907 | if not (isinstance(node.step, ast.Name) and 908 | node.step.id == 'None'): 909 | self.visit(node.step) 910 | 911 | def visit_Index(self, node): 912 | with self.delimit(node) as delimiters: 913 | set_precedence(delimiters.p, node.value) 914 | self.visit(node.value) 915 | 916 | def visit_ExtSlice(self, node): 917 | dims = node.dims 918 | set_precedence(node, *dims) 919 | self.comma_list(dims, len(dims) == 1) 920 | 921 | def visit_Yield(self, node): 922 | with self.delimit(node): 923 | set_precedence(get_op_precedence(node) + 1, node.value) 924 | self.write('yield') 925 | self.conditional_write(' ', node.value) 926 | 927 | # new for Python 3.3 928 | def visit_YieldFrom(self, node): 929 | with self.delimit(node): 930 | self.write('yield from ', node.value) 931 | 932 | # new for Python 3.5 933 | def visit_Await(self, node): 934 | with self.delimit(node): 935 | self.write('await ', node.value) 936 | 937 | def visit_Lambda(self, node): 938 | with self.delimit(node) as delimiters: 939 | set_precedence(delimiters.p, node.body) 940 | self.write('lambda ') 941 | self.visit_arguments(node.args) 942 | self.write(': ', node.body) 943 | 944 | def visit_Ellipsis(self, node): 945 | self.write('...') 946 | 947 | def visit_ListComp(self, node): 948 | with self.delimit('[]'): 949 | self.write(node.elt, *node.generators) 950 | 951 | def visit_GeneratorExp(self, node): 952 | with self.delimit(node) as delimiters: 953 | if delimiters.pp == Precedence.call_one_arg: 954 | delimiters.discard = True 955 | set_precedence(Precedence.Comma, node.elt) 956 | self.write(node.elt, *node.generators) 957 | 958 | def visit_SetComp(self, node): 959 | with self.delimit('{}'): 960 | self.write(node.elt, *node.generators) 961 | 962 | def visit_DictComp(self, node): 963 | with self.delimit('{}'): 964 | self.write(node.key, ': ', node.value, *node.generators) 965 | 966 | def visit_IfExp(self, node): 967 | with self.delimit(node) as delimiters: 968 | set_precedence(delimiters.p + 1, node.body, node.test) 969 | set_precedence(delimiters.p, node.orelse) 970 | self.write(node.body, ' if ', node.test, ' else ', node.orelse) 971 | 972 | def visit_Starred(self, node): 973 | self.write('*', node.value) 974 | 975 | def visit_Repr(self, node): 976 | # XXX: python 2.6 only 977 | with self.delimit('``'): 978 | self.visit(node.value) 979 | 980 | def visit_Module(self, node): 981 | self.write(*node.body) 982 | 983 | visit_Interactive = visit_Module 984 | 985 | def visit_Expression(self, node): 986 | self.visit(node.body) 987 | 988 | # Helper Nodes 989 | 990 | def visit_arg(self, node): 991 | self.write(node.arg) 992 | self.conditional_write(': ', node.annotation) 993 | 994 | def visit_alias(self, node): 995 | self.write(node.name) 996 | self.conditional_write(' as ', node.asname) 997 | 998 | def visit_comprehension(self, node): 999 | set_precedence(node, node.iter, *node.ifs) 1000 | set_precedence(Precedence.comprehension_target, node.target) 1001 | stmt = ' async for ' if self.get_is_async(node) else ' for ' 1002 | self.write(stmt, node.target, ' in ', node.iter) 1003 | for if_ in node.ifs: 1004 | self.write(' if ', if_) 1005 | -------------------------------------------------------------------------------- /tests/test_code_gen.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | """ 3 | Part of the astor library for Python AST manipulation 4 | 5 | License: 3-clause BSD 6 | 7 | Copyright (c) 2014 Berker Peksag 8 | Copyright (c) 2015 Patrick Maupin 9 | """ 10 | 11 | import ast 12 | import math 13 | import sys 14 | import textwrap 15 | 16 | try: 17 | import unittest2 as unittest 18 | except ImportError: 19 | import unittest 20 | 21 | import astor 22 | 23 | 24 | def canonical(srctxt): 25 | return textwrap.dedent(srctxt).strip() 26 | 27 | def astorexpr(x): 28 | return eval(astor.to_source(ast.Expression(body=x))) 29 | 30 | def astornum(x): 31 | return astorexpr(ast.Num(n=x)) 32 | 33 | class Comparisons(object): 34 | 35 | to_source = staticmethod(astor.to_source) 36 | 37 | def assertAstEqual(self, ast1, ast2): 38 | dmp1 = astor.dump_tree(ast1) 39 | dmp2 = astor.dump_tree(ast2) 40 | self.assertEqual(dmp1, dmp2) 41 | 42 | def assertAstEqualsSource(self, tree, source): 43 | self.assertEqual(self.to_source(tree).rstrip(), source) 44 | 45 | def assertAstRoundtrips(self, srctxt): 46 | """This asserts that the reconstituted source 47 | code can be compiled into the exact same AST 48 | as the original source code. 49 | """ 50 | srctxt = canonical(srctxt) 51 | srcast = ast.parse(srctxt) 52 | dsttxt = self.to_source(srcast) 53 | dstast = ast.parse(dsttxt) 54 | self.assertAstEqual(srcast, dstast) 55 | 56 | def assertAstRoundtripsGtVer(self, source, min_should_work, 57 | max_should_error=None): 58 | if max_should_error is None: 59 | max_should_error = min_should_work[0], min_should_work[1] - 1 60 | if sys.version_info >= min_should_work: 61 | self.assertAstRoundtrips(source) 62 | elif sys.version_info <= max_should_error: 63 | self.assertRaises(SyntaxError, ast.parse, source) 64 | 65 | def assertSrcRoundtrips(self, srctxt): 66 | """This asserts that the reconstituted source 67 | code is identical to the original source code. 68 | This is a much stronger statement than assertAstRoundtrips, 69 | which may not always be appropriate. 70 | """ 71 | srctxt = canonical(srctxt) 72 | self.assertEqual(self.to_source(ast.parse(srctxt)).rstrip(), srctxt) 73 | 74 | def assertSrcDoesNotRoundtrip(self, srctxt): 75 | srctxt = canonical(srctxt) 76 | self.assertNotEqual(self.to_source(ast.parse(srctxt)).rstrip(), 77 | srctxt) 78 | 79 | def assertSrcRoundtripsGtVer(self, source, min_should_work, 80 | max_should_error=None): 81 | if max_should_error is None: 82 | max_should_error = min_should_work[0], min_should_work[1] - 1 83 | if sys.version_info >= min_should_work: 84 | self.assertSrcRoundtrips(source) 85 | elif sys.version_info <= max_should_error: 86 | self.assertRaises(SyntaxError, ast.parse, source) 87 | 88 | 89 | class CodegenTestCase(unittest.TestCase, Comparisons): 90 | 91 | def test_imports(self): 92 | source = "import ast" 93 | self.assertSrcRoundtrips(source) 94 | source = "import operator as op" 95 | self.assertSrcRoundtrips(source) 96 | source = "from math import floor" 97 | self.assertSrcRoundtrips(source) 98 | source = "from .. import foobar" 99 | self.assertSrcRoundtrips(source) 100 | source = "from ..aaa import foo, bar as bar2" 101 | self.assertSrcRoundtrips(source) 102 | 103 | def test_empty_iterable_literals(self): 104 | self.assertSrcRoundtrips('()') 105 | self.assertSrcRoundtrips('[]') 106 | self.assertSrcRoundtrips('{}') 107 | # Python has no literal for empty sets, but code_gen should produce an 108 | # expression that evaluates to one. 109 | self.assertEqual(astorexpr(ast.Set([])), set()) 110 | 111 | def test_dictionary_literals(self): 112 | source = "{'a': 1, 'b': 2}" 113 | self.assertSrcRoundtrips(source) 114 | another_source = "{'nested': ['structures', {'are': 'important'}]}" 115 | self.assertSrcRoundtrips(another_source) 116 | 117 | def test_try_expect(self): 118 | source = """ 119 | try: 120 | 'spam'[10] 121 | except IndexError: 122 | pass""" 123 | self.assertAstRoundtrips(source) 124 | 125 | source = """ 126 | try: 127 | 'spam'[10] 128 | except IndexError as exc: 129 | sys.stdout.write(exc)""" 130 | self.assertAstRoundtrips(source) 131 | 132 | source = """ 133 | try: 134 | 'spam'[10] 135 | except IndexError as exc: 136 | sys.stdout.write(exc) 137 | else: 138 | pass 139 | finally: 140 | pass""" 141 | self.assertAstRoundtrips(source) 142 | source = """ 143 | try: 144 | size = len(iterable) 145 | except (TypeError, AttributeError): 146 | pass 147 | else: 148 | if n >= size: 149 | return sorted(iterable, key=key, reverse=True)[:n]""" 150 | self.assertAstRoundtrips(source) 151 | 152 | def test_del_statement(self): 153 | source = "del l[0]" 154 | self.assertSrcRoundtrips(source) 155 | source = "del obj.x" 156 | self.assertSrcRoundtrips(source) 157 | 158 | def test_arguments(self): 159 | source = """ 160 | j = [1, 2, 3] 161 | 162 | 163 | def test(a1, a2, b1=j, b2='123', b3={}, b4=[]): 164 | pass""" 165 | self.assertSrcRoundtrips(source) 166 | 167 | @unittest.skipUnless(sys.version_info >= (3, 8, 0, "alpha", 4), 168 | "positional only arguments introduced in Python 3.8") 169 | def test_positional_only_arguments(self): 170 | source = """ 171 | def test(a, b, /, c, *, d, **kwargs): 172 | pass 173 | 174 | 175 | def test(a=3, b=4, /, c=7): 176 | pass 177 | 178 | 179 | def test(a, b=4, /, c=8, d=9): 180 | pass 181 | """ 182 | self.assertSrcRoundtrips(source) 183 | 184 | def test_pass_arguments_node(self): 185 | source = canonical(""" 186 | j = [1, 2, 3] 187 | 188 | 189 | def test(a1, a2, b1=j, b2='123', b3={}, b4=[]): 190 | pass""") 191 | root_node = ast.parse(source) 192 | arguments_node = [n for n in ast.walk(root_node) 193 | if isinstance(n, ast.arguments)][0] 194 | self.assertEqual(self.to_source(arguments_node).rstrip(), 195 | "a1, a2, b1=j, b2='123', b3={}, b4=[]") 196 | source = """ 197 | def call(*popenargs, timeout=None, **kwargs): 198 | pass""" 199 | # Probably also works on < 3.4, but doesn't work on 2.7... 200 | self.assertSrcRoundtripsGtVer(source, (3, 4), (2, 7)) 201 | 202 | def test_attribute(self): 203 | self.assertSrcRoundtrips("x.foo") 204 | self.assertSrcRoundtrips("(5).foo") 205 | 206 | def test_matrix_multiplication(self): 207 | for source in ("(a @ b)", "a @= b"): 208 | self.assertAstRoundtripsGtVer(source, (3, 5)) 209 | 210 | def test_multiple_call_unpackings(self): 211 | source = """ 212 | my_function(*[1], *[2], **{'three': 3}, **{'four': 'four'})""" 213 | self.assertSrcRoundtripsGtVer(source, (3, 5)) 214 | 215 | def test_right_hand_side_dictionary_unpacking(self): 216 | source = """ 217 | our_dict = {'a': 1, **{'b': 2, 'c': 3}}""" 218 | self.assertSrcRoundtripsGtVer(source, (3, 5)) 219 | 220 | def test_dictionary_unpacking_parens(self): 221 | self.assertSrcRoundtripsGtVer("f(**x)", (3, 5)) 222 | self.assertSrcRoundtripsGtVer("{**x}", (3, 5)) 223 | self.assertSrcRoundtripsGtVer("f(**([] or 5))", (3, 5)) 224 | self.assertSrcRoundtripsGtVer("{**([] or 5)}", (3, 5)) 225 | 226 | def test_async_def_with_for(self): 227 | source = """ 228 | async def read_data(db): 229 | async with connect(db) as db_cxn: 230 | data = await db_cxn.fetch('SELECT foo FROM bar;') 231 | async for datum in data: 232 | if quux(datum): 233 | return datum""" 234 | self.assertSrcRoundtripsGtVer(source, (3, 5)) 235 | 236 | def test_double_await(self): 237 | source = """ 238 | async def foo(): 239 | return await (await bar())""" 240 | self.assertSrcRoundtripsGtVer(source, (3, 5)) 241 | 242 | def test_class_definition_with_starbases_and_kwargs(self): 243 | source = """ 244 | class TreeFactory(*[FactoryMixin, TreeBase], **{'metaclass': Foo}): 245 | pass""" 246 | self.assertSrcRoundtripsGtVer(source, (3, 0)) 247 | 248 | def test_yield(self): 249 | source = "yield" 250 | self.assertAstRoundtrips(source) 251 | source = """ 252 | def dummy(): 253 | yield""" 254 | self.assertAstRoundtrips(source) 255 | source = "foo((yield bar))" 256 | self.assertAstRoundtrips(source) 257 | source = "(yield bar)()" 258 | self.assertAstRoundtrips(source) 259 | source = "return (yield 1)" 260 | self.assertAstRoundtrips(source) 261 | source = "return (yield from sam())" 262 | self.assertAstRoundtripsGtVer(source, (3, 3)) 263 | source = "((yield a) for b in c)" 264 | self.assertAstRoundtrips(source) 265 | source = "[(yield)]" 266 | self.assertAstRoundtrips(source) 267 | source = "if (yield): pass" 268 | self.assertAstRoundtrips(source) 269 | source = "if (yield from foo): pass" 270 | self.assertAstRoundtripsGtVer(source, (3, 3)) 271 | source = "(yield from (a, b))" 272 | self.assertAstRoundtripsGtVer(source, (3, 3)) 273 | source = "yield from sam()" 274 | self.assertSrcRoundtripsGtVer(source, (3, 3)) 275 | 276 | def test_with(self): 277 | source = """ 278 | with foo: 279 | pass 280 | """ 281 | self.assertSrcRoundtrips(source) 282 | source = """ 283 | with foo as bar: 284 | pass 285 | """ 286 | self.assertSrcRoundtrips(source) 287 | source = """ 288 | with foo as bar, mary, william as bill: 289 | pass 290 | """ 291 | self.assertAstRoundtripsGtVer(source, (2, 7)) 292 | 293 | def test_huge_int(self): 294 | for n in (10**1000, 295 | 0xdfa21cd2a530ccc8c870aa60d9feb3b35deeab81c3215a96557abbd683d21f4600f38e475d87100da9a4404220eeb3bb5584e5a2b5b48ffda58530ea19104a32577d7459d91e76aa711b241050f4cc6d5327ccee254f371bcad3be56d46eb5919b73f20dbdb1177b700f00891c5bf4ed128bb90ed541b778288285bcfa28432ab5cbcb8321b6e24760e998e0daa519f093a631e44276d7dd252ce0c08c75e2ab28a7349ead779f97d0f20a6d413bf3623cd216dc35375f6366690bcc41e3b2d5465840ec7ee0dc7e3f1c101d674a0c7dbccbc3942788b111396add2f8153b46a0e4b50d66e57ee92958f1c860dd97cc0e40e32febff915343ed53573142bdf4b): 296 | self.assertEqual(astornum(n), n) 297 | 298 | def test_complex(self): 299 | source = """ 300 | (3) + (4j) + (1+2j) + (1+0j) 301 | """ 302 | self.assertAstRoundtrips(source) 303 | 304 | self.assertIsInstance(astornum(1+0j), complex) 305 | 306 | def test_inf(self): 307 | source = """ 308 | (1e1000) + (-1e1000) + (1e1000j) + (-1e1000j) 309 | """ 310 | self.assertAstRoundtrips(source) 311 | # We special case infinities in code_gen. So we will 312 | # return the same AST construction but it won't 313 | # roundtrip to 'source'. See the SourceGenerator.visit_Num 314 | # method for details. (#82) 315 | source = 'a = 1e400' 316 | self.assertAstRoundtrips(source) 317 | # Returns 'a = 1e1000'. 318 | self.assertSrcDoesNotRoundtrip(source) 319 | 320 | self.assertIsInstance(astornum((1e1000+1e1000)+0j), complex) 321 | 322 | def test_nan(self): 323 | self.assertTrue(math.isnan(astornum(float('nan')))) 324 | 325 | v = astornum(complex(-1e1000, float('nan'))) 326 | self.assertEqual(v.real, -1e1000) 327 | self.assertTrue(math.isnan(v.imag)) 328 | 329 | v = astornum(complex(float('nan'), -1e1000)) 330 | self.assertTrue(math.isnan(v.real)) 331 | self.assertEqual(v.imag, -1e1000) 332 | 333 | def test_unary(self): 334 | source = """ 335 | -(1) + ~(2) + +(3) 336 | """ 337 | self.assertAstRoundtrips(source) 338 | 339 | def test_pow(self): 340 | source = """ 341 | (-2) ** (-3) 342 | """ 343 | self.assertAstRoundtrips(source) 344 | source = """ 345 | (+2) ** (+3) 346 | """ 347 | self.assertAstRoundtrips(source) 348 | source = """ 349 | 2 ** 3 ** 4 350 | """ 351 | self.assertAstRoundtrips(source) 352 | source = """ 353 | -2 ** -3 354 | """ 355 | self.assertAstRoundtrips(source) 356 | source = """ 357 | -2 ** -3 ** -4 358 | """ 359 | self.assertAstRoundtrips(source) 360 | source = """ 361 | -((-1) ** other._sign) 362 | (-1) ** self._sign 363 | """ 364 | self.assertAstRoundtrips(source) 365 | 366 | def test_comprehension(self): 367 | source = """ 368 | ((x,y) for x,y in zip(a,b) if a) 369 | """ 370 | self.assertAstRoundtrips(source) 371 | source = """ 372 | fields = [(a, _format(b)) for (a, b) in iter_fields(node) if a] 373 | """ 374 | self.assertAstRoundtrips(source) 375 | source = """ 376 | ra = np.fromiter(((i * 3, i * 2) for i in range(10)), 377 | n, dtype='i8,f8') 378 | """ 379 | self.assertAstRoundtrips(source) 380 | 381 | def test_async_comprehension(self): 382 | source = """ 383 | async def f(): 384 | [(await x) async for x in y] 385 | [(await i) for i in b if await c] 386 | (await x async for x in y) 387 | {i for i in b async for i in a if await i for b in i} 388 | """ 389 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 390 | 391 | def test_tuple_corner_cases(self): 392 | source = """ 393 | a = () 394 | """ 395 | self.assertAstRoundtrips(source) 396 | source = """ 397 | assert (a, b), (c, d) 398 | """ 399 | self.assertAstRoundtrips(source) 400 | source = """ 401 | return UUID(fields=(time_low, time_mid, time_hi_version, 402 | clock_seq_hi_variant, clock_seq_low, node), version=1) 403 | """ 404 | self.assertAstRoundtrips(source) 405 | source = """ 406 | raise(os.error, ('multiple errors:', errors)) 407 | """ 408 | self.assertAstRoundtrips(source) 409 | source = """ 410 | exec(expr, global_dict, local_dict) 411 | """ 412 | self.assertAstRoundtrips(source) 413 | source = """ 414 | with (a, b) as (c, d): 415 | pass 416 | """ 417 | self.assertAstRoundtrips(source) 418 | self.assertAstRoundtrips(source) 419 | source = """ 420 | with (a, b) as (c, d), (e,f) as (h,g): 421 | pass 422 | """ 423 | self.assertAstRoundtripsGtVer(source, (2, 7)) 424 | source = """ 425 | Pxx[..., (0,-1)] = xft[..., (0,-1)]**2 426 | """ 427 | self.assertAstRoundtripsGtVer(source, (2, 7)) 428 | source = """ 429 | responses = { 430 | v: (v.phrase, v.description) 431 | for v in HTTPStatus.__members__.values() 432 | } 433 | """ 434 | self.assertAstRoundtripsGtVer(source, (2, 7)) 435 | 436 | def test_output_formatting(self): 437 | source = """ 438 | __all__ = ['ArgumentParser', 'ArgumentError', 'ArgumentTypeError', 439 | 'FileType', 'HelpFormatter', 'ArgumentDefaultsHelpFormatter', 440 | 'RawDescriptionHelpFormatter', 'RawTextHelpFormatter', 'Namespace', 441 | 'Action', 'ONE_OR_MORE', 'OPTIONAL', 'PARSER', 'REMAINDER', 'SUPPRESS', 442 | 'ZERO_OR_MORE'] 443 | """ # NOQA 444 | self.maxDiff = 2000 445 | self.assertSrcRoundtrips(source) 446 | 447 | def test_elif(self): 448 | source = """ 449 | if a: 450 | b 451 | elif c: 452 | d 453 | elif e: 454 | f 455 | else: 456 | g 457 | """ 458 | self.assertSrcRoundtrips(source) 459 | 460 | def test_fstrings(self): 461 | source = """ 462 | x = f'{x}' 463 | x = f'{x.y}' 464 | x = f'{int(x)}' 465 | x = f'a{b:c}d' 466 | x = f'a{b!s:c{d}e}f' 467 | x = f'{x + y}' 468 | x = f'""' 469 | x = f'"\\'' 470 | """ 471 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 472 | source = """ 473 | a_really_long_line_will_probably_break_things = ( 474 | f'a{b!s:c{d}e}fghijka{b!s:c{d}e}a{b!s:c{d}e}a{b!s:c{d}e}') 475 | """ 476 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 477 | source = """ 478 | return f"functools.{qualname}({', '.join(args)})" 479 | """ 480 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 481 | 482 | def test_assignment_expr(self): 483 | cases = ( 484 | "(x := 3)", 485 | "1 + (x := y)", 486 | "x = (y := 0)", 487 | "1 + (p := 1 if 2 else 3)", 488 | "[y := f(x), y**2, y**3]", 489 | "(2 ** 3 * 4 + 5 and 6, x := 2 ** 3 * 4 + 5 and 6)", 490 | "foo(x := 3, cat='vector')", 491 | "foo(x=(y := f(x)))", 492 | "any(len(longline := line) >= 100 for line in lines)", 493 | "[[y := f(x), x/y] for x in range(5)]", 494 | "lambda: (x := 1)", 495 | "def foo(answer=(p := 42)): pass", 496 | "def foo(answer: (p := 42) = 5): pass", 497 | "if reductor := dispatch_table.get(cls): pass", 498 | "while line := fp.readline(): pass", 499 | "while (command := input('> ')) != 'quit': pass") 500 | for case in cases: 501 | self.assertAstRoundtripsGtVer(case, (3, 8)) 502 | 503 | @unittest.skipUnless(sys.version_info <= (3, 3), 504 | "ast.Name used for True, False, None until Python 3.4") 505 | def test_deprecated_constants_as_name(self): 506 | self.assertAstEqualsSource( 507 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Name(id='True')), 508 | "spam = True") 509 | 510 | self.assertAstEqualsSource( 511 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Name(id='False')), 512 | "spam = False") 513 | 514 | self.assertAstEqualsSource( 515 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Name(id='None')), 516 | "spam = None") 517 | 518 | @unittest.skipUnless(sys.version_info >= (3, 4), 519 | "ast.NameConstant introduced in Python 3.4") 520 | def test_deprecated_name_constants(self): 521 | self.assertAstEqualsSource( 522 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.NameConstant(value=True)), 523 | "spam = True") 524 | 525 | self.assertAstEqualsSource( 526 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.NameConstant(value=False)), 527 | "spam = False") 528 | 529 | self.assertAstEqualsSource( 530 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.NameConstant(value=None)), 531 | "spam = None") 532 | 533 | def test_deprecated_constant_nodes(self): 534 | self.assertAstEqualsSource( 535 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(3)), 536 | "spam = 3") 537 | 538 | self.assertAstEqualsSource( 539 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(-93)), 540 | "spam = -93") 541 | 542 | self.assertAstEqualsSource( 543 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(837.3888)), 544 | "spam = 837.3888") 545 | 546 | self.assertAstEqualsSource( 547 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(-0.9877)), 548 | "spam = -0.9877") 549 | 550 | self.assertAstEqualsSource(ast.Ellipsis(), "...") 551 | 552 | if sys.version_info >= (3, 0): 553 | self.assertAstEqualsSource( 554 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Bytes(b"Bytes")), 555 | "spam = b'Bytes'") 556 | 557 | self.assertAstEqualsSource( 558 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Str("String")), 559 | "spam = 'String'") 560 | 561 | @unittest.skipUnless(sys.version_info >= (3, 6), 562 | "ast.Constant introduced in Python 3.6") 563 | def test_constant_nodes(self): 564 | self.assertAstEqualsSource( 565 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=3)), 566 | "spam = 3") 567 | 568 | self.assertAstEqualsSource( 569 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=-93)), 570 | "spam = -93") 571 | 572 | self.assertAstEqualsSource( 573 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=837.3888)), 574 | "spam = 837.3888") 575 | 576 | self.assertAstEqualsSource( 577 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=-0.9877)), 578 | "spam = -0.9877") 579 | 580 | self.assertAstEqualsSource( 581 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=True)), 582 | "spam = True") 583 | 584 | self.assertAstEqualsSource( 585 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=False)), 586 | "spam = False") 587 | 588 | self.assertAstEqualsSource( 589 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value=None)), 590 | "spam = None") 591 | 592 | self.assertAstEqualsSource(ast.Constant(value=Ellipsis), "...") 593 | 594 | self.assertAstEqualsSource( 595 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(b"Bytes")), 596 | "spam = b'Bytes'") 597 | 598 | self.assertAstEqualsSource( 599 | ast.Assign(targets=[ast.Name(id='spam')], value=ast.Constant(value="String")), 600 | "spam = 'String'") 601 | 602 | def test_annassign(self): 603 | source = """ 604 | a: int 605 | (a): int 606 | a.b: int 607 | (a.b): int 608 | b: Tuple[int, str, ...] 609 | c.d[e].f: Any 610 | q: 3 = (1, 2, 3) 611 | t: Tuple[int, ...] = (1, 2, 3) 612 | some_list: List[int] = [] 613 | (a): int = 0 614 | a:int = 0 615 | (a.b): int = 0 616 | a.b: int = 0 617 | """ 618 | self.assertAstRoundtripsGtVer(source, (3, 6)) 619 | 620 | @unittest.skipUnless(sys.version_info >= (3, 6), 621 | "typing and annotated assignment was introduced in " 622 | "Python 3.6") 623 | def test_function_typing(self): 624 | source = canonical(""" 625 | def foo(x : int) ->str: 626 | i : str = '3' 627 | return i 628 | """) 629 | target = canonical(""" 630 | def foo(x: int) -> str: 631 | i: str = '3' 632 | return i 633 | """) 634 | self.assertAstEqualsSource(ast.parse(source), target) 635 | 636 | def test_compile_types(self): 637 | code = '(a + b + c) * (d + e + f)\n' 638 | for mode in 'exec eval single'.split(): 639 | srcast = compile(code, 'dummy', mode, ast.PyCF_ONLY_AST) 640 | dsttxt = self.to_source(srcast) 641 | if code.strip() != dsttxt.strip(): 642 | self.assertEqual('(%s)' % code.strip(), dsttxt.strip()) 643 | 644 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 645 | "match statement introduced in Python 3.10") 646 | def test_match_sequence(self): 647 | source = canonical(""" 648 | match command.split(): 649 | case ['quit']: 650 | ... 651 | # sequence pattern 652 | case [1 | 2]: 653 | ... 654 | # group pattern 655 | case (1 | 2): 656 | ... 657 | """) 658 | target = canonical(""" 659 | match command.split(): 660 | case ['quit']: 661 | ... 662 | case [1 | 2]: 663 | ... 664 | case 1 | 2: 665 | ... 666 | """) 667 | self.assertAstEqualsSource(ast.parse(source), target) 668 | 669 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 670 | "match statement introduced in Python 3.10") 671 | def test_match_sequence_brackets(self): 672 | # There is no way to tell if brackets or parentheses were used 673 | # from the AST. Syntactically they are identical. 674 | source = canonical(""" 675 | match point: 676 | case (Point(x1, y1), Point(x2, y2) as p2): 677 | ... 678 | case [Point(x1, y1), Point(x2, y2) as p2]: 679 | ... 680 | 681 | """) 682 | target = canonical(""" 683 | match point: 684 | case [Point(x1, y1), Point(x2, y2) as p2]: 685 | ... 686 | case [Point(x1, y1), Point(x2, y2) as p2]: 687 | ... 688 | """) 689 | self.assertAstEqualsSource(ast.parse(source), target) 690 | 691 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 692 | "match statement introduced in Python 3.10") 693 | def test_match_singleton(self): 694 | source = canonical(""" 695 | match x: 696 | case 1: 697 | print('Goodbye!') 698 | quit_game() 699 | """) 700 | target = canonical(""" 701 | match x: 702 | case 1: 703 | print('Goodbye!') 704 | quit_game() 705 | """) 706 | self.assertAstEqualsSource(ast.parse(source), target) 707 | 708 | 709 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 710 | "match statement introduced in Python 3.10") 711 | def test_match_star(self): 712 | source = canonical(""" 713 | match x: 714 | case [1, 2, *rest]: 715 | print('Goodbye!') 716 | quit_game() 717 | case [*_]: 718 | return 'seq' 719 | """) 720 | target = canonical(""" 721 | match x: 722 | case [1, 2, *rest]: 723 | print('Goodbye!') 724 | quit_game() 725 | case [*_]: 726 | return 'seq' 727 | """) 728 | self.assertAstEqualsSource(ast.parse(source), target) 729 | 730 | 731 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 732 | "match statement introduced in Python 3.10") 733 | def test_match_mapping(self): 734 | source = canonical(""" 735 | match x: 736 | case {'text': message, 'color': c, **rest}: 737 | pass 738 | case {1: _, 2: _}: 739 | print('You won!') 740 | win_game() 741 | case {**rest}: 742 | print('You Lose!') 743 | lose_game() 744 | """) 745 | target = canonical(""" 746 | match x: 747 | case {'text': message, 'color': c, **rest}: 748 | pass 749 | case {1: _, 2: _}: 750 | print('You won!') 751 | win_game() 752 | case {**rest}: 753 | print('You Lose!') 754 | lose_game() 755 | """) 756 | self.assertAstEqualsSource(ast.parse(source), target) 757 | 758 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 759 | "match statement introduced in Python 3.10") 760 | def test_match_class(self): 761 | source = canonical(""" 762 | match point: 763 | case Point(x=0, y=0): 764 | print('Origin') 765 | case Point(x=0, y=y): 766 | print(f'Y={y}') 767 | case Point(x=x, y=0): 768 | print(f'X={x}') 769 | case Point(1, y=1): 770 | print('1, y=1') 771 | case Point(): 772 | print('Somewhere else') 773 | case A.B.C.D: 774 | ... 775 | case _: 776 | print('Not a point') 777 | """) 778 | target = canonical(""" 779 | match point: 780 | case Point(x=0, y=0): 781 | print('Origin') 782 | case Point(x=0, y=y): 783 | print(f'Y={y}') 784 | case Point(x=x, y=0): 785 | print(f'X={x}') 786 | case Point(1, y=1): 787 | print('1, y=1') 788 | case Point(): 789 | print('Somewhere else') 790 | case A.B.C.D: 791 | ... 792 | case _: 793 | print('Not a point') 794 | """) 795 | self.assertAstEqualsSource(ast.parse(source), target) 796 | 797 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 798 | "match statement introduced in Python 3.10") 799 | def test_match_guard(self): 800 | source = canonical(""" 801 | match point: 802 | case Point(x, y) if x == y: 803 | print(f'Y=X at {x}') 804 | case Point(x, y) if x in (1, 2, 3): 805 | print(f'Not on the diagonal') 806 | case Point(x, y) if (x := x[:0]): 807 | ... 808 | 809 | """) 810 | target = canonical(""" 811 | match point: 812 | case Point(x, y) if x == y: 813 | print(f'Y=X at {x}') 814 | case Point(x, y) if x in (1, 2, 3): 815 | print(f'Not on the diagonal') 816 | case Point(x, y) if (x := x[:0]): 817 | ... 818 | """) 819 | self.assertAstEqualsSource(ast.parse(source), target) 820 | 821 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 822 | "match statement introduced in Python 3.10") 823 | def test_match_capture(self): 824 | # For now there is no way to check if there were parentheses around 825 | # pattern or not, syntactically they are identical 826 | source = canonical(""" 827 | match point: 828 | case [Point(x1, y1), Point(x2, y2) as p2]: 829 | print('p2') 830 | case (0 as z) | (1 as z) | (2 as z): 831 | ... 832 | 833 | """) 834 | target = canonical(""" 835 | match point: 836 | case [Point(x1, y1), Point(x2, y2) as p2]: 837 | print('p2') 838 | case 0 as z | 1 as z | 2 as z: 839 | ... 840 | """) 841 | self.assertAstEqualsSource(ast.parse(source), target) 842 | 843 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 844 | "match statement introduced in Python 3.10") 845 | def test_match_or(self): 846 | source = canonical(""" 847 | match point: 848 | case [x] | y: 849 | ... 850 | case [x, y] | [z]: 851 | ... 852 | case [x] as y: 853 | ... 854 | case {0: [1, 2, {}] | True} | {1: [[]]} | {0: [1, 2, {}]} | [] | 'X' | {}: 855 | ... 856 | """) 857 | target = canonical(""" 858 | match point: 859 | case [x] | y: 860 | ... 861 | case [x, y] | [z]: 862 | ... 863 | case [x] as y: 864 | ... 865 | case {0: [1, 2, {}] | True} | {1: [[]]} | {0: [1, 2, {}]} | [] | 'X' | {}: 866 | ... 867 | """) 868 | self.assertAstEqualsSource(ast.parse(source), target) 869 | 870 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 871 | "match statement introduced in Python 3.10") 872 | def test_match_nested(self): 873 | source = canonical(""" 874 | match match: 875 | case case: 876 | match match: 877 | case case: 878 | pass 879 | """) 880 | target = canonical(""" 881 | match match: 882 | case case: 883 | match match: 884 | case case: 885 | pass 886 | """) 887 | self.assertAstEqualsSource(ast.parse(source), target) 888 | 889 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 890 | "match statement introduced in Python 3.10") 891 | def test_match_call(self): 892 | source = canonical(""" 893 | match Seq(): 894 | case bool(z): 895 | y = 0 896 | match [match.group('grade'), match.group('material')]: 897 | case ['MD' | 'HD', 'SS' as code]: 898 | print('You will get here') 899 | """) 900 | target = canonical(""" 901 | match Seq(): 902 | case bool(z): 903 | y = 0 904 | match [match.group('grade'), match.group('material')]: 905 | case ['MD' | 'HD', 'SS' as code]: 906 | print('You will get here') 907 | """) 908 | self.assertAstEqualsSource(ast.parse(source), target) 909 | 910 | @unittest.skipUnless(sys.version_info >= (3, 10, 0), 911 | "match statement introduced in Python 3.10") 912 | def test_match_num(self): 913 | source = canonical(""" 914 | match 3: 915 | case 0 | 1 | 2 | 3: 916 | 1 917 | 918 | """) 919 | target = canonical(""" 920 | match 3: 921 | case 0 | 1 | 2 | 3: 922 | 1 923 | """) 924 | self.assertAstEqualsSource(ast.parse(source), target) 925 | 926 | def test_unicode_literals(self): 927 | source = """ 928 | from __future__ import (print_function, unicode_literals) 929 | x = b'abc' 930 | y = u'abc' 931 | """ 932 | self.assertAstRoundtrips(source) 933 | 934 | def test_slicing(self): 935 | source = """ 936 | x[1,2] 937 | x[...,...] 938 | x[1,...] 939 | x[...,3] 940 | x[:,:] 941 | x[:,] 942 | x[1,:] 943 | x[:,2] 944 | x[1:2,] 945 | x[3:4,...] 946 | x[5:6,7:8] 947 | x[1:2,3:4] 948 | x[5:6:7,] 949 | x[1:2:-3,] 950 | x[4:5:6,...] 951 | x[7:8:-9,...] 952 | x[1:2:3,4:5] 953 | x[6:7:-8,9:0] 954 | x[...,1:2] 955 | x[1:2,3:4] 956 | x[...,1:2:3] 957 | x[...,4:5:-6] 958 | x[1:2,3:4:5] 959 | x[1:2,3:4:-5] 960 | """ 961 | self.assertAstRoundtrips(source) 962 | 963 | def test_non_string_leakage(self): 964 | source = ''' 965 | tar_compression = {'gzip': 'gz', None: ''} 966 | ''' 967 | self.assertAstRoundtrips(source) 968 | 969 | def test_fstring_trailing_newline(self): 970 | source = ''' 971 | x = f"""{host}\n\t{port}\n""" 972 | ''' 973 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 974 | source = ''' 975 | if 1: 976 | x = f'{host}\\n\\t{port}\\n' 977 | ''' 978 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 979 | 980 | def test_fstring_escaped_braces(self): 981 | source = ''' 982 | x = f'{{hello world}}' 983 | ''' 984 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 985 | source = ''' 986 | x = f'{f.name}={{self.{f.name}!r}}' 987 | ''' 988 | self.assertSrcRoundtripsGtVer(source, (3, 6)) 989 | 990 | @unittest.skipUnless(sys.version_info >= (3, 8, 0, "alpha", 4), 991 | "f-string debugging introduced in Python 3.8") 992 | def test_fstring_debugging(self): 993 | source = """ 994 | x = f'{5=}' 995 | y = f'{5=!r}' 996 | z = f'{3*x+15=}' 997 | f'{x=:}' 998 | f'{x=:.2f}' 999 | f'alpha α {pi=} ω omega' 1000 | """ 1001 | self.assertAstRoundtripsGtVer(source, (3, 8)) 1002 | 1003 | def test_docstring_function(self): 1004 | source = ''' 1005 | def f(arg): 1006 | """ 1007 | docstring 1008 | """ 1009 | return 3 1010 | ''' 1011 | self.assertSrcRoundtrips(source) 1012 | 1013 | def test_docstring_class(self): 1014 | source = ''' 1015 | class Class: 1016 | """ 1017 | docstring 1018 | """ 1019 | pass 1020 | ''' 1021 | self.assertSrcRoundtrips(source) 1022 | 1023 | def test_docstring_method(self): 1024 | source = ''' 1025 | class Class: 1026 | 1027 | def f(arg): 1028 | """ 1029 | docstring 1030 | """ 1031 | return 3 1032 | ''' 1033 | self.assertSrcRoundtrips(source) 1034 | 1035 | def test_docstring_module(self): 1036 | source = ''' 1037 | """ 1038 | docstring1 1039 | """ 1040 | 1041 | 1042 | class Class: 1043 | 1044 | def f(arg): 1045 | pass 1046 | ''' 1047 | self.assertSrcRoundtrips(source) 1048 | 1049 | @unittest.skipUnless(sys.version_info >= (3, 12, 0), 1050 | "type parameter introduced in Python 3.12") 1051 | def test_type_parameter_function(self): 1052 | source = ''' 1053 | def f[T](arg: T) -> T: 1054 | return arg 1055 | 1056 | 1057 | def f[*V](*args: *V) -> tuple[*V,]: 1058 | return args 1059 | 1060 | 1061 | def f[**P](*args: P.args, **kwargs: P.kwargs): 1062 | pass 1063 | ''' 1064 | self.assertSrcRoundtrips(source) 1065 | 1066 | @unittest.skipUnless(sys.version_info >= (3, 12, 0), 1067 | "type parameter introduced in Python 3.12") 1068 | def test_type_parameter_class(self): 1069 | source = ''' 1070 | class Class[T]: 1071 | pass 1072 | 1073 | 1074 | class Class[*V]: 1075 | pass 1076 | 1077 | 1078 | class Class[**P]: 1079 | pass 1080 | ''' 1081 | self.assertSrcRoundtrips(source) 1082 | 1083 | @unittest.skipUnless(sys.version_info >= (3, 12, 0), 1084 | "type alias statement introduced in Python 3.12") 1085 | def test_type_alias(self): 1086 | source = ''' 1087 | type A = int 1088 | type B[T] = T 1089 | type C[*V] = tuple[*V,] 1090 | ''' 1091 | self.assertSrcRoundtrips(source) 1092 | 1093 | 1094 | if __name__ == '__main__': 1095 | unittest.main() 1096 | --------------------------------------------------------------------------------