├── .gitignore ├── README.md ├── __transformers__ ├── __init__.py ├── __main__.py ├── ellipsis_partial.py ├── loader.py └── matmul_pipe.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── module_for_test.py ├── test_ellipsis_partial.py ├── test_loader.py └── test_matmul_pipe.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | .env 60 | .idea 61 | 62 | # vim temporary files 63 | .*.swp 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Experimental module for AST transformations 2 | 3 | ## Install 4 | 5 | ```bash 6 | pip install git+https://github.com/nvbn/__transformers__ 7 | ``` 8 | 9 | ## Usage 10 | 11 | Available transformations: 12 | 13 | * `ellipsis_partial` – `map(lambda x: x + 1, ...)` to `lambda y: map(lambda x: x + 1, y)`; 14 | * `matmul_pipe` – `"hello world" @ print` to `print("hello world")`. 15 | 16 | Enable transformations in code: 17 | 18 | ```python 19 | from __transformers__ import ellipsis_partial, matmul_pipe 20 | 21 | range(10) @ map(lambda x: x ** 2, ...) @ list @ print 22 | ``` 23 | 24 | Run code with transformations: 25 | 26 | ```bash 27 | ➜ python -m __transformers__ -m python_module 28 | [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] 29 | 30 | ➜ python -m __transformers__ python/module/path.py 31 | [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] 32 | ``` 33 | 34 | Manually enable transformations from code: 35 | 36 | ```python 37 | from __transformers__ import setup 38 | 39 | setup() 40 | 41 | import module_that_use_transformers 42 | ``` 43 | 44 | ## Rationale 45 | 46 | * [Idea](https://nvbn.github.io/2016/08/09/partial-piping/); 47 | * [AST transformations](https://nvbn.github.io/2016/08/09/partial-piping-ast/); 48 | * [AST transformations on import](https://nvbn.github.io/2016/08/17/ast-import/). 49 | 50 | ## License MIT 51 | -------------------------------------------------------------------------------- /__transformers__/__init__.py: -------------------------------------------------------------------------------- 1 | from .loader import setup 2 | 3 | __all__ = ['setup'] 4 | -------------------------------------------------------------------------------- /__transformers__/__main__.py: -------------------------------------------------------------------------------- 1 | from runpy import run_module 2 | from pathlib import Path 3 | import sys 4 | 5 | from . import setup 6 | 7 | setup() 8 | 9 | del sys.argv[0] 10 | 11 | if sys.argv[0] == '-m': 12 | del sys.argv[0] 13 | run_module(sys.argv[0]) 14 | else: 15 | # rnupy.run_path ignores meta_path for first import 16 | path = Path(sys.argv[0]).parent.as_posix() 17 | module_name = Path(sys.argv[0]).name[:-3] 18 | sys.path.insert(0, path) 19 | run_module(module_name) 20 | -------------------------------------------------------------------------------- /__transformers__/ellipsis_partial.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | 4 | class EllipsisPartialTransformer(ast.NodeTransformer): 5 | def __init__(self): 6 | self._counter = 0 7 | 8 | def _get_arg_name(self): 9 | """Return unique argument name for lambda.""" 10 | try: 11 | return '__ellipsis_partial_arg_{}'.format(self._counter) 12 | finally: 13 | self._counter += 1 14 | 15 | def _is_ellipsis(self, arg): 16 | return isinstance(arg, ast.Ellipsis) 17 | 18 | def _replace_argument(self, node, arg_name): 19 | """Replace ellipsis with argument.""" 20 | replacement = ast.Name(id=arg_name, 21 | ctx=ast.Load()) 22 | node.args = [replacement if self._is_ellipsis(arg) else arg 23 | for arg in node.args] 24 | return node 25 | 26 | def _wrap_in_lambda(self, node): 27 | """Wrap call in lambda and replace ellipsis with argument.""" 28 | arg_name = self._get_arg_name() 29 | node = self._replace_argument(node, arg_name) 30 | return ast.Lambda( 31 | args=ast.arguments(args=[ast.arg(arg=arg_name, annotation=None)], 32 | vararg=None, kwonlyargs=[], kw_defaults=[], 33 | kwarg=None, defaults=[]), 34 | body=node) 35 | 36 | def visit_Call(self, node): 37 | if any(self._is_ellipsis(arg) for arg in node.args): 38 | node = self._wrap_in_lambda(node) 39 | node = ast.fix_missing_locations(node) 40 | 41 | return self.generic_visit(node) 42 | 43 | 44 | transformer = EllipsisPartialTransformer() 45 | -------------------------------------------------------------------------------- /__transformers__/loader.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from importlib import import_module 3 | from importlib.machinery import PathFinder, SourceFileLoader 4 | import sys 5 | 6 | 7 | class NodeVisitor(ast.NodeVisitor): 8 | def __init__(self): 9 | self._found = [] 10 | 11 | def visit_ImportFrom(self, node): 12 | if node.module == '__transformers__': 13 | self._found += [name.name for name in node.names 14 | if name.name not in ('_loader', 'setup')] 15 | 16 | @classmethod 17 | def get_transformers(cls, tree): 18 | visitor = cls() 19 | visitor.visit(tree) 20 | return visitor._found 21 | 22 | 23 | def transform(tree): 24 | """Apply transformations to ast.""" 25 | transformers = NodeVisitor.get_transformers(tree) 26 | 27 | for module_name in transformers: 28 | module = import_module('.{}'.format(module_name), '__transformers__') 29 | tree = module.transformer.visit(tree) 30 | 31 | return tree 32 | 33 | 34 | class Finder(PathFinder): 35 | @classmethod 36 | def find_spec(cls, fullname, path=None, target=None): 37 | spec = super(Finder, cls).find_spec(fullname, path, target) 38 | if spec is None: 39 | return None 40 | 41 | spec.loader = Loader(spec.loader.name, spec.loader.path) 42 | return spec 43 | 44 | 45 | class Loader(SourceFileLoader): 46 | def source_to_code(self, data, path, *, _optimize=-1): 47 | tree = ast.parse(data) 48 | tree = transform(tree) 49 | return compile(tree, path, 'exec', 50 | dont_inherit=True, optimize=_optimize) 51 | 52 | 53 | def setup(): 54 | sys.meta_path.insert(0, Finder) 55 | -------------------------------------------------------------------------------- /__transformers__/matmul_pipe.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | 4 | class MatMulPipeTransformer(ast.NodeTransformer): 5 | def _replace_with_call(self, node): 6 | """Call right part of operation with left part as an argument.""" 7 | return ast.Call(func=node.right, args=[node.left], keywords=[]) 8 | 9 | def visit_BinOp(self, node): 10 | if isinstance(node.op, ast.MatMult): 11 | node = self._replace_with_call(node) 12 | node = ast.fix_missing_locations(node) 13 | 14 | return self.generic_visit(node) 15 | 16 | 17 | transformer = MatMulPipeTransformer() 18 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup, find_packages 3 | 4 | setup(name='transformers', 5 | version='0.1', 6 | description="Experimental module for AST transformations.", 7 | author='Vladimir Iakovlev', 8 | author_email='nvbn.rm@gmail.com', 9 | url='https://github.com/nvbn/__transformers__', 10 | license='MIT', 11 | packages=find_packages(exclude=['ez_setup', 'examples', 12 | 'tests', 'tests.*', 'release'])) 13 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nvbn/__transformers__/a294eeec34ec4adcd986e047ecbbf28e6de73eb4/tests/__init__.py -------------------------------------------------------------------------------- /tests/module_for_test.py: -------------------------------------------------------------------------------- 1 | from __transformers__ import matmul_pipe, ellipsis_partial 2 | 3 | 4 | def test(x): 5 | return x @ range(...) @ map(lambda x: x + 1, ...) @ sum 6 | -------------------------------------------------------------------------------- /tests/test_ellipsis_partial.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from __transformers__.ellipsis_partial import transformer 3 | 4 | 5 | def test_transformer(): 6 | code = 'result = map(lambda x: x + 1, ...)(range(5))' 7 | tree = ast.parse(code) 8 | tree = transformer.visit(tree) 9 | code = compile(tree, '', 'exec') 10 | exec(code) 11 | assert list(locals()['result']) == [1, 2, 3, 4, 5] 12 | -------------------------------------------------------------------------------- /tests/test_loader.py: -------------------------------------------------------------------------------- 1 | from __transformers__.loader import setup 2 | 3 | 4 | def test(): 5 | setup() 6 | from .module_for_test import test 7 | 8 | assert test(10) == 55 9 | -------------------------------------------------------------------------------- /tests/test_matmul_pipe.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from __transformers__.matmul_pipe import transformer 3 | 4 | 5 | def test_transformer(): 6 | code = 'result = range(10) @ sum' 7 | tree = ast.parse(code) 8 | tree = transformer.visit(tree) 9 | code = compile(tree, '', 'exec') 10 | exec(code) 11 | assert locals()['result'] == 45 12 | --------------------------------------------------------------------------------