├── .gitignore ├── .meta_version ├── .travis.yml ├── .vscode ├── settings.json └── tasks.json ├── ChangeLog.md ├── LICENSE ├── README.md ├── benchmark.py ├── manager.py ├── moshmosh ├── __init__.py ├── ast_compat.py ├── ctx_fix.py ├── extension.py ├── extension_register.py ├── extensions │ ├── __init__.py │ ├── lazy_import │ │ ├── __init__.py │ │ ├── main.py │ │ └── runtime.py │ ├── pattern_matching │ │ ├── __init__.py │ │ ├── core.py │ │ ├── core.pyi │ │ ├── main.py │ │ └── runtime.py │ ├── pipelines.py │ ├── quick_lambdas.py │ ├── scoped_operators.py │ └── template_python.py ├── repl_apis.py └── rewrite_helper.py ├── moshmosh_ipy.py ├── requirements.txt ├── setup.py ├── static ├── img1.png ├── img2.png ├── main.png └── upack.png ├── tests ├── __init__.py ├── __main__.py ├── ast_shape.py ├── asts.py ├── complex_match.py ├── complex_match_py34.py ├── debugging.py ├── incremental.py ├── lazy_import.py ├── list_view.py ├── match.py ├── pipelines.py ├── py35_unparse.py ├── quick_lambdas.py ├── scoped_operators.py ├── template_python.py └── test_all.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /.meta_version: -------------------------------------------------------------------------------- 1 | method: autoinc 2 | current: 0.0.1 -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "3.5" 5 | - "3.6" 6 | - "3.7" 7 | 8 | install: pip install tox-travis 9 | script: tox 10 | 11 | after_success: 12 | - codecov 13 | - coveralls -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.linting.enabled": false, 3 | "python.pythonPath": "/home/redbq/Software/Anaconda/bin/python" 4 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "runtests", 8 | "type": "shell", 9 | "command": "python -m tests", 10 | "group": { 11 | "kind": "build", 12 | "isDefault": true 13 | } 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | Unreleased Notes 2 | ----------------- 3 | 4 | 2019/07/13: 5 | commit: "compat Python3.x" 6 | 7 | 2019/10/08 8 | - removed the use of uncompyle, instead, we introduced the mechanism from future-fstrings 9 | 10 | 2019/10/11 11 | - removed the mechanism from future-fstrings due to the redundant manipulations, use Python import mechanism since then. 12 | - more extensions: 13 | - `quick-lambdas` 14 | - `scoped-operators` 15 | - `pipelines` 16 | 17 | 2019/10/13: 18 | - removed locations in quotations. 19 | - added unpacking support for list and tuple patterns. 20 | 21 | 22 | Released Notes 23 | ----------------- 24 | 25 | 2019/10/13: 26 | - First released version: 0.2, supporting 3.5+ 27 | 28 | 2019/10/14: 29 | - `lazy-import` extension. 30 | - IPython support 31 | 32 | 2019/10/15: 33 | - guards 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | Copyright (c) 2019 thautwarm 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 19 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 21 | OR OTHER DEALINGS IN THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Notes 3 | 4 | This project is now under inactive development. 5 | 6 | Want to take over? Email me. 7 | 8 | # Moshmosh 9 | 10 | [![Build](https://travis-ci.com/thautwarm/moshmosh.svg?branch=master)](https://travis-ci.com/thautwarm/moshmosh) [![Support](https://img.shields.io/badge/PyPI- 3\.5~3\.7-Orange.svg?style=flat)](https://pypi.org/project/moshmosh-base) [![codecov](https://codecov.io/gh/thautwarm/moshmosh/branch/master/graph/badge.svg)](https://codecov.io/gh/thautwarm/moshmosh) 11 | 12 | An advanced syntax extension system implemented in pure python. 13 | 14 | ``` 15 | pip install -U moshmosh-base --no-compile 16 | ``` 17 | 18 | Note that `--no-compile` is required. 19 | 20 | # Preview 21 | 22 | ## Working with IPython 23 | 24 | You should copy [moshmosh_ipy.py](https://raw.githubusercontent.com/thautwarm/moshmosh/master/moshmosh_ipy.py) 25 | to `$USER/.ipython/profile_default/startup/`. 26 | 27 | If this directory does not exist, use command `ipython profile create` to instantiate. 28 | 29 | Some examples about pattern matching, pipelines and quick lambdas: 30 | 31 | ![IPython example 1](https://raw.githubusercontent.com/thautwarm/moshmosh/master/static/img1.png) 32 | 33 | Some examples about the scoped operators: 34 | 35 | ![IPython example 2](https://raw.githubusercontent.com/thautwarm/moshmosh/master/static/img2.png) 36 | 37 | ## Working with regular Python files 38 | 39 | Import `moshmosh` in your main module: 40 | 41 | ![Main.py](https://raw.githubusercontent.com/thautwarm/moshmosh/master/static/main.png) 42 | 43 | Then, in `mypackage.py`, start coding with a pragma comment `# moshmosh?`, then you can use moshmosh extension system. 44 | 45 | ![Upack.py](https://raw.githubusercontent.com/thautwarm/moshmosh/master/static/upack.png) 46 | 47 | ## Case Study : Pattern Matching 48 | 49 | The matching protocol which stems from Python-ideas mailing list is introduced in, 50 | which means you can define your own patterns conveniently. 51 | The link is [here](https://mail.python.org/pipermail/python-ideas/2015-April/032920.html). 52 | 53 | ```python 54 | # moshmosh? 55 | # +pattern-matching 56 | 57 | class GreaterThan: 58 | def __init__(self, v): 59 | self.v = v 60 | 61 | def __match__(self, cnt: int, to_match): 62 | if isinstance(to_match, int) and cnt is 0 and to_match > self.v: 63 | return () # matched 64 | # 'return None' indicates 'unmatched' 65 | 66 | with match(114, 514): 67 | if (GreaterThan(42)() and a, b): 68 | print(b, a) 69 | # 514 114 70 | ``` 71 | 72 | Note that the matching clauses should be exhaustive, 73 | otherwise, a `moshmosh.extensions.pattern_matching.runtime.NotExhaustive` 74 | might get raised. 75 | 76 | The supported Patterns are listed here, which is 77 | of course much more powerful than most programming languages. 78 | 79 | - And pattern: `pat1 and pat2 and pat3 ...` 80 | - Or pattern: `pat1 or pat2 or pat3...` 81 | - Pin pattern: `pin(value)`, this is quite useful. See [Elixir Pin Operator](https://elixir-lang.org/getting-started/pattern-matching.html#the-pin-operator) 82 | - Literal pattern: `1, "str", 1+2j, (1, 2)` 83 | - As pattern: `a, var` 84 | - Wildcard: `_` 85 | - Guard: `when(cond1, cond2, cond3)` 86 | - Nested patterns: 87 | - Tuple: `(pat1, pat2, pat3), (pat1, *pat2, pat3)` 88 | - List: `[pat1, pat2, pat3], [pat1, *pat2, pat3]` 89 | - Recogniser: `Cons(pat1, pat2, pat3)`, note that, 90 | the function `Cons.__match__(, value_to_match)` is exact the protocol. 91 | 92 | The pattern matching should be more efficient than those hand-written codes without 93 | ugly optimizations. 94 | 95 | Besides, Moshmosh's pattern matching is orders of magnitude faster than 96 | any other alternatives. 97 | 98 | ## Case Study : Template-Python 99 | 100 | This is relatively a simple quasiquote implementation, inspired by MetaOCaml. 101 | It does not support manual splices or nested quotations, but the function arguments 102 | are automatically spliced. 103 | 104 | ```python 105 | # moshmosh? 106 | # +template-python 107 | 108 | @quote 109 | def f(x): 110 | x + 1 111 | x = y + 1 112 | 113 | from moshmosh.ast_compat import ast 114 | from astpretty import pprint 115 | 116 | stmts = f(ast.Name("a")) 117 | pprint(ast.fix_missing_locations(stmts[0])) 118 | pprint(ast.fix_missing_locations(stmts[1])) 119 | 120 | # => 121 | Expr( 122 | lineno=7, 123 | col_offset=4, 124 | value=BinOp( 125 | lineno=7, 126 | col_offset=4, 127 | left=Name(lineno=7, col_offset=4, id='a', ctx=Load()), 128 | op=Add(), 129 | right=Num(lineno=7, col_offset=8, n=1), 130 | ), 131 | ) 132 | Assign( 133 | lineno=8, 134 | col_offset=4, 135 | targets=[Name(lineno=8, col_offset=4, id='a', ctx=Store())], 136 | value=BinOp( 137 | lineno=8, 138 | col_offset=8, 139 | left=Name(lineno=8, col_offset=8, id='y', ctx=Load()), 140 | op=Add(), 141 | right=Num(lineno=8, col_offset=12, n=1), 142 | ), 143 | ) 144 | ``` 145 | 146 | ## Case Study: Lazy Import 147 | 148 | ```python 149 | # moshmosh? 150 | # +lazy-import 151 | import numpy as np 152 | # -lazy-import 153 | 154 | # in fact numpy is not imported here, 155 | # and once you use it, it gets imported. 156 | 157 | def arr10(): 158 | # The first time call 159 | # arr10 will enforce the import of numpy. 160 | return np.zeros(10) 161 | ``` 162 | 163 | After the lazy modules are actually imported, there's 164 | no overhead to access their members. 165 | 166 | However, please only import modules when using `lazy-import`. 167 | 168 | The use case is about the necessary cross-import when you want to 169 | organise your codebase in a more fine-grained way. 170 | 171 | 172 | ## Acknowledgements 173 | 174 | - [future-fstrings](https://github.com/asottile/future-fstrings) 175 | - Pattern matching in Python 176 | - [older implementations](http://www.grantjenks.com/docs/patternmatching/#alternative-packages) 177 | - search "pattern matching" at [Python-ideas](https://mail.python.org/archives/list/python-ideas@python.org/). 178 | -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | from moshmosh.extension import perform_extension 2 | from inspect import getsource 3 | from pampy import match, _ 4 | 5 | 6 | def rewrite_fn(f): 7 | src = getsource(f) 8 | src = perform_extension(src) 9 | exec(src, f.__globals__) 10 | return f.__globals__[f.__name__] 11 | 12 | def test_pampy(data): 13 | for datum in data: 14 | match(datum, [_, str, _], lambda a, b, c: "%s(%s)%s" % (a, b, c), 15 | (str, int), lambda a, b: a * b, (int, int), lambda a, b: "%d%d" % 16 | (a, b)) 17 | 18 | 19 | def mk(_isinstance): 20 | def test_mm(data): 21 | isinstance = _isinstance 22 | # +pattern-matching 23 | for d in data: 24 | with match(d): 25 | if [a, isinstance(str) and b, c]: 26 | "%s(%s)%s" % (a, b, c) 27 | if (isinstance(str) and s, isinstance(int) and i): 28 | s * i 29 | if (isinstance(int) and i1, isinstance(int) and i2): 30 | "%d%d" % (i1, i2) 31 | return test_mm 32 | 33 | rewrite_fn(mk) 34 | 35 | data = [("xx", 3), ("yyy", 2), (1, 2), (5, 6), (1000, 2000)] 36 | 37 | test_mm = mk(isinstance) 38 | test_pampy(data) 39 | test_mm(data) 40 | 41 | from timeit import timeit 42 | 43 | time1 = timeit("test(data)", globals=dict(data=data, test=test_pampy), number=10000) 44 | time2 = timeit("test(data)", globals=dict(data=data, test=test_mm), number=10000) 45 | # In [1]: %timeit test_pampy(data) 46 | # 56.6 µs ± 824 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each) 47 | 48 | # In [2]: %timeit test_mm(data) 49 | # 3.93 µs ± 86.5 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each) 50 | 51 | assert time1 / time2 > 10 52 | print("pampy : {}\nmoshmosh: {}\npampy/moshmosh: {}".format(time1, time2, time1/time2)) 53 | -------------------------------------------------------------------------------- /manager.py: -------------------------------------------------------------------------------- 1 | import moshmosh 2 | import sys 3 | from importlib import import_module 4 | from wisepy2 import wise 5 | 6 | 7 | def runtests(): 8 | from tests.test_all import Test 9 | Test().test_extensions() 10 | 11 | @wise 12 | def cli(*, test: bool = False, file=''): 13 | if test: 14 | runtests() 15 | if file: 16 | import_module("tests." + file) 17 | 18 | if __name__ == '__main__': 19 | cli(sys.argv[1:]) -------------------------------------------------------------------------------- /moshmosh/__init__.py: -------------------------------------------------------------------------------- 1 | from .ast_compat import ast 2 | from .extension_register import * 3 | from .extensions import template_python 4 | from .extensions import lazy_import 5 | import warnings 6 | with warnings.catch_warnings(): 7 | warnings.simplefilter("ignore", category=SyntaxWarning) 8 | from .extensions import pattern_matching 9 | from .extensions import scoped_operators 10 | from .extensions import pipelines 11 | from .extensions import quick_lambdas 12 | -------------------------------------------------------------------------------- /moshmosh/ast_compat.py: -------------------------------------------------------------------------------- 1 | import ast as _ast 2 | from types import ModuleType 3 | from sys import version_info 4 | 5 | from numbers import Number 6 | 7 | 8 | class SupertypeMeta(type): 9 | def __instancecheck__(self, other): 10 | return isinstance(other, self._sup_cls) 11 | 12 | 13 | class ConsistentConstant(metaclass=SupertypeMeta): 14 | _sup_cls = (_ast.Num, _ast.NameConstant, _ast.Str) 15 | 16 | def __new__(self, i): 17 | if isinstance(i, Number): 18 | return ast.Num(i) 19 | if isinstance(i, str): 20 | return ast.Str(i) 21 | if isinstance(i, tuple) and version_info < (3, 6): 22 | return ast.Tuple(elts=list(map(ConsistentConstant, i)), ctx=ast.Load()) 23 | return ast.NameConstant(i) 24 | 25 | if version_info < (3, 7): 26 | ast = ModuleType("ast", _ast.__doc__) 27 | 28 | def make_new_init(supercls, fields_): 29 | def init(self, *args, **kwargs): 30 | _undef = object() 31 | supercls.__init__(self) 32 | fields = iter(fields_) 33 | for arg in args: 34 | field = next(fields) 35 | setattr(self, field, arg) 36 | for field in fields: 37 | v = kwargs.get(field, _undef) 38 | if v is not _undef: 39 | setattr(self, field, v) 40 | 41 | return init 42 | 43 | for k, v in _ast.__dict__.items(): 44 | if isinstance(v, type) and issubclass(v, _ast.AST): 45 | ns = {"__init__": make_new_init(v, v._fields), "_sup_cls": v} 46 | v = SupertypeMeta(k, (v, ), ns) 47 | 48 | setattr(ast, k, v) 49 | 50 | if not hasattr(_ast, "Constant"): 51 | 52 | class Constant(ConsistentConstant): 53 | pass 54 | 55 | ast.Constant = Constant 56 | def get_constant(n: Constant): 57 | if isinstance(n, ast.Num): 58 | return n.n 59 | elif isinstance(n, ast.Str): 60 | return n.s 61 | return n.value 62 | else: 63 | def get_constant(n: ast.Constant): 64 | return n.value 65 | 66 | if not hasattr(_ast, "Starred"): 67 | 68 | class Starred: 69 | def __init__(self, *args, **kwargs): 70 | raise NotImplementedError 71 | 72 | ast.Starred = Starred 73 | 74 | if not hasattr(_ast, "AnnAssign"): 75 | 76 | class AnnAssign: 77 | def __init__(self, *args, **kwargs): 78 | raise NotImplementedError 79 | 80 | ast.AnnAssign = AnnAssign 81 | else: 82 | ast = _ast 83 | def get_constant(n: ast.Constant): 84 | return n.value -------------------------------------------------------------------------------- /moshmosh/ctx_fix.py: -------------------------------------------------------------------------------- 1 | from moshmosh.ast_compat import ast 2 | 3 | 4 | class ExprContextWriter(ast.NodeVisitor): 5 | def __init__(self, ctx): 6 | self.ctx = ctx 7 | 8 | def _store_simply(self, node): 9 | node.ctx = self.ctx 10 | 11 | def _store_recursively(self, node): 12 | node.ctx = self.ctx 13 | self.generic_visit(node) 14 | 15 | visit_Name = _store_simply 16 | visit_Subscript = _store_simply 17 | visit_Attribute = _store_simply 18 | visit_Tuple = _store_recursively 19 | visit_List = _store_recursively 20 | visit_Starred = _store_recursively 21 | 22 | 23 | _store_writer = ExprContextWriter(ast.Store()) 24 | _del_writer = ExprContextWriter(ast.Del()) 25 | 26 | 27 | class ExprContextFixer(ast.NodeVisitor): 28 | def visit_AnnAssign(self, n: ast.AnnAssign): 29 | _store_writer.visit(n.target) 30 | self.generic_visit(n.annotation) 31 | if n.value: 32 | self.generic_visit(n.value) 33 | 34 | def visit_AugAssign(self, n: ast.AugAssign): 35 | _store_writer.visit(n.target) 36 | self.generic_visit(n.value) 37 | 38 | def visit_Assign(self, n: ast.Assign): 39 | for target in n.targets: 40 | _store_writer.visit(target) 41 | self.generic_visit(n.value) 42 | 43 | def visit_Delete(self, n: ast.Delete): 44 | for target in n.targets: 45 | _del_writer.visit(target) 46 | 47 | def visit_Name(self, n: ast.Name): 48 | if not hasattr(n, 'ctx') or n.ctx is None: 49 | n.ctx = ast.Load() 50 | 51 | def visit_For(self, n: ast.For): 52 | _store_writer.visit(n.target) 53 | self.generic_visit(n) 54 | -------------------------------------------------------------------------------- /moshmosh/extension.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import typing as t 3 | import re 4 | import traceback 5 | from io import StringIO 6 | from moshmosh.rewrite_helper import ast_to_literal 7 | from moshmosh.ast_compat import ast 8 | 9 | _extension_token_b = re.compile(b"#\s*moshmosh\?\s*?") 10 | _extension_token_u = re.compile(r"#\s*moshmosh\?\s*?") 11 | 12 | _extension_pragma_re_u = re.compile( 13 | r'\s*#\s*(?P[+-])(?P[^(\s]+)\s*(\((?P.*)\))?[^\S\n]*?') 14 | 15 | class Registered: 16 | extensions = {} # type: t.Dict[str, t.Type[Extension]] 17 | 18 | class Activation: 19 | """This sort of instances tell us 20 | whether an extension is enabled at a specific line number""" 21 | 22 | def __init__(self): 23 | self.intervals = [] 24 | 25 | def enable(self, line: int): 26 | intervals = self.intervals 27 | if not intervals: 28 | intervals.append(line) 29 | return 30 | if isinstance(intervals[-1], int): 31 | # already enabled 32 | return 33 | 34 | intervals.append(line) 35 | 36 | def disable(self, line: int): 37 | intervals = self.intervals 38 | if not intervals or isinstance(intervals[-1], range): 39 | # already disabled 40 | return 41 | enable_line = intervals.pop() 42 | intervals.append(range(enable_line, line)) 43 | 44 | def __contains__(self, item): 45 | for each in self.intervals: 46 | if isinstance(each, int): 47 | if item >= each: 48 | return True 49 | else: 50 | assert isinstance(each, range) 51 | if item in each: 52 | return True 53 | return False 54 | 55 | 56 | class ExtensionNotFoundError(Exception): 57 | pass 58 | 59 | 60 | class ExtensionMeta(type): 61 | def __new__(mcs, name, bases, ns: dict): 62 | if ns.get('_root', False): 63 | return super().__new__(mcs, name, bases, ns) 64 | 65 | bases = tuple(filter(lambda it: Extension is not it, bases)) 66 | 67 | # All extensions need instantiating the its activation info. 68 | __init__ = ns.get('__init__', None) 69 | if __init__: 70 | def init(self, *args, **kwargs): 71 | self.activation = Activation() 72 | __init__(self, *args, **kwargs) 73 | else: 74 | def init(self): 75 | self.activation = Activation() 76 | if 'activation' not in ns: 77 | ns['activation'] = None 78 | 79 | assert 'identifier' in ns, "An extension should have its identifier." 80 | 81 | ns['__init__'] = init 82 | ns = { 83 | **Extension.__dict__, 84 | **ns 85 | } 86 | ret = type(name, bases, ns) # type: t.Type[RealExtension] 87 | Registered.extensions[ret.identifier] = ret 88 | 89 | return ret 90 | 91 | def __instancecheck__(self, other): 92 | return other in Registered.extensions.values() 93 | 94 | class Extension(metaclass=ExtensionMeta): 95 | """automatically extension""" 96 | 97 | _root = True 98 | 99 | @property 100 | def activation(self) -> Activation: 101 | raise NotImplemented 102 | 103 | @property 104 | @abc.abstractmethod 105 | def identifier(cls): 106 | "A string to indicate the class of extension instance." 107 | raise NotImplemented 108 | 109 | def pre_rewrite_src(self, io: StringIO): 110 | pass 111 | 112 | @abc.abstractmethod 113 | def rewrite_ast(self, node: ast.AST): 114 | "A function to perform AST level rewriting" 115 | raise NotImplemented 116 | 117 | def post_rewrite_src(self, io: StringIO): 118 | pass 119 | 120 | def __gt__(self, other): 121 | return False 122 | 123 | def __lt__(self, other): 124 | return False 125 | 126 | 127 | class RequirementNotResolved(Exception): 128 | pass 129 | 130 | def solve_deps(exts): 131 | deps = {ext: set() for ext in exts} 132 | # build dependencies 133 | for ext in exts: 134 | for other in exts: 135 | if ext is other: 136 | continue 137 | if ext > other or other < ext: 138 | deps[ext].add(other) 139 | groups = [] 140 | while deps: 141 | group = set.union(*deps.values()).difference(deps.keys()) 142 | to_del_deps = [] 143 | for k, v in deps.items(): 144 | if not v: 145 | to_del_deps.append(k) 146 | group.update(to_del_deps) 147 | if not group: 148 | raise RequirementNotResolved 149 | 150 | for k in to_del_deps: 151 | del deps[k] 152 | 153 | for v in deps.values(): 154 | v.difference_update(group) 155 | groups.append(group) 156 | return groups 157 | 158 | 159 | def extract_pragmas(lines): 160 | """ 161 | Traverse the source codes and extract out the scope of 162 | every extension. 163 | """ 164 | # bind to local for faster visiting in the loop 165 | extension_pragma_re = _extension_pragma_re_u 166 | registered = Registered.extensions 167 | extension_builder = {} # type: t.Dict[object, Extension] 168 | 169 | for i, line in enumerate(lines): 170 | pragma = extension_pragma_re.match(line) 171 | if pragma: 172 | pragma = pragma.groupdict() 173 | action = pragma['action'] 174 | extension = pragma['ext'] 175 | params = pragma['params'] or '' 176 | params = (param.strip() for param in params.split(',')) 177 | params = tuple(i for i in params if i) 178 | try: 179 | ext_cls = registered[extension] 180 | except KeyError: 181 | # TODO: add source code position info 182 | raise ExtensionNotFoundError(extension) 183 | key = (ext_cls, params) 184 | 185 | ext = extension_builder.get(key, None) 186 | if ext is None: 187 | try: 188 | ext = extension_builder[key] = ext_cls(*params) 189 | except Exception as e: 190 | raise 191 | 192 | lineno = i + 1 193 | if action == "+": 194 | ext.activation.enable(lineno) 195 | else: 196 | ext.activation.disable(lineno) 197 | 198 | return list(extension_builder.values()) 199 | 200 | 201 | def _stack_exc(f): 202 | def apply(source_code, filename=''): 203 | try: 204 | return f(source_code, filename) 205 | except Exception as e: 206 | traceback.print_tb(e.__traceback__) 207 | raise e 208 | 209 | return apply 210 | 211 | 212 | def check_if_use_moshmosh_sys(source_code): 213 | str_type = type(source_code) 214 | extension_token = _extension_token_b if str_type is bytes else _extension_token_u 215 | return bool(extension_token.match(source_code)) 216 | 217 | 218 | @_stack_exc 219 | def perform_extension(source_code, filename): 220 | 221 | str_type = type(source_code) 222 | 223 | node = ast.parse(source_code, filename) 224 | if str_type is bytes: 225 | source_code = source_code.decode('utf8') 226 | 227 | extensions = extract_pragmas(StringIO(source_code)) 228 | extensions = sum(map(list, solve_deps(extensions)), []) 229 | 230 | string_io = StringIO() 231 | for each in extensions: 232 | each.pre_rewrite_src(string_io) 233 | 234 | for each in extensions: 235 | node = each.rewrite_ast(node) 236 | ast.fix_missing_locations(node) 237 | 238 | literal = ast_to_literal(node) 239 | string_io.write("import ast as _ast\n") 240 | string_io.write("from moshmosh.rewrite_helper import literal_to_ast as _literal_to_ast\n") 241 | string_io.write('\n') 242 | string_io.write('__literal__ = ') 243 | string_io.write(repr(literal)) 244 | string_io.write('\n') 245 | string_io.write("__ast__ = _literal_to_ast(__literal__)\n") 246 | 247 | string_io.write('__code__ = compile') 248 | string_io.write('(__ast__, ') 249 | string_io.write('__import__("os").path.abspath("") if __name__ == "__main__" else __file__,') 250 | string_io.write('"exec")\n') 251 | 252 | string_io.write('exec(__code__, globals())\n') 253 | 254 | for each in extensions: 255 | each.post_rewrite_src(string_io) 256 | 257 | code = string_io.getvalue() 258 | if str_type is bytes: 259 | code = bytes(code, encoding='utf8') 260 | return code 261 | -------------------------------------------------------------------------------- /moshmosh/extension_register.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from moshmosh.extension import perform_extension, check_if_use_moshmosh_sys 3 | from importlib._bootstrap import ModuleSpec 4 | from importlib._bootstrap_external import (PathFinder, FileLoader, 5 | SourceFileLoader, 6 | SourcelessFileLoader, 7 | ExtensionFileLoader) 8 | 9 | __all__ = [] 10 | 11 | 12 | class ProxySourceFileLoader(SourceFileLoader): 13 | def __init__(self, loader: SourceFileLoader): 14 | SourceFileLoader.__init__(self, loader.name, loader.path) 15 | 16 | def get_data(self, path: str): 17 | data = SourceFileLoader.get_data(self, path) 18 | if check_if_use_moshmosh_sys(data): 19 | return perform_extension(data, self.path) 20 | return data 21 | 22 | 23 | class ProxySourcelessLoader(SourcelessFileLoader): 24 | def __init__(self, loader: SourcelessFileLoader): 25 | SourcelessFileLoader.__init__(self, loader.name, loader.path) 26 | 27 | def get_data(self, path: str): 28 | data = SourcelessFileLoader.get_data(self, path) 29 | if check_if_use_moshmosh_sys(data): 30 | return perform_extension(data, self.path) 31 | return data 32 | 33 | 34 | class MoshmoshFinder(PathFinder): 35 | @classmethod 36 | def find_spec(cls, fullname, path=None, target=None): 37 | spec = PathFinder.find_spec(fullname, path, target) 38 | if spec and spec.loader and isinstance(spec.loader, FileLoader): 39 | loader = spec.loader 40 | loader_ty = loader.__class__ 41 | loader_ = { 42 | SourcelessFileLoader: lambda: ProxySourcelessLoader(loader), 43 | SourceFileLoader: lambda: ProxySourceFileLoader(loader), 44 | ExtensionFileLoader: lambda: loader 45 | }[loader_ty]() 46 | spec.loader = loader_ 47 | return spec 48 | 49 | 50 | sys.meta_path.insert(0, MoshmoshFinder) 51 | -------------------------------------------------------------------------------- /moshmosh/extensions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/moshmosh/extensions/__init__.py -------------------------------------------------------------------------------- /moshmosh/extensions/lazy_import/__init__.py: -------------------------------------------------------------------------------- 1 | from .runtime import * 2 | from .main import * 3 | -------------------------------------------------------------------------------- /moshmosh/extensions/lazy_import/main.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +template-python 3 | from moshmosh.extension import Extension 4 | from moshmosh.ast_compat import ast 5 | from moshmosh.extensions.lazy_import.runtime import LazyModule 6 | 7 | runtime_lazy_mod = "_moshmosh_lazy_module" 8 | 9 | 10 | class LazyImportVisitor(ast.NodeTransformer): 11 | def __init__(self, act): 12 | self.act = act 13 | 14 | def visit_Import(self, imp: ast.Import): 15 | if imp.lineno not in self.act: 16 | return imp 17 | 18 | @quote 19 | def f(lazy_module, name, asname, names_var): 20 | for name, asname, in names_var: 21 | lazy_module(globals(), name, asname) 22 | 23 | names_var = ast.Constant(tuple((name.name, name.asname) for name in imp.names)) 24 | name = ast.Name(".name") 25 | asname = ast.Name(".asname") 26 | return f(ast.Name(runtime_lazy_mod, ast.Load()), name, asname, names_var) 27 | 28 | def visit_ImportFrom(self, imp: ast.ImportFrom): 29 | if imp.lineno not in self.act or imp.names[0].name == '*': 30 | return imp 31 | if imp.module is None: 32 | from_mod = ast.Name(".from_mod") 33 | @quote 34 | def mk_from_mod(from_mod_n, level): 35 | from_mod_n = '.'.join(__name__.split('.')[:-level]) 36 | mk_from_mod = mk_from_mod(from_mod, ast.Constant(imp.level)) 37 | else: 38 | mk_from_mod = [] 39 | from_mod = ast.Constant(imp.module) 40 | 41 | names_var = ast.Constant(tuple((name.name, name.asname) for name in imp.names)) 42 | name = ast.Name(".name") 43 | asname = ast.Name(".asname") 44 | @quote 45 | def f(lazy_module, from_mod, name, asname, names_var): 46 | for name, asname, in names_var: 47 | lazy_module(globals(), name, asname, from_mod) 48 | stmts = f( 49 | ast.Name(runtime_lazy_mod, ast.Load()), 50 | from_mod, 51 | name, 52 | asname, 53 | names_var 54 | ) 55 | mk_from_mod.extend(stmts) 56 | return stmts 57 | 58 | 59 | class LazyImport(Extension): 60 | identifier = "lazy-import" 61 | 62 | def __init__(self): 63 | self.visitor = LazyImportVisitor(self.activation) 64 | 65 | def rewrite_ast(self, node): 66 | node = ast.fix_missing_locations(self.visitor.visit(node)) 67 | # from astpretty import pprint 68 | # pprint(node) 69 | return node 70 | 71 | def pre_rewrite_src(self, io): 72 | io.write("from {} import {} as {}\n".format(__name__, LazyModule.__name__, runtime_lazy_mod)) 73 | -------------------------------------------------------------------------------- /moshmosh/extensions/lazy_import/runtime.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from types import ModuleType 3 | 4 | def import_module_plus(name, from_mod=None, package=None): 5 | if from_mod is not None: 6 | mod = import_module(from_mod, package) 7 | try: 8 | return getattr(mod, name) 9 | except AttributeError: 10 | path = mod.__file__ 11 | exc = ImportError() 12 | exc.msg = "Module {} cannot import {}".format( 13 | from_mod, 14 | name 15 | ) 16 | exc.path = path 17 | exc.name = from_mod 18 | raise exc 19 | else: 20 | return import_module(name, package) 21 | 22 | def import_and_replace(globals, name, asname, from_mod, package): 23 | mod = import_module_plus(name, from_mod, package) 24 | if asname: 25 | globals[asname] = mod 26 | else: 27 | globals[name] = mod 28 | return mod 29 | 30 | class LazyModule(ModuleType): 31 | __imp_info__ = () 32 | 33 | def __init__(self, globals, name, asname=None, from_mod=None, package=None): 34 | object.__setattr__(self, '__imp_info__', (globals, name, asname, from_mod, package)) 35 | if asname: 36 | globals[asname] = self 37 | else: 38 | globals[name] = self 39 | 40 | def __getattr__(self, name): 41 | imp_info = object.__getattribute__(self, '__imp_info__') 42 | mod = import_and_replace(*imp_info) 43 | return getattr(mod, name) 44 | 45 | def __delattr__(self, name): 46 | imp_info = object.__getattribute__(self, '__imp_info__') 47 | mod = import_and_replace(*imp_info) 48 | return delattr(mod, name) 49 | 50 | def __setattr__(self, name, value): 51 | imp_info = object.__getattribute__(self, '__imp_info__') 52 | mod = import_and_replace(*imp_info) 53 | return setattr(mod, name, value) 54 | -------------------------------------------------------------------------------- /moshmosh/extensions/pattern_matching/__init__.py: -------------------------------------------------------------------------------- 1 | from .main import * 2 | from .core import * 3 | -------------------------------------------------------------------------------- /moshmosh/extensions/pattern_matching/core.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +template-python 3 | import typing as t 4 | from moshmosh.ast_compat import ast, get_constant 5 | from moshmosh.extensions.pattern_matching.runtime import NotExhaustive 6 | from toolz import compose 7 | T = t.TypeVar('T') 8 | G = t.TypeVar('G') 9 | H = t.TypeVar('H') 10 | 11 | not_exhaustive_err_type = ast.Name(NotExhaustive.__name__, ast.Load()) 12 | 13 | 14 | def quote(_): 15 | raise NotImplemented 16 | 17 | 18 | class Names(dict): 19 | def __missing__(self, key): 20 | v = self[key] = len(self) 21 | return v 22 | 23 | 24 | class Gensym: 25 | names = Names() 26 | 27 | def __init__(self, base_name): 28 | self.base = base_name 29 | 30 | def gen(self) -> ast.Name: 31 | i = self.names[self.base] 32 | self.names[self.base] += 1 33 | return ast.Name('{}.{}'.format(self.base, i), ast.Load()) 34 | 35 | 36 | class Expr(t.Generic[T]): 37 | value = None # type: ast.expr 38 | 39 | def __init__(self, mk: ast.expr): 40 | self.value = mk 41 | 42 | 43 | class Stmts(t.Generic[T]): 44 | suite = None # type: t.List[ast.stmt] 45 | 46 | def __init__(self, mk: t.List[ast.stmt]): 47 | self.suite = mk 48 | 49 | 50 | class Symbol(t.Generic[T]): 51 | def __init__(self, name, lineno, col_offset): 52 | self.name = name 53 | self.lineno = lineno 54 | self.col_offset = col_offset 55 | 56 | def to_name(self): 57 | return ast.Name( 58 | self.name, 59 | ast.Load(), 60 | lineno=self.lineno, 61 | col_offset=self.col_offset) 62 | 63 | 64 | class Pattern(t.Generic[T, G]): 65 | def __init__(self, f): 66 | self.f = f 67 | 68 | def apply(self, i: Expr[T], remain: Stmts[G]) -> Stmts[G]: 69 | return self.f(i, remain) 70 | 71 | 72 | def dyn_check(f): 73 | def func(a, b): 74 | assert isinstance(a, Expr), (f, type(a)) 75 | assert isinstance(b, Stmts), (f, type(b)) 76 | c = f(a, b) 77 | assert isinstance(c, Stmts), (f, type(c)) 78 | return c 79 | 80 | return func 81 | 82 | 83 | class CaseCompilation(t.Generic[G]): 84 | def __init__(self, ret_sym: str = '.RET'): 85 | self.ret = ast.Name(ret_sym, ast.Load()) 86 | 87 | def literal(self, v): 88 | # noinspection PyStatementEffect PyUnusedLocal 89 | @quote 90 | def quote_eq(ret, v, expr, stmts): 91 | if v == expr: 92 | stmts 93 | else: 94 | ret = None # failed 95 | 96 | @dyn_check 97 | def pat(target: Expr, remain: Stmts): 98 | stmts = quote_eq(self.ret, v, target.value, 99 | remain.suite) 100 | return Stmts(stmts) 101 | 102 | return Pattern(pat) 103 | 104 | def pin(self, s: Expr): 105 | # noinspection PyStatementEffect PyUnusedLocal 106 | @quote 107 | def quote_eq(ret, v, expr, stmts): 108 | if v == expr: 109 | stmts 110 | else: 111 | ret = None 112 | 113 | @dyn_check 114 | def pat(target: Expr, remain: Stmts): 115 | stmts = quote_eq(self.ret, s.value, target.value, 116 | remain.suite) 117 | return Stmts(stmts) 118 | 119 | return Pattern(pat) 120 | 121 | def guard(self, s: Expr): 122 | # noinspection PyStatementEffect PyUnusedLocal 123 | @quote 124 | def quote_when(ret, expr, stmts): 125 | if expr: 126 | stmts 127 | else: 128 | ret = None 129 | 130 | @dyn_check 131 | def pat(target: Expr, remain: Stmts): 132 | stmts = quote_when(self.ret, s.value, remain.suite) 133 | return Stmts(stmts) 134 | 135 | return Pattern(pat) 136 | 137 | def wildcard(_): 138 | @dyn_check 139 | def pat(_, remain): 140 | return remain 141 | 142 | return Pattern(pat) 143 | 144 | # noinspection PyStatementEffect 145 | def capture(_, sym: Symbol): 146 | lhs = sym.to_name() 147 | 148 | # noinspection PyStatementEffect PyUnusedLocal 149 | @quote 150 | def quote_capture(lhs, target, remain): 151 | lhs = target 152 | remain 153 | 154 | @dyn_check 155 | def pat(target: Expr, remain: Stmts): 156 | stmts = quote_capture(lhs, target.value, remain.suite) 157 | return Stmts(stmts) 158 | 159 | return Pattern(pat) 160 | 161 | def intersect(_, ps): 162 | @dyn_check 163 | def pat(target, body): 164 | for p in reversed(ps): 165 | body = p.apply(target, body) 166 | return body 167 | 168 | return Pattern(pat) 169 | 170 | def alternative(self, ps): 171 | # noinspection PyStatementEffect PyUnusedLocal 172 | @quote 173 | def quote_alt(now_stmts, then_stmts, ret): 174 | now_stmts 175 | if ret is None: 176 | then_stmts 177 | 178 | @dyn_check 179 | def pat(target, body): 180 | then_code = Stmts([ast.Pass()]) 181 | for each in reversed(ps): 182 | now_code = each.apply(target, body) # type: Stmts 183 | assert isinstance(now_code, Stmts) 184 | stmts = quote_alt(now_code.suite, then_code.suite, 185 | self.ret) 186 | then_code = Stmts(stmts) 187 | return then_code 188 | 189 | return Pattern(pat) 190 | 191 | def recog(self, maker, item): 192 | def pmk(elts: t.List[Pattern]): 193 | @dyn_check 194 | def pat(target, body): 195 | n = len(elts) 196 | last = body 197 | mid = Gensym("recog").gen() 198 | for i in reversed(range(n)): 199 | sub_tag = item(target, i) # type: Stmts 200 | assert isinstance(sub_tag, Expr) 201 | last = elts[i].apply(Expr(mid), last) # type: : Stmts 202 | last = Stmts( 203 | [ast.Assign([mid], sub_tag.value), *last.suite]) 204 | return last 205 | 206 | return maker(Pattern(pat)) 207 | 208 | return pmk 209 | 210 | def recog2(self, ctor: Expr, elts: t.List[Pattern]): 211 | """ 212 | for deconstructors 213 | """ 214 | 215 | @dyn_check 216 | def pat(target: Expr, body: Stmts): 217 | mid = Gensym("decons").gen() 218 | n = len(elts) 219 | 220 | # noinspection PyStatementEffect PyUnusedLocal 221 | @quote 222 | def decons(mid, ret, ctor, n, tag, body): 223 | mid = ctor.__match__(n, tag) 224 | if mid is None: 225 | ret = None 226 | else: 227 | body 228 | 229 | inner = self.seq_n(tuple, elts) 230 | stmts = inner.apply(Expr(mid), body) 231 | # noinspection PyTypeChecker 232 | suite = decons(mid, self.ret, ctor.value, 233 | ast.Constant(n), target.value, stmts.suite) 234 | return Stmts(suite) 235 | 236 | return Pattern(pat) 237 | 238 | def instance_of(self, ty): 239 | # noinspection PyStatementEffect PyUnusedLocal 240 | @quote 241 | def quote_tychk(ret, tag, type, stmts): 242 | if isinstance(tag, type): 243 | stmts 244 | else: 245 | ret = None 246 | 247 | @dyn_check 248 | def pat(target: Expr, remain: Stmts): 249 | stmts = quote_tychk(self.ret, target.value, ty.value, 250 | remain.suite) 251 | return Stmts(stmts) 252 | 253 | return Pattern(pat) 254 | 255 | def type_as(self, ty): 256 | if isinstance(ty, type): 257 | ty = ty.__name__ 258 | ty = ast.Name(ty, ast.Load()) 259 | 260 | def then(pattern): 261 | # noinspection PyStatementEffect PyUnusedLocal 262 | @quote 263 | def quote_tychk(ret, tag, ty, stmts): 264 | if isinstance(tag, ty): 265 | stmts 266 | else: 267 | ret = None 268 | 269 | @dyn_check 270 | def pat(target: Expr, remain: Stmts): 271 | remain = pattern.apply(target, remain) 272 | stmts = quote_tychk(self.ret, target.value, ty, 273 | remain.suite) 274 | return Stmts(stmts) 275 | 276 | return Pattern(pat) 277 | 278 | return then 279 | 280 | def size_is(self, n: int): 281 | n = ast.Constant(n) 282 | 283 | def then(pattern): 284 | if -5 < get_constant(n) < 256: 285 | # noinspection PyStatementEffect PyUnusedLocal 286 | @quote 287 | def quote_size_chk(ret, tag, n, stmts): 288 | if len(tag) is n: 289 | stmts 290 | else: 291 | ret = None 292 | else: 293 | # noinspection PyStatementEffect PyUnusedLocal 294 | @quote 295 | def quote_size_chk(ret, tag, n, stmts): 296 | if len(tag) == n: 297 | stmts 298 | else: 299 | ret = None 300 | 301 | @dyn_check 302 | def pat(target: Expr, remain: Stmts): 303 | remain = pattern.apply(target, remain) 304 | # noinspection PyTypeChecker 305 | stmts = quote_size_chk(self.ret, target.value, n, 306 | remain.suite) 307 | return Stmts(stmts) 308 | 309 | return Pattern(pat) 310 | 311 | return then 312 | 313 | def size_ge(self, n: int): 314 | n = ast.Constant(n) 315 | 316 | def then(pattern): 317 | @quote 318 | def quote_size_chk(ret, tag, n, stmts): 319 | if len(tag) >= n: 320 | stmts 321 | else: 322 | ret = None 323 | 324 | @dyn_check 325 | def pat(target: Expr, remain: Stmts): 326 | remain = pattern.apply(target, remain) 327 | # noinspection PyTypeChecker 328 | stmts = quote_size_chk(self.ret, target.value, n, 329 | remain.suite) 330 | return Stmts(stmts) 331 | 332 | return Pattern(pat) 333 | 334 | return then 335 | 336 | def seq_n(self, type, elts): 337 | return self.recog( 338 | compose(self.type_as(type), self.size_is(len(elts))), 339 | self.item)(elts) 340 | 341 | def seq_m_star_n(self, type, elts1, star, elts2): 342 | n1 = len(elts1) 343 | n2 = len(elts2) 344 | n = n1 + n2 345 | if n2 is 0: 346 | # when elts2 is empty, 347 | # use a[-1:None] instead of a[-1:-0] 348 | end = None 349 | else: 350 | end = -n2 351 | 352 | def item(expr: Expr, i): 353 | if i < n1: 354 | return self.item(expr, i) 355 | if i > n1: 356 | # return self.item(expr, i - x) 357 | # where: 358 | # (n1 + n2 + 1 - 1) - x = -1 359 | # x = n1 + n2 + 1 360 | 361 | return self.item(expr, i - n - 1) 362 | 363 | @quote 364 | def get_item(value, start, end): 365 | # noinspection PyStatementEffect 366 | value[start:end] 367 | 368 | # noinspection PyTypeChecker 369 | expr = get_item(expr.value, ast.Constant(n1), ast.Constant(end))[0] # type: ast.Expr 370 | return Expr(expr.value) 371 | 372 | return self.recog( 373 | compose(self.type_as(type), self.size_ge(n)), 374 | item 375 | )(elts1 + [star] + elts2) 376 | 377 | def item(self, expr: Expr, i): 378 | @quote 379 | def get_item(value, ith): 380 | # noinspection PyStatementEffect 381 | value[ith] 382 | 383 | # noinspection PyTypeChecker 384 | expr = get_item(expr.value, ast.Constant(i))[0] # type: ast.Expr 385 | return Expr(expr.value) 386 | 387 | def match(self, pairs): 388 | @quote 389 | def quote_alt(now_stmts, then_stmts, ret): 390 | now_stmts 391 | if ret is None: 392 | then_stmts 393 | 394 | @quote 395 | def quote_match_failed(not_exhaustive_err_type): 396 | raise not_exhaustive_err_type 397 | 398 | def pat(target): 399 | then_code = Stmts(quote_match_failed(not_exhaustive_err_type)) 400 | for each, body in reversed(pairs): 401 | 402 | suite = body.suite 403 | suite.reverse() 404 | suite.append(ast.Assign([self.ret], ast.Constant(()))) 405 | suite.reverse() 406 | 407 | now_code = each.apply(target, body) # type: Stmts 408 | assert isinstance(now_code, Stmts) 409 | stmts = quote_alt(now_code.suite, then_code.suite, 410 | self.ret) 411 | then_code = Stmts(stmts) 412 | return then_code 413 | 414 | return pat 415 | -------------------------------------------------------------------------------- /moshmosh/extensions/pattern_matching/core.pyi: -------------------------------------------------------------------------------- 1 | import ast 2 | import typing as t 3 | import typing_extensions as te 4 | from dataclasses import dataclass 5 | T = t.TypeVar('T') 6 | G = t.TypeVar('G') 7 | H = t.TypeVar('H') 8 | 9 | runtime_match_failed = "_match_failed" 10 | runtime_match_succeeded = "_match_succeeded" 11 | 12 | @dataclass 13 | class Stmts(t.Generic[T]): 14 | suite: t.List[ast.stmt] 15 | 16 | 17 | @dataclass 18 | class Expr(t.Generic[T]): 19 | value: ast.expr 20 | 21 | 22 | @dataclass 23 | class Symbol(t.Generic[T]): 24 | name: str 25 | lineno: int 26 | col_offset: int 27 | 28 | def to_name(self) -> ast.Name: 29 | ... 30 | 31 | 32 | class Pattern(t.Generic[T, G]): 33 | def __init__(self, f): 34 | self.f = f 35 | 36 | def apply(self, i: Expr[T], remain: Stmts[G]) -> Stmts[G]: 37 | ... 38 | 39 | 40 | class CaseCompilation(t.Generic[G]): 41 | ret: ast.Name 42 | 43 | def __init__(self, ret_sym: str = ...): 44 | ... 45 | 46 | def literal(cls, v: T) -> Pattern[T, G]: 47 | ... 48 | 49 | def pin(self, sym: Expr[T]) -> Pattern[T, G]: 50 | ... 51 | 52 | def wildcard(cls) -> Pattern[t.Any, G]: 53 | ... 54 | 55 | def capture(cls, sym: Symbol[T]) -> Pattern[T, G]: 56 | ... 57 | 58 | def intersect(cls, ps: t.List[Pattern[T, G]]) -> Pattern[T, G]: 59 | ... 60 | 61 | def alternative(cls, ps: t.List[Pattern[T, G]]) -> Pattern[T, G]: 62 | ... 63 | 64 | def recog(cls, pattern_maker, item): 65 | # type: (t.Callable[[Pattern[T, G]], Pattern[T, G]], t.Callable[[Expr[G], int], Expr[H]]) -> t.Callable[[t.List[Pattern[H, G]]], Pattern[T, G]] 66 | ... 67 | 68 | def recog2(cls, ctor: Expr, elts: t.List[Pattern]) -> Pattern: 69 | ... 70 | 71 | def size_is(self, n: int) -> t.Callable[[Pattern], Pattern]: 72 | ... 73 | 74 | def instance_of(self, ty: Expr[T]) -> Pattern[T, G]: 75 | ... 76 | 77 | @classmethod 78 | def type_as(cls, a: t.Type[T]): 79 | def g(b): 80 | # type: (Pattern[T, G]) -> Pattern[T, G] 81 | ... 82 | 83 | return g 84 | 85 | @classmethod 86 | def match(cls, pairs: t.List[t.Tuple[Pattern[T, G], Stmts[G]]] 87 | ) -> t.Callable[[Expr[T]], Stmts[G]]: 88 | ... 89 | 90 | def tuple_n(self, elts: t.List[Pattern]) -> Pattern: 91 | ... 92 | 93 | def list_n(self, elts: t.List[Pattern]) -> Pattern: 94 | ... 95 | 96 | def item(self, expr: Expr, i) -> Expr: 97 | ... 98 | -------------------------------------------------------------------------------- /moshmosh/extensions/pattern_matching/main.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | from moshmosh.ast_compat import ast 3 | from moshmosh.ast_compat import ConsistentConstant 4 | from moshmosh.extensions.pattern_matching.core import * 5 | from moshmosh.extension import Extension, Activation 6 | from moshmosh.extensions.pattern_matching.runtime import NotExhaustive 7 | from moshmosh.ctx_fix import ExprContextFixer 8 | 9 | 10 | class SyntacticPatternBinding: 11 | def __init__(self, case_comp: CaseCompilation): 12 | self.case_comp = case_comp 13 | 14 | def visit_Name(self, n: ast.Name): 15 | if n.id == "_": 16 | return self.case_comp.wildcard() 17 | return self.case_comp.capture(Symbol(n.id, n.lineno, n.col_offset)) 18 | 19 | def visit_Call(self, n: ast.Call): 20 | if n.keywords: 21 | raise NotImplementedError(n) 22 | 23 | if isinstance(n.func, ast.Name): 24 | if n.func.id == 'pin' and len( 25 | n.args) == 1: 26 | return self.case_comp.pin(Expr(n.args[0])) 27 | 28 | if n.func.id == 'isinstance': 29 | if len(n.args) == 1: 30 | expr = Expr(n.args[0]) 31 | else: 32 | expr = Expr(ast.Tuple(n.args, ast.Load())) 33 | return self.case_comp.instance_of(expr) 34 | if n.func.id == 'when': 35 | if len(n.args) == 1: 36 | expr = Expr(n.args[0]) 37 | else: 38 | expr = Expr(ast.BoolOp(op=ast.And(), values=n.args)) 39 | return self.case_comp.guard(expr) 40 | 41 | return self.case_comp.recog2( 42 | Expr(n.func), [self.visit(elt) for elt in n.args]) 43 | 44 | def visit_BoolOp(self, n: ast.BoolOp): 45 | 46 | if isinstance(n.op, ast.And): 47 | cases = list(map(self.visit, n.values)) 48 | return self.case_comp.intersect(cases) 49 | if isinstance(n.op, ast.Or): 50 | cases = list(map(self.visit, n.values)) 51 | return self.case_comp.alternative(cases) 52 | 53 | raise NotImplementedError(n) 54 | 55 | def _visit_seq(self, type, n): 56 | def find_star(elts): 57 | for i, elt in enumerate(elts): 58 | if isinstance(elt, ast.Starred): 59 | return i 60 | return -1 61 | star_idx = find_star(n.elts) 62 | if star_idx is -1: 63 | elts = list(map(self.visit, n.elts)) 64 | return self.case_comp.seq_n(type, elts) 65 | 66 | ast_elts = n.elts 67 | elts1 = list(map(self.visit, ast_elts[:star_idx])) 68 | star = self.visit(ast_elts[star_idx].value) 69 | elts2 = list(map(self.visit, ast_elts[star_idx+1:])) 70 | return self.case_comp.seq_m_star_n(type, elts1, star, elts2) 71 | 72 | 73 | def visit_Tuple(self, n: ast.Tuple): 74 | return self._visit_seq(tuple, n) 75 | 76 | def visit_List(self, n: ast.List): 77 | return self._visit_seq(list, n) 78 | 79 | def visit(self, node): 80 | """Visit a node.""" 81 | if isinstance(node, ConsistentConstant): 82 | return self.case_comp.literal(node) 83 | method = 'visit_' + node.__class__.__name__ 84 | visitor = getattr(self, method, None) 85 | if visitor is None: 86 | raise TypeError(node) 87 | return visitor(node) 88 | 89 | 90 | class GenMatch(ast.NodeTransformer): 91 | def __init__(self, token: str, activation): 92 | self.token = token 93 | self.activation = activation 94 | 95 | def id_gen(): 96 | i = 0 97 | while True: 98 | yield "PM%d.%d" % (id(self.activation), i) 99 | i += 1 100 | 101 | self.local_id_generator = id_gen() 102 | 103 | @property 104 | def next_id(self): 105 | return next(self.local_id_generator) 106 | 107 | def visit_With(self, node: ast.With): 108 | if node.lineno not in self.activation: 109 | return self.generic_visit(node) 110 | 111 | if not len(node.items): 112 | return self.generic_visit(node) 113 | 114 | item = node.items[0].context_expr 115 | if not isinstance(item, ast.Call): 116 | return self.generic_visit(node) 117 | 118 | fn = item.func 119 | if not isinstance(fn, ast.Name) or fn.id != self.token: 120 | return self.generic_visit(node) 121 | 122 | assert not item.keywords 123 | 124 | assert all(isinstance(stmt, ast.If) for stmt in node.body) 125 | 126 | if len(item.args) is not 1: 127 | val_to_match = ast.Tuple(item.args, ast.Load()) 128 | else: 129 | val_to_match = item.args[0] 130 | 131 | cached = Symbol(self.next_id, node.lineno, node.col_offset).to_name() 132 | 133 | ifs = node.body # type: t.List[ast.If] 134 | for if_ in ifs: 135 | assert not if_.orelse 136 | 137 | case_comp = CaseCompilation() 138 | spb = SyntacticPatternBinding(case_comp) 139 | 140 | pairs = [] 141 | for if_ in ifs: 142 | case = spb.visit(if_.test) 143 | stmts = Stmts([self.visit(each) for each in if_.body]) 144 | pairs.append((case, stmts)) 145 | 146 | res = case_comp.match(pairs)(Expr(cached)) 147 | suite = res.suite 148 | suite.reverse() 149 | suite.append(ast.Assign([cached], val_to_match)) 150 | suite.reverse() 151 | return suite 152 | 153 | class PatternMatching(Extension): 154 | identifier = 'pattern-matching' 155 | 156 | def pre_rewrite_src(self, io: StringIO): 157 | io.write('from {} import {}\n'.format(__name__, 158 | NotExhaustive.__name__)) 159 | 160 | def rewrite_ast(self, node: ast.AST): 161 | node = GenMatch(self.token, self.activation).visit(node) 162 | ExprContextFixer().visit(node) 163 | return node 164 | 165 | def __init__(self, token='match'): 166 | self.token = token 167 | -------------------------------------------------------------------------------- /moshmosh/extensions/pattern_matching/runtime.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | import itertools 3 | 4 | 5 | class NotExhaustive(Exception): 6 | pass 7 | 8 | class ListViewProspectiveGrowError(Exception): 9 | pass 10 | 11 | 12 | class ListView(list): 13 | # noinspection PyMissingConstructor 14 | def __init__(self, src, view_indices): 15 | self.src = src 16 | self.view_indices = view_indices 17 | 18 | def count(self): 19 | return len(self.view_indices) 20 | 21 | def __getitem__(self, item): 22 | return self.src[self.view_indices[item]] 23 | 24 | def __setitem__(self, key, value): 25 | self.src[self.view_indices[key]] = value 26 | 27 | def __iter__(self): 28 | src = self.src 29 | for each in self.view_indices: 30 | yield src[each] 31 | 32 | def __contains__(self, item): 33 | return item in iter(self) 34 | 35 | def __add__(self, other): 36 | return list(self) + other 37 | 38 | def copy(self): 39 | return ListView(self.src, self.view_indices) 40 | 41 | def append(self, _): 42 | raise ListViewProspectiveGrowError 43 | 44 | def extend(self, _): 45 | raise ListViewProspectiveGrowError 46 | 47 | def pop(self, _=...): 48 | raise ListViewProspectiveGrowError 49 | 50 | def clear(self): 51 | raise ListViewProspectiveGrowError 52 | 53 | def remove(self, obj): 54 | raise ListViewProspectiveGrowError 55 | 56 | def reverse(self): 57 | view_indices = self.view_indices 58 | for i in range(len(view_indices) // 2): 59 | view_indices[i], view_indices[-i] = view_indices[-i], view_indices[ 60 | i] 61 | 62 | def insert(self, _, __): 63 | raise ListViewProspectiveGrowError 64 | 65 | def index(self, obj, start=0, stop=None): 66 | src = self.src 67 | view_indices = self.view_indices 68 | n = len(view_indices) 69 | if stop is None: 70 | stop = n 71 | else: 72 | stop = min(n, stop) 73 | 74 | for i in range(start, stop): 75 | if src[view_indices[i]] == obj: 76 | return i 77 | raise ValueError 78 | 79 | def sort(self, *, key=None, reverse=False): 80 | src = self.src 81 | if key: 82 | 83 | def key_(i): 84 | return key(src[i]) 85 | else: 86 | 87 | def key_(i): 88 | return src[i] 89 | 90 | if isinstance(self.view_indices, list): 91 | self.view_indices.sort(key=key_, reverse=reverse) 92 | else: 93 | self.view_indices = sorted( 94 | self.view_indices, key=key_, reverse=reverse) 95 | 96 | def __eq__(self, other): 97 | xs_l = iter(self) 98 | xs_r = iter(other) 99 | for l, r in zip(xs_l, xs_r): 100 | if l == r: 101 | continue 102 | return False 103 | try: 104 | next(xs_r) 105 | except StopIteration: 106 | return True 107 | return False 108 | 109 | def __repr__(self): 110 | return 'View{}'.format(list(self)) 111 | -------------------------------------------------------------------------------- /moshmosh/extensions/pipelines.py: -------------------------------------------------------------------------------- 1 | from moshmosh.extension import Extension 2 | from moshmosh.ast_compat import ast 3 | 4 | 5 | class PipelineVisitor(ast.NodeTransformer): 6 | """ 7 | `a | f -> f(a)`, recursively 8 | """ 9 | def __init__(self, activation): 10 | self.activation = activation 11 | 12 | def visit_BinOp(self, n: ast.BinOp): 13 | if n.lineno in self.activation and isinstance(n.op, ast.BitOr): 14 | return ast.Call( 15 | self.visit(n.right), 16 | [self.visit(n.left)], 17 | [], 18 | lineno=n.lineno, 19 | col_offset=n.col_offset 20 | ) 21 | return self.generic_visit(n) 22 | 23 | class Pipeline(Extension): 24 | identifier = "pipeline" 25 | def __init__(self): 26 | self.visitor = PipelineVisitor(self.activation) 27 | 28 | def rewrite_ast(self, node): 29 | return self.visitor.visit(node) 30 | -------------------------------------------------------------------------------- /moshmosh/extensions/quick_lambdas.py: -------------------------------------------------------------------------------- 1 | # +lazy-import 2 | from moshmosh.extensions import pattern_matching, scoped_operators, pipelines, template_python 3 | # -lazy-import 4 | from moshmosh.extension import Extension 5 | from moshmosh.ast_compat import ast 6 | import re 7 | 8 | class LambdaCollector(ast.NodeTransformer): 9 | def __init__(self, pattern: 're.Pattern', mk_arg): # re.Pattern might not found 10 | self.mk_arg = mk_arg 11 | self.pattern = pattern 12 | self.max_arg_index = -1 13 | 14 | def found_quick_lambda(self): 15 | return self.max_arg_index is not -1 16 | 17 | def visit_Name(self, n: ast.Name): 18 | match = self.pattern.match(n.id) 19 | if match: 20 | ith = match.group('ith') 21 | if ith: 22 | ith = int(ith) 23 | self.max_arg_index = max(self.max_arg_index, ith) 24 | n.id = '.' + n.id 25 | else: 26 | self.max_arg_index = max(0, self.max_arg_index) 27 | n.id = '.' + self.mk_arg(0) 28 | 29 | return n 30 | 31 | class CollectorDelegate: 32 | def __init__(self, cls, token, mk_arg): 33 | pattern = re.compile(mk_arg("(?P\d*)") + '$') 34 | self.collector = cls(pattern, mk_arg) 35 | self._mk_new = lambda : cls(pattern, mk_arg) 36 | 37 | def mk_new_(self): 38 | self.collector = self._mk_new() 39 | 40 | def get(self): 41 | return self.collector 42 | 43 | class QuickLambdaDetector(ast.NodeTransformer): 44 | """ 45 | scala-style lambdas, not recursively processed. 46 | """ 47 | def __init__(self, activation, token: str): 48 | self.arg_col = CollectorDelegate( 49 | LambdaCollector, 50 | token, 51 | lambda x: "{}{}".format(token, x) 52 | ) 53 | 54 | self.placeholder_col = CollectorDelegate( 55 | LambdaCollector, 56 | token, 57 | lambda x: "{}{}_".format(token, x) 58 | ) 59 | self.activation = activation 60 | 61 | def visit_Call(self, call: ast.Call): 62 | if call.lineno in self.activation: 63 | def mk_quick_lam(col: CollectorDelegate, arg: ast.AST): 64 | if arg.lineno in self.activation: 65 | assert not isinstance(arg, ast.Starred) 66 | cur_collector = col.get() 67 | arg = cur_collector.visit(arg) 68 | if cur_collector.found_quick_lambda(): 69 | col.mk_new_() 70 | argcount = cur_collector.max_arg_index + 1 71 | return ast.Lambda( 72 | ast.arguments( 73 | posonlyargs=[], 74 | args=[ 75 | ast.arg( 76 | '.' + cur_collector.mk_arg(i), 77 | annotation=None 78 | ) 79 | for i in range(argcount) 80 | ], 81 | vararg=None, 82 | kwonlyargs=[], 83 | kw_defaults=[], 84 | kwarg=None, 85 | defaults=[] 86 | ), 87 | arg 88 | ) 89 | return arg 90 | call.func = self.visit(call.func) 91 | call.args = [mk_quick_lam(self.arg_col, arg) for arg in call.args] 92 | call.keywords = [mk_quick_lam(self.arg_col, arg) for arg in call.keywords] 93 | return mk_quick_lam(self.placeholder_col, call) 94 | return self.generic_visit(call) 95 | 96 | class QuickLambda(Extension): 97 | identifier = "quick-lambda" 98 | def __init__(self, token: str=None): 99 | token = token or '_' 100 | self.token = token 101 | self.detector = QuickLambdaDetector(self.activation, token) 102 | 103 | def rewrite_ast(self, node): 104 | node = self.detector.visit(node) 105 | # from rbnf_rts.unparse import Unparser 106 | # Unparser(node) 107 | return node 108 | 109 | def __gt__(self, other): 110 | return isinstance(other, pattern_matching.PatternMatching) 111 | 112 | def __lt__(self, other): 113 | return isinstance( 114 | other, 115 | ( 116 | template_python.Template, 117 | pipelines.Pipeline, 118 | scoped_operators.ScopedOperator 119 | ) 120 | ) 121 | -------------------------------------------------------------------------------- /moshmosh/extensions/scoped_operators.py: -------------------------------------------------------------------------------- 1 | from moshmosh.extension import Extension 2 | from moshmosh.ast_compat import ast 3 | 4 | # https://github.com/python/cpython/blob/master/Parser/Python.asdl#L102 5 | opname_map = { 6 | "+": 'Add', 7 | '-': 'Sub', 8 | '*': 'Mult', 9 | '/': 'Div', 10 | '%': 'Mod', 11 | '**': 'Pow', 12 | '<<': 'LShift', 13 | '>>': 'RShift', 14 | '|': 'BitOr', 15 | '^': 'BitXor', 16 | '&': 'BitAnd', 17 | '//': 'FloorDiv' 18 | } 19 | 20 | 21 | class ScopedOperatorVisitor(ast.NodeTransformer): 22 | """ 23 | `a op b -> func(a, b)`, recursively. 24 | The `op => func` pair is specified by users. 25 | """ 26 | def __init__(self, activation, op_name: str, func_name: str): 27 | self.pair = (op_name, func_name) 28 | self.activation = activation 29 | 30 | def visit_BinOp(self, n: ast.BinOp): 31 | if n.lineno in self.activation: 32 | name = n.op.__class__.__name__ 33 | pair = self.pair 34 | if name == pair[0]: 35 | fn = ast.Name(pair[1], ast.Load()) 36 | return ast.Call( 37 | fn, 38 | [self.visit(n.left), self.visit(n.right)], 39 | [], 40 | lineno=n.lineno, 41 | col_offset=n.col_offset 42 | ) 43 | 44 | return self.generic_visit(n) 45 | 46 | 47 | class ScopedOperator(Extension): 48 | identifier = "scoped-operator" 49 | 50 | def __init__(self, op_name: str, func_name: str): 51 | self.op_name = opname_map.get(op_name, op_name) 52 | self.func_name = func_name 53 | self.visitor = ScopedOperatorVisitor( 54 | self.activation, 55 | self.op_name, 56 | self.func_name 57 | ) 58 | 59 | def rewrite_ast(self, node): 60 | return self.visitor.visit(node) 61 | -------------------------------------------------------------------------------- /moshmosh/extensions/template_python.py: -------------------------------------------------------------------------------- 1 | from moshmosh.ast_compat import ast 2 | from moshmosh.extension import Extension, Activation 3 | from moshmosh.rewrite_helper import ast_to_literal_without_locations 4 | from moshmosh.ctx_fix import ExprContextFixer 5 | 6 | runtime_ast_build = '_ast_build' 7 | runtime_ast_copy = '_ast_copy' 8 | _fix_ast_ctx = ExprContextFixer().visit 9 | 10 | __all__ = ['build_ast'] 11 | 12 | 13 | def build_ast(d): 14 | nodes = literal_build_ast(d) 15 | nodes = fix_ast_ctx(nodes) 16 | 17 | mock = ast.Module(nodes) 18 | 19 | return mock.body 20 | 21 | 22 | def fix_ast_ctx(node): 23 | if isinstance(node, list): 24 | for each in node: 25 | fix_ast_ctx(each) 26 | else: 27 | _fix_ast_ctx(node) 28 | return node 29 | 30 | 31 | def literal_build_ast(literal): 32 | """ 33 | Convert a python literal to an AST. 34 | """ 35 | if isinstance(literal, dict): 36 | ctor = literal.pop('constructor') 37 | ctor = getattr(ast, ctor) 38 | return ctor(**{k: literal_build_ast(v) for k, v in literal.items()}) 39 | 40 | if isinstance(literal, list): 41 | res = [] 42 | for each in literal: 43 | each = literal_build_ast(each) 44 | if isinstance(each, list): 45 | res.extend(each) 46 | elif isinstance(each, ast.Expr) and isinstance(each.value, list): 47 | res.extend(each.value) 48 | else: 49 | res.append(each) 50 | return res 51 | 52 | return literal 53 | 54 | 55 | class Symbol: 56 | def __init__(self, id): 57 | self.id = id 58 | 59 | def __repr__(self): 60 | return self.id 61 | 62 | def __iter__(self): 63 | yield self 64 | 65 | 66 | class Splicing(ast.NodeTransformer): 67 | def __init__(self, syms): 68 | self.syms = { 69 | k: Symbol('{}({})'.format(runtime_ast_copy, k)) 70 | for k in syms 71 | } 72 | 73 | def visit_Name(self, n: ast.Name): 74 | return self.syms.get(n.id, n) 75 | 76 | 77 | class ArgumentCollector(ast.NodeVisitor): 78 | def __init__(self): 79 | self.args = set() 80 | 81 | def visit_arg(self, arg: ast.arg): 82 | self.args.add(arg.arg) 83 | 84 | 85 | class MacroTransform(ast.NodeTransformer): 86 | def __init__(self, activation: Activation, token: str): 87 | self.activation = activation 88 | self.token = token 89 | 90 | def visit_FunctionDef(self, fn: ast.FunctionDef): 91 | if fn.lineno not in self.activation: 92 | return self.generic_visit(fn) 93 | 94 | if len(fn.decorator_list) is not 1: 95 | return self.generic_visit(fn) 96 | 97 | deco = fn.decorator_list[0] 98 | 99 | if not (isinstance(deco, ast.Name) and deco.id == self.token): 100 | return self.generic_visit(fn) 101 | 102 | assert fn.body 103 | fn.decorator_list.pop() 104 | arg_collector = ArgumentCollector() 105 | arg_collector.visit(fn) 106 | args = arg_collector.args 107 | splicing = Splicing(args) 108 | 109 | body_head = fn.body[-1] 110 | lineno, col_offset = body_head.lineno, body_head.col_offset 111 | new_body = [] 112 | for stmt in fn.body: 113 | stmt = splicing.visit(stmt) 114 | new_body.append(stmt) 115 | 116 | mod = ast.parse(repr(ast_to_literal_without_locations(new_body))) # type: ast.Module 117 | ast.fix_missing_locations(mod) 118 | expr = mod.body[0] # type: ast.Expr 119 | value = expr.value 120 | fn.body = [ 121 | ast.Return( 122 | ast.Call(ast.Name(runtime_ast_build, ast.Load()), [value], []), 123 | lineno=lineno, 124 | col_offset=col_offset) 125 | ] 126 | return fn 127 | 128 | 129 | class Template(Extension): 130 | identifier = "template-python" 131 | 132 | def __init__(self, token="quote"): 133 | self.token = token 134 | self.visitor = MacroTransform(self.activation, self.token) 135 | self.ctx_fixer = ExprContextFixer() 136 | 137 | def pre_rewrite_src(self, io): 138 | io.write('from {} import build_ast as {}\n'.format( 139 | __name__, runtime_ast_build)) 140 | io.write('from copy import deepcopy as {}\n'.format(runtime_ast_copy)) 141 | 142 | def rewrite_ast(self, node: ast.AST): 143 | node = self.visitor.visit(node) 144 | self.ctx_fixer.visit(node) 145 | return node 146 | -------------------------------------------------------------------------------- /moshmosh/repl_apis.py: -------------------------------------------------------------------------------- 1 | from moshmosh.extension import * 2 | from moshmosh.extension import _extension_pragma_re_u 3 | 4 | def update_pragmas(extension_builder: t.Dict[object, Extension], lines): 5 | """ 6 | Traverse the source codes and extract out the scope of 7 | every extension. Incrementally. 8 | """ 9 | # bind to local for faster visiting in the loop 10 | extension_pragma_re = _extension_pragma_re_u 11 | registered = Registered.extensions 12 | 13 | # for new cells of IPython, refresh the scoping info 14 | # of the extensions 15 | for ext in extension_builder.values(): 16 | intervals = ext.activation.intervals 17 | if intervals: 18 | last = intervals.pop() 19 | intervals.clear() 20 | if type(last) is int: 21 | intervals.append(0) 22 | 23 | for i, line in enumerate(lines): 24 | pragma = extension_pragma_re.match(line) 25 | if pragma: 26 | pragma = pragma.groupdict() 27 | action = pragma['action'] 28 | extension = pragma['ext'] 29 | params = pragma['params'] or '' 30 | params = (param.strip() for param in params.split(',')) 31 | params = tuple(i for i in params if i) 32 | try: 33 | ext_cls = registered[extension] 34 | except KeyError: 35 | # TODO: add source code position info 36 | raise ExtensionNotFoundError(extension) 37 | key = (ext_cls, params) 38 | 39 | ext = extension_builder.get(key, None) 40 | if ext is None: 41 | try: 42 | ext = extension_builder[key] = ext_cls(*params) 43 | except Exception as e: 44 | raise 45 | 46 | lineno = i + 1 47 | if action == "+": 48 | ext.activation.enable(lineno) 49 | else: 50 | ext.activation.disable(lineno) 51 | 52 | return list(extension_builder.values()) 53 | 54 | def perform_extension_incr( 55 | extension_builder: t.Dict[object, Extension], 56 | source_code, 57 | filename 58 | ): 59 | 60 | str_type = type(source_code) 61 | 62 | node = ast.parse(source_code, filename) 63 | if str_type is bytes: 64 | source_code = source_code.decode('utf8') 65 | 66 | extensions = update_pragmas(extension_builder, StringIO(source_code)) 67 | extensions = sum(map(list, solve_deps(extensions)), []) 68 | 69 | string_io = StringIO() 70 | for each in extensions: 71 | each.pre_rewrite_src(string_io) 72 | 73 | for each in extensions: 74 | node = each.rewrite_ast(node) 75 | ast.fix_missing_locations(node) 76 | 77 | literal = ast_to_literal(node) 78 | string_io.write("import ast as _ast\n") 79 | string_io.write("from moshmosh.rewrite_helper import literal_to_ast as _literal_to_ast\n") 80 | string_io.write('\n') 81 | string_io.write('__literal__ = ') 82 | string_io.write(repr(literal)) 83 | string_io.write('\n') 84 | string_io.write("__ast__ = _literal_to_ast(__literal__)\n") 85 | 86 | string_io.write('__code__ = compile') 87 | string_io.write('(__ast__, ') 88 | string_io.write('__import__("os").path.abspath("") if __name__ == "__main__" else __file__,') 89 | string_io.write('"exec")\n') 90 | 91 | string_io.write('exec(__code__, globals())\n') 92 | 93 | for each in extensions: 94 | each.post_rewrite_src(string_io) 95 | 96 | code = string_io.getvalue() 97 | if str_type is bytes: 98 | code = bytes(code, encoding='utf8') 99 | return code 100 | 101 | 102 | class IPythonSupport: 103 | def __init__(self, builder: t.Dict[object, Extension]): 104 | self.builder = builder 105 | self.tmp_exts = None 106 | 107 | def input_transform(self, lines): 108 | self.tmp_exts = update_pragmas(self.builder, lines) 109 | return lines 110 | 111 | def ast_transform(self, node): 112 | extensions = self.tmp_exts 113 | extensions = sum(map(list, solve_deps(extensions)), []) 114 | 115 | prev_io = StringIO() 116 | for each in extensions: 117 | each.pre_rewrite_src(prev_io) 118 | 119 | prev_io.write("import ast as _ast\n") 120 | prev_io.write("from moshmosh.rewrite_helper import literal_to_ast as _literal_to_ast\n") 121 | prev_stmts = ast.parse(prev_io.getvalue()).body 122 | 123 | for each in extensions: 124 | node = each.rewrite_ast(node) 125 | ast.fix_missing_locations(node) 126 | 127 | post_io = StringIO() 128 | for each in extensions: 129 | each.post_rewrite_src(post_io) 130 | 131 | post_stmts = ast.parse(post_io.getvalue()).body 132 | node.body = prev_stmts + node.body + post_stmts 133 | return node 134 | -------------------------------------------------------------------------------- /moshmosh/rewrite_helper.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | 4 | def ast_to_literal(node): 5 | """ 6 | Convert an AST to a python literal. 7 | We avoid the use of comprehension expressions here 8 | to get more human-friendly call stacks. 9 | """ 10 | if isinstance(node, ast.AST): 11 | field_names = node._fields 12 | res = {'constructor': node.__class__.__name__} 13 | for field_name in field_names: 14 | field = getattr(node, field_name, None) 15 | field = ast_to_literal(field) 16 | res[field_name] = field 17 | if hasattr(node, 'lineno'): 18 | res['lineno'] = node.lineno 19 | if hasattr(node, 'col_offset'): 20 | res['col_offset'] = node.col_offset 21 | 22 | return res 23 | if isinstance(node, list): 24 | res = [] 25 | for each in node: 26 | res.append(ast_to_literal(each)) 27 | return res 28 | return node 29 | 30 | def ast_to_literal_without_locations(node): 31 | if isinstance(node, ast.AST): 32 | field_names = node._fields 33 | res = {'constructor': node.__class__.__name__} 34 | for field_name in field_names: 35 | field = getattr(node, field_name, None) 36 | field = ast_to_literal_without_locations(field) 37 | res[field_name] = field 38 | 39 | return res 40 | if isinstance(node, list): 41 | res = [] 42 | for each in node: 43 | res.append(ast_to_literal_without_locations(each)) 44 | return res 45 | return node 46 | 47 | def literal_to_ast(literal): 48 | """ 49 | Convert a python literal to an AST. 50 | """ 51 | if isinstance(literal, dict): 52 | ctor = literal.pop('constructor') 53 | ctor = getattr(ast, ctor) 54 | 55 | return ctor(**{k: literal_to_ast(v) for k, v in literal.items()}) 56 | 57 | if isinstance(literal, list): 58 | return list(map(literal_to_ast, literal)) 59 | 60 | return literal 61 | -------------------------------------------------------------------------------- /moshmosh_ipy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Move this script to the directory 3 | $USER/.ipython/profile_default/startup/ , 4 | and you can use moshmosh syntax in IPython 5 | """ 6 | 7 | import sys 8 | import os 9 | import ast 10 | from IPython import get_ipython 11 | from IPython.core.interactiveshell import InputRejected 12 | 13 | _moshmosh_ipy = None 14 | def moshmosh_input_transf(lines): 15 | global _moshmosh_ipy 16 | if _moshmosh_ipy is None: 17 | from moshmosh.repl_apis import IPythonSupport 18 | _moshmosh_ipy = IPythonSupport({}) 19 | return _moshmosh_ipy.input_transform(lines) 20 | 21 | class MoshmoshASTTransf(ast.NodeTransformer): 22 | @staticmethod 23 | def visit(node): 24 | try: 25 | node = _moshmosh_ipy.ast_transform(node) 26 | return node 27 | except Exception as e: 28 | raise InputRejected 29 | 30 | ip = get_ipython() 31 | ip.input_transformers_post.append(moshmosh_input_transf) 32 | ip.ast_transformers.append(MoshmoshASTTransf) 33 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | toolz -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open('README.md') as readme: 4 | readme = readme.read() 5 | 6 | version = "0.3.4" 7 | 8 | setup( 9 | name='moshmosh-base', 10 | version=version if isinstance(version, str) else str(version), 11 | keywords="syntax, semantics, extension, macro, pattern matching", # keywords of your project separated by comma "," 12 | description="advanced syntax&semantics extension system for Python", # a concise introduction of your project 13 | long_description=readme, 14 | long_description_content_type="text/markdown", 15 | license='mit', 16 | url='https://github.com/thautwarm/moshmosh', 17 | author='thautwarm', 18 | author_email='twshere@outlook.com', 19 | packages=find_packages(), 20 | python_requires='>=3.5', 21 | entry_points={"console_scripts": []}, 22 | # above option specifies commands to be installed, 23 | # e.g: entry_points={"console_scripts": ["yapypy=yapypy.cmd.compiler"]} 24 | install_requires=["toolz"], 25 | platforms="any", 26 | classifiers=[ 27 | "Programming Language :: Python :: 3.5", 28 | "Programming Language :: Python :: 3.6", 29 | "Programming Language :: Python :: 3.7", 30 | "Programming Language :: Python :: Implementation :: CPython", 31 | ], 32 | zip_safe=False, 33 | ) -------------------------------------------------------------------------------- /static/img1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/static/img1.png -------------------------------------------------------------------------------- /static/img2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/static/img2.png -------------------------------------------------------------------------------- /static/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/static/main.png -------------------------------------------------------------------------------- /static/upack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/static/upack.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/tests/__init__.py -------------------------------------------------------------------------------- /tests/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thautwarm/moshmosh/12435ac6288e88b42ea13d59825b90b37e297f38/tests/__main__.py -------------------------------------------------------------------------------- /tests/ast_shape.py: -------------------------------------------------------------------------------- 1 | from astpretty import pprint 2 | from ast import parse 3 | # from moshmosh.extensions import pattern_matching 4 | import ast 5 | print_ast = ast.Name("print", ast.Load()) 6 | 7 | 8 | pprint(parse(""" 9 | 1 and 2 10 | """)) 11 | -------------------------------------------------------------------------------- /tests/asts.py: -------------------------------------------------------------------------------- 1 | 2 | from ast import * 3 | a = Module( 4 | body=[ 5 | Assign( 6 | lineno=1, 7 | col_offset=0, 8 | targets=[Name(lineno=4, col_offset=0, id='PM140085921707568.0', ctx=Store())], 9 | value=Tuple( 10 | lineno=1, 11 | col_offset=0, 12 | elts=[ 13 | Num(lineno=4, col_offset=11, n=1), 14 | Num(lineno=4, col_offset=14, n=2), 15 | ], 16 | ctx=Load(), 17 | ), 18 | ), 19 | Expr( 20 | lineno=314, 21 | col_offset=12, 22 | value=[ 23 | If( 24 | lineno=242, 25 | col_offset=16, 26 | test=Call( 27 | lineno=242, 28 | col_offset=19, 29 | func=Name(lineno=242, col_offset=19, id='isinstance', ctx=Load()), 30 | args=[ 31 | Name(lineno=4, col_offset=0, id='PM140085921707568.0', ctx=Load()), 32 | Name(lineno=242, col_offset=19, id='tuple', ctx=Load()), 33 | ], 34 | keywords=[], 35 | ), 36 | body=[ 37 | Expr( 38 | lineno=243, 39 | col_offset=20, 40 | value=[ 41 | 42 | ], 43 | ), 44 | ], 45 | 46 | ), 47 | ], 48 | ), 49 | 50 | ], 51 | ) 52 | 53 | compile(a, "x", "exec") -------------------------------------------------------------------------------- /tests/complex_match.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +pattern-matching 3 | # +quick-lambda 4 | # +pipeline 5 | 6 | class C: 7 | @classmethod 8 | def __match__(cls, i, x): 9 | if i is not 2: 10 | return None 11 | return x, x 12 | 13 | def f1(x): 14 | with match(x): 15 | if C(C(a, b), C(c, d)): return (a, b, c, d) 16 | 17 | assert f1(1) == (1, ) * 4 18 | 19 | def f2(x, r=1): 20 | with match(x): 21 | if 0: return r 22 | if x: return f2(x-1, r * x) 23 | 24 | assert f2(5) == 120 25 | 26 | def f3(x): 27 | # +pattern-matching 28 | with match(x): 29 | if [1, *x]: 30 | print(x) 31 | return sum(x) 32 | if (1, x, y): return x + y 33 | if _: return 0 34 | 35 | 36 | assert f3([1, 2, 3, 4]) == 9 37 | assert f3((1, 2, 3)) == 5 38 | assert f3((1, 2, 3, 4)) == 0 39 | assert f3(1) == 0 40 | 41 | with match([4, 2, 3]): 42 | if [hd, *tl]: 43 | res = map(_ + hd, tl) | list 44 | 45 | assert res == [6, 7] 46 | 47 | with match(2, 1, 2, 3): 48 | if (a, b, c) or [a, b, c]: 49 | res = (a + b + c) 50 | if (hd, *tl): 51 | res = (hd, tl) 52 | 53 | assert res == (2, (1, 2, 3)) 54 | 55 | 56 | def test_fn(data): 57 | with match(data): 58 | if (e, isinstance(int) and count): 59 | res = [e] * count 60 | if (a, b, _) or [_, a, b]: 61 | res = (a + b) 62 | if (hd, *tl) or [hd, *tl]: 63 | res = (hd, tl) 64 | if "42": 65 | res = 42 66 | return res 67 | 68 | assert test_fn([1, 2, 3]) == 5 69 | assert test_fn((1, 2, 3)) == 3 70 | assert test_fn((1, 2)) == [1, 1] 71 | assert test_fn((object, 3)) == [object, object, object] 72 | assert test_fn((object, 3, 4, 5)) == (object, (3, 4, 5)) 73 | assert test_fn("42") == 42 74 | from moshmosh.extensions.pattern_matching import NotExhaustive 75 | 76 | try: 77 | test_fn(1) 78 | except NotExhaustive: 79 | pass 80 | 81 | with match(1): 82 | if a and when(a != 1): 83 | res = 100 84 | if a and when(a == 1): 85 | res = 200 86 | 87 | assert res == 200 88 | 89 | with match(1, 2, 3): 90 | if (a, *b, c): 91 | pass 92 | 93 | print('abc', a, b, c) 94 | assert a == 1 95 | assert b == (2,) 96 | assert c == 3 97 | -------------------------------------------------------------------------------- /tests/complex_match_py34.py: -------------------------------------------------------------------------------- 1 | from moshmosh.extensions.pattern_matching import * 2 | class MatchError(Exception): 3 | pass 4 | 5 | class C: 6 | @classmethod 7 | def __match__(cls, i, x): 8 | if i is not 2: 9 | return None 10 | return x, x 11 | 12 | def f1(x): 13 | with match(x): 14 | if C(C(a, b), C(c, d)): return (a, b, c, d) 15 | 16 | assert f1(1) == (1, ) * 4 17 | 18 | def f2(x, r=1): 19 | with match(x): 20 | if 0: return r 21 | if x: return f2(x-1, r * x) 22 | 23 | assert f2(5) == 120 24 | 25 | def f3(x): 26 | with match(x): 27 | if [1, x]: 28 | print(x) 29 | return sum(x) 30 | if (1, x, y): return x + y 31 | if _: return 0 32 | 33 | assert f3([1, 2, 3, 4]) == 9 34 | 35 | assert f3((1, 2, 3)) == 5 36 | assert f3((1, 2, 3, 4)) == 0 37 | assert f3(1) == 0 38 | 39 | with match(2, 1): 40 | if (a, b, c) or [a, b, c]: 41 | res = (a + b + c) 42 | if (hd, tl): 43 | res = (hd, tl) 44 | 45 | assert res == (2, 1) 46 | 47 | 48 | with match(1): 49 | if a and when(a != 1): 50 | res = 100 51 | if a and when(a == 1): 52 | res = 200 53 | 54 | assert res == 200 55 | -------------------------------------------------------------------------------- /tests/debugging.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +pattern-matching 3 | from rbnf_rts.unparse import Unparser 4 | Unparser(__ast__) 5 | with match(2, 1, 2, 3): 6 | if (a, b, c) or [a, b, c]: 7 | print(a + b + c) 8 | if (hd, *tl): 9 | print(hd, tl) 10 | 11 | -------------------------------------------------------------------------------- /tests/incremental.py: -------------------------------------------------------------------------------- 1 | from moshmosh.repl_apis import * 2 | 3 | ext_builder = {} 4 | 5 | src = perform_extension_incr(ext_builder, """ 6 | # +pattern-matching 7 | 8 | with match(1): 9 | if 1: res = 114 10 | 11 | 12 | # -pattern-matching 13 | # +pipeline 14 | """, "yoyoyo") 15 | 16 | scope = {'__file__': __file__} 17 | exec(src, scope) 18 | assert scope['res'] == 114 19 | 20 | src = perform_extension_incr(ext_builder, """ 21 | with match(1): 22 | if 1: res = 114 23 | """, "yoyoyo") 24 | 25 | res = None 26 | try: 27 | exec(src, scope) 28 | except NameError as e: 29 | res = e.args[0] 30 | 31 | assert "name 'match' is not defined" == res 32 | 33 | 34 | 35 | src = perform_extension_incr(ext_builder, """ 36 | [1] | print 37 | """, "yoyoyo") 38 | 39 | res = None 40 | exec(src, scope) -------------------------------------------------------------------------------- /tests/lazy_import.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | 3 | from moshmosh.extensions.lazy_import import LazyModule 4 | # +lazy-import 5 | import io 6 | 7 | assert issubclass(type(io), LazyModule) 8 | assert isinstance(io, LazyModule) 9 | 10 | repr(io) 11 | 12 | assert not issubclass(type(io), LazyModule) 13 | assert not isinstance(io, LazyModule) 14 | # -lazy-import 15 | import subprocess 16 | assert not issubclass(type(subprocess), LazyModule) 17 | assert not isinstance(subprocess, LazyModule) 18 | -------------------------------------------------------------------------------- /tests/list_view.py: -------------------------------------------------------------------------------- 1 | from moshmosh.extensions.pattern_matching.runtime import ListView 2 | v = ListView([1, 2, 3, 5], range(1, 4)) 3 | v.sort(reverse=True) 4 | assert v == [5, 3, 2] 5 | 6 | v.sort(reverse=False) 7 | assert v == [2, 3, 5] 8 | 9 | v.sort(reverse=False, key=lambda x: -x) 10 | 11 | assert v == [5, 3, 2] 12 | 13 | assert isinstance(v, list) -------------------------------------------------------------------------------- /tests/match.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +pattern-matching 3 | 4 | with match(1, 2): 5 | if (a, pin(3)): 6 | print(a) 7 | if (_, pin(2)) and (pin(1), _): 8 | print(10) 9 | if _: 10 | print(5) 11 | 12 | with match([1, 2, 3, 4]): 13 | if [1, 2, a, b]: 14 | print(a + b) 15 | 16 | 17 | class Succ: 18 | @classmethod 19 | def __match__(cls, n, tag): 20 | if n == 0 or not isinstance(tag, int): 21 | return None # failed 22 | 23 | return tuple(tag + 1 for i in range(n)) 24 | 25 | 26 | def f(val): 27 | with match(val): 28 | if (4, Succ(3, x, y, z)) or (3, Succ(4, x, y, z)): 29 | print(x + y + z) 30 | if _: 31 | print('otherwise') 32 | 33 | 34 | f((4, 2)) 35 | f((3, 3)) 36 | 37 | 38 | class GreaterThan: 39 | def __init__(self, v): 40 | self.v = v 41 | 42 | def __match__(self, cnt: int, to_match): 43 | if isinstance(to_match, int) and cnt is 0 and to_match > self.v: 44 | return () # matched 45 | 46 | 47 | with match(114, 514): 48 | if (GreaterThan(42)() and a, b): 49 | print(b, a) 50 | -------------------------------------------------------------------------------- /tests/pipelines.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | 3 | # +pipeline 4 | assert "pipeline enabled" | len == len("pipeline enabled") 5 | # -pipeline 6 | _ = 1 | 2 7 | 8 | # +pipeline 9 | assert [1, 2, 3] | (lambda x: map(lambda x: x + 1, x)) | list == [2, 3, 4] -------------------------------------------------------------------------------- /tests/py35_unparse.py: -------------------------------------------------------------------------------- 1 | "Usage: unparse.py " 2 | import sys 3 | import ast 4 | import tokenize 5 | import io 6 | import os 7 | 8 | # Large float and imaginary literals get turned into infinities in the AST. 9 | # We unparse those infinities to INFSTR. 10 | INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) 11 | 12 | def interleave(inter, f, seq): 13 | """Call f on each item in seq, calling inter() in between. 14 | """ 15 | seq = iter(seq) 16 | try: 17 | f(next(seq)) 18 | except StopIteration: 19 | pass 20 | else: 21 | for x in seq: 22 | inter() 23 | f(x) 24 | 25 | class Unparser: 26 | """Methods in this class recursively traverse an AST and 27 | output source code for the abstract syntax; original formatting 28 | is disregarded. """ 29 | 30 | def __init__(self, tree, file = sys.stdout): 31 | """Unparser(tree, file=sys.stdout) -> None. 32 | Print the source for tree to file.""" 33 | self.f = file 34 | self._indent = 0 35 | self.dispatch(tree) 36 | print("", file=self.f) 37 | self.f.flush() 38 | 39 | def fill(self, text = ""): 40 | "Indent a piece of text, according to the current indentation level" 41 | self.f.write("\n"+" "*self._indent + text) 42 | 43 | def write(self, text): 44 | "Append a piece of text to the current line." 45 | self.f.write(text) 46 | 47 | def enter(self): 48 | "Print ':', and increase the indentation." 49 | self.write(":") 50 | self._indent += 1 51 | 52 | def leave(self): 53 | "Decrease the indentation level." 54 | self._indent -= 1 55 | 56 | def dispatch(self, tree): 57 | "Dispatcher function, dispatching tree type T to method _T." 58 | if isinstance(tree, list): 59 | for t in tree: 60 | self.dispatch(t) 61 | return 62 | meth = getattr(self, "_"+tree.__class__.__name__) 63 | meth(tree) 64 | 65 | 66 | ############### Unparsing methods ###################### 67 | # There should be one method per concrete grammar type # 68 | # Constructors should be grouped by sum type. Ideally, # 69 | # this would follow the order in the grammar, but # 70 | # currently doesn't. # 71 | ######################################################## 72 | 73 | def _Module(self, tree): 74 | for stmt in tree.body: 75 | self.dispatch(stmt) 76 | 77 | # stmt 78 | def _Expr(self, tree): 79 | self.fill() 80 | self.dispatch(tree.value) 81 | 82 | def _Import(self, t): 83 | self.fill("import ") 84 | interleave(lambda: self.write(", "), self.dispatch, t.names) 85 | 86 | def _ImportFrom(self, t): 87 | self.fill("from ") 88 | self.write("." * t.level) 89 | if t.module: 90 | self.write(t.module) 91 | self.write(" import ") 92 | interleave(lambda: self.write(", "), self.dispatch, t.names) 93 | 94 | def _Assign(self, t): 95 | self.fill() 96 | for target in t.targets: 97 | self.dispatch(target) 98 | self.write(" = ") 99 | self.dispatch(t.value) 100 | 101 | def _AugAssign(self, t): 102 | self.fill() 103 | self.dispatch(t.target) 104 | self.write(" "+self.binop[t.op.__class__.__name__]+"= ") 105 | self.dispatch(t.value) 106 | 107 | def _Return(self, t): 108 | self.fill("return") 109 | if t.value: 110 | self.write(" ") 111 | self.dispatch(t.value) 112 | 113 | def _Pass(self, t): 114 | self.fill("pass") 115 | 116 | def _Break(self, t): 117 | self.fill("break") 118 | 119 | def _Continue(self, t): 120 | self.fill("continue") 121 | 122 | def _Delete(self, t): 123 | self.fill("del ") 124 | interleave(lambda: self.write(", "), self.dispatch, t.targets) 125 | 126 | def _Assert(self, t): 127 | self.fill("assert ") 128 | self.dispatch(t.test) 129 | if t.msg: 130 | self.write(", ") 131 | self.dispatch(t.msg) 132 | 133 | def _Global(self, t): 134 | self.fill("global ") 135 | interleave(lambda: self.write(", "), self.write, t.names) 136 | 137 | def _Nonlocal(self, t): 138 | self.fill("nonlocal ") 139 | interleave(lambda: self.write(", "), self.write, t.names) 140 | 141 | def _Await(self, t): 142 | self.write("(") 143 | self.write("await") 144 | if t.value: 145 | self.write(" ") 146 | self.dispatch(t.value) 147 | self.write(")") 148 | 149 | def _Yield(self, t): 150 | self.write("(") 151 | self.write("yield") 152 | if t.value: 153 | self.write(" ") 154 | self.dispatch(t.value) 155 | self.write(")") 156 | 157 | def _YieldFrom(self, t): 158 | self.write("(") 159 | self.write("yield from") 160 | if t.value: 161 | self.write(" ") 162 | self.dispatch(t.value) 163 | self.write(")") 164 | 165 | def _Raise(self, t): 166 | self.fill("raise") 167 | if not t.exc: 168 | assert not t.cause 169 | return 170 | self.write(" ") 171 | self.dispatch(t.exc) 172 | if t.cause: 173 | self.write(" from ") 174 | self.dispatch(t.cause) 175 | 176 | def _Try(self, t): 177 | self.fill("try") 178 | self.enter() 179 | self.dispatch(t.body) 180 | self.leave() 181 | for ex in t.handlers: 182 | self.dispatch(ex) 183 | if t.orelse: 184 | self.fill("else") 185 | self.enter() 186 | self.dispatch(t.orelse) 187 | self.leave() 188 | if t.finalbody: 189 | self.fill("finally") 190 | self.enter() 191 | self.dispatch(t.finalbody) 192 | self.leave() 193 | 194 | def _ExceptHandler(self, t): 195 | self.fill("except") 196 | if t.type: 197 | self.write(" ") 198 | self.dispatch(t.type) 199 | if t.name: 200 | self.write(" as ") 201 | self.write(t.name) 202 | self.enter() 203 | self.dispatch(t.body) 204 | self.leave() 205 | 206 | def _ClassDef(self, t): 207 | self.write("\n") 208 | for deco in t.decorator_list: 209 | self.fill("@") 210 | self.dispatch(deco) 211 | self.fill("class "+t.name) 212 | self.write("(") 213 | comma = False 214 | for e in t.bases: 215 | if comma: self.write(", ") 216 | else: comma = True 217 | self.dispatch(e) 218 | for e in t.keywords: 219 | if comma: self.write(", ") 220 | else: comma = True 221 | self.dispatch(e) 222 | self.write(")") 223 | 224 | self.enter() 225 | self.dispatch(t.body) 226 | self.leave() 227 | 228 | def _FunctionDef(self, t): 229 | self.__FunctionDef_helper(t, "def") 230 | 231 | def _AsyncFunctionDef(self, t): 232 | self.__FunctionDef_helper(t, "async def") 233 | 234 | def __FunctionDef_helper(self, t, fill_suffix): 235 | self.write("\n") 236 | for deco in t.decorator_list: 237 | self.fill("@") 238 | self.dispatch(deco) 239 | def_str = fill_suffix+" "+t.name + "(" 240 | self.fill(def_str) 241 | self.dispatch(t.args) 242 | self.write(")") 243 | if t.returns: 244 | self.write(" -> ") 245 | self.dispatch(t.returns) 246 | self.enter() 247 | self.dispatch(t.body) 248 | self.leave() 249 | 250 | def _For(self, t): 251 | self.__For_helper("for ", t) 252 | 253 | def _AsyncFor(self, t): 254 | self.__For_helper("async for ", t) 255 | 256 | def __For_helper(self, fill, t): 257 | self.fill(fill) 258 | self.dispatch(t.target) 259 | self.write(" in ") 260 | self.dispatch(t.iter) 261 | self.enter() 262 | self.dispatch(t.body) 263 | self.leave() 264 | if t.orelse: 265 | self.fill("else") 266 | self.enter() 267 | self.dispatch(t.orelse) 268 | self.leave() 269 | 270 | def _If(self, t): 271 | self.fill("if ") 272 | self.dispatch(t.test) 273 | self.enter() 274 | self.dispatch(t.body) 275 | self.leave() 276 | # collapse nested ifs into equivalent elifs. 277 | while (t.orelse and len(t.orelse) == 1 and 278 | isinstance(t.orelse[0], ast.If)): 279 | t = t.orelse[0] 280 | self.fill("elif ") 281 | self.dispatch(t.test) 282 | self.enter() 283 | self.dispatch(t.body) 284 | self.leave() 285 | # final else 286 | if t.orelse: 287 | self.fill("else") 288 | self.enter() 289 | self.dispatch(t.orelse) 290 | self.leave() 291 | 292 | def _While(self, t): 293 | self.fill("while ") 294 | self.dispatch(t.test) 295 | self.enter() 296 | self.dispatch(t.body) 297 | self.leave() 298 | if t.orelse: 299 | self.fill("else") 300 | self.enter() 301 | self.dispatch(t.orelse) 302 | self.leave() 303 | 304 | def _With(self, t): 305 | self.fill("with ") 306 | interleave(lambda: self.write(", "), self.dispatch, t.items) 307 | self.enter() 308 | self.dispatch(t.body) 309 | self.leave() 310 | 311 | def _AsyncWith(self, t): 312 | self.fill("async with ") 313 | interleave(lambda: self.write(", "), self.dispatch, t.items) 314 | self.enter() 315 | self.dispatch(t.body) 316 | self.leave() 317 | 318 | # expr 319 | def _Constant(self, t): 320 | self.write(repr(t.s)) 321 | 322 | def _Bytes(self, t): 323 | self.write(repr(t.s)) 324 | 325 | def _Str(self, tree): 326 | self.write(repr(tree.s)) 327 | 328 | def _Name(self, t): 329 | self.write(t.id) 330 | 331 | def _NameConstant(self, t): 332 | self.write(repr(t.value)) 333 | 334 | def _Num(self, t): 335 | # Substitute overflowing decimal literal for AST infinities. 336 | self.write(repr(t.n).replace("inf", INFSTR)) 337 | 338 | def _List(self, t): 339 | self.write("[") 340 | interleave(lambda: self.write(", "), self.dispatch, t.elts) 341 | self.write("]") 342 | 343 | def _ListComp(self, t): 344 | self.write("[") 345 | self.dispatch(t.elt) 346 | for gen in t.generators: 347 | self.dispatch(gen) 348 | self.write("]") 349 | 350 | def _GeneratorExp(self, t): 351 | self.write("(") 352 | self.dispatch(t.elt) 353 | for gen in t.generators: 354 | self.dispatch(gen) 355 | self.write(")") 356 | 357 | def _SetComp(self, t): 358 | self.write("{") 359 | self.dispatch(t.elt) 360 | for gen in t.generators: 361 | self.dispatch(gen) 362 | self.write("}") 363 | 364 | def _DictComp(self, t): 365 | self.write("{") 366 | self.dispatch(t.key) 367 | self.write(": ") 368 | self.dispatch(t.value) 369 | for gen in t.generators: 370 | self.dispatch(gen) 371 | self.write("}") 372 | 373 | def _comprehension(self, t): 374 | self.write(" for ") 375 | self.dispatch(t.target) 376 | self.write(" in ") 377 | self.dispatch(t.iter) 378 | for if_clause in t.ifs: 379 | self.write(" if ") 380 | self.dispatch(if_clause) 381 | 382 | def _IfExp(self, t): 383 | self.write("(") 384 | self.dispatch(t.body) 385 | self.write(" if ") 386 | self.dispatch(t.test) 387 | self.write(" else ") 388 | self.dispatch(t.orelse) 389 | self.write(")") 390 | 391 | def _Set(self, t): 392 | assert(t.elts) # should be at least one element 393 | self.write("{") 394 | interleave(lambda: self.write(", "), self.dispatch, t.elts) 395 | self.write("}") 396 | 397 | def _Dict(self, t): 398 | self.write("{") 399 | def write_key_value_pair(k, v): 400 | self.dispatch(k) 401 | self.write(": ") 402 | self.dispatch(v) 403 | 404 | def write_item(item): 405 | k, v = item 406 | if k is None: 407 | # for dictionary unpacking operator in dicts {**{'y': 2}} 408 | # see PEP 448 for details 409 | self.write("**") 410 | self.dispatch(v) 411 | else: 412 | write_key_value_pair(k, v) 413 | interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values)) 414 | self.write("}") 415 | 416 | def _Tuple(self, t): 417 | self.write("(") 418 | if len(t.elts) == 1: 419 | (elt,) = t.elts 420 | self.dispatch(elt) 421 | self.write(",") 422 | else: 423 | interleave(lambda: self.write(", "), self.dispatch, t.elts) 424 | self.write(")") 425 | 426 | unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} 427 | def _UnaryOp(self, t): 428 | self.write("(") 429 | self.write(self.unop[t.op.__class__.__name__]) 430 | self.write(" ") 431 | self.dispatch(t.operand) 432 | self.write(")") 433 | 434 | binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", 435 | "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", 436 | "FloorDiv":"//", "Pow": "**"} 437 | def _BinOp(self, t): 438 | self.write("(") 439 | self.dispatch(t.left) 440 | self.write(" " + self.binop[t.op.__class__.__name__] + " ") 441 | self.dispatch(t.right) 442 | self.write(")") 443 | 444 | cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", 445 | "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} 446 | def _Compare(self, t): 447 | self.write("(") 448 | self.dispatch(t.left) 449 | for o, e in zip(t.ops, t.comparators): 450 | self.write(" " + self.cmpops[o.__class__.__name__] + " ") 451 | self.dispatch(e) 452 | self.write(")") 453 | 454 | boolops = {ast.And: 'and', ast.Or: 'or'} 455 | def _BoolOp(self, t): 456 | self.write("(") 457 | s = " %s " % self.boolops[t.op.__class__] 458 | interleave(lambda: self.write(s), self.dispatch, t.values) 459 | self.write(")") 460 | 461 | def _Attribute(self,t): 462 | self.dispatch(t.value) 463 | # Special case: 3.__abs__() is a syntax error, so if t.value 464 | # is an integer literal then we need to either parenthesize 465 | # it or add an extra space to get 3 .__abs__(). 466 | if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): 467 | self.write(" ") 468 | self.write(".") 469 | self.write(t.attr) 470 | 471 | def _Call(self, t): 472 | self.dispatch(t.func) 473 | self.write("(") 474 | comma = False 475 | for e in t.args: 476 | if comma: self.write(", ") 477 | else: comma = True 478 | self.dispatch(e) 479 | for e in t.keywords: 480 | if comma: self.write(", ") 481 | else: comma = True 482 | self.dispatch(e) 483 | self.write(")") 484 | 485 | def _Subscript(self, t): 486 | self.dispatch(t.value) 487 | self.write("[") 488 | self.dispatch(t.slice) 489 | self.write("]") 490 | 491 | def _Starred(self, t): 492 | self.write("*") 493 | self.dispatch(t.value) 494 | 495 | # slice 496 | def _Ellipsis(self, t): 497 | self.write("...") 498 | 499 | def _Index(self, t): 500 | self.dispatch(t.value) 501 | 502 | def _Slice(self, t): 503 | if t.lower: 504 | self.dispatch(t.lower) 505 | self.write(":") 506 | if t.upper: 507 | self.dispatch(t.upper) 508 | if t.step: 509 | self.write(":") 510 | self.dispatch(t.step) 511 | 512 | def _ExtSlice(self, t): 513 | interleave(lambda: self.write(', '), self.dispatch, t.dims) 514 | 515 | # argument 516 | def _arg(self, t): 517 | self.write(t.arg) 518 | if t.annotation: 519 | self.write(": ") 520 | self.dispatch(t.annotation) 521 | 522 | # others 523 | def _arguments(self, t): 524 | first = True 525 | # normal arguments 526 | defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults 527 | for a, d in zip(t.args, defaults): 528 | if first:first = False 529 | else: self.write(", ") 530 | self.dispatch(a) 531 | if d: 532 | self.write("=") 533 | self.dispatch(d) 534 | 535 | # varargs, or bare '*' if no varargs but keyword-only arguments present 536 | if t.vararg or t.kwonlyargs: 537 | if first:first = False 538 | else: self.write(", ") 539 | self.write("*") 540 | if t.vararg: 541 | self.write(t.vararg.arg) 542 | if t.vararg.annotation: 543 | self.write(": ") 544 | self.dispatch(t.vararg.annotation) 545 | 546 | # keyword-only arguments 547 | if t.kwonlyargs: 548 | for a, d in zip(t.kwonlyargs, t.kw_defaults): 549 | if first:first = False 550 | else: self.write(", ") 551 | self.dispatch(a), 552 | if d: 553 | self.write("=") 554 | self.dispatch(d) 555 | 556 | # kwargs 557 | if t.kwarg: 558 | if first:first = False 559 | else: self.write(", ") 560 | self.write("**"+t.kwarg.arg) 561 | if t.kwarg.annotation: 562 | self.write(": ") 563 | self.dispatch(t.kwarg.annotation) 564 | 565 | def _keyword(self, t): 566 | if t.arg is None: 567 | self.write("**") 568 | else: 569 | self.write(t.arg) 570 | self.write("=") 571 | self.dispatch(t.value) 572 | 573 | def _Lambda(self, t): 574 | self.write("(") 575 | self.write("lambda ") 576 | self.dispatch(t.args) 577 | self.write(": ") 578 | self.dispatch(t.body) 579 | self.write(")") 580 | 581 | def _alias(self, t): 582 | self.write(t.name) 583 | if t.asname: 584 | self.write(" as "+t.asname) 585 | 586 | def _withitem(self, t): 587 | self.dispatch(t.context_expr) 588 | if t.optional_vars: 589 | self.write(" as ") 590 | self.dispatch(t.optional_vars) 591 | 592 | def roundtrip(filename, output=sys.stdout): 593 | with open(filename, "rb") as pyfile: 594 | encoding = tokenize.detect_encoding(pyfile.readline)[0] 595 | with open(filename, "r", encoding=encoding) as pyfile: 596 | source = pyfile.read() 597 | tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) 598 | Unparser(tree, output) 599 | 600 | 601 | 602 | def testdir(a): 603 | try: 604 | names = [n for n in os.listdir(a) if n.endswith('.py')] 605 | except OSError: 606 | print("Directory not readable: %s" % a, file=sys.stderr) 607 | else: 608 | for n in names: 609 | fullname = os.path.join(a, n) 610 | if os.path.isfile(fullname): 611 | output = io.StringIO() 612 | print('Testing %s' % fullname) 613 | try: 614 | roundtrip(fullname, output) 615 | except Exception as e: 616 | print(' Failed to compile, exception is %s' % repr(e)) 617 | elif os.path.isdir(fullname): 618 | testdir(fullname) 619 | 620 | def main(args): 621 | if args[0] == '--testdir': 622 | for a in args[1:]: 623 | testdir(a) 624 | else: 625 | for a in args: 626 | roundtrip(a) 627 | 628 | if __name__=='__main__': 629 | main(sys.argv[1:]) -------------------------------------------------------------------------------- /tests/quick_lambdas.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +quick-lambda 3 | # +pipeline 4 | from tests.py35_unparse import Unparser 5 | Unparser(__ast__) 6 | from functools import reduce 7 | print() 8 | print() 9 | x = map(_ + 1, _0_) 10 | assert isinstance(x([1, 2, 3]), map) 11 | 12 | x = [1, 2, 3] | map(_ + 1, _0_) | list 13 | assert x == [2, 3, 4] 14 | 15 | x = [3, 4, 5] | map(_ % 3, _0_) | list 16 | assert x == [0, 1, 2] 17 | 18 | assert reduce(_0 + _1, [10, 41, 59]) == 110 19 | 20 | assert map(_1_, _0_)([1, 4, 3], _ + 2) | list == [3, 6, 5] 21 | -------------------------------------------------------------------------------- /tests/scoped_operators.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +scoped-operator(+, myadd) 3 | 4 | 5 | def myadd(a, b): 6 | return (a, b) 7 | 8 | assert (1 + 5) == (1, 5) 9 | 10 | # -scoped-operator(+, myadd) 11 | assert (1 + 5) == 6 12 | -------------------------------------------------------------------------------- /tests/template_python.py: -------------------------------------------------------------------------------- 1 | # moshmosh? 2 | # +template-python 3 | 4 | @quote 5 | def f(x): 6 | x + 1 7 | x = y + 1 8 | 9 | from moshmosh.ast_compat import ast 10 | from astpretty import pprint 11 | 12 | stmts = f(ast.Name("a")) 13 | 14 | pprint(ast.fix_missing_locations(stmts[0])) 15 | pprint(ast.fix_missing_locations(stmts[1])) 16 | -------------------------------------------------------------------------------- /tests/test_all.py: -------------------------------------------------------------------------------- 1 | import moshmosh.extension_register 2 | import unittest 3 | import sys 4 | 5 | class Test(unittest.TestCase): 6 | def test_extensions(self): 7 | import tests.lazy_import 8 | import tests.match 9 | import tests.template_python 10 | import tests.pipelines 11 | import tests.quick_lambdas 12 | import tests.scoped_operators 13 | if sys.version_info < (3, 5): 14 | import tests.complex_match_py34 15 | else: 16 | import tests.complex_match 17 | import tests.list_view 18 | import tests.incremental -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, py36, py37 3 | 4 | [testenv] 5 | passenv = TOXENV CI TRAVIS TRAVIS_* 6 | deps= 7 | pytest 8 | coverage 9 | astpretty 10 | codecov>=1.4.0 11 | aenum>=2.0;python_version<"3.6" 12 | commands= 13 | coverage run --source=moshmosh -m pytest 14 | coverage report 15 | 16 | [flake8] 17 | # E501 line too long (88 > 79 characters) 18 | # W503 line break before binary operator 19 | ignore = E501,W503 --------------------------------------------------------------------------------