├── .gitignore ├── .travis.install ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── docs └── Customization.md ├── lambdex ├── __init__.py ├── _aliases.py ├── _config.py ├── _exports.py ├── _features.py ├── ast_parser.py ├── compiler │ ├── __init__.py │ ├── asm │ │ ├── __init__.py │ │ ├── core.py │ │ └── frontend.py │ ├── cache.py │ ├── clauses.py │ ├── context.py │ ├── core.py │ ├── dispatcher.py │ ├── error.py │ └── rules.py ├── fmt │ ├── __init__.py │ ├── __main__.py │ ├── adapters │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── black.py │ │ ├── dummy.py │ │ └── yapf.py │ ├── cli │ │ ├── __init__.py │ │ ├── main.py │ │ ├── mock.py │ │ └── opts.py │ ├── core │ │ ├── __init__.py │ │ ├── _stream_base.py │ │ ├── api.py │ │ ├── definitions │ │ │ ├── __init__.py │ │ │ ├── actions.py │ │ │ ├── annotation.py │ │ │ ├── btstream.py │ │ │ ├── context.py │ │ │ ├── state.py │ │ │ ├── token.py │ │ │ └── token_info.py │ │ ├── tkutils │ │ │ ├── __init__.py │ │ │ ├── builtins │ │ │ │ ├── __init__.py │ │ │ │ ├── token.py │ │ │ │ └── tokenize.py │ │ │ ├── rules │ │ │ │ ├── __init__.py │ │ │ │ ├── matcher.py │ │ │ │ └── rules.py │ │ │ └── tokenize.py │ │ └── transforms │ │ │ ├── AnnotateLeadingWhitespace.py │ │ │ ├── AsCode.py │ │ │ ├── CollectComments.py │ │ │ ├── DropToken.py │ │ │ ├── InsertNewline.py │ │ │ ├── NormalizeWhitespaceBeforeComments.py │ │ │ ├── NormalizeWhitespaceBeforeToken.py │ │ │ ├── Reindent.py │ │ │ ├── SuppressWhitespaces.py │ │ │ └── __init__.py │ ├── jobs_meta.py │ └── utils │ │ ├── __init__.py │ │ ├── colored.py │ │ ├── importlib.py │ │ ├── io.py │ │ └── logger.py ├── keywords.py ├── repl.py └── utils │ ├── __init__.py │ ├── ast.py │ ├── compat.py │ ├── ops.py │ ├── registry.py │ ├── repl_compat │ ├── __init__.py │ ├── builtin.py │ ├── checks.py │ └── idle.py │ └── sysinfo.py ├── setup.py ├── tea.yaml └── tests ├── asm ├── __init__.py ├── sample.py ├── test_find_blocks.py ├── test_frontend.py └── test_transpile.py ├── compiler ├── aliases │ ├── .lambdex.cfg │ ├── __init__.py │ └── test_aliases.py ├── await_attribute │ ├── __init__.py │ ├── disabled │ │ ├── .lambdex.cfg │ │ ├── __init__.py │ │ └── test_await_attribute.py │ └── enabled │ │ ├── .lambdex.cfg │ │ ├── __init__.py │ │ └── test_await_attribute.py ├── implicit_return │ ├── __init__.py │ ├── disabled │ │ ├── .lambdex.cfg │ │ ├── __init__.py │ │ └── test_implicit_return.py │ └── enabled │ │ ├── .lambdex.cfg │ │ ├── __init__.py │ │ └── test_implicit_return.py ├── test_ast.py ├── test_cache.py ├── test_renames.py ├── test_scoping.py └── test_syntax_error.py ├── fmt ├── fmt_samples │ ├── aliases_1 │ │ ├── .lambdex.cfg │ │ ├── test_demo.dst.py │ │ └── test_demo.src.py │ ├── test_augassign.dst.py │ ├── test_augassign.src.py │ ├── test_demo.dst.py │ ├── test_demo.src.py │ ├── test_lxfmt_directive.dst.py │ └── test_lxfmt_directive.src.py └── test_fmt_result.py └── repl ├── _cases.py ├── test_builtin.py ├── test_idle.py └── test_ipython.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Project specific files 132 | /csrc 133 | /test.py 134 | .clang-format 135 | /.vscode/ -------------------------------------------------------------------------------- /.travis.install: -------------------------------------------------------------------------------- 1 | set -euvx 2 | 3 | DEPS=("astcheck" "pexpect") 4 | 5 | if [ "${TARGET}"x == "test_repl_ipython"x ]; then 6 | DEPS+=("ipython") 7 | fi 8 | 9 | python3 -m pip install ${DEPS[@]} -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.5" 4 | - "3.6" 5 | - "3.7" 6 | - "3.8" 7 | - "3.9" 8 | - "3.10-dev" 9 | services: 10 | - xvfb 11 | env: 12 | - "TARGET=test_repl_builtin" 13 | - "TARGET=test_repl_ipython" 14 | - "TARGET=test_repl_idle" 15 | - "TARGET=test" 16 | - "TARGET=test_fmt" 17 | - "TARGET=test_asm" 18 | jobs: 19 | exclude: 20 | - python: "3.5" 21 | env: "TARGET=test_asm" 22 | install: 23 | - bash .travis.install 24 | script: 25 | - make ${TARGET} 26 | cache: 27 | directories: 28 | - /home/travis/.cache/ 29 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v1.0.0 2 | 3 | ## What's New 4 | 5 | - Support all augmented assignments using the syntax `op_<`, such as `+_<` for `+=`. Formatter is also enhanced to eliminate whitespaces among the operators. 6 | 7 | # v0.8.0 8 | 9 | ## What's New 10 | 11 | - The newly added function/module-level bytecode optimization can make lambdexes running faster. 12 | - Add support for the missing `del_` keyword. 13 | 14 | ## BugFix 15 | 16 | ### Compiler 17 | 18 | - `lambdex.utils.ast::ast_from_source` should not modify lines list returned from `inspect.getsourcelines()` inplacely. ([b3a1bd3](../../commit/b3a1bd3a11e7f340034930a55ea61300c9f333ae)) 19 | - Compiler should not assume that closure variables order of code object before and after compilation are compatible. Instead, compiler should be repsonsible to re-order the closure tuple when wrapping a code object. ([61685b3](../../commit/61685b33c487570ff128351c32d2e7abd7cf7064)) 20 | - `lambda.ast_parser::_shallow_match_ast` should further walk down to default value expressions once a Lambda node matched. ([61685b3](../../commit/61685b33c487570ff128351c32d2e7abd7cf7064)) 21 | 22 | ### Formatter 23 | 24 | - IN_LBDX_LAMBDA state should end at the rightmost colon, instead of intermediate one. ([a8c73a9](../../commit/a8c73a9bd3bcc10f0ab9d429abd44cbef177d3ad)) 25 | 26 | # v0.7.0 27 | 28 | ## What's New 29 | 30 | - **lambdex** now supports function renaming! The feature could enhance readability for error traceback. 31 | - **lxfmt** now supports `# lxfmt: on` and `# lxfmt: off` for temporarily turning on/off formatting. 32 | 33 | # v0.6.0 34 | 35 | ## What's New 36 | 37 | - Finally comes the corontine syntax! **lambdex** now supports the keywords `async_def_`, `async_with_`, `async_for_` and `await_`. 38 | - Customized language features are now available! You can enable specific features to fit a preferrable style. 39 | 40 | # v0.5.0 41 | 42 | ## What's New 43 | 44 | Now we can access to the current lambdex via the `callee_` keyword! 45 | 46 | ## BugFix 47 | 48 | ### Compiler 49 | 50 | - `lambdex.utils.ast::source_from_ast` should consider the probability that garbadge remains at the end of the source. ([accc5a7](../../commit/accc5a7a1cf59c7d60558ed527b226fc3814b37a)) 51 | 52 | # v0.4.1 53 | 54 | ## BugFix 55 | 56 | ### Formatter 57 | 58 | - Windows batch scripts generated by **lxfmt-mock** should export `LXALIAS=on`. ([d4c585e](../../commit/d4c585ee7f4c436f37af978515a70db0ee0ea53f)) 59 | 60 | # v0.4.0 61 | 62 | ## What's New 63 | 64 | **lambdex** is now supporting keyword and operator customization! See the [docs](../docs/Customization.md) for more details. 65 | 66 | ## BugFix 67 | 68 | ### Formatter 69 | 70 | - `lambdex.fmt` should ensure NEWLINE at the end when writing to stdout. ([3cbc0fc](../../commit/3cbc0fceca803aa9fb5e227274d47b2c40ab0a7a)) 71 | 72 | ### CI 73 | 74 | - Fix missing target `test_repl_ipython`. ([bb7e9d2](../../commit/bb7e9d205bfa7722b00a7427947b7159c2ed04c6)) 75 | 76 | # v0.3.0 77 | 78 | ## What's New 79 | 80 | **lambdex** is now able to run in REPL! Currently three environments **built-in Python REPL**, **IDLE** and **IPython (Jupyter)** are supported. 81 | 82 | ## BugFix 83 | 84 | ### Compiler 85 | 86 | - Top-level script checking in `lambdex/__init__.py` should check the first frame that contains no `'importlib'`. ([6e1dfb8](../../commit/6e1dfb86ab77f5160bcc4d9fe9b5c2eeef862e8e)) 87 | - Top-level script checking in `lambdex/__init__.py` should not fail if `site.getusersitepackages` not available. ([f174187](../../commit/f174187cccf4614d8a4afc4bcc328145c1bb4ded)) 88 | - `lambdex.utils.ast::ast_from_source` should not assume that lines are ending with line separators. ([e6aa950](../../commit/e6aa9507abdade2479167abc1dec1c7cb5b4dbe5)) 89 | 90 | # v0.2.0 91 | 92 | ## What's New 93 | 94 | - Support and tested on Python 3.5, 3.6, 3.7, 3.8, 3.9, 3.10-dev. 95 | - Improve the compile-time error messages. 96 | 97 | ## BugFix 98 | 99 | ### Compiler 100 | 101 | - Use `sys` instead of `inspect` to check current running environment (as executable or module) in `lambdex/__init__.py`. `inspect` has far more dependencies than `sys`, which may cause module name conflicts in **lxfmt**. ([8a43346](../../commit/8a43346c087db4f6eb1bc158e6a5554dfce640a1)) 102 | - `lambdex.utils.ast::is_lvalue` should check recursively. L-value checking is also removed in `lambdex.utils.ast::cast_to_lvalue`. ([55fbfb6](../../commit/55fbfb6351778db9f41ea04cec9e7b6be3ec115c), [8c801bd](../../commit/8c801bd1bb65b1611c3847e101088c88288bf6cd)) 103 | - `lambdex.utils.ast::check_compare` should ensure that argument `node` is of type `ast.Compare`. ([5fbe3d5](../../commit/5fbe3d52b3dd93dc4e5e6754ebcb365b8015eda7)) 104 | - Comparisons other than assignments should not raise a `SyntaxError` in body. This allows expressions like `a > b` to exist in body. ([73c228d](../../commit/73c228d7e252a11562684032a31bf1326452eb34)) 105 | - Default `except_` should be the last exception handler. Otherwise a `SyntaxError` raised. ([9091565](../../commit/9091565b688cd9af550db83cee45f82c1c965ee1)) 106 | - Should raise a `SyntaxError` when `finally_` has a clause head. ([ee70df0](../../commit/ee70df030e9a07304821b31b0fcb9fa0ddb681be)) 107 | - Should raise a `SyntaxError` when unknown clause encountered in a `try_` block. ([16edff4](../../commit/16edff4b2b70b47452e369e31cf5f0127d7b9009)) 108 | - Should raise a `SyntaxError` when `Slice` node found in `ExtSlice` node. This disallows code like `if_[a:b][...]`. ([77d796f](../../commit/77d796fb1e2a952deba18532fd760f36704e4d49)) 109 | 110 | # v0.1.0 111 | 112 | ## What's New 113 | 114 | ### Compiler 115 | 116 | - Add detailed compile-time and runtime error messages. 117 | 118 | # v0.0.1 119 | 120 | ## What's New 121 | 122 | ### Syntax Features 123 | 124 | - `<` assignments 125 | - `if_`, `elif_`, `else_` 126 | - `for_`, `else_` 127 | - `while_`, `else_` 128 | - `try_`, `except_`, `else_`, `finally_` 129 | - `with_` 130 | - `nonlocal_`, `global_` 131 | - `pass_` 132 | - `return_` 133 | - `raise_`, `from_` 134 | - `yield_`, `yield_from_` 135 | - Nested lambdex 136 | 137 | ### Compiler 138 | 139 | - Bytecode caching 140 | 141 | ### Formatter 142 | 143 | - CLI tool `lxfmt` 144 | - CLI tool `lxfmt-mock` 145 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifeq (${PY}, ) 2 | PY=python3 3 | endif 4 | 5 | ifeq (${OPT}, ) 6 | OPT= 7 | endif 8 | 9 | clear_cache: 10 | find tests -name '*.py[co]' -exec rm -rf '{}' \; 11 | 12 | test: clear_cache 13 | ${PY} -m unittest discover tests/compiler/ ${OPT} 14 | test_fmt: clear_cache 15 | ${PY} -m unittest discover tests/fmt/ ${OPT} 16 | test_repl_builtin: clear_cache 17 | ${PY} -m unittest tests/repl/*builtin* ${OPT} 18 | test_repl_ipython: clear_cache 19 | ${PY} -m unittest tests/repl/*ipython* ${OPT} 20 | test_repl_idle: clear_cache 21 | ${PY} -m unittest tests/repl/*idle* ${OPT} 22 | test_asm: clear_cache 23 | ${PY} -m unittest discover tests/asm/ ${OPT} -------------------------------------------------------------------------------- /docs/Customization.md: -------------------------------------------------------------------------------- 1 | # Customization 2 | 3 | **lambdex** supports user customization for both the compiler and the formatter. To customize, one should provide a config file named `.lambdex.cfg` in the file system. The file is written in INI format. **Once configured, the settings will never be changed at runtime**. 4 | 5 | If you want to completely disable customization, simply set environment variable `LXNOCFG=1`. 6 | 7 | List of environment variables that affect the behavior of **lambdex**: 8 | 9 | - `LXNOCFG=1` disables any customization; 10 | - `LXALIAS=1` enableds keyword and operator aliasing. 11 | 12 | ## Config File Resolving 13 | 14 | **lambdex** will search for the config file in the directories specified below: 15 | 16 | 1. All parents of the input file, if **lambdex** is run as formatter and not reading from stdin; 17 | 2. All parents of the importer module, if **lambdex** is imported by a user script / module; 18 | 3. CWD and all its parents. 19 | 20 | The first one matched will be adopted. Below are several examples for better understanding. 21 | 22 | ### Example 1 23 | 24 | Suppose **lambdex** is run as formatter (such as **lxfmt**), and two input files `/path1/to/file1.py`, `/path2/to/file2.py` supplied, and CWD is `/path3/cwd/`. The config file will be searched from (_note that the two inputs may not share a config file_): 25 | 26 | ```bash 27 | # Search paths for input 1 28 | /path1/to/ 29 | /path1/ 30 | / 31 | /path3/cwd/ 32 | /path3/ 33 | / 34 | 35 | # Search paths for input 2 36 | /path2/to/ 37 | /path2/ 38 | / 39 | /path3/cwd/ 40 | /path3/ 41 | / 42 | ``` 43 | 44 | ### Example 2 45 | 46 | Suppose **lambdex** is imported by `/path/to/file.py`, and CWD is `/path3/cwd/`. The config file will be searched from: 47 | 48 | ```bash 49 | /path/to/ 50 | /path/ 51 | / 52 | /path3/cwd/ 53 | /path3/ 54 | / 55 | ``` 56 | 57 | ### Example 3 58 | 59 | Suppose **lambdex** is imported in an REPL, and CWD is `/path3/cwd/`. The config file will be searched from: 60 | 61 | ```bash 62 | /path3/cwd/ 63 | /path3/ 64 | / 65 | ``` 66 | 67 | ## Keyword and Operator Aliasing 68 | 69 | Aliasing allows you to use keywords and operetors other than the default ones. By default, aliasing is **disabled**. One should set environment variable `LXALIAS=1` to enable it. 70 | 71 | Aliasing should be specified in the `[aliases]` section in `.lambdex.cfg`, e.g. 72 | 73 | ```ini 74 | # .lambdex.cfg 75 | [aliases] 76 | def_ = Def 77 | if_ = If 78 | else_ = Else 79 | return_ = Return 80 | Assignment = <= 81 | ``` 82 | 83 | The configuration above allows you to write code like 84 | 85 | ```python 86 | from lambdex import Def # <-- Note here 87 | 88 | max_plus_one = Def(lambda a, b: [ 89 | If[a > b] [ 90 | larger <= a, 91 | ].Else [ 92 | larger <= b, 93 | ], 94 | Return[larger + 1] 95 | ]) 96 | ``` 97 | 98 | Full list of alias entries goes here 99 | 100 | ```ini 101 | # .lambdex.cfg 102 | [aliases] 103 | # Keywords 104 | def_ = def_ 105 | if_ = if_ 106 | elif_ = elif_ 107 | else_ = else_ 108 | for_ = for_ 109 | while_ = while_ 110 | with_ = with_ 111 | try_ = try_ 112 | except_ = except_ 113 | finally_ = finally_ 114 | yield_ = yield_ 115 | yield_from_ = yield_from_ 116 | pass_ = pass_ 117 | return_ = return_ 118 | from_ = from_ 119 | raise_ = raise_ 120 | global_ = global_ 121 | nonlocal_ = nonlocal_ 122 | del_ = del_ 123 | break_ = break_ 124 | continue_ = continue_ 125 | callee_ = callee_ 126 | async_def_ = async_def_ 127 | async_with_ = async_with_ 128 | async_for_ = async_for_ 129 | await_ = await_ 130 | 131 | # Operators 132 | # From now on, the fields should start with captialized letters 133 | Assignment = < 134 | As = > 135 | ``` 136 | 137 | ## Language Extension 138 | 139 | Extensions should be specified in the `[features]` section in `.lambdex.cfg`. To turn on a specific feature, simply set the entry to `ON`. 140 | 141 | Full list of extensions goes here: 142 | 143 | ```ini 144 | [features] 145 | await_attribute = OFF 146 | implicit_return = OFF 147 | ``` 148 | -------------------------------------------------------------------------------- /lambdex/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | 5 | def _is_run_as_script(): 6 | """ 7 | Check that whether lambdex is run as top-level script. 8 | 9 | We iterate from the stack bottom to find the first frame that contains 10 | no 'importlib' string, which should be the importer of lambdex. 11 | 12 | If the importer has any system prefixes, we assert that lambdex is run 13 | as top-level script. 14 | """ 15 | from os.path import dirname 16 | from .utils.sysinfo import get_importer_path, get_site_paths 17 | 18 | importer_path = get_importer_path() 19 | 20 | if not importer_path: 21 | return False 22 | 23 | for sitepath in get_site_paths(): 24 | prefix = dirname(dirname(dirname(sitepath))) # such as '/usr/local' 25 | if importer_path.startswith(prefix): 26 | return True 27 | 28 | return False 29 | 30 | 31 | # If run as top-level script, user is happened to use lxfmt. 32 | # We remove CWD from sys.path, so that the files user editting will not 33 | # cause name conflicts with built-in modules. 34 | if _is_run_as_script(): 35 | if sys.path and sys.path[0] == os.getcwd(): 36 | sys.path = sys.path[1:] 37 | # Otherwise, we import keywords as normal 38 | else: 39 | from ._exports import * 40 | from ._exports import __all__ 41 | 42 | del os, sys, _is_run_as_script 43 | 44 | __version__ = "1.0.0" 45 | -------------------------------------------------------------------------------- /lambdex/_aliases.py: -------------------------------------------------------------------------------- 1 | """ 2 | Manage the mapping between user-defined keywords/operators and internal 3 | used symbols. 4 | """ 5 | import os 6 | from keyword import iskeyword 7 | from collections import namedtuple 8 | 9 | from ._config import get_parser, ParsingError 10 | from .utils.ops import COMPARATORS 11 | 12 | _Aliases = namedtuple( 13 | "_Aliases", 14 | [ 15 | "def_", 16 | "if_", 17 | "elif_", 18 | "else_", 19 | "for_", 20 | "while_", 21 | "with_", 22 | "try_", 23 | "except_", 24 | "finally_", 25 | "yield_", 26 | "yield_from_", 27 | "pass_", 28 | "return_", 29 | "from_", 30 | "raise_", 31 | "global_", 32 | "nonlocal_", 33 | "del_", 34 | "break_", 35 | "continue_", 36 | "callee_", 37 | # Coroutines related 38 | "async_def_", 39 | "async_with_", 40 | "async_for_", 41 | "await_", 42 | # From now on, the fields should start with captialized letters 43 | "Assignment", 44 | "As", 45 | ], 46 | ) 47 | 48 | # Mapping between operator names and their string representations 49 | _DEFAULT_OPS = { 50 | "Assignment": "<", 51 | "As": ">", 52 | } 53 | 54 | _aliases = None 55 | 56 | 57 | def get_declarers(): 58 | return {_aliases.def_, _aliases.async_def_} 59 | 60 | 61 | def _validate_aliases(aliases: _Aliases): 62 | """ 63 | Check that given `aliases` is valid. 64 | """ 65 | for name, value in aliases._asdict().items(): 66 | 67 | # If `name` is a keyword symbol 68 | if name[0].islower(): 69 | # Ensure that it's an identifier 70 | if not value.isidentifier(): 71 | raise ParsingError( 72 | "alias for {!r} should be an identifier, got {!r}".format( 73 | name, value 74 | ) 75 | ) 76 | # Ensure that it's not a keyword 77 | if iskeyword(value): 78 | raise ParsingError( 79 | "alias for {!r} should not be a keyowrd, got {!r}".format( 80 | name, value 81 | ) 82 | ) 83 | 84 | # If `name` is an operator symbol 85 | else: 86 | # Ensure that it's a valid comparator 87 | if value not in COMPARATORS: 88 | raise ParsingError( 89 | "alias for {!r} should be one of {}, got {!r}".format( 90 | name, " ".join(map(repr, COMPARATORS)), value 91 | ) 92 | ) 93 | 94 | 95 | def get_aliases(userpaths=(), reinit=False) -> _Aliases: 96 | """ 97 | Return or rebuild an _Aliases instance. 98 | 99 | `userpaths` and `reinit` are for building the config parser. 100 | """ 101 | global _aliases 102 | if _aliases is not None and not reinit: 103 | return _aliases 104 | 105 | # Initialize the default building arguments 106 | build_kwargs = {} 107 | for name in _Aliases._fields: 108 | if name[0].islower(): 109 | build_kwargs[name] = name 110 | else: 111 | build_kwargs[name] = _DEFAULT_OPS[name] 112 | 113 | if os.getenv("LXALIAS") is not None: 114 | parser = get_parser(userpaths, reinit=reinit) 115 | if parser.has_section("aliases"): 116 | for name in build_kwargs: 117 | if name in parser["aliases"]: 118 | build_kwargs[name] = parser["aliases"][name] 119 | 120 | _aliases = _Aliases(**build_kwargs) 121 | _validate_aliases(_aliases) 122 | 123 | return _aliases 124 | -------------------------------------------------------------------------------- /lambdex/_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define the config parser for universal lambdex configuration, e.g., keyword 3 | aliases or styles. 4 | """ 5 | 6 | from typing import List, Optional 7 | 8 | import os 9 | import pathlib 10 | import configparser 11 | 12 | from .utils.sysinfo import get_importer_path 13 | 14 | 15 | def _is_readable_file(path): 16 | """ 17 | Check if a given path is a file and is readable. 18 | """ 19 | path = str(path) 20 | return os.access(path, os.R_OK) and os.path.isfile(path) 21 | 22 | 23 | def _walk_parents(paths: List[pathlib.Path]): 24 | """ 25 | A genrator that yields all parents of path in `paths`. Each path yielded 26 | will appear only once. 27 | """ 28 | visited = set() 29 | for path in paths: 30 | path = pathlib.Path(path).absolute() 31 | 32 | # If path is a file, walk upwards 33 | if _is_readable_file(path): 34 | path = path.parent 35 | 36 | # Walk upwards until reach the root 37 | while True: 38 | # If a path is visited, all of its parents should also be 39 | if path in visited: 40 | break 41 | 42 | visited.add(path) 43 | yield path 44 | 45 | # If reach the root, try another path 46 | if path.parent == path: 47 | break 48 | 49 | path = path.parent 50 | 51 | 52 | def _find_config_file( 53 | userpaths: List[str], filename=".lambdex.cfg" 54 | ) -> Optional[pathlib.Path]: 55 | """ 56 | Search for the first file with filename `filename` in the order below: 57 | 58 | - Path of the direct importer (if any); 59 | - Paths in `userpaths`; 60 | - CWD. 61 | 62 | For each path, we try all of its parents until reach the root. 63 | 64 | If envvar LXNOCFG set, simply return None (don't use any config files). 65 | """ 66 | if os.getenv("LXNOCFG") is not None: 67 | return None 68 | 69 | paths = [] 70 | 71 | # Append importer path if available. 72 | importer_path = get_importer_path() 73 | if importer_path is not None: 74 | paths.append(importer_path) 75 | 76 | # Append userpaths 77 | paths.extend(userpaths) 78 | 79 | # Apeend CWD 80 | paths.append(os.getcwd()) 81 | 82 | for path in _walk_parents(paths): 83 | cfg_file = path / filename 84 | # Check if the file exists and is readable 85 | if _is_readable_file(cfg_file): 86 | return cfg_file 87 | 88 | return None 89 | 90 | 91 | _parser = None 92 | _config_path = None 93 | 94 | 95 | def get_parser(userpaths: List[str], reinit=False) -> configparser.ConfigParser: 96 | """ 97 | Build and return a config parser. 98 | 99 | `userpaths` is for searching config file. If a config file found, the parser 100 | will read from it. 101 | 102 | If `reinit` is True, the function will always try to build a new parser; 103 | otherwise the parser will be cached. 104 | """ 105 | global _parser, _config_path 106 | if _parser is None or reinit: 107 | _config_path = _find_config_file(userpaths) 108 | _parser = configparser.ConfigParser() 109 | if _config_path is not None: 110 | with _config_path.open("r", encoding="utf-8") as fd: 111 | _parser.read_file(fd) 112 | 113 | return _parser 114 | 115 | 116 | def get_config_path() -> Optional[pathlib.Path]: 117 | """ 118 | Return the config file path that current parser reads from. 119 | """ 120 | return _config_path 121 | 122 | 123 | ParsingError = configparser.ParsingError 124 | -------------------------------------------------------------------------------- /lambdex/_exports.py: -------------------------------------------------------------------------------- 1 | from .keywords import * 2 | from .keywords import __all__ 3 | 4 | from .compiler.asm.frontend import asmopt 5 | 6 | __all__ = __all__ + ["asmopt"] 7 | -------------------------------------------------------------------------------- /lambdex/_features.py: -------------------------------------------------------------------------------- 1 | """ 2 | Manage the enability of language features. 3 | """ 4 | import os 5 | from keyword import iskeyword 6 | from collections import namedtuple 7 | 8 | from ._config import get_parser, ParsingError 9 | 10 | _Features = namedtuple( 11 | "_Features", 12 | [ 13 | "await_attribute", 14 | "implicit_return", 15 | ], 16 | ) 17 | 18 | _DEFAULT = _Features( 19 | await_attribute=False, 20 | implicit_return=False, 21 | ) 22 | 23 | _features = None 24 | 25 | 26 | def _parse_flag(flag: str): 27 | """ 28 | Parse "y", "on" as True, and "n", "off" as False in a case-insensitive way. 29 | 30 | If invalid value encountered, return None. 31 | """ 32 | flag = flag.lower() 33 | if flag in {"y", "on"}: 34 | return True 35 | if flag in {"n", "off"}: 36 | return False 37 | return None 38 | 39 | 40 | def get_features(userpaths=(), reinit=False) -> _Features: 41 | """ 42 | Return or rebuild an _Features instance. 43 | 44 | `userpaths` and `reinit` are for building the config parser. 45 | """ 46 | global _features 47 | if _features is not None and not reinit: 48 | return _features 49 | 50 | _features = _DEFAULT 51 | build_kwargs = {} 52 | parser = get_parser(userpaths, reinit=reinit) 53 | if parser.has_section("features"): 54 | for name in _Features._fields: 55 | if name in parser["features"]: 56 | flag = parser["features"][name] 57 | flag = _parse_flag(flag) 58 | if flag is None: 59 | raise ParsingError("unknown option '{} = {}'".format(name, flag)) 60 | build_kwargs[name] = flag 61 | 62 | _features = _features._replace(**build_kwargs) 63 | return _features 64 | -------------------------------------------------------------------------------- /lambdex/ast_parser.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, Dict, Optional, Union 2 | 3 | import ast 4 | import types 5 | import inspect 6 | import textwrap 7 | import linecache 8 | from itertools import chain 9 | 10 | from lambdex.utils.ast import ast_from_source 11 | from lambdex.compiler import error 12 | 13 | from lambdex._aliases import get_aliases, get_declarers 14 | 15 | __all__ = [ 16 | "lambda_to_ast", 17 | "find_lambdex_ast_in_code", 18 | "LambdexASTLookupKey", 19 | "LambdexASTLookupTable", 20 | ] 21 | 22 | 23 | def _shallow_match_ast(node, match, *, yield_node_only=True): 24 | """ 25 | Yields all children of `node` that fulfill `match` in a shallow manner. 26 | """ 27 | 28 | match_result = match(node) 29 | if match_result is not None: 30 | yield (node if yield_node_only else (node, match_result)) 31 | lambda_args = node.args[0].args 32 | children = enumerate(chain(lambda_args.kw_defaults, lambda_args.defaults)) 33 | else: 34 | children = ast.iter_fields(node) 35 | 36 | # Adapted from `ast.walk` 37 | for _, value in children: 38 | if isinstance(value, list): 39 | for item in value: 40 | if isinstance(item, ast.AST): 41 | yield from _shallow_match_ast( 42 | item, match, yield_node_only=yield_node_only 43 | ) 44 | elif isinstance(value, ast.AST): 45 | yield from _shallow_match_ast(value, match, yield_node_only=yield_node_only) 46 | 47 | 48 | def _make_pattern(keyword: str, identifier: str): 49 | """ 50 | Returns a function that matches a node of form `.(...)` 51 | or `(...)` if `identifier` is empty. 52 | """ 53 | 54 | if keyword is None: 55 | assert identifier is None 56 | get_aliases() 57 | declarers = get_declarers() 58 | 59 | def _match(node): 60 | if node.__class__ is ast.Name and node.id in declarers: 61 | return (node.id, None) 62 | elif ( 63 | node.__class__ is ast.Attribute 64 | and node.value.__class__ is ast.Name 65 | and node.value.id in declarers 66 | ): 67 | return (node.value.id, node.attr) 68 | else: 69 | return None 70 | 71 | elif not identifier: 72 | 73 | def _match(node): 74 | if node.__class__ is ast.Name and keyword == node.id: 75 | return (keyword, None) 76 | return None 77 | 78 | else: 79 | 80 | def _match(node): 81 | if ( 82 | node.__class__ is ast.Attribute 83 | and node.value.__class__ is ast.Name 84 | and keyword == node.value.id 85 | and identifier == node.attr 86 | ): 87 | return (keyword, identifier) 88 | 89 | return None 90 | 91 | def _pattern(node: ast.AST) -> bool: 92 | if ( 93 | node.__class__ is not ast.Call 94 | or len(node.args) != 1 95 | or node.args[0].__class__ != ast.Lambda 96 | ): 97 | return None 98 | 99 | return _match(node.func) 100 | 101 | return _pattern 102 | 103 | 104 | def _raise_ambiguity(node, filename, keyword, identifier): 105 | """ 106 | Raise SyntaxError reporting an ambiguious declaration. 107 | """ 108 | decl = keyword if not identifier else keyword + "." + identifier 109 | error.assert_(False, "ambiguious declaration {!r}".format(decl), node, filename) 110 | 111 | 112 | def lambda_to_ast( 113 | lambda_object: types.FunctionType, *, keyword: str, identifier: str = "" 114 | ): 115 | """ 116 | Returns the AST of `lambda_object`. 117 | """ 118 | tree = ast_from_source(lambda_object, keyword) 119 | if isinstance(tree, ast.Expr): 120 | assert not isinstance(tree.value, ast.Lambda) 121 | 122 | pattern = _make_pattern(keyword, identifier) 123 | matched = list(_shallow_match_ast(tree, pattern)) 124 | 125 | if not len(matched): 126 | raise SyntaxError("cannot parse lambda for unknown reason") 127 | 128 | if len(matched) > 1: 129 | _raise_ambiguity( 130 | matched[0], lambda_object.__code__.co_filename, keyword, identifier 131 | ) 132 | 133 | assert isinstance(matched[0], ast.Call) 134 | 135 | return matched[0] 136 | 137 | 138 | LambdexASTLookupKey = Tuple[int, str, Optional[str]] 139 | LambdexASTLookupTable = Dict[LambdexASTLookupKey, ast.AST] 140 | 141 | 142 | def find_lambdex_ast_in_code( 143 | code: types.CodeType, ismod: bool 144 | ) -> LambdexASTLookupTable: 145 | """ 146 | Find out all possible lambdex declaration AST nodes within the source of `code`. 147 | """ 148 | if ismod: 149 | lines = linecache.getlines(code.co_filename) 150 | else: 151 | lines, lnum = inspect.getsourcelines(code) 152 | lines = ["\n"] * (lnum - 1) + lines 153 | lines = textwrap.dedent("".join(lines)) 154 | ast_node = ast.parse(lines) 155 | table = {} 156 | iterator = _shallow_match_ast( 157 | ast_node, _make_pattern(None, None), yield_node_only=False 158 | ) 159 | for lambdex_node, (keyword, identifier) in iterator: 160 | key = (lambdex_node.lineno, keyword, identifier) 161 | if key in table: 162 | _raise_ambiguity(lambdex_node, code.co_filename, keyword, identifier) 163 | 164 | table[key] = lambdex_node 165 | 166 | return table 167 | -------------------------------------------------------------------------------- /lambdex/compiler/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import compile_lambdex 2 | -------------------------------------------------------------------------------- /lambdex/compiler/asm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/compiler/asm/__init__.py -------------------------------------------------------------------------------- /lambdex/compiler/asm/frontend.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import types 5 | import linecache 6 | import importlib 7 | import threading 8 | import traceback 9 | 10 | if sys.version_info > (3, 6, float("inf")): 11 | from queue import SimpleQueue 12 | from py_compile import PycInvalidationMode 13 | else: 14 | import enum 15 | from queue import Queue as SimpleQueue 16 | 17 | class PycInvalidationMode(enum.Enum): 18 | TIMESTAMP = 1 19 | CHECKED_HASH = 2 20 | UNCHECKED_HASH = 3 21 | 22 | 23 | if sys.version_info < (3, 7, 2): 24 | 25 | def _get_default_invalidation_mode(): 26 | if os.environ.get("SOURCE_DATE_EPOCH"): 27 | return PycInvalidationMode.CHECKED_HASH 28 | else: 29 | return PycInvalidationMode.TIMESTAMP 30 | 31 | 32 | else: 33 | from py_compile import _get_default_invalidation_mode 34 | 35 | RE_LAMBDEX_REWRITE = re.compile(rb"#\s*lambdex:\s*modopt") 36 | 37 | 38 | def _transpile_file(file, optimize=-1, invalidation_mode=None): 39 | """ 40 | Given a source filename, try to transpile the bytecodes and write to corresponding 41 | .pyc file. 42 | 43 | Adapted from py_compile.py. 44 | """ 45 | if invalidation_mode is None: 46 | invalidation_mode = _get_default_invalidation_mode() 47 | 48 | if optimize >= 0: 49 | optimization = optimize if optimize >= 1 else "" 50 | cfile = importlib.util.cache_from_source(file, optimization=optimization) 51 | else: 52 | cfile = importlib.util.cache_from_source(file) 53 | 54 | if os.path.islink(cfile): 55 | msg = ( 56 | "{} is a symlink and will be changed into a regular file if " 57 | "import writes a byte-compiled file to it" 58 | ) 59 | raise FileExistsError(msg.format(cfile)) 60 | elif os.path.exists(cfile) and not os.path.isfile(cfile): 61 | msg = ( 62 | "{} is a non-regular file and will be changed into a regular " 63 | "one if import writes a byte-compiled file to it" 64 | ) 65 | raise FileExistsError(msg.format(cfile)) 66 | 67 | loader = importlib.machinery.SourceFileLoader("", file) 68 | lines = linecache.getlines(file) 69 | if lines: 70 | source_bytes = "".join(lines).encode("utf-8") 71 | else: 72 | source_bytes = loader.get_data(file) 73 | if RE_LAMBDEX_REWRITE.search(source_bytes) is None: 74 | return None 75 | 76 | code = loader.source_to_code(source_bytes, file, _optimize=optimize) 77 | from lambdex.compiler.asm.core import transpile 78 | 79 | code = transpile(code, ismod=True) 80 | 81 | try: 82 | dirname = os.path.dirname(cfile) 83 | if dirname: 84 | os.makedirs(dirname) 85 | except FileExistsError: 86 | pass 87 | if invalidation_mode == PycInvalidationMode.TIMESTAMP: 88 | source_stats = loader.path_stats(file) 89 | if sys.version_info < (3, 6, float("inf")): 90 | serialize = importlib._bootstrap_external._code_to_bytecode 91 | else: 92 | serialize = importlib._bootstrap_external._code_to_timestamp_pyc 93 | 94 | bytecode = serialize(code, source_stats["mtime"], source_stats["size"]) 95 | else: 96 | source_hash = importlib.util.source_hash(source_bytes) 97 | bytecode = importlib._bootstrap_external._code_to_hash_pyc( 98 | code, 99 | source_hash, 100 | (invalidation_mode == PycInvalidationMode.CHECKED_HASH), 101 | ) 102 | mode = importlib._bootstrap_external._calc_mode(file) 103 | importlib._bootstrap_external._write_atomic(cfile, bytecode, mode) 104 | return cfile 105 | 106 | 107 | # We do the module bytecodes transpilation in another thread, so that it will 108 | # not block the main thread. This thread is looping forever and not daemonic. 109 | # 110 | # To ensure that the transpilation thread would gracefully exit when the program 111 | # ends, we start another monitor thread, which is daemonic and will send a 112 | # signal to the former one after the main thread ends. 113 | # (Reference: https://stackoverflow.com/questions/58910372/) 114 | _job_thread = _monitor_thread = None 115 | _job_history = set() 116 | _job_queue = SimpleQueue() 117 | _done_queue = None 118 | 119 | 120 | def _transpilation_target(): 121 | """ 122 | Transpilation thread. 123 | """ 124 | while True: 125 | file = _job_queue.get() # blocking 126 | 127 | if file is None: # exit sentinel 128 | return 129 | 130 | if file in _job_history: # skip if handled 131 | continue 132 | 133 | _job_history.add(file) 134 | if file[0] == "<" and file[-1] == ">": # skip if not a file 135 | continue 136 | 137 | try: 138 | _transpile_file(file) 139 | if _done_queue is not None: 140 | _done_queue.put((file, "ok")) 141 | except Exception as exc: 142 | # fail silently 143 | if os.getenv("LXBC_DEBUG") is not None: 144 | traceback.print_exception(*sys.exc_info()) 145 | if _done_queue is not None: 146 | _done_queue.put((file, "fail")) 147 | 148 | 149 | def _monitor_target(): 150 | """ 151 | Monitor thread. 152 | 153 | Block until main thread exits, then signal the transpilation thread with None. 154 | """ 155 | main_thread = threading.main_thread() 156 | main_thread.join() 157 | _job_queue.put(None) 158 | 159 | 160 | # We only do bytecode transpilation for Python 3.6+ 161 | if sys.version_info < (3, 5, float("inf")): 162 | 163 | def transpile_file(modname: str): 164 | """ 165 | Not available in Python 3.5 or below. 166 | """ 167 | pass 168 | 169 | def asmopt(func): 170 | """ 171 | Not available in Python 3.5 or below. 172 | """ 173 | return func 174 | 175 | 176 | else: 177 | 178 | def transpile_file(modname: str): 179 | """ 180 | Asynchronously transpile the .pyc file of module `modname`. 181 | """ 182 | global _job_thread, _monitor_thread 183 | 184 | mod = sys.modules.get(modname) 185 | if not hasattr(mod, "__file__"): 186 | return 187 | 188 | _job_queue.put(mod.__file__) 189 | if _job_thread is None: 190 | if _monitor_thread is None: 191 | _monitor_thread = threading.Thread(target=_monitor_target, daemon=True) 192 | _monitor_thread.start() 193 | _job_thread = threading.Thread(target=_transpilation_target, daemon=False) 194 | _job_thread.start() 195 | 196 | def asmopt(func: types.FunctionType) -> types.FunctionType: 197 | """ 198 | Optimize the bytecodes of `func` by eliminating runtime lambdex transpilation. 199 | """ 200 | from lambdex.compiler.asm.core import transpile 201 | 202 | code = transpile(func.__code__, ismod=False) 203 | new_func = types.FunctionType( 204 | code, 205 | func.__globals__, 206 | func.__name__, 207 | func.__defaults__, 208 | func.__closure__, 209 | ) 210 | new_func.__annotations__ = func.__annotations__ 211 | return new_func 212 | -------------------------------------------------------------------------------- /lambdex/compiler/cache.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "get", 3 | "set", 4 | "set_enabled", 5 | "is_enabled", 6 | ] 7 | 8 | _cache = {} 9 | __enabled__ = True 10 | 11 | 12 | def get(declarer): 13 | """ 14 | Return the cached code object corresponding to `declarer`. 15 | 16 | If cache not enabled or not hit, return `None`. 17 | """ 18 | if not __enabled__: 19 | return 20 | return _cache.get(declarer.get_key(), None) 21 | 22 | 23 | def set(declarer, value): 24 | """ 25 | Store `value` into the cache with `declarer` as key. 26 | 27 | If the key exists in cache, raise an error. 28 | """ 29 | if not __enabled__: 30 | return 31 | key = declarer.get_key() 32 | assert key not in _cache 33 | _cache[key] = value 34 | 35 | 36 | def set_enabled(value: bool): 37 | """ 38 | Enable or disable the cache. 39 | """ 40 | global __enabled__ 41 | __enabled__ = value 42 | 43 | 44 | def is_enabled() -> bool: 45 | """ 46 | Check whether the cache is enabled. 47 | """ 48 | return __enabled__ 49 | -------------------------------------------------------------------------------- /lambdex/compiler/clauses.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from collections import namedtuple 3 | 4 | from lambdex.utils.ast import value_from_subscript 5 | 6 | 7 | class Clause(namedtuple("_Clause", "node name head body")): 8 | def no_head(self) -> bool: 9 | """ 10 | Check that whether the head is None. 11 | """ 12 | return self.head is None 13 | 14 | def single_body(self) -> bool: 15 | """ 16 | Check that whether the body has only one item. 17 | """ 18 | return len(self.body) == 1 19 | 20 | def try_tuple_body(self, ctx=ast.Load()): 21 | """ 22 | If body has single item, return it. Otherwise, wrap the items in 23 | an `ast.Tuple` and return. 24 | """ 25 | if self.single_body(): 26 | return self.body[0] 27 | 28 | return ast.Tuple(elts=self.body, ctx=ctx) 29 | 30 | def unwrap_body(self): 31 | """ 32 | Check that the body has only one item, and return it. 33 | """ 34 | assert self.single_body() 35 | return self.body[0] 36 | 37 | def single_head(self) -> bool: 38 | """ 39 | Check whether the head has only one item. 40 | """ 41 | return self.head is not None and len(self.head) == 1 42 | 43 | def unwrap_head(self): 44 | """ 45 | Check that the head has only one item, and return it. 46 | """ 47 | assert self.single_head() 48 | return self.head[0] 49 | 50 | 51 | class Clauses(list): 52 | def single(self) -> bool: 53 | """ 54 | Check that whether only one `Clause` exists in self. 55 | """ 56 | return len(self) == 1 57 | 58 | 59 | def match_clauses(node: ast.Subscript, raise_) -> Clauses: 60 | """ 61 | Extract info from a `node` with lambdex compound statement syntax pattern. 62 | """ 63 | 64 | # Clauses in `node` appears in an reversed order. 65 | # 66 | # e.g., `if_[][].else_[]`` has a structure 67 | # Subscript( 68 | # slice=..., 69 | # value=Attribute( 70 | # attr='else_', 71 | # value=Subscript( 72 | # slice=..., 73 | # value=Subscript( 74 | # slice=..., 75 | # value=Name('if_') 76 | # ) 77 | # ) 78 | # ) 79 | # ) 80 | 81 | results = [] 82 | body = head = name = None 83 | while node is not None: 84 | # If body is None, we are matching a new clause 85 | if isinstance(node, ast.Subscript) and body is None: 86 | body = value_from_subscript(node, force_list=True, raise_=raise_) 87 | node = node.value 88 | continue 89 | 90 | # Otherwise, the body has matched, we expect a head or a keyword 91 | 92 | # Match a head 93 | if isinstance(node, ast.Subscript) and head is None: 94 | head = value_from_subscript(node, force_list=True, raise_=raise_) 95 | node = node.value 96 | continue 97 | 98 | # Otherwise, the body and head are matched. We expect a keyword. 99 | # The keyword may appear as a `Name` (first clause) or `Attribute` (sub-clause) 100 | 101 | # Match a sub-clause keyword 102 | if isinstance(node, ast.Attribute) and name is None: 103 | name = node.attr 104 | next_node = node.value 105 | 106 | # Match a first clause keyword 107 | if isinstance(node, ast.Name) and name is None: 108 | name = node.id 109 | next_node = None 110 | 111 | # If everything is OK, we construct and store a clause 112 | if name is not None: 113 | results.append(Clause(node, name, head, body)) 114 | body = head = name = None 115 | node = next_node 116 | continue 117 | 118 | # Otherwise, return None as unmatched symbol 119 | return 120 | 121 | # Reverse the clauses so that they accord with definition order 122 | return Clauses(reversed(results)) 123 | -------------------------------------------------------------------------------- /lambdex/compiler/context.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import random 3 | from functools import partial 4 | 5 | from lambdex.utils import compat 6 | from lambdex.utils.ast import is_lvalue, is_coroutine_ast 7 | 8 | from . import error 9 | 10 | __all__ = ["Context", "ContextFlag"] 11 | 12 | 13 | def random_hex(nbits=32) -> str: 14 | """ 15 | Return a random number with `nbits` bits in hex string format. 16 | """ 17 | return hex(random.randint(0, 1 << nbits))[2:].zfill(nbits // 4) 18 | 19 | 20 | auto = compat.enum_auto() 21 | 22 | 23 | class ContextFlag(enum.Enum): 24 | 25 | # Expect an `ast.stmt` 26 | should_be_stmt = auto() 27 | 28 | # Expect an `ast.expr` 29 | should_be_expr = auto() 30 | 31 | # Indicate that this should be the outermost lambdex 32 | outermost_lambdex = auto() 33 | 34 | 35 | del auto 36 | 37 | 38 | class Frame: 39 | __slots__ = ["detached_functions", "name", "is_async"] 40 | 41 | def __init__(self): 42 | self.name = None 43 | self.is_async = False 44 | self.detached_functions = [] 45 | 46 | 47 | EM_HEAD_FOUND = "expect only one group of '[]'" 48 | EM_HEAD_MISSING = "expect another group of '[]'" 49 | EM_TOO_MANY_ITEMS = "expect only one item inside '[]'" 50 | EM_UNEXPECTED_CLAUSE = "unexpected clause" 51 | EM_NOT_LVALUE = "cannot be assigned" 52 | 53 | 54 | class Context: 55 | """ 56 | A `Context` object is passed among rules for sharing global informations. 57 | 58 | Attributes: 59 | - `compile`: a shorthand for `compile_node(..., self)` 60 | - `globals`: a dict containing globalvars of currently compiling lambdex 61 | - `used_names`: a set containing currently occupied names 62 | - `frames`: current `Frame` stack 63 | """ 64 | 65 | __slots__ = ["compile", "globals", "used_names", "frames", "filename", "renames"] 66 | 67 | def __init__(self, compile_fn, globals_dict, filename): 68 | self.compile = partial(compile_fn, ctx=self) 69 | self.globals = globals_dict 70 | self.used_names = set(globals_dict) 71 | self.frames = [] 72 | self.filename = filename 73 | self.renames = {} 74 | 75 | def select_name(self, prefix): 76 | """ 77 | Return a name with prefix `prefix` that is not contained in 78 | `self.used_names`. 79 | """ 80 | while True: 81 | name = "{}_{}".format(prefix, random_hex()) 82 | if name not in self.used_names: 83 | return name 84 | 85 | def select_name_and_use(self, prefix): 86 | """ 87 | Return a name with prefix `prefix` that is not contained in 88 | `self.used_names`. The name will be add to `self.used_names` 89 | before returned. 90 | """ 91 | name = self.select_name(prefix) 92 | self.used_names.add(name) 93 | return name 94 | 95 | def push_frame(self): 96 | """ 97 | Push a new `Frame` instance to the stack. 98 | """ 99 | self.frames.append(Frame()) 100 | return self.frames[-1] 101 | 102 | def pop_frame(self): 103 | """ 104 | Pop the top `Frame` instance from the stack. 105 | """ 106 | self.frames.pop() 107 | 108 | @property 109 | def frame(self): 110 | """ 111 | The top-most `Frame` instance on the stack. 112 | """ 113 | return self.frames[-1] 114 | 115 | # Below are helper functions for compile-time assertion 116 | 117 | def assert_(self, cond: bool, msg: str, node): 118 | error.assert_(cond, msg, node, self.filename) 119 | 120 | def raise_(self, msg: str, node): 121 | error.assert_(False, msg, node, self.filename) 122 | 123 | def assert_is_instance(self, node, type_: type, msg): 124 | self.assert_(isinstance(node, type_), msg, node) 125 | 126 | def assert_single_head(self, clause): 127 | self.assert_(clause.single_head(), EM_TOO_MANY_ITEMS, lambda: clause.head[1]) 128 | 129 | def assert_single_body(self, clause): 130 | self.assert_(clause.single_body(), EM_TOO_MANY_ITEMS, lambda: clause.body[1]) 131 | 132 | def assert_clause_num_at_most(self, clauses, num: int): 133 | self.assert_( 134 | len(clauses) <= num, EM_UNEXPECTED_CLAUSE, lambda: clauses[num].node 135 | ) 136 | 137 | def assert_no_head(self, clause): 138 | self.assert_(clause.no_head(), EM_HEAD_FOUND, lambda: clause.node) 139 | 140 | def assert_head(self, clause): 141 | self.assert_(clause.head, EM_HEAD_MISSING, lambda: clause.node) 142 | 143 | def assert_name_equals(self, clause, name: str): 144 | self.assert_(clause.name == name, "expect {!r}".format(name), clause.node) 145 | 146 | def assert_name_in(self, clause, names): 147 | self.assert_( 148 | clause.name in names, "expect " + " or ".join(map(repr, names)), clause.node 149 | ) 150 | 151 | def assert_lvalue(self, node, msg=None): 152 | check_result, failed_at = is_lvalue(node) 153 | self.assert_(check_result, msg or EM_NOT_LVALUE, failed_at) 154 | 155 | def check_coroutine(self, x, node, keyword): 156 | if is_coroutine_ast(x): 157 | self.assert_( 158 | self.frame.is_async, 159 | "{!r} outside async function".format(keyword), 160 | node, 161 | ) 162 | -------------------------------------------------------------------------------- /lambdex/compiler/core.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import types 3 | import typing 4 | import inspect 5 | import functools 6 | 7 | from ..utils import compat 8 | from .rules import Rules 9 | from .context import Context, ContextFlag 10 | from .dispatcher import Dispatcher 11 | from . import cache 12 | from .asm.frontend import transpile_file 13 | 14 | from lambdex.utils.ast import pformat, empty_arguments, None_node 15 | from lambdex.utils import compat 16 | 17 | __all__ = ["compile_lambdex"] 18 | 19 | # This flag is used internally. when turned on: 20 | # - compiled lambdex will have attribute `__ast__` 21 | # - verbose message are printed when error occurs during compiling 22 | __DEBUG__ = False 23 | 24 | 25 | def compile_node(node, ctx, *, flag=ContextFlag.should_be_expr): 26 | """ 27 | Compile an AST node `node` to transpile lambdex syntax to Python lambdex. 28 | """ 29 | if node is None: 30 | return None 31 | 32 | dispatcher = Dispatcher.get(node.__class__) 33 | rule_id, extra_args = dispatcher(node, ctx, flag) 34 | rule = Rules.get(rule_id, None) 35 | 36 | if rule is not None: 37 | return rule(node, ctx, *extra_args) 38 | 39 | # If no rule found, recursively compile children of `node` 40 | for field, old_value in ast.iter_fields(node): 41 | if isinstance(old_value, list): 42 | new_values = [] 43 | for value in old_value: 44 | if isinstance(value, ast.AST): 45 | value = compile_node(value, ctx) 46 | 47 | if value is None: 48 | # Discard from `new_values` 49 | continue 50 | elif not isinstance(value, ast.AST): 51 | new_values.extend(value) 52 | continue 53 | new_values.append(value) 54 | old_value[:] = new_values 55 | elif isinstance(old_value, ast.AST): 56 | new_node = compile_node(old_value, ctx) 57 | if new_node is None: 58 | delattr(node, field) 59 | else: 60 | setattr(node, field, new_node) 61 | 62 | return node 63 | 64 | 65 | def _wrap_code_object( 66 | code_obj: types.CodeType, 67 | lambda_func: types.FunctionType, 68 | lambdex_ast_node: ast.AST, 69 | fvmapping: typing.Sequence[int], 70 | ) -> types.FunctionType: 71 | """ 72 | Construct a function using `code_obj`. 73 | 74 | To ensure the two functions have same context, the returned function 75 | copies `__globals__`, `__defaults__`, and rebuilds `__closure__` from 76 | `lambda_func`. 77 | """ 78 | 79 | # Trick: Obtain a cell object referencing current function, by 80 | # constructing a new function and extract its closure. 81 | callee_ref_cell = (lambda: ret).__closure__[0] 82 | 83 | # Rebuild the closure 84 | new_closure = tuple( 85 | lambda_func.__closure__[i] if i >= 0 else callee_ref_cell for i in fvmapping 86 | ) 87 | 88 | ret = types.FunctionType( 89 | code=code_obj, 90 | globals=lambda_func.__globals__, 91 | name=code_obj.co_name, 92 | argdefs=lambda_func.__defaults__, 93 | closure=tuple(new_closure), 94 | ) 95 | 96 | if __DEBUG__: 97 | ret.__ast__ = lambdex_ast_node 98 | 99 | return ret 100 | 101 | 102 | def _rename_code_object(code, ctx: Context): 103 | """ 104 | Recursively rename the `co_name` field in all code objects. 105 | """ 106 | 107 | kwargs = {} 108 | 109 | new_name = ctx.renames.get(code.co_name) 110 | if new_name is not None: 111 | kwargs["co_name"] = new_name 112 | 113 | new_consts = [] 114 | for const in code.co_consts: 115 | if inspect.iscode(const): 116 | const = _rename_code_object(const, ctx) 117 | new_consts.append(const) 118 | kwargs["co_consts"] = tuple(new_consts) 119 | 120 | return compat.code_replace(code, **kwargs) 121 | 122 | 123 | def _resolve_freevars_mapping( 124 | old_freevars: typing.Sequence[str], 125 | new_freevars: typing.Sequence[str], 126 | ) -> typing.List[int]: 127 | """ 128 | Return a list `m` such that new_freevars[i] == old_freevars[m[i]]. 129 | 130 | m[i] will be -1 if new_freevars[i] not in old_freevars. 131 | """ 132 | mapping = {v: k for k, v in enumerate(old_freevars)} 133 | return [mapping.get(varname, -1) for varname in new_freevars] 134 | 135 | 136 | def _compile( 137 | ast_node: ast.AST, 138 | filename: str, 139 | freevars: typing.Sequence[str], 140 | globals: typing.Optional[dict] = None, 141 | ) -> typing.Tuple[types.CodeType, ast.AST, typing.Sequence[int]]: 142 | """ 143 | An internal function that do the compilation. 144 | """ 145 | if globals is None: 146 | globals = {} 147 | 148 | context = Context( 149 | compile_node, 150 | globals, 151 | filename, 152 | ) 153 | lambdex_node = compile_node( 154 | ast_node, 155 | ctx=context, 156 | flag=ContextFlag.outermost_lambdex, 157 | ) 158 | 159 | # A name in `lambdex_node` should be compiled as nonlocal instead of 160 | # global (default) if it appears in `freevars`. 161 | # 162 | # This is done by wrapping `lambdex_node` in another FunctionDef, and 163 | # let names in `freevars` become local variables in the wrapper. 164 | wrapper_name = context.select_name_and_use("wrapper") 165 | if freevars: 166 | wrapper_body = [ 167 | ast.Assign( 168 | targets=[ast.Name(id=name, ctx=ast.Store()) for name in freevars], 169 | value=None_node, 170 | ), 171 | lambdex_node, 172 | ] 173 | else: 174 | wrapper_body = [lambdex_node] 175 | 176 | wrapper_node = ast.FunctionDef( 177 | name=wrapper_name, 178 | args=empty_arguments, 179 | body=wrapper_body, 180 | decorator_list=[], 181 | returns=None, 182 | ) 183 | module_node = ast.Module( 184 | body=[wrapper_node], 185 | type_ignores=[], 186 | ) 187 | module_node = ast.fix_missing_locations(module_node) 188 | 189 | if __DEBUG__: 190 | try: 191 | module_code = compile(module_node, filename, "exec") 192 | except Exception as e: 193 | raise SyntaxError(pformat(module_node)) from e 194 | else: 195 | module_code = compile(module_node, filename, "exec") 196 | 197 | # unwrap the outer FunctionDef. 198 | # since no other definition in the module, it should be co_consts[0] 199 | wrapper_code = module_code.co_consts[0] 200 | 201 | # the desired code object should be in `module_code.co_consts` 202 | # we use `.co_name` to identify 203 | for obj in wrapper_code.co_consts: 204 | if inspect.iscode(obj) and obj.co_name == lambdex_node.name: 205 | lambdex_code = obj 206 | break 207 | 208 | # Append code object name to its co_freevars, so that lambdex 209 | # can always access itself via its name `anonymous_...` 210 | callee_name = lambdex_code.co_name 211 | try: 212 | callee_index = lambdex_code.co_freevars.index(callee_name) 213 | except ValueError: 214 | callee_index = len(lambdex_code.co_freevars) 215 | lambdex_code = compat.code_replace( 216 | lambdex_code, 217 | co_freevars=(*lambdex_code.co_freevars, callee_name), 218 | ) 219 | freevars_mapping = _resolve_freevars_mapping(freevars, lambdex_code.co_freevars) 220 | 221 | lambdex_code = _rename_code_object(lambdex_code, context) 222 | 223 | return lambdex_code, lambdex_node, freevars_mapping 224 | 225 | 226 | def compile_lambdex(declarer) -> types.FunctionType: 227 | """ 228 | Compile a lambda object given by `declarer` into a function. 229 | 230 | Multiple calls with a same declarer yield functions with same code object, 231 | whilst there closure and globals may be different. 232 | """ 233 | # If cache hit, simply update metadata and return 234 | cached_value = cache.get(declarer) 235 | if cached_value is not None: 236 | code_obj, lambdex_ast_node, fvmapping = cached_value 237 | return _wrap_code_object(code_obj, declarer.func, lambdex_ast_node, fvmapping) 238 | 239 | # Otherwise, we have to compile from scratch 240 | 241 | lambda_ast = declarer.get_ast() 242 | lambda_func = declarer.func 243 | 244 | lambdex_code, lambdex_node, fvmapping = _compile( 245 | lambda_ast, 246 | lambda_func.__code__.co_filename, 247 | lambda_func.__code__.co_freevars, 248 | lambda_func.__globals__, 249 | ) 250 | 251 | cache.set(declarer, (lambdex_code, lambdex_node, fvmapping)) 252 | transpile_file(lambda_func.__module__) 253 | return _wrap_code_object(lambdex_code, lambda_func, lambdex_node, fvmapping) 254 | -------------------------------------------------------------------------------- /lambdex/compiler/dispatcher.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from collections import namedtuple 3 | 4 | from lambdex._aliases import get_aliases 5 | from lambdex._features import get_features 6 | 7 | aliases = get_aliases() 8 | features = get_features() 9 | 10 | from lambdex.utils.registry import FunctionRegistry 11 | from .clauses import match_clauses 12 | from .context import ContextFlag, Context 13 | 14 | __all__ = ["Dispatcher"] 15 | 16 | RuleMeta = namedtuple("RuleMeta", ["id", "args"]) 17 | EMPTY_RULE = RuleMeta(None, ()) 18 | Dispatcher = FunctionRegistry("Dispatcher").set_default(lambda *_: EMPTY_RULE) 19 | 20 | 21 | @Dispatcher.register(ast.Lambda) 22 | def disp_Lambda(node: ast.Lambda, ctx: Context, flag: ContextFlag): 23 | if flag != ContextFlag.outermost_lambdex: 24 | return RuleMeta(ast.Lambda, ()) 25 | 26 | return RuleMeta((ast.Lambda, flag), ()) 27 | 28 | 29 | @Dispatcher.register(ast.Call) 30 | def disp_Call(node: ast.Call, ctx: Context, flag: ContextFlag): 31 | func = node.func 32 | 33 | if isinstance(func, ast.Name): 34 | name = func.id 35 | func_name = None 36 | elif isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): 37 | name = func.value.id 38 | func_name = func.attr 39 | else: 40 | name = None 41 | func_name = None 42 | 43 | ast_type = { 44 | aliases.def_: ast.FunctionDef, 45 | aliases.async_def_: ast.AsyncFunctionDef, 46 | }.get(name) 47 | return RuleMeta((ast_type, flag), (func_name,)) 48 | 49 | 50 | @Dispatcher.register(ast.Name) 51 | def disp_Name(node: ast.Name, ctx: Context, flag: ContextFlag): 52 | if node.id == aliases.callee_: 53 | return RuleMeta("callee", ()) 54 | 55 | if flag == ContextFlag.should_be_expr: 56 | mapping = { 57 | aliases.yield_: ast.Yield, 58 | } 59 | elif flag == ContextFlag.should_be_stmt: 60 | mapping = { 61 | aliases.continue_: ast.Continue, 62 | aliases.break_: ast.Break, 63 | aliases.pass_: ast.Pass, 64 | aliases.yield_: ast.Yield, 65 | aliases.raise_: ast.Raise, 66 | aliases.return_: ast.Return, 67 | } 68 | 69 | rule_type = mapping.get(node.id) 70 | 71 | if rule_type is not None: 72 | return RuleMeta("single_keyword_stmt", (rule_type,)) 73 | 74 | return EMPTY_RULE 75 | 76 | 77 | @Dispatcher.register(ast.Subscript) 78 | def disp_Subscript(node: ast.Subscript, ctx: Context, flag: ContextFlag): 79 | clauses = match_clauses(node, ctx.raise_) 80 | if clauses is None: 81 | return EMPTY_RULE 82 | 83 | ast_type = { 84 | aliases.return_: ast.Return, 85 | aliases.if_: ast.If, 86 | aliases.for_: ast.For, 87 | aliases.while_: ast.While, 88 | aliases.with_: ast.With, 89 | aliases.raise_: ast.Raise, 90 | aliases.try_: ast.Try, 91 | aliases.yield_: ast.Yield, 92 | aliases.yield_from_: ast.YieldFrom, 93 | aliases.global_: ast.Global, 94 | aliases.nonlocal_: ast.Nonlocal, 95 | aliases.async_for_: ast.AsyncFor, 96 | aliases.async_with_: ast.AsyncWith, 97 | aliases.await_: ast.Await, 98 | aliases.del_: ast.Delete, 99 | }.get(clauses[0].name) 100 | 101 | ctx.check_coroutine(ast_type, clauses[0].node, clauses[0].name) 102 | 103 | return RuleMeta(ast_type, (clauses,)) 104 | 105 | 106 | if features.await_attribute: 107 | 108 | @Dispatcher.register(ast.Attribute) 109 | def disp_Attribute(node: ast.Attribute, ctx: Context, flag: ContextFlag): 110 | if node.attr != aliases.await_: 111 | return EMPTY_RULE 112 | 113 | ctx.check_coroutine(ast.Await, node, aliases.await_) 114 | return RuleMeta((ast.Await, ast.Attribute), ()) 115 | 116 | 117 | @Dispatcher.register(ast.Compare) 118 | def disp_Compare(node: ast.Compare, ctx: Context, flag: ContextFlag): 119 | if flag != ContextFlag.should_be_stmt: 120 | return EMPTY_RULE 121 | 122 | return RuleMeta(ast.Assign, ()) 123 | -------------------------------------------------------------------------------- /lambdex/compiler/error.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import ast 3 | import linecache 4 | 5 | if sys.version_info < (3, 8): 6 | import re 7 | from os import linesep 8 | 9 | def _get_end_lineinfo(node: ast.AST, filename: str): 10 | """ 11 | Fallback function to get endling lineno and col_offset of `node`. 12 | """ 13 | 14 | def _get_last_child_loc(): 15 | """ 16 | Return the lineinfo for child of `node` with maximum lineno and col_offset. 17 | """ 18 | last_child_loc = None 19 | for n in ast.walk(node): 20 | if n is node: 21 | continue 22 | 23 | if isinstance(n, (ast.Name, ast.Attribute)): 24 | # If n is Name or Attribute, use its ending location 25 | loc = _get_end_lineinfo(n, filename) 26 | elif "lineno" in n._fields: 27 | # Otherwise, use its starting location 28 | loc = (n.lineno, n.col_offset) 29 | else: 30 | # If no attribute `lineno`, simply ignore it 31 | continue 32 | 33 | if not last_child_loc or last_child_loc < loc: 34 | last_child_loc = loc 35 | 36 | return last_child_loc 37 | 38 | if isinstance(node, ast.Name): 39 | return (node.lineno, node.col_offset + len(node.id)) 40 | elif isinstance(node, ast.Attribute): 41 | regexp = re.compile( 42 | r""" 43 | \. # a single dot 44 | ( # following with arbitary whitespaces / comments: 45 | \s* # zero or more leading whitespaces 46 | ( # zero or one ending tokens 47 | \\ # CONTINUE 48 | |\#.* # COMMENT 49 | )? # 50 | \n+ # one or more newlines 51 | )* # 52 | \s* # zero or more whitespaces 53 | {} # the target attribute name 54 | """.format( 55 | node.attr 56 | ), 57 | re.MULTILINE | re.VERBOSE, 58 | ) 59 | lines = linecache.getlines(filename) 60 | lineno, col_offset = _get_last_child_loc() 61 | text_after = lines[lineno - 1][col_offset:] + "".join(lines[lineno:]) 62 | text_after = text_after.replace(linesep, "\n") 63 | matched = re.search(regexp, text_after) 64 | matched_lines = text_after[: matched.end()].split("\n") 65 | if len(matched_lines) == 1: 66 | # On the same line 67 | return lineno, col_offset + matched.end() 68 | else: 69 | # On different line 70 | return lineno + len(matched_lines) - 1, len(matched_lines[-1]) 71 | else: 72 | raise RuntimeError 73 | 74 | 75 | else: 76 | 77 | def _get_end_lineinfo(node: ast.AST, filename: str): 78 | return node.end_lineno, node.end_col_offset 79 | 80 | 81 | def assert_(cond: bool, msg: str, node: ast.AST, filename: str): 82 | if cond: 83 | return 84 | 85 | if callable(node): 86 | node = node() 87 | 88 | if isinstance(node, (ast.Attribute, ast.Name)): 89 | lineno, offset = _get_end_lineinfo(node, filename) 90 | else: 91 | lineno = node.lineno 92 | offset = node.col_offset + 1 93 | 94 | raise SyntaxError( 95 | msg, 96 | ( 97 | filename, 98 | lineno, 99 | offset, 100 | linecache.getline(filename, lineno), 101 | ), 102 | ) 103 | -------------------------------------------------------------------------------- /lambdex/fmt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/fmt/__init__.py -------------------------------------------------------------------------------- /lambdex/fmt/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from .cli.main import main 3 | 4 | sys.exit(main()) 5 | -------------------------------------------------------------------------------- /lambdex/fmt/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from ._base import BaseAdapter 4 | from .yapf import YapfAdapter 5 | from .dummy import DummyAdapter 6 | from .black import BlackAdapter 7 | 8 | mapping = { 9 | "yapf": YapfAdapter, 10 | "dummy": DummyAdapter, 11 | "black": BlackAdapter, 12 | } 13 | 14 | 15 | def build(name: str, *args) -> BaseAdapter: 16 | return mapping[name](*args) 17 | -------------------------------------------------------------------------------- /lambdex/fmt/adapters/_base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import argparse 3 | import subprocess 4 | from functools import partial 5 | from typing import Sequence 6 | 7 | from lambdex.fmt.jobs_meta import JobsMeta 8 | from lambdex.fmt.core.api import FormatCode 9 | from lambdex.fmt.utils.logger import getLogger 10 | from lambdex.fmt.utils.io import StdinResource, FileResource, _ResourceBase 11 | 12 | logger = getLogger(__name__) 13 | 14 | 15 | class Result: 16 | 17 | __slots__ = ["success", "output"] 18 | 19 | def __init__(self, success: bool, output: bytes): 20 | self.success = success 21 | self.output = output 22 | 23 | 24 | class BaseAdapter(abc.ABC): 25 | def __init__(self, opts: argparse.Namespace, backend_argv: Sequence[str]): 26 | self.opts = opts 27 | self.backend_argv = backend_argv 28 | 29 | self.jobs_meta = self._make_jobs_meta() 30 | 31 | @abc.abstractmethod 32 | def _make_jobs_meta(self) -> JobsMeta: 33 | pass 34 | 35 | @abc.abstractmethod 36 | def _get_backend_cmd_for_resource(self, resource: _ResourceBase) -> Sequence[str]: 37 | pass 38 | 39 | def _reset_aliases(self, filename): 40 | from lambdex.fmt.core.tkutils.rules import matcher 41 | 42 | if filename is None: 43 | matcher.reset_aliases() 44 | else: 45 | matcher.reset_aliases(filename) 46 | 47 | def _create_resource(self, filename) -> _ResourceBase: 48 | if filename in {None, "-"}: 49 | resource = StdinResource(self.jobs_meta) 50 | else: 51 | resource = FileResource(self.jobs_meta, filename) 52 | 53 | return resource 54 | 55 | def _job(self, filename=None) -> bool: 56 | self._reset_aliases(filename) 57 | resource = self._create_resource(filename) 58 | 59 | cmd = self._get_backend_cmd_for_resource(resource) 60 | backend_result = self.call_backend(cmd, resource.source) 61 | if not backend_result.success: 62 | logger.error("backend exits unexpectedly") 63 | resource.set_backend_output(backend_result.output) 64 | 65 | formatted_code = FormatCode(resource.backend_output_stream.readline) 66 | resource.write_formatted_code(formatted_code) 67 | 68 | return resource.is_changed(formatted_code) 69 | 70 | def get_jobs(self): 71 | if not self.jobs_meta.files: 72 | yield self._job 73 | else: 74 | yield from ( 75 | partial(self._job, filename) for filename in self.jobs_meta.files 76 | ) 77 | 78 | def call_backend(self, cmd: Sequence[str], stdin: bytes) -> Result: 79 | process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) 80 | output, _ = process.communicate(input=stdin) 81 | return Result( 82 | success=process.returncode == 0, 83 | output=output, 84 | ) 85 | -------------------------------------------------------------------------------- /lambdex/fmt/adapters/black.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | import os 4 | import sys 5 | from collections import namedtuple 6 | from contextlib import contextmanager 7 | 8 | from lambdex.fmt.jobs_meta import JobsMeta 9 | from lambdex.fmt.utils.io import _ResourceBase, StringResource 10 | from lambdex.fmt.utils.logger import getLogger 11 | from lambdex.fmt.utils.importlib import silent_import 12 | 13 | from ._base import BaseAdapter 14 | 15 | logger = getLogger(__name__) 16 | 17 | 18 | StringSourceCode = namedtuple("StringSourceCode", "code") 19 | 20 | 21 | @contextmanager 22 | def _black_context(argv): 23 | black_main = silent_import("black", ["main"]) 24 | click_Exit = silent_import("click.exceptions", ["Exit"]) 25 | 26 | try: 27 | with black_main.make_context("black", argv) as ctx: 28 | yield ctx 29 | except click_Exit as exc: 30 | sys.exit(exc.exit_code) 31 | 32 | 33 | class BlackAdapter(BaseAdapter): 34 | def _make_jobs_meta(self) -> JobsMeta: 35 | with _black_context(self.backend_argv) as ctx: 36 | return self._do_make_jobs_meta(ctx) 37 | 38 | def _do_make_jobs_meta(self, ctx) -> JobsMeta: 39 | black_get_sources = silent_import("black", ["get_sources"]) 40 | 41 | bopts = self._backend_opts = ctx.params 42 | 43 | meta = JobsMeta(adapter="black") 44 | meta.print_diff = bopts["diff"] 45 | meta.quiet = False 46 | 47 | if bopts["code"] is not None: 48 | sources = [StringSourceCode(code=bopts["code"])] 49 | else: 50 | Report = silent_import("black.report", ["Report"]) 51 | report = Report( 52 | check=bopts["check"], 53 | diff=bopts["diff"], 54 | quiet=bopts["quiet"], 55 | verbose=bopts["verbose"], 56 | ) 57 | sources = black_get_sources( 58 | ctx=ctx, 59 | src=bopts["src"], 60 | quiet=bopts["quiet"], 61 | verbose=bopts["verbose"], 62 | include=bopts["include"], 63 | exclude=bopts["exclude"], 64 | extend_exclude=bopts["extend_exclude"], 65 | force_exclude=bopts["force_exclude"], 66 | report=report, 67 | stdin_filename=bopts["stdin_filename"], 68 | ) 69 | sources = list(map(str, sources)) 70 | 71 | meta.files = sources 72 | meta.parallel = len(sources) > 1 73 | meta.in_place = not meta.print_diff and not any( 74 | f == "-" or isinstance(f, StringSourceCode) for f in sources 75 | ) 76 | 77 | return meta 78 | 79 | def _create_resource(self, filename) -> _ResourceBase: 80 | if isinstance(filename, StringSourceCode): 81 | return StringResource(self.jobs_meta, filename.code) 82 | 83 | return super()._create_resource(filename) 84 | 85 | def _get_backend_cmd_for_resource(self, resource: _ResourceBase) -> Sequence[str]: 86 | cmd = [self.opts.executable or "black", "-", "-q"] 87 | bopts = self._backend_opts 88 | 89 | if bopts["config"] is not None: 90 | cmd.extend(["--config", bopts["config"]]) 91 | 92 | cmd.extend(["--line-length", str(bopts["line_length"])]) 93 | 94 | if bopts["skip_magic_trailing_comma"]: 95 | cmd.append("--skip-magic-trailing-comma") 96 | 97 | if bopts["skip_string_normalization"]: 98 | cmd.append("--skip-string-normalization") 99 | 100 | return cmd 101 | 102 | # for linespec in bopts.lines or []: 103 | # cmd.extend(["-l", linespec]) 104 | # if bopts.no_local_style: 105 | # cmd.append("--no-local-style") 106 | # if bopts.style is not None: 107 | # cmd.extend(["--style", bopts.style]) 108 | 109 | # return cmd 110 | -------------------------------------------------------------------------------- /lambdex/fmt/adapters/dummy.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | import os 4 | import argparse 5 | 6 | from lambdex.fmt.jobs_meta import JobsMeta 7 | from lambdex.fmt.utils.io import FileResource, StdinResource 8 | from lambdex.fmt.utils.logger import getLogger 9 | from lambdex.fmt.utils.importlib import silent_import 10 | from lambdex.fmt.core.api import FormatCode 11 | 12 | from ._base import BaseAdapter 13 | 14 | logger = getLogger(__name__) 15 | 16 | 17 | def _build_argument_parser(): 18 | parser = argparse.ArgumentParser(description="Default formatter for lambdex") 19 | diff_inplace_quiet_group = parser.add_mutually_exclusive_group() 20 | diff_inplace_quiet_group.add_argument( 21 | "-d", 22 | "--diff", 23 | action="store_true", 24 | help="print the diff for the fixed source", 25 | ) 26 | diff_inplace_quiet_group.add_argument( 27 | "-i", 28 | "--in-place", 29 | action="store_true", 30 | help="make changes to files in place", 31 | ) 32 | diff_inplace_quiet_group.add_argument( 33 | "-q", 34 | "--quiet", 35 | action="store_true", 36 | help="output nothing and set return value", 37 | ) 38 | 39 | parser.add_argument( 40 | "-p", 41 | "--parallel", 42 | action="store_true", 43 | help="run in parallel when formatting multiple files.", 44 | ) 45 | 46 | parser.add_argument( 47 | "files", nargs="*", help="reads from stdin when no files are specified." 48 | ) 49 | return parser 50 | 51 | 52 | class DummyAdapter(BaseAdapter): 53 | def _make_jobs_meta(self) -> JobsMeta: 54 | parser = _build_argument_parser() 55 | 56 | bopts = parser.parse_args(self.backend_argv) 57 | 58 | if (bopts.in_place or bopts.diff) and not bopts.files: 59 | logger.error( 60 | "cannot use --in-place or --diff flags when reading from stdin" 61 | ) 62 | 63 | meta = JobsMeta(adapter="yapf") 64 | meta.in_place = bopts.in_place 65 | meta.parallel = bopts.parallel 66 | meta.print_diff = bopts.diff 67 | meta.quiet = bopts.quiet 68 | meta.files = bopts.files 69 | 70 | return meta 71 | 72 | def _job(self, filename=None) -> bool: 73 | self._reset_aliases(filename) 74 | if filename is None: 75 | resource = StdinResource(self.jobs_meta) 76 | else: 77 | resource = FileResource(self.jobs_meta, filename) 78 | 79 | resource.set_backend_output(resource.source) 80 | 81 | formatted_code = FormatCode(resource.backend_output_stream.readline) 82 | resource.write_formatted_code(formatted_code) 83 | 84 | return resource.is_changed(formatted_code) 85 | 86 | def _get_backend_cmd_for_resource(self, resource) -> Sequence[str]: 87 | pass 88 | -------------------------------------------------------------------------------- /lambdex/fmt/adapters/yapf.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | import os 4 | 5 | from lambdex.fmt.jobs_meta import JobsMeta 6 | from lambdex.fmt.utils.io import _ResourceBase 7 | from lambdex.fmt.utils.logger import getLogger 8 | from lambdex.fmt.utils.importlib import silent_import 9 | 10 | from ._base import BaseAdapter 11 | 12 | logger = getLogger(__name__) 13 | 14 | 15 | class YapfAdapter(BaseAdapter): 16 | def _make_jobs_meta(self) -> JobsMeta: 17 | _ParseArguments = silent_import("yapf", ["_ParseArguments", "_BuildParser"]) 18 | if _ParseArguments.__name__ == "_BuildParser": 19 | _ParseArguments = (lambda f: lambda args: f().parse_args(args[1:]))( 20 | _ParseArguments 21 | ) 22 | file_resources = silent_import("yapf.yapflib.file_resources") 23 | 24 | bopts = self._backend_opts = _ParseArguments([" "] + self.backend_argv) 25 | 26 | if bopts.lines and len(bopts.files) > 1: 27 | logger.error("cannot use -l/--lines with more than one file") 28 | 29 | if (bopts.in_place or bopts.diff) and not bopts.files: 30 | logger.error( 31 | "cannot use --in-place or --diff flags when reading from stdin" 32 | ) 33 | 34 | meta = JobsMeta(adapter="yapf") 35 | meta.in_place = bopts.in_place 36 | meta.parallel = bopts.parallel 37 | meta.print_diff = bopts.diff 38 | meta.quiet = bopts.quiet 39 | 40 | exclude_patterns_from_ignore_file = file_resources.GetExcludePatternsForDir( 41 | os.getcwd() 42 | ) 43 | files = file_resources.GetCommandLineFiles( 44 | bopts.files, 45 | bopts.recursive, 46 | (bopts.exclude or []) + exclude_patterns_from_ignore_file, 47 | ) 48 | 49 | meta.files = files 50 | 51 | return meta 52 | 53 | def _get_backend_cmd_for_resource(self, resource: _ResourceBase) -> Sequence[str]: 54 | cmd = [self.opts.executable or "yapf"] 55 | 56 | bopts = self._backend_opts 57 | for linespec in bopts.lines or []: 58 | cmd.extend(["-l", linespec]) 59 | if bopts.no_local_style: 60 | cmd.append("--no-local-style") 61 | if bopts.style is not None: 62 | cmd.extend(["--style", bopts.style]) 63 | 64 | return cmd 65 | -------------------------------------------------------------------------------- /lambdex/fmt/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/fmt/cli/__init__.py -------------------------------------------------------------------------------- /lambdex/fmt/cli/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from lambdex.fmt import adapters 4 | from .opts import split_argv, build_parser 5 | 6 | 7 | def main() -> int: 8 | backend_argv, argv = split_argv() 9 | opts = build_parser().parse_args(argv) 10 | 11 | adapter = adapters.build(opts.adapter, opts, backend_argv) 12 | 13 | changed = False 14 | if adapter.jobs_meta.parallel: 15 | import multiprocessing 16 | import concurrent.futures 17 | 18 | with concurrent.futures.ProcessPoolExecutor( 19 | multiprocessing.cpu_count() 20 | ) as executor: 21 | future_formats = [executor.submit(job) for job in adapter.get_jobs()] 22 | for future in concurrent.futures.as_completed(future_formats): 23 | changed |= future.result() 24 | else: 25 | for job in adapter.get_jobs(): 26 | changed |= job() 27 | 28 | return ( 29 | 1 30 | if changed and (adapter.jobs_meta.print_diff or adapter.jobs_meta.quiet) 31 | else 0 32 | ) 33 | -------------------------------------------------------------------------------- /lambdex/fmt/cli/mock.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import argparse 5 | from pathlib import Path 6 | from textwrap import dedent 7 | 8 | import inquirer 9 | 10 | LAMBDEX_ROOT = Path(__file__).parent.parent.parent.parent 11 | AUTO_GENERATED_COMMENT = b"auto generated by lambdex.fmt.cli.mock" 12 | PREFIX = "original_" 13 | 14 | if os.name == "nt": 15 | SCRIPT_TEMPLATE = """ 16 | @echo off 17 | REM {comment} 18 | set PYTHONPATH={lambdex_root};%PYTHONPATH% 19 | set LXALIAS=on 20 | {py_interpreter} -m lambdex.fmt %* -- -b {formatter_type} -e {formatter_path} 21 | @echo on 22 | """ 23 | else: 24 | SCRIPT_TEMPLATE = """ 25 | #!/bin/sh 26 | # {comment} 27 | if [ -z $PYTHONPATH ]; then 28 | export PYTHONPATH={lambdex_root} 29 | else 30 | export PYTHONPATH={lambdex_root}:$PYTHONPATH 31 | fi 32 | export LXALIAS=on 33 | {py_interpreter} -m lambdex.fmt $@ -- -b {formatter_type} -e {formatter_path} 34 | """ 35 | 36 | 37 | def _script(formatter_type: str, formatter_path: Path) -> str: 38 | output = SCRIPT_TEMPLATE.format( 39 | py_interpreter=sys.executable, 40 | formatter_type=formatter_type, 41 | formatter_path=str(formatter_path), 42 | comment=AUTO_GENERATED_COMMENT.decode("utf-8"), 43 | lambdex_root=LAMBDEX_ROOT, 44 | ) 45 | return dedent(output).strip() 46 | 47 | 48 | def _save_script(path: Path, content: str): 49 | if os.name == "nt": 50 | name = path.name.replace(".exe", ".cmd") 51 | path = path.parent / name 52 | 53 | path.write_text(content) 54 | 55 | if os.name != "nt": 56 | path.chmod(0o775) 57 | 58 | 59 | def _is_generated_script(path: Path) -> bool: 60 | with path.open("rb") as fd: 61 | content = fd.read() 62 | return AUTO_GENERATED_COMMENT in content 63 | 64 | 65 | def _backup_executable(path: Path) -> Path: 66 | parent, name = path.parent, path.name 67 | new_name = PREFIX + name 68 | shutil.move(path, parent / new_name) 69 | return parent / new_name 70 | 71 | 72 | def _restore_executable(path: Path): 73 | parent, name = path.parent, path.name 74 | if os.name == "nt": 75 | name = name.replace(".cmd", ".exe") 76 | exe_name = PREFIX + name 77 | shutil.move(parent / exe_name, parent / name) 78 | 79 | 80 | def _whereis(command: str): 81 | if os.name == "nt": 82 | p = os.popen("where {}".format(command)) 83 | output = p.read().strip().split() 84 | else: 85 | p = os.popen("whereis {}".format(command)) 86 | output = p.read().strip().split()[1:] 87 | 88 | p.close() 89 | return output 90 | 91 | 92 | def main(): 93 | parser = argparse.ArgumentParser( 94 | "lxfmt-mock", description="Mock or reset specified formater backend" 95 | ) 96 | parser.add_argument( 97 | "BACKEND", 98 | help="The backend to be mocked/reset", 99 | ) 100 | parser.add_argument( 101 | "-r", 102 | "--reset", 103 | action="store_true", 104 | help="If specified, the selected command will be reset", 105 | ) 106 | opts = parser.parse_args() 107 | 108 | command = opts.BACKEND 109 | commands = [ 110 | cmd 111 | for cmd in _whereis(command) 112 | if _is_generated_script(Path(cmd)) == opts.reset 113 | ] 114 | 115 | questions = [ 116 | inquirer.List( 117 | "path", 118 | message="Which one do you want to {}?".format( 119 | "reset" if opts.reset else "mock" 120 | ), 121 | choices=commands, 122 | ), 123 | ] 124 | 125 | answers = inquirer.prompt(questions, theme=inquirer.themes.GreenPassion()) 126 | if answers is None: 127 | return 128 | path = Path(answers["path"]) 129 | 130 | if not opts.reset: 131 | backup_path = _backup_executable(path) 132 | _save_script(path, _script(opts.BACKEND, backup_path)) 133 | else: 134 | _restore_executable(path) 135 | 136 | 137 | if __name__ == "__main__": 138 | main() 139 | -------------------------------------------------------------------------------- /lambdex/fmt/cli/opts.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import argparse 3 | 4 | from lambdex.fmt import adapters 5 | from lambdex.fmt.utils.logger import getLogger 6 | 7 | logger = getLogger(__name__) 8 | 9 | DELIMITTER = "--" 10 | 11 | 12 | def split_argv(): 13 | argv = sys.argv[1:] 14 | idx_delimitters = [i for i, arg in enumerate(argv) if arg == DELIMITTER] 15 | num_delimitters = len(idx_delimitters) 16 | if num_delimitters > 2: 17 | logger.error("Too many '--' found, expected less than 3") 18 | 19 | if num_delimitters == 0: 20 | return argv, () 21 | elif num_delimitters == 1: 22 | idx = idx_delimitters[0] 23 | return argv[:idx], argv[idx + 1 :] 24 | else: 25 | start, end = idx_delimitters 26 | return argv[:start] + argv[end + 1 :], argv[start + 1 : end] 27 | 28 | 29 | def build_parser(): 30 | parser = argparse.ArgumentParser( 31 | "lxfmt [ARGS OF BACKEND] --", 32 | description="Lambdex formatter as a post-processor for specific backend", 33 | ) 34 | parser.add_argument( 35 | "-b", 36 | "--backend", 37 | metavar="BACKEND", 38 | dest="adapter", 39 | default="dummy", 40 | choices=list(adapters.mapping), 41 | help="name of formatter backend (default: dummy)", 42 | ) 43 | parser.add_argument( 44 | "-e", 45 | "--executable", 46 | help="executable of backend", 47 | ) 48 | return parser 49 | -------------------------------------------------------------------------------- /lambdex/fmt/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/fmt/core/__init__.py -------------------------------------------------------------------------------- /lambdex/fmt/core/_stream_base.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | import abc 4 | 5 | from lambdex.fmt.utils.logger import getLogger, IS_DEBUG 6 | from .definitions import TokenInfo, actions 7 | 8 | 9 | class _StreamBase(abc.ABC): 10 | def __init__(self, tokenseq: Sequence[TokenInfo]): 11 | self.buffering = False 12 | self.last_token = TokenInfo.fake 13 | self.buffer = [] 14 | self.action = actions.Default() 15 | 16 | self.tokenseq = tokenseq 17 | 18 | self._init() 19 | 20 | def _init(self): 21 | pass 22 | 23 | def __iter__(self): 24 | def _stream(): 25 | token = None 26 | iterator = iter(self.tokenseq) 27 | while True: 28 | if not self.action.dont_consume: 29 | try: 30 | token = next(iterator) 31 | except StopIteration: 32 | break 33 | yield token 34 | 35 | for token in _stream(): 36 | self.action = actions.Default() 37 | yield from self._handle_token(token) 38 | self.action = actions.BaseAction.from_(self.action) 39 | yield from self._handle_action(token, self.action) 40 | 41 | @abc.abstractmethod 42 | def _handle_token(self, token: TokenInfo): 43 | pass 44 | 45 | def _update_last_token(self, token: TokenInfo): 46 | if token.annotation is not None: 47 | self.last_token = token 48 | 49 | def _handle_action( 50 | self, token: TokenInfo, action: actions.BaseAction 51 | ) -> Sequence[TokenInfo]: 52 | 53 | if actions.Default.is_class_of(action): 54 | yield from self._handle_default(token) 55 | elif actions.StartBuffer.is_class_of(action): 56 | yield from self._handle_start_buffer(token) 57 | elif actions.StopBuffer.is_class_of(action): 58 | yield from self._handle_stop_buffer(token) 59 | else: 60 | yield from self._handle_unknown_action(token, action) 61 | 62 | def _append_buffer(self, token): 63 | self.buffer.append(token) 64 | 65 | def _handle_default(self, token): 66 | if self.buffering: 67 | assert self.action.no_special, self.action 68 | self._append_buffer(token) 69 | return 70 | 71 | if self.action.no_special: 72 | yield token 73 | 74 | self._update_last_token(token) 75 | 76 | def _handle_start_buffer(self, token): 77 | assert not self.buffering 78 | self.buffering = True 79 | self._append_buffer(token) 80 | self._update_last_token(token) 81 | return () 82 | 83 | def _handle_stop_buffer(self, token): 84 | assert self.buffering 85 | self.buffering = False 86 | if not self.action.dont_yield_buffer: 87 | yield from self.buffer 88 | if self.action.no_special: 89 | yield token 90 | self.buffer.clear() 91 | self._update_last_token(token) 92 | return () 93 | 94 | def _handle_unknown_action(self, token, action): 95 | return () 96 | 97 | 98 | class _StreamWithLog(_StreamBase): 99 | def __iter__(self): 100 | logger = getLogger(__name__) 101 | for token in super().__iter__(): 102 | yield token 103 | logger.debug(token) 104 | 105 | 106 | if not IS_DEBUG: 107 | _StreamWithLog = _StreamBase 108 | -------------------------------------------------------------------------------- /lambdex/fmt/core/api.py: -------------------------------------------------------------------------------- 1 | from .tkutils.tokenize import tokenize 2 | from .transforms import transform, AsCode 3 | 4 | 5 | def FormatCode(source): 6 | 7 | seq = tokenize(source) 8 | output = AsCode(transform(seq)) 9 | 10 | return output 11 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/__init__.py: -------------------------------------------------------------------------------- 1 | from . import token as tk 2 | 3 | from .state import State 4 | from .context import Context 5 | from .btstream import BTStream 6 | from .token_info import TokenInfo 7 | from .annotation import Annotation as A 8 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/actions.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.core.definitions import State 2 | 3 | 4 | class BaseAction: 5 | 6 | __slots__ = ["dont_consume", "dont_store", "dont_yield_buffer"] 7 | 8 | @classmethod 9 | def is_class_of(cls, action): 10 | return action.__class__ is cls 11 | 12 | @classmethod 13 | def from_(cls, value) -> "BaseAction": 14 | if value is None: 15 | return Default() 16 | 17 | assert isinstance(value, BaseAction) 18 | return value 19 | 20 | def __init__( 21 | self, 22 | *, 23 | dont_consume: bool = False, 24 | dont_store: bool = False, 25 | dont_yield_buffer: bool = False 26 | ): 27 | self.dont_consume = dont_consume 28 | self.dont_store = dont_store 29 | self.dont_yield_buffer = dont_yield_buffer 30 | 31 | @property 32 | def no_special(self): 33 | return not (self.dont_consume or self.dont_store) 34 | 35 | def __repr__(self): 36 | attrs = ("{}={}".format(name, getattr(self, name)) for name in self.__slots__) 37 | return "<{}: {}>".format( 38 | self.__class__.__name__, 39 | ", ".join(attrs), 40 | ) 41 | 42 | 43 | class Default(BaseAction): 44 | pass 45 | 46 | 47 | class StartBuffer(BaseAction): 48 | pass 49 | 50 | 51 | class StopBuffer(BaseAction): 52 | pass 53 | 54 | 55 | class Backtrace(BaseAction): 56 | 57 | __slots__ = BaseAction.__slots__ + ["new_state"] 58 | 59 | def state(self, new_state: State) -> "Backtrace": 60 | self.new_state = new_state 61 | return self 62 | 63 | 64 | # default = BaseAction() 65 | # dont_store = BaseAction() 66 | # start_buffer = BaseAction() 67 | # stop_buffer = BaseAction() 68 | # dont_consume = BaseAction() 69 | # stop_buffer_and_dont_consume = BaseAction() 70 | # stop_buffer_and_dont_store = BaseAction() 71 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/annotation.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | from lambdex.utils import compat 4 | 5 | auto = compat.enum_auto() 6 | 7 | 8 | class Annotation(enum.Enum): 9 | 10 | DECL = auto() 11 | DECL_LPAR = auto() 12 | DECL_LAMBDA = auto() 13 | 14 | BODY_LSQB = auto() 15 | BODY_RSQB = auto() 16 | 17 | CLS_DOT = auto() 18 | CLS_DECL = auto() 19 | 20 | CLS_HEAD_LSQB = auto() 21 | CLS_HEAD_RSQB = auto() 22 | 23 | CLS_BODY_LSQB = auto() 24 | CLS_BODY_RSQB = auto() 25 | 26 | STMT_COMMA = auto() 27 | STMT_START = auto() 28 | STMT_END = auto() 29 | 30 | DECL_ARG_COMMA = auto() 31 | DECL_RPAR = auto() 32 | 33 | # Just behind the last comma or the last STMT_END 34 | LAST_STMT_WITH_COMMA = auto() 35 | LAST_STMT_WITHOUT_COMMA = auto() 36 | 37 | LAST_NL_BEFORE_RSQB = auto() 38 | 39 | # Aug-assign related 40 | AUGASSIGN_START = auto() 41 | AUGASSIGN_END = auto() 42 | 43 | def __repr__(self) -> str: 44 | return self.name 45 | 46 | 47 | del auto 48 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/btstream.py: -------------------------------------------------------------------------------- 1 | from typing import List, Sequence, Optional 2 | 3 | from collections import deque 4 | 5 | from lambdex.fmt.utils.logger import getLogger 6 | 7 | from .token_info import TokenInfo 8 | 9 | logger = getLogger(__name__) 10 | 11 | 12 | class BufferFrame: 13 | __slots__ = ["is_backtracing", "buffer"] 14 | 15 | def __init__(self): 16 | self.is_backtracing = False 17 | self.buffer = deque() 18 | 19 | 20 | class BTStream: 21 | def __init__(self, tokenseq: Sequence[TokenInfo]): 22 | self.stack = [] 23 | self._backtrace_invoked = False 24 | self.tokenseq = iter(tokenseq) 25 | 26 | def start_buffer(self): 27 | assert not self.stack or self.stack[-1].is_backtracing 28 | logger.debug("== Start Buffer ==") 29 | self.stack.append(BufferFrame()) 30 | 31 | def stop_buffer(self): 32 | assert self.last_is_buffering(), self.stack 33 | yield from self.stack[-1].buffer 34 | if logger.is_debug: 35 | logger.debug("== Stop Buffer ==") 36 | for token in self.stack[-1].buffer: 37 | logger.debug(repr(token)) 38 | self.stack.pop() 39 | 40 | def backtrace(self): 41 | self.stack[-1].is_backtracing = True 42 | self._backtrace_invoked = True 43 | 44 | if logger.is_debug: 45 | logger.debug("") 46 | logger.debug("Backtracing!") 47 | for token in self.stack[-1].buffer: 48 | logger.debug("{}".format(token)) 49 | logger.debug("") 50 | 51 | def last_is_buffering(self) -> bool: 52 | return self.stack and not self.stack[-1].is_backtracing 53 | 54 | def _last_bt_frame(self) -> BufferFrame: 55 | for idx in range(len(self.stack) - 1, -1, -1): 56 | if self.stack[idx].is_backtracing: 57 | return self.stack[idx] 58 | 59 | return None 60 | 61 | def _get_next_token(self) -> Optional[TokenInfo]: 62 | frame = self._last_bt_frame() 63 | if frame is None: 64 | try: 65 | token = next(self.tokenseq) 66 | except StopIteration: 67 | return None 68 | elif frame.buffer: 69 | token = frame.buffer.popleft() 70 | 71 | if frame is not None and not frame.buffer: 72 | if frame is not self.stack[-1]: 73 | logger.debug("ERROR: frame is not at the last") 74 | for idx, frame in enumerate(self.stack): 75 | logger.debug( 76 | "== Frame {}: backtracing: {}".format(idx, frame.is_backtracing) 77 | ) 78 | for token in frame.buffer: 79 | logger.debug(repr(token)) 80 | logger.debug("== End of Frame {}".format(idx)) 81 | raise RuntimeError 82 | 83 | self.stack.remove(frame) 84 | 85 | return token 86 | 87 | def __iter__(self): 88 | 89 | while True: 90 | 91 | token = self._get_next_token() 92 | if token is None: 93 | break 94 | 95 | yield token 96 | 97 | if self.last_is_buffering() or self._backtrace_invoked: 98 | self.stack[-1].buffer.append(token) 99 | self._backtrace_invoked = False 100 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/context.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | from lambdex.fmt.utils.logger import getLogger 4 | 5 | from .state import State 6 | from .token_info import TokenInfo 7 | 8 | logger = getLogger(__name__) 9 | 10 | 11 | class Context: 12 | def __init__(self): 13 | self.ret = [] 14 | self.op_stack = [] 15 | self.state_stack = [State.UNKNOWN] 16 | self.cache = None 17 | 18 | @property 19 | def last_op(self) -> TokenInfo: 20 | return self.op_stack[-1] 21 | 22 | @property 23 | def last_state(self) -> State: 24 | return self.state_stack[-1] 25 | 26 | def debug(self, text): 27 | if not logger.is_debug: 28 | return 29 | 30 | import sys 31 | 32 | frame = sys._getframe(2) 33 | logger.debug("====> {}".format(text)) 34 | logger.debug("==> {} {}".format(frame.f_code.co_filename, frame.f_lineno)) 35 | logger.debug("==> {}".format(frame.f_locals["token"])) 36 | 37 | logger.debug("{:^60s}".format("--- OP Stack ---")) 38 | for op, state in self.op_stack: 39 | logger.debug( 40 | "{:>20s}{:>20s}{:>20s}".format( 41 | op.string, 42 | op.annotation.name if op.annotation is not None else "", 43 | state.name, 44 | ) 45 | ) 46 | logger.debug("{:^60s}".format("--- ST Stack ---")) 47 | for x in self.state_stack: 48 | logger.debug("{:>60s}".format(x.name)) 49 | 50 | def error(self): 51 | self.debug("ERROR") 52 | raise RuntimeError 53 | 54 | def push_ret(self, v: TokenInfo) -> None: 55 | self.ret.append(v) 56 | 57 | def push_op(self, v: TokenInfo) -> None: 58 | self.op_stack.append((v, self.last_state)) 59 | self.debug("PUSH OP AFTER") 60 | 61 | def pop_op(self) -> Tuple[TokenInfo, State]: 62 | ret = self.op_stack.pop() 63 | self.debug("POP OP AFTER") 64 | return ret 65 | 66 | def push_state(self, v: State) -> None: 67 | self.state_stack.append(v) 68 | self.debug("PUSH State AFTER") 69 | 70 | def pop_state(self) -> State: 71 | popped = self.state_stack.pop() 72 | self.debug("POP State AFTER") 73 | return popped 74 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/state.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | from lambdex.utils import compat 4 | 5 | auto = compat.enum_auto() 6 | 7 | 8 | class State(enum.Enum): 9 | DISABLED = auto() 10 | UNKNOWN = auto() 11 | 12 | EXPECT_LBDX_LPAR = auto() 13 | EXPECT_LBDX_RPAR = auto() 14 | MUST_LBDX_LPAR = auto() 15 | EXPECT_LBDX_NAME = auto() 16 | 17 | IN_LBDX_CALL = auto() 18 | IN_LBDX_LAMBDA = auto() 19 | IN_LBDX_BODY_LIST = auto() 20 | IN_LBDX_CLS_HEAD = auto() 21 | IN_LBDX_CLS_BODY = auto() 22 | 23 | EXPECT_LBDX_LSQB = auto() 24 | 25 | EXPECT_CLS_HEAD_LSQB = auto() 26 | EXPECT_CLS_BODY_LSQB = auto() 27 | EXPECT_SUBCLS_DOT = auto() 28 | EXPECT_SUBCLS_NAME = auto() 29 | MUST_SUBCLS_DOT_WITH_HEAD = auto() 30 | MUST_SUBCLS_DOT_WITH_BODY = auto() 31 | 32 | MUST_SUBCLS_NAME_WITH_HEAD = auto() 33 | MUST_SUBCLS_NAME_WITH_BODY = auto() 34 | 35 | EXPECT_CLS_HEAD_OR_BODY_LSQB = auto() 36 | EXPECT_CLS_HEAD_OR_BODY_RSQB = auto() 37 | 38 | EXPECT_CLS_MAYBE_BODY_LSQB = auto() 39 | 40 | EXPECT_AUGASSIGN_DASH = auto() 41 | EXPECT_AUGASSIGN_ASSIGN = auto() 42 | 43 | 44 | del auto 45 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/token.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.core.tkutils.builtins import token as bltk 2 | 3 | __all__ = bltk.__all__ + ["ISMATCHED", "EXACT_TOKEN_TYPES", "WHITESPACE", "SENTINEL"] 4 | 5 | 6 | def ISMATCHED(l, r): 7 | return (l.string, r.string) in { 8 | ("(", ")"), 9 | ("[", "]"), 10 | ("{", "}"), 11 | } 12 | 13 | 14 | class auto: 15 | _counter = bltk.NT_OFFSET 16 | 17 | def __new__(cls): 18 | cls._counter += 1 19 | return cls._counter 20 | 21 | 22 | SENTINEL = auto() 23 | WHITESPACE = auto() 24 | 25 | bltk.tok_name.update( 26 | { 27 | value: name 28 | for name, value in globals().items() 29 | if isinstance(value, int) and not name.startswith("_") 30 | } 31 | ) 32 | 33 | from lambdex.fmt.core.tkutils.builtins.token import * 34 | from lambdex.fmt.core.tkutils.builtins.token import EXACT_TOKEN_TYPES 35 | 36 | del auto, bltk 37 | -------------------------------------------------------------------------------- /lambdex/fmt/core/definitions/token_info.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple, FrozenSet 2 | 3 | import re 4 | 5 | from lambdex.fmt.utils.colored import colored, colorful 6 | 7 | from . import token as tk 8 | from .annotation import Annotation as A 9 | 10 | 11 | class TokenInfo: 12 | 13 | __slots__ = [ 14 | "type", 15 | "string", 16 | "start", 17 | "end", 18 | "line", 19 | "annotation", 20 | "leading_whitespace", 21 | ] 22 | 23 | def __init__( 24 | self, 25 | type: int, 26 | string: str, 27 | start: Optional[Tuple[int, int]] = None, 28 | end: Optional[Tuple[int, int]] = None, 29 | line: Optional[str] = None, 30 | annotation: Optional[A] = None, 31 | leading_whitespace: Optional[str] = None, 32 | ): 33 | self.type = type 34 | self.string = string 35 | self.start = start 36 | self.end = end 37 | self.line = line 38 | self.annotation = annotation 39 | self.leading_whitespace = leading_whitespace 40 | 41 | def visualize(self, *, repr=False) -> Optional[str]: 42 | def _(string): 43 | return string.__repr__()[1:-1] if repr else string 44 | 45 | if self.start is None or self.end is None or self.line is None: 46 | return None 47 | 48 | string = self.line 49 | if not colorful: 50 | return _(string) 51 | 52 | start, end = self.start[1], self.end[1] 53 | 54 | before = colored(_(string[:start]), "yellow", attrs=["underline", "dark"]) 55 | after = colored(_(string[end:]), "yellow", attrs=["underline", "dark"]) 56 | 57 | middle = self.string 58 | if not middle: 59 | middle = "\u2591" 60 | suffix = "" 61 | else: 62 | middle = _(middle) 63 | suffix = " " 64 | 65 | middle = colored(middle, "yellow", attrs=["underline", "bold"]) 66 | return before + middle + after + suffix 67 | 68 | def __repr__(self) -> str: 69 | annotated_type = "%d (%s)" % (self.type, tk.tok_name[self.type]) 70 | visualized = self.visualize(repr=True) 71 | 72 | if visualized is None: 73 | return "TokenInfo({}, type={:>17s}, A={:>20s}, LWS={})".format( 74 | repr(self.string), 75 | annotated_type, 76 | repr(self.annotation), 77 | repr(self.leading_whitespace), 78 | ) 79 | else: 80 | return ( 81 | "TokenInfo({}, type={:>17s}, A={:>20s}, LWS={}, lineno={}:{})".format( 82 | visualized, 83 | annotated_type, 84 | repr(self.annotation), 85 | repr(self.leading_whitespace), 86 | self.start[0], 87 | self.end[0], 88 | ) 89 | ) 90 | 91 | @property 92 | def exact_type(self): 93 | if self.type == tk.OP and self.string in tk.EXACT_TOKEN_TYPES: 94 | return tk.EXACT_TOKEN_TYPES[self.string] 95 | else: 96 | return self.type 97 | 98 | @classmethod 99 | def new_sentinel_after(cls, token, annotation): 100 | return cls( 101 | type=tk.SENTINEL, 102 | start=token.end, 103 | end=token.end, 104 | string="", 105 | line=token.line, 106 | annotation=annotation, 107 | ) 108 | 109 | @classmethod 110 | def new_sentinel_before(cls, token, annotation): 111 | return cls( 112 | type=tk.SENTINEL, 113 | start=token.start, 114 | end=token.start, 115 | string="", 116 | line=token.line, 117 | annotation=annotation, 118 | ) 119 | 120 | def lxfmt_directive(self) -> Optional[str]: 121 | if not self.is_CMT: 122 | return None 123 | matched = re.match(r"#.*\blxfmt:\s*(?Pon|off)", self.string) 124 | if matched is not None: 125 | return matched.group("directive") 126 | return None 127 | 128 | def __eq__(self, rhs): 129 | if isinstance(rhs, A): 130 | return self.annotation == rhs 131 | return super().__eq__(rhs) 132 | 133 | WS = frozenset([tk.WHITESPACE]) 134 | NL = frozenset([tk.NL, tk.NEWLINE]) 135 | CMT = frozenset([tk.COMMENT, tk.TYPE_IGNORE, tk.TYPE_COMMENT]) 136 | 137 | WS_NL = WS | NL 138 | NL_CMT = NL | CMT 139 | WS_NL_CMT = WS_NL | CMT 140 | 141 | def _build_property(types: FrozenSet): 142 | return property(lambda self: self.type in types) 143 | 144 | for name, value in list(locals().items()): 145 | if not isinstance(value, frozenset): 146 | continue 147 | locals()["is_{}".format(name)] = _build_property(value) 148 | del locals()[name] 149 | 150 | del name, value 151 | 152 | 153 | TokenInfo.fake = TokenInfo(type=-1, string=None, annotation=object()) 154 | -------------------------------------------------------------------------------- /lambdex/fmt/core/tkutils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/fmt/core/tkutils/__init__.py -------------------------------------------------------------------------------- /lambdex/fmt/core/tkutils/builtins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/fmt/core/tkutils/builtins/__init__.py -------------------------------------------------------------------------------- /lambdex/fmt/core/tkutils/builtins/token.py: -------------------------------------------------------------------------------- 1 | """Token constants.""" 2 | # Auto-generated by Tools/scripts/generate_token.py 3 | 4 | __all__ = ["tok_name", "ISTERMINAL", "ISNONTERMINAL", "ISEOF"] 5 | 6 | ENDMARKER = 0 7 | NAME = 1 8 | NUMBER = 2 9 | STRING = 3 10 | NEWLINE = 4 11 | INDENT = 5 12 | DEDENT = 6 13 | LPAR = 7 14 | RPAR = 8 15 | LSQB = 9 16 | RSQB = 10 17 | COLON = 11 18 | COMMA = 12 19 | SEMI = 13 20 | PLUS = 14 21 | MINUS = 15 22 | STAR = 16 23 | SLASH = 17 24 | VBAR = 18 25 | AMPER = 19 26 | LESS = 20 27 | GREATER = 21 28 | EQUAL = 22 29 | DOT = 23 30 | PERCENT = 24 31 | LBRACE = 25 32 | RBRACE = 26 33 | EQEQUAL = 27 34 | NOTEQUAL = 28 35 | LESSEQUAL = 29 36 | GREATEREQUAL = 30 37 | TILDE = 31 38 | CIRCUMFLEX = 32 39 | LEFTSHIFT = 33 40 | RIGHTSHIFT = 34 41 | DOUBLESTAR = 35 42 | PLUSEQUAL = 36 43 | MINEQUAL = 37 44 | STAREQUAL = 38 45 | SLASHEQUAL = 39 46 | PERCENTEQUAL = 40 47 | AMPEREQUAL = 41 48 | VBAREQUAL = 42 49 | CIRCUMFLEXEQUAL = 43 50 | LEFTSHIFTEQUAL = 44 51 | RIGHTSHIFTEQUAL = 45 52 | DOUBLESTAREQUAL = 46 53 | DOUBLESLASH = 47 54 | DOUBLESLASHEQUAL = 48 55 | AT = 49 56 | ATEQUAL = 50 57 | RARROW = 51 58 | ELLIPSIS = 52 59 | COLONEQUAL = 53 60 | OP = 54 61 | AWAIT = 55 62 | ASYNC = 56 63 | TYPE_IGNORE = 57 64 | TYPE_COMMENT = 58 65 | # These aren't used by the C tokenizer but are needed for tokenize.py 66 | ERRORTOKEN = 59 67 | COMMENT = 60 68 | NL = 61 69 | ENCODING = 62 70 | N_TOKENS = 63 71 | # Special definitions for cooperation with parser 72 | NT_OFFSET = 256 73 | 74 | tok_name = { 75 | value: name 76 | for name, value in globals().items() 77 | if isinstance(value, int) and not name.startswith("_") 78 | } 79 | __all__.extend(tok_name.values()) 80 | 81 | EXACT_TOKEN_TYPES = { 82 | "!=": NOTEQUAL, 83 | "%": PERCENT, 84 | "%=": PERCENTEQUAL, 85 | "&": AMPER, 86 | "&=": AMPEREQUAL, 87 | "(": LPAR, 88 | ")": RPAR, 89 | "*": STAR, 90 | "**": DOUBLESTAR, 91 | "**=": DOUBLESTAREQUAL, 92 | "*=": STAREQUAL, 93 | "+": PLUS, 94 | "+=": PLUSEQUAL, 95 | ",": COMMA, 96 | "-": MINUS, 97 | "-=": MINEQUAL, 98 | "->": RARROW, 99 | ".": DOT, 100 | "...": ELLIPSIS, 101 | "/": SLASH, 102 | "//": DOUBLESLASH, 103 | "//=": DOUBLESLASHEQUAL, 104 | "/=": SLASHEQUAL, 105 | ":": COLON, 106 | ":=": COLONEQUAL, 107 | ";": SEMI, 108 | "<": LESS, 109 | "<<": LEFTSHIFT, 110 | "<<=": LEFTSHIFTEQUAL, 111 | "<=": LESSEQUAL, 112 | "=": EQUAL, 113 | "==": EQEQUAL, 114 | ">": GREATER, 115 | ">=": GREATEREQUAL, 116 | ">>": RIGHTSHIFT, 117 | ">>=": RIGHTSHIFTEQUAL, 118 | "@": AT, 119 | "@=": ATEQUAL, 120 | "[": LSQB, 121 | "]": RSQB, 122 | "^": CIRCUMFLEX, 123 | "^=": CIRCUMFLEXEQUAL, 124 | "{": LBRACE, 125 | "|": VBAR, 126 | "|=": VBAREQUAL, 127 | "}": RBRACE, 128 | "~": TILDE, 129 | } 130 | 131 | 132 | def ISTERMINAL(x): 133 | return x < NT_OFFSET 134 | 135 | 136 | def ISNONTERMINAL(x): 137 | return x >= NT_OFFSET 138 | 139 | 140 | def ISEOF(x): 141 | return x == ENDMARKER 142 | -------------------------------------------------------------------------------- /lambdex/fmt/core/tkutils/rules/__init__.py: -------------------------------------------------------------------------------- 1 | from . import rules 2 | from .matcher import matcher 3 | -------------------------------------------------------------------------------- /lambdex/fmt/core/tkutils/rules/matcher.py: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | from lambdex.fmt.core.definitions import tk, TokenInfo 3 | 4 | # A sentinel indicating an empty argument for `_make_key()` 5 | _empty = object() 6 | 7 | 8 | def _make_key(exact_type, string, last_state, *, strict=True): 9 | if exact_type == tk.NAME: 10 | return (exact_type, string, last_state) 11 | else: 12 | if strict: 13 | assert string is _empty 14 | return (exact_type, last_state) 15 | 16 | 17 | def _generate_queries(token, last_state, keyword_to_symbol): 18 | def _keys(): 19 | # Use both `token` and `last_state` 20 | yield _make_key(exact_type, symbol, last_state, strict=False) 21 | 22 | # Use `token` and `last_state`, but ignore `symbol` 23 | # This is for the case you want to capture a `tk.NAME`, but don't care about its content 24 | yield _make_key(exact_type, _empty, last_state, strict=False) 25 | 26 | # Use `token` and 'ALL' as `last_state` 27 | # NOTE that this is different from only `token` 28 | yield _make_key(exact_type, _empty, "ALL", strict=False) 29 | 30 | # Use only `last_state` 31 | yield _make_key(_empty, _empty, last_state, strict=False) 32 | 33 | # Use only `token` 34 | yield _make_key(exact_type, symbol, _empty, strict=False) 35 | 36 | symbol = keyword_to_symbol.get(token.string, token.string) 37 | 38 | exact_type = keyword_to_symbol.get(token.string) 39 | if exact_type is not None: 40 | yield from _keys() 41 | exact_type = token.exact_type 42 | yield from _keys() 43 | 44 | 45 | class Matcher: 46 | def __init__(self): 47 | self._mapping = {} 48 | self._keyword_to_symbol = {} 49 | 50 | def reset_aliases(self, *userpaths): 51 | from lambdex._aliases import _Aliases, get_aliases 52 | 53 | aliases = get_aliases(userpaths, reinit=True) 54 | 55 | self._keyword_to_symbol = {} 56 | for name, value in aliases._asdict().items(): 57 | self._keyword_to_symbol[value] = getattr(_Aliases, name) 58 | 59 | def __call__(self, *, exact_type=_empty, string=_empty, last_state=_empty): 60 | def _key_combinations(): 61 | iters = [] 62 | for condition in (exact_type, string, last_state): 63 | iters.append( 64 | condition if isinstance(condition, (tuple, list)) else [condition] 65 | ) 66 | return (_make_key(*args) for args in product(*iters)) 67 | 68 | def _inner(f): 69 | for key in _key_combinations(): 70 | assert key not in self._mapping 71 | self._mapping[key] = f 72 | return f 73 | 74 | return _inner 75 | 76 | def dispatch(self, ctx, token): 77 | for query in _generate_queries(token, ctx.last_state, self._keyword_to_symbol): 78 | if query in self._mapping: 79 | return self._mapping[query](ctx, token) 80 | 81 | else: 82 | # If all queries fail, pass through 83 | return None 84 | 85 | 86 | matcher = Matcher() 87 | -------------------------------------------------------------------------------- /lambdex/fmt/core/tkutils/tokenize.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from lambdex.fmt.core._stream_base import _StreamWithLog 4 | from lambdex.fmt.core.definitions import TokenInfo, tk, Context, A, actions, BTStream 5 | 6 | from .rules import matcher 7 | from .builtins import tokenize as bltokenize 8 | 9 | 10 | class AddWhitespace(_StreamWithLog): 11 | def _handle_token(self, token: TokenInfo): 12 | ws_start = ws_end = None 13 | last_token = self.last_token 14 | if last_token is TokenInfo.fake or last_token.type == tk.ENCODING: 15 | pass 16 | elif last_token.is_WS_NL: # NEWLINE or CONTINUE 17 | assert token.start[0] == last_token.end[0] + 1 18 | if token.start[1] != 0: 19 | ws_start = (token.start[0], 0) 20 | ws_end = token.start 21 | elif last_token.end != token.start: 22 | assert last_token.end[0] == token.start[0] 23 | ws_start = last_token.end 24 | ws_end = token.start 25 | 26 | if ws_start is not None: 27 | whitespace_token = TokenInfo( 28 | tk.WHITESPACE, 29 | token.line[ws_start[1] : ws_end[1]], 30 | ws_start, 31 | ws_end, 32 | token.line, 33 | ) 34 | yield whitespace_token 35 | 36 | self.last_token = token 37 | yield token 38 | 39 | self.action = actions.Default(dont_store=True) 40 | 41 | 42 | class RearrangeSentinel(_StreamWithLog): 43 | def _init(self): 44 | self.stmt_start_in_buffer = None 45 | 46 | def _handle_token(self, token: TokenInfo): 47 | if token.is_WS_NL_CMT or token == A.STMT_START: 48 | if not self.buffering: 49 | self.action = actions.StartBuffer() 50 | elif self.buffering: 51 | if self.stmt_start_in_buffer is not None: 52 | yield from self.buffer 53 | 54 | # collapse STMT_START and STMT_END if adjcent 55 | if token != A.STMT_END: 56 | yield self.stmt_start_in_buffer 57 | yield token 58 | elif token == A.STMT_END: 59 | yield token 60 | yield from self.buffer 61 | else: 62 | yield from self.buffer 63 | yield token 64 | self.stmt_start_in_buffer = None 65 | self.action = actions.StopBuffer(dont_yield_buffer=True, dont_store=True) 66 | 67 | def _append_buffer(self, token): 68 | if token == A.STMT_START: 69 | self.stmt_start_in_buffer = token 70 | else: 71 | self.buffer.append(token) 72 | 73 | 74 | class HandleLastSTMT(_StreamWithLog): 75 | def _init(self): 76 | self.insert_last_stmt_at = None 77 | 78 | def _append_buffer(self, token): 79 | self.buffer.append(token) 80 | if token == A.STMT_COMMA: 81 | self.insert_last_stmt_at = len(self.buffer) 82 | 83 | def _handle_token(self, token: TokenInfo): 84 | if token.annotation == A.STMT_END and not self.buffering: 85 | self.action = actions.StartBuffer() 86 | elif self.buffering and token.annotation in ( 87 | A.BODY_RSQB, 88 | A.CLS_BODY_RSQB, 89 | A.STMT_START, 90 | ): 91 | 92 | if token.annotation != A.STMT_START: 93 | pos = self.insert_last_stmt_at 94 | if pos is None: 95 | annotation = A.LAST_STMT_WITHOUT_COMMA 96 | else: 97 | annotation = A.LAST_STMT_WITH_COMMA 98 | pos = pos or 1 99 | 100 | yield from self.buffer[:pos] 101 | yield TokenInfo.new_sentinel_after(self.buffer[pos - 1], annotation) 102 | yield from self.buffer[pos:] 103 | else: 104 | yield from self.buffer 105 | 106 | self.insert_last_stmt_at = None 107 | self.action = actions.StopBuffer(dont_yield_buffer=True) 108 | elif self.buffering and token == A.STMT_START: 109 | self.insert_last_stmt_at = None 110 | self.action = actions.StopBuffer() 111 | 112 | 113 | class Annotate(_StreamWithLog): 114 | def _init(self): 115 | assert isinstance(self.tokenseq, BTStream) 116 | 117 | self.context = Context() 118 | self.context.is_buffering = lambda: self.tokenseq.last_is_buffering() 119 | 120 | def _handle_token(self, token): 121 | self.context.tokenseq = self.tokenseq 122 | self.action = matcher.dispatch(self.context, token) 123 | return () 124 | 125 | def _handle_default(self, token): 126 | if self.action.dont_store: 127 | assert not self.tokenseq.last_is_buffering() 128 | yield from self.context.ret 129 | self.context.ret.clear() 130 | return 131 | 132 | assert not self.context.ret 133 | 134 | if self.tokenseq.last_is_buffering(): 135 | return 136 | 137 | if self.action.dont_consume: 138 | return 139 | 140 | yield token 141 | 142 | def _handle_start_buffer(self, token): 143 | self.tokenseq.start_buffer() 144 | return () 145 | 146 | def _handle_stop_buffer(self, token): 147 | yield from self.tokenseq.stop_buffer() 148 | 149 | if self.action.no_special: 150 | yield token 151 | elif self.action.dont_store: 152 | yield from self.context.ret 153 | self.context.ret.clear() 154 | 155 | def _handle_unknown_action(self, token, action): 156 | 157 | assert actions.Backtrace.is_class_of(action) 158 | if action.new_state is not None: 159 | self.context.state_stack[-1] = action.new_state 160 | self.tokenseq.backtrace() 161 | return () 162 | 163 | 164 | def tokenize(readline): 165 | seq = bltokenize.tokenize(readline) 166 | seq = AddWhitespace(seq) 167 | seq = BTStream(seq) 168 | seq = Annotate(seq) 169 | seq = RearrangeSentinel(seq) 170 | seq = HandleLastSTMT(seq) 171 | 172 | return seq 173 | 174 | 175 | if __name__ == "__main__": 176 | import sys 177 | 178 | with open(sys.argv[1], "rb") as fd: 179 | tokenize(fd.__next__) 180 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/AnnotateLeadingWhitespace.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 2 | from lambdex.fmt.core._stream_base import _StreamWithLog 3 | 4 | 5 | class AnnotateLeadingWhitespace(_StreamWithLog): 6 | def _init(self): 7 | self.newlined = False 8 | self.last_leading_whitespace = "" 9 | 10 | def _handle_token(self, token: TokenInfo): 11 | if self.newlined: 12 | if token.is_WS and self.last_token.annotation in ( 13 | A.STMT_START, 14 | A.CLS_HEAD_LSQB, 15 | ): 16 | token.leading_whitespace = self.last_leading_whitespace 17 | elif token.is_WS: 18 | self.last_leading_whitespace = token.string 19 | else: 20 | self.last_leading_whitespace = "" 21 | self.newlined = False 22 | 23 | if token.is_NL: 24 | self.newlined = True 25 | 26 | return () 27 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/AsCode.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence, Union 2 | 3 | from lambdex.fmt.core.definitions import TokenInfo, tk 4 | 5 | 6 | def AsCode(tokenseq: Sequence[TokenInfo], *, encode=False) -> Union[str, bytes]: 7 | encoding = "" 8 | token_strings = [] 9 | for token in tokenseq: 10 | if token.type == tk.ENCODING: 11 | encoding = token.string 12 | else: 13 | token_strings.append(token.string) 14 | result = "".join(token_strings) 15 | if encode: 16 | result = result.encode(encoding) 17 | return result 18 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/CollectComments.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.utils.logger import getLogger 2 | 3 | from lambdex.fmt.core._stream_base import _StreamWithLog 4 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 5 | 6 | logger = getLogger(__name__) 7 | 8 | BEFORE = 1 9 | AFTER = 2 10 | 11 | 12 | def _match_rule(pattern, rules): 13 | pattern = tuple(pattern) 14 | for rule in rules: 15 | if len(pattern) > len(rule.pattern): 16 | continue 17 | if pattern == rule.pattern[: len(pattern)]: 18 | return True, rule, len(pattern) == len(rule.pattern) 19 | 20 | return False, None, False 21 | 22 | 23 | class _CollectRule: 24 | __slots__ = ["pattern", "insert_at"] 25 | 26 | def __init__(self, *, insert_at: int, pattern: tuple): 27 | self.pattern = pattern 28 | self.insert_at = insert_at 29 | 30 | 31 | COLLECT_BACKWARD = [ 32 | _CollectRule( 33 | insert_at=AFTER, 34 | pattern=(A.DECL, A.DECL_LPAR, A.DECL_LAMBDA, A.BODY_LSQB), 35 | ), 36 | _CollectRule( 37 | insert_at=AFTER, 38 | pattern=(A.CLS_HEAD_RSQB, A.CLS_BODY_LSQB), 39 | ), 40 | _CollectRule( 41 | insert_at=AFTER, 42 | pattern=(A.STMT_END, A.STMT_COMMA, A.LAST_STMT_WITH_COMMA), 43 | ), 44 | _CollectRule( 45 | insert_at=BEFORE, 46 | pattern=(A.STMT_END, A.STMT_COMMA, A.STMT_START), 47 | ), 48 | ] 49 | 50 | COLLECT_FORWARD = [ 51 | _CollectRule( 52 | insert_at=BEFORE, 53 | pattern=(A.BODY_RSQB, A.DECL_RPAR), 54 | ), 55 | _CollectRule( 56 | insert_at=BEFORE, 57 | pattern=(A.CLS_BODY_RSQB, A.CLS_DOT, A.CLS_DECL, A.CLS_HEAD_LSQB), 58 | ), 59 | _CollectRule( 60 | insert_at=BEFORE, 61 | pattern=(A.CLS_BODY_RSQB, A.CLS_DOT, A.CLS_DECL, A.CLS_BODY_LSQB), 62 | ), 63 | ] 64 | 65 | START_TOKENS = frozenset(x.pattern[0] for x in COLLECT_FORWARD + COLLECT_BACKWARD) 66 | 67 | 68 | class CollectComments(_StreamWithLog): 69 | def _split_buffer(self): 70 | comments, others = [], [] 71 | iterator = iter(self.buffer) 72 | 73 | def _next(): 74 | try: 75 | return next(iterator) 76 | except StopIteration: 77 | return TokenInfo.fake 78 | 79 | while True: 80 | token = _next() 81 | if token is TokenInfo.fake: 82 | break 83 | 84 | if token.is_CMT: 85 | comments.append(token) 86 | token = _next() 87 | assert token.is_NL 88 | comments.append(token) 89 | else: 90 | others.append(token) 91 | 92 | return comments, others 93 | 94 | def _handle_token(self, token): 95 | if token.annotation is None: 96 | return 97 | 98 | if not self.buffering: 99 | if token.annotation in START_TOKENS: 100 | self.action = actions.StartBuffer() 101 | self.pattern = [token.annotation] 102 | return 103 | 104 | self.pattern.append(token.annotation) 105 | 106 | matched, rule, exhausted = _match_rule(self.pattern, COLLECT_BACKWARD) 107 | if matched and exhausted: 108 | comments, others = self._split_buffer() 109 | 110 | yield from others 111 | if rule.insert_at == BEFORE: 112 | yield from comments 113 | yield token 114 | else: 115 | yield token 116 | yield from comments 117 | 118 | self.action = actions.StopBuffer(dont_yield_buffer=True, dont_store=True) 119 | return 120 | elif matched: 121 | return 122 | 123 | matched, rule, exhausted = _match_rule(self.pattern, COLLECT_FORWARD) 124 | if matched and exhausted: 125 | comments, others = self._split_buffer() 126 | assert rule.insert_at == BEFORE 127 | 128 | yield from comments 129 | yield from others 130 | yield token 131 | 132 | self.action = actions.StopBuffer(dont_yield_buffer=True, dont_store=True) 133 | return 134 | elif matched: 135 | return 136 | 137 | self.action = actions.StopBuffer(dont_consume=True) 138 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/DropToken.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 4 | from lambdex.fmt.core._stream_base import _StreamWithLog 5 | 6 | ANNOTATIONS_TO_DROP = frozenset( 7 | [ 8 | A.DECL_ARG_COMMA, 9 | ] 10 | ) 11 | 12 | 13 | class DropToken(_StreamWithLog): 14 | def _handle_token(self, token: TokenInfo): 15 | if token.annotation in ANNOTATIONS_TO_DROP: 16 | self.action = actions.Default(dont_store=True) 17 | return () 18 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/InsertNewline.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 2 | from lambdex.fmt.core._stream_base import _StreamWithLog 3 | 4 | INSERT_BETWEEN = frozenset( 5 | { 6 | (A.BODY_LSQB, A.STMT_START), 7 | (A.BODY_LSQB, A.BODY_RSQB), 8 | (A.STMT_COMMA, A.STMT_START), 9 | (A.LAST_STMT_WITH_COMMA, A.BODY_RSQB), 10 | (A.LAST_STMT_WITHOUT_COMMA, A.BODY_RSQB), 11 | # 12 | (A.CLS_BODY_LSQB, A.STMT_START), 13 | (A.CLS_BODY_LSQB, A.CLS_BODY_RSQB), 14 | (A.STMT_COMMA, A.STMT_START), 15 | (A.LAST_STMT_WITH_COMMA, A.CLS_BODY_RSQB), 16 | (A.LAST_STMT_WITHOUT_COMMA, A.CLS_BODY_RSQB), 17 | } 18 | ) 19 | 20 | START_TOKENS = frozenset(x[0] for x in INSERT_BETWEEN) 21 | 22 | 23 | class InsertNewline(_StreamWithLog): 24 | def _init(self): 25 | self.last_has_newline = False 26 | self.last_NL = None 27 | 28 | def _ensure_NL_exists_in_buffer(self): 29 | if not self.last_has_newline: 30 | self.last_NL = TokenInfo(tk.NL, "\n") 31 | self.buffer.append(self.last_NL) 32 | 33 | def _annotate_NL_before_RSQB(self, token): 34 | if token.annotation in {A.BODY_RSQB, A.CLS_BODY_RSQB}: 35 | self.last_NL.annotation = A.LAST_NL_BEFORE_RSQB 36 | 37 | def _memorize_NL(self, token): 38 | if token.is_NL: 39 | self.last_has_newline = True 40 | self.last_NL = token 41 | 42 | def _reset(self): 43 | self.last_has_newline = False 44 | 45 | def _handle_token(self, token): 46 | if token.annotation is None: 47 | self._memorize_NL(token) 48 | return () 49 | 50 | if token.annotation in START_TOKENS: 51 | if self.buffering: 52 | self.action = actions.StopBuffer(dont_consume=True) 53 | return () 54 | self.action = actions.StartBuffer() 55 | return () 56 | 57 | if (self.last_token.annotation, token.annotation) in INSERT_BETWEEN: 58 | self._ensure_NL_exists_in_buffer() 59 | self._annotate_NL_before_RSQB(token) 60 | 61 | self._reset() 62 | self.action = actions.StopBuffer() 63 | return () 64 | 65 | self._reset() 66 | return () 67 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/NormalizeWhitespaceBeforeComments.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.utils.logger import getLogger 2 | 3 | from lambdex.fmt.core._stream_base import _StreamWithLog 4 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 5 | 6 | logger = getLogger(__name__) 7 | 8 | 9 | class NormalizeWhitespaceBeforeComments(_StreamWithLog): 10 | def _init(self): 11 | self.newlined = False 12 | self.leading = False 13 | self.scope_stack = [] 14 | 15 | def _handle_token(self, token: TokenInfo): 16 | if token.annotation == A.DECL_LPAR: 17 | self.scope_stack.append(token) 18 | elif token.annotation == A.DECL_RPAR: 19 | self.scope_stack.pop() 20 | elif not self.scope_stack: 21 | return 22 | 23 | if token.is_NL: 24 | self.newlined = True 25 | return 26 | 27 | if token.is_WS: 28 | if self.newlined: 29 | self.leading = True 30 | 31 | if not self.buffering: 32 | self.action = actions.StartBuffer() 33 | self.newlined = False 34 | 35 | return 36 | 37 | if not token.is_CMT: 38 | if self.buffering: 39 | self.action = actions.StopBuffer() 40 | self.leading = False 41 | self.newlined = False 42 | 43 | return 44 | 45 | if not self.buffering and not self.newlined: 46 | yield TokenInfo(tk.WHITESPACE, " ") 47 | yield token 48 | self.action = actions.Default(dont_store=True) 49 | return 50 | 51 | if self.buffering: 52 | if any("\\" in x.string for x in self.buffer) or self.leading: 53 | self.action = actions.StopBuffer() 54 | else: 55 | yield TokenInfo(tk.WHITESPACE, " ") 56 | yield token 57 | self.action = actions.StopBuffer( 58 | dont_store=True, dont_yield_buffer=True 59 | ) 60 | 61 | self.leading = False 62 | self.newlined = False 63 | return 64 | self.newlined = False 65 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/NormalizeWhitespaceBeforeToken.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.utils.logger import getLogger 2 | 3 | from lambdex.fmt.core._stream_base import _StreamWithLog 4 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 5 | 6 | logger = getLogger(__name__) 7 | 8 | NORMALIZE_WHITESPACE_BEFORE = { 9 | A.BODY_LSQB: " ", 10 | A.CLS_HEAD_LSQB: "", 11 | A.CLS_BODY_LSQB: " ", 12 | } 13 | 14 | 15 | class NormalizeWhitespaceBeforeToken(_StreamWithLog): 16 | def _init(self): 17 | self.scope_stack = [] 18 | self.prev_non_ws_token = None 19 | 20 | def _handle_token(self, token: TokenInfo): 21 | if token.annotation == A.DECL_LPAR: 22 | self.scope_stack.append(token) 23 | elif token.annotation == A.DECL_RPAR: 24 | self.scope_stack.pop() 25 | elif not self.scope_stack: 26 | return 27 | 28 | if token.is_WS_NL: 29 | if not self.buffering: 30 | self.action = actions.StartBuffer() 31 | return 32 | 33 | if token.annotation in NORMALIZE_WHITESPACE_BEFORE: 34 | replacement = NORMALIZE_WHITESPACE_BEFORE[token.annotation] 35 | if ( 36 | self.prev_non_ws_token.annotation != A.CLS_HEAD_RSQB 37 | and token.annotation == A.CLS_BODY_LSQB 38 | ): 39 | replacement = "" 40 | whitespace = TokenInfo( 41 | type=tk.WHITESPACE, 42 | string=replacement, 43 | ) 44 | yield whitespace 45 | 46 | if self.buffering: 47 | self.action = actions.StopBuffer(dont_yield_buffer=True) 48 | elif self.buffering: 49 | self.action = actions.StopBuffer() 50 | 51 | self.prev_non_ws_token = token -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/Reindent.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 2 | from lambdex.fmt.core._stream_base import _StreamWithLog 3 | 4 | REPLACE = 1 5 | INSERT = 2 6 | 7 | 8 | class Scope: 9 | 10 | __slots__ = ["leading_whitespace", "indent_level"] 11 | 12 | def __init__(self, leading_whitespace, *, indent_level=0): 13 | self.leading_whitespace = leading_whitespace 14 | self.indent_level = indent_level 15 | 16 | 17 | class Reindent(_StreamWithLog): 18 | 19 | SPACES_PER_TAB = 4 20 | 21 | def _init(self): 22 | self.indent_initialized = False 23 | self.orig_indent_str = " " 24 | self.spaced_indent_str = " " 25 | 26 | self.str_newline = None 27 | 28 | self.newlined = False 29 | self.last_leading_whitespace = "" 30 | self.scopes = [] 31 | 32 | def _to_spaced(self, string: str) -> str: 33 | return string.replace("\t", " " * self.SPACES_PER_TAB) 34 | 35 | def _restore_tabbed(self, string: str) -> str: 36 | return string.replace(self.spaced_indent_str, self.orig_indent_str) 37 | 38 | def _store_constant(self, token: TokenInfo): 39 | if token.type == tk.INDENT and not self.indent_initialized: 40 | self.indent_initialized = True 41 | self.orig_indent_str = token.string 42 | self.spaced_indent_str = self._to_spaced(token.string) 43 | elif token.is_NL and self.str_newline is None: 44 | self.str_newline = token.string 45 | 46 | def _process_leading_whitespace(self, token: TokenInfo): 47 | if not self.scopes: 48 | if token.is_WS_NL or token.type == tk.INDENT: 49 | return token, REPLACE 50 | else: 51 | return TokenInfo(type=tk.WHITESPACE, string=""), INSERT 52 | 53 | if token.is_WS: 54 | orig_lws = token.string 55 | action = REPLACE 56 | elif not token.is_NL: 57 | orig_lws = "" 58 | action = INSERT 59 | else: 60 | return token, REPLACE 61 | 62 | indentation = ( 63 | self.spaced_indent_str * self.scopes[-1].indent_level 64 | + self.scopes[-1].leading_whitespace 65 | ) 66 | orig_lws = self._to_spaced(orig_lws) 67 | 68 | if token.leading_whitespace is not None: 69 | orig_parent_lws = self._to_spaced(token.leading_whitespace) 70 | if orig_lws.startswith(orig_parent_lws): 71 | indentation += orig_lws[len(orig_parent_lws) :] 72 | else: 73 | indentation = indentation[: len(orig_lws) - len(orig_parent_lws)] 74 | 75 | indentation = self._restore_tabbed(indentation) 76 | 77 | return TokenInfo(type=tk.WHITESPACE, string=indentation), action 78 | 79 | def _handle_token(self, token): 80 | if token.type == tk.ENCODING: 81 | return 82 | self._store_constant(token) 83 | 84 | if self.newlined: 85 | new_whitespace, action = self._process_leading_whitespace(token) 86 | if action == REPLACE: 87 | token = new_whitespace 88 | elif action == INSERT: 89 | yield new_whitespace 90 | 91 | self.last_leading_whitespace = new_whitespace.string 92 | self.newlined = False 93 | 94 | if token.annotation == A.DECL_LPAR: 95 | self.scopes.append(Scope(self.last_leading_whitespace)) 96 | elif token.annotation == A.DECL_RPAR: 97 | self.scopes.pop() 98 | 99 | if token.annotation in (A.BODY_LSQB, A.CLS_BODY_LSQB): 100 | self.scopes[-1].indent_level += 1 101 | 102 | if token.annotation in (A.LAST_NL_BEFORE_RSQB,): 103 | self.scopes[-1].indent_level -= 1 104 | 105 | if token.is_NL: 106 | self.newlined = True 107 | 108 | yield token 109 | self.action = actions.Default(dont_store=True) 110 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/SuppressWhitespaces.py: -------------------------------------------------------------------------------- 1 | from lambdex.fmt.core.definitions import tk, A, TokenInfo, actions 2 | from lambdex.fmt.core._stream_base import _StreamWithLog 3 | 4 | SUPPRESS_WHITESPACE_AFTER = frozenset( 5 | [ 6 | A.DECL, 7 | A.DECL_LPAR, 8 | A.BODY_RSQB, 9 | A.CLS_BODY_RSQB, 10 | A.CLS_DOT, 11 | A.CLS_DECL, 12 | A.STMT_END, 13 | A.AUGASSIGN_START, 14 | ] 15 | ) 16 | 17 | 18 | class SuppressWhitespaces(_StreamWithLog): 19 | def _handle_token(self, token): 20 | if self.last_token.annotation in SUPPRESS_WHITESPACE_AFTER and token.is_WS_NL: 21 | self.action = actions.Default(dont_store=True) 22 | return () 23 | -------------------------------------------------------------------------------- /lambdex/fmt/core/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence, Union 2 | 3 | from lambdex.fmt.core.definitions import TokenInfo 4 | 5 | from .AsCode import AsCode 6 | from .Reindent import Reindent 7 | from .DropToken import DropToken 8 | from .InsertNewline import InsertNewline 9 | from .CollectComments import CollectComments 10 | from .SuppressWhitespaces import SuppressWhitespaces 11 | from .AnnotateLeadingWhitespace import AnnotateLeadingWhitespace 12 | from .NormalizeWhitespaceBeforeToken import NormalizeWhitespaceBeforeToken 13 | from .NormalizeWhitespaceBeforeComments import NormalizeWhitespaceBeforeComments 14 | 15 | 16 | def transform(tokenseq: Sequence[TokenInfo]) -> Sequence[TokenInfo]: 17 | seq = DropToken(tokenseq) 18 | seq = AnnotateLeadingWhitespace(seq) 19 | seq = CollectComments(seq) 20 | seq = SuppressWhitespaces(seq) 21 | seq = InsertNewline(seq) 22 | seq = Reindent(seq) 23 | seq = NormalizeWhitespaceBeforeComments(seq) 24 | seq = NormalizeWhitespaceBeforeToken(seq) 25 | return seq 26 | 27 | 28 | if __name__ == "__main__": 29 | import sys 30 | from .tkutils.tokenize import tokenize 31 | 32 | with open(sys.argv[1], "rb") as fd: 33 | tokenseq = tokenize(fd.__next__) 34 | 35 | seq = transform(tokenseq) 36 | code = AsCode(seq) 37 | 38 | print(code) 39 | -------------------------------------------------------------------------------- /lambdex/fmt/jobs_meta.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from .utils.logger import getLogger 4 | 5 | logger = getLogger(__name__) 6 | 7 | 8 | class JobsMeta: 9 | 10 | __slots__ = ["adapter", "quiet", "in_place", "print_diff", "parallel", "files"] 11 | 12 | def __init__(self, adapter: str): 13 | self.adapter = adapter 14 | self.quiet = False 15 | self.in_place = False 16 | self.print_diff = False 17 | self.parallel = False 18 | self.files = [] 19 | 20 | @property 21 | def from_stdin(self): 22 | return not self.files 23 | -------------------------------------------------------------------------------- /lambdex/fmt/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/fmt/utils/__init__.py -------------------------------------------------------------------------------- /lambdex/fmt/utils/colored.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | colorful = os.getenv("NOCOLOR") is None 5 | if colorful: 6 | try: 7 | from termcolor import colored 8 | except ImportError: 9 | colorful = False 10 | 11 | if not colorful: 12 | colored = lambda msg, *_, **__: msg 13 | -------------------------------------------------------------------------------- /lambdex/fmt/utils/importlib.py: -------------------------------------------------------------------------------- 1 | import typing 2 | import importlib 3 | 4 | from lambdex.utils.compat import ModuleNotFoundError 5 | from .logger import getLogger 6 | 7 | logger = getLogger(__name__) 8 | 9 | 10 | def silent_import( 11 | dotted_name: str, 12 | names: typing.Optional[typing.Sequence[str]] = None, 13 | ) -> typing.Optional[typing.Any]: 14 | try: 15 | mod = importlib.import_module(dotted_name) 16 | except ModuleNotFoundError: 17 | logger.warning("module {} not found".format(dotted_name)) 18 | return None 19 | 20 | if names is None: 21 | return mod 22 | 23 | for name in names: 24 | if hasattr(mod, name): 25 | return getattr(mod, name) 26 | 27 | logger.warning( 28 | "attribute {} not found on module {}".format(", ".join(names), dotted_name) 29 | ) 30 | return None 31 | -------------------------------------------------------------------------------- /lambdex/fmt/utils/io.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import sys 3 | import difflib 4 | import pathlib 5 | from io import BytesIO 6 | 7 | from lambdex.fmt.jobs_meta import JobsMeta 8 | 9 | 10 | def get_stdin() -> bytes: 11 | contents = [] 12 | while True: 13 | if sys.stdin.closed: 14 | break 15 | try: 16 | content = sys.stdin.buffer.read() 17 | if not content: 18 | break 19 | contents.append(content) 20 | except EOFError: 21 | break 22 | except KeyboardInterrupt: 23 | sys.exit(1) 24 | 25 | return b"".join(contents) 26 | 27 | 28 | class _ResourceBase(abc.ABC): 29 | def __init__(self, jobs_meta: JobsMeta): 30 | self._meta = jobs_meta 31 | self._source = self._get_source() 32 | self._backend_output_stream = None 33 | 34 | @abc.abstractmethod 35 | def _get_source(self) -> bytes: 36 | ... 37 | 38 | @abc.abstractmethod 39 | def _display_filename(self) -> str: 40 | ... 41 | 42 | @property 43 | def source(self): 44 | return self._source 45 | 46 | def set_backend_output(self, output: bytes): 47 | self._backend_output_stream = BytesIO(output) 48 | 49 | @property 50 | def backend_output_stream(self) -> BytesIO: 51 | assert self._backend_output_stream is not None 52 | return self._backend_output_stream 53 | 54 | def is_changed(self, formatted_code: str) -> bool: 55 | return self._source.decode("utf-8") != formatted_code 56 | 57 | def write_formatted_code(self, formatted_code: str): 58 | content = formatted_code 59 | if self._meta.print_diff: 60 | before = self._source.decode("utf-8").splitlines() 61 | after = formatted_code.splitlines() 62 | content = ( 63 | "\n".join( 64 | difflib.unified_diff( 65 | before, 66 | after, 67 | self._display_filename(), 68 | self._display_filename(), 69 | "(original)", 70 | "(reformatted)", 71 | lineterm="", 72 | ) 73 | ) 74 | + "\n" 75 | ) 76 | elif not self._meta.in_place and not content.endswith("\n"): 77 | content += "\n" 78 | 79 | self._write_content(content) 80 | 81 | 82 | class StdinResource(_ResourceBase): 83 | def _get_source(self) -> bytes: 84 | return get_stdin() 85 | 86 | def _display_filename(self) -> str: 87 | return "" 88 | 89 | def _write_content(self, content: str): 90 | assert not self._meta.in_place 91 | if not self._meta.quiet: 92 | sys.stdout.write(content) 93 | 94 | 95 | class StringResource(_ResourceBase): 96 | def __init__(self, jobs_meta: JobsMeta, content: str): 97 | self._source = content 98 | super().__init__(jobs_meta) 99 | 100 | def _get_source(self) -> bytes: 101 | return self._source.encode() 102 | 103 | def _display_filename(self) -> str: 104 | return "<-c>" 105 | 106 | def _write_content(self, content: str): 107 | StdinResource._write_content(self, content) 108 | 109 | 110 | class FileResource(_ResourceBase): 111 | def __init__(self, jobs_meta: JobsMeta, filename: str): 112 | self._filename = filename 113 | self._filepath = pathlib.Path(filename) 114 | super(FileResource, self).__init__(jobs_meta) 115 | 116 | def _get_source(self) -> bytes: 117 | return self._filepath.read_bytes() 118 | 119 | def _display_filename(self) -> str: 120 | return self._filename 121 | 122 | def _write_content(self, content: str): 123 | if self._meta.in_place: 124 | self._filepath.write_text(content) 125 | elif not self._meta.quiet: 126 | sys.stdout.write(content) 127 | -------------------------------------------------------------------------------- /lambdex/fmt/utils/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | 5 | from .colored import colored 6 | 7 | IS_DEBUG = os.getenv("DEBUG") is not None 8 | 9 | 10 | class _Formatter(logging.Formatter): 11 | def __init__(self, *args, **kwargs): 12 | kwargs["style"] = "{" 13 | kwargs.setdefault( 14 | "fmt", 15 | colored("[{funcName}] ", "green") + "{message}", 16 | ) 17 | super(_Formatter, self).__init__(*args, **kwargs) 18 | 19 | def formatMessage(self, record): 20 | log = super(_Formatter, self).formatMessage(record) 21 | if record.levelno == logging.WARNING: 22 | prefix = colored("WARNING", "red", attrs=["blink"]) 23 | elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: 24 | prefix = colored("ERROR", "red", attrs=["blink", "underline"]) 25 | else: 26 | return log 27 | return prefix + " " + log 28 | 29 | 30 | class _Logger(logging.Logger): 31 | def findCaller(self, stack_info=False, stacklevel=1): 32 | """ 33 | Find the stack frame of the caller so that we can note the source 34 | file name, line number and function name. 35 | """ 36 | f = logging.currentframe() 37 | # On some versions of IronPython, currentframe() returns None if 38 | # IronPython isn't run with -X:Frames. 39 | if f is not None: 40 | f = f.f_back 41 | orig_f = f 42 | while f and stacklevel > 1: 43 | f = f.f_back 44 | stacklevel -= 1 45 | if not f: 46 | f = orig_f 47 | rv = "(unknown file)", 0, "(unknown function)", None 48 | while hasattr(f, "f_code"): 49 | co = f.f_code 50 | filename = os.path.normcase(co.co_filename) 51 | if filename == logging._srcfile: 52 | f = f.f_back 53 | continue 54 | sinfo = None 55 | if stack_info: 56 | sio = io.StringIO() 57 | sio.write("Stack (most recent call last):\n") 58 | traceback.print_stack(f, file=sio) 59 | sinfo = sio.getvalue() 60 | if sinfo[-1] == "\n": 61 | sinfo = sinfo[:-1] 62 | sio.close() 63 | 64 | localvars = f.f_locals 65 | classname_prefix = "" 66 | if "self" in localvars: 67 | classname_prefix = localvars["self"].__class__.__name__ + "::" 68 | 69 | rv = (co.co_filename, f.f_lineno, classname_prefix + co.co_name, sinfo) 70 | break 71 | return rv 72 | 73 | @property 74 | def is_debug(self): 75 | return IS_DEBUG 76 | 77 | def error(self, *args, **kwargs): 78 | super(_Logger, self).error(*args, **kwargs) 79 | sys.exit(2) 80 | 81 | 82 | def getLogger(name: str) -> _Logger: 83 | logger = _Logger(name) 84 | 85 | formatter = _Formatter() 86 | 87 | handler = logging.StreamHandler(sys.stderr) 88 | handler.setFormatter(formatter) 89 | handler.setLevel(logging.WARNING) 90 | logger.addHandler(handler) 91 | 92 | handler = logging.StreamHandler(sys.stdout) 93 | handler.setFormatter(formatter) 94 | handler.setLevel(logging.DEBUG if IS_DEBUG else logging.INFO) 95 | logger.addHandler(handler) 96 | 97 | return logger 98 | -------------------------------------------------------------------------------- /lambdex/keywords.py: -------------------------------------------------------------------------------- 1 | from . import ast_parser, compiler, _aliases 2 | 3 | aliases = _aliases.get_aliases() 4 | 5 | __all__ = [aliases.def_, aliases.async_def_] 6 | 7 | 8 | class Declarer: 9 | """ 10 | This class serves as an entry of defining (transpiling) a lambdex. Instances 11 | of `Declarer` can be attributed or called to form the syntax `()` 12 | or `.()`. 13 | """ 14 | 15 | __slots__ = ["__keyword", "__identifier", "func"] 16 | 17 | def __init__(self, keyword): 18 | self.__identifier = None 19 | self.__keyword = keyword 20 | self.func = None 21 | 22 | def __getattr__(self, identifier: str): 23 | """ 24 | Create a new `Declarer` instance with `self.__keyword` as keyword and `identifier` 25 | as identifier. 26 | """ 27 | if self.__identifier is not None: 28 | raise SyntaxError( 29 | "Duplicated name {!r} and {!r}".format(identifier, self.__identifier) 30 | ) 31 | 32 | if not identifier.isidentifier(): 33 | raise SyntaxError("{!r} is not valid identifier".format(identifier)) 34 | 35 | ret = Declarer(self.__keyword) 36 | ret.__identifier = identifier 37 | 38 | return ret 39 | 40 | def get_ast(self): 41 | """ 42 | Return the AST of `self.func`. 43 | 44 | This process requires `self.__keyword` and `self.__identifier` and thus can not be 45 | performed outside. 46 | """ 47 | return ast_parser.lambda_to_ast( 48 | self.func, keyword=self.__keyword, identifier=self.__identifier 49 | ) 50 | 51 | def __call__(self, f): 52 | """ 53 | Transpile `f` into ordinary function and returns it. 54 | """ 55 | self.func = f 56 | return compiler.compile_lambdex(self) 57 | 58 | def get_key(self): 59 | """ 60 | Construct a unique key for `self.func`. 61 | """ 62 | extra = () 63 | if self.func is not None: 64 | code_obj = self.func.__code__ 65 | extra = (code_obj.co_filename, code_obj.co_firstlineno, code_obj.co_code) 66 | 67 | return (self.__keyword, self.__identifier, *extra) 68 | 69 | 70 | globals()[aliases.def_] = Declarer(aliases.def_) 71 | globals()[aliases.async_def_] = Declarer(aliases.async_def_) 72 | -------------------------------------------------------------------------------- /lambdex/repl.py: -------------------------------------------------------------------------------- 1 | from .utils.repl_compat import patch 2 | 3 | # Export all keywords 4 | from ._exports import * 5 | from ._exports import __all__ 6 | 7 | # Patch the current REPL environment 8 | patch() 9 | -------------------------------------------------------------------------------- /lambdex/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/lambdex/utils/__init__.py -------------------------------------------------------------------------------- /lambdex/utils/ast.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import ast 3 | import inspect 4 | import textwrap 5 | 6 | from .ops import COMPARATORS_S2A 7 | 8 | try: 9 | import astpretty 10 | except ImportError: 11 | astpretty = None 12 | 13 | __all__ = [ 14 | "pprint", 15 | "pformat", 16 | "check", 17 | "value_from_subscript", 18 | "ast_from_source", 19 | "recursively_set_attr", 20 | "copy_lineinfo", 21 | "is_lvalue", 22 | "cast_to_ctx", 23 | "check_compare", 24 | "check_as", 25 | "is_coroutine_ast", 26 | "empty_arguments", 27 | "None_node", 28 | ] 29 | 30 | 31 | def pprint(ast_node) -> None: 32 | """ 33 | Pretty-print an AST node `ast_node`. 34 | """ 35 | recursively_set_attr(ast_node, "type_comment", "") 36 | astpretty.pprint(ast_node, show_offsets=False) 37 | 38 | 39 | def pformat(ast_node) -> str: 40 | """ 41 | Pretty-format an AST node `ast_node`, and return the string. 42 | """ 43 | recursively_set_attr(ast_node, "type_comment", "") 44 | return astpretty.pformat(ast_node, show_offsets=False) 45 | 46 | 47 | def check(node, ast_type): 48 | """ 49 | If `node` is not instance of `ast_type`, raise an error. 50 | """ 51 | assert isinstance(node, ast_type) 52 | 53 | 54 | def value_from_subscript(node: ast.Subscript, *, force_list=False, raise_=None): 55 | """ 56 | Extract value(s) from the brackets of `node`. 57 | 58 | If `force_list` is `True`, result will be guaranteed as a list. 59 | Otherwise the original value will be returned. 60 | """ 61 | 62 | def _raise_slice_error(): 63 | message = "':' is not allowed in '[]'" 64 | if callable(raise_): 65 | raise_(message, node.value) 66 | else: 67 | raise SyntaxError(message) 68 | 69 | slice_ = node.slice 70 | if isinstance(slice_, ast.Index): 71 | ret = slice_.value 72 | elif isinstance(slice_, ast.ExtSlice): 73 | for dim in slice_.dims: 74 | if isinstance(dim, ast.Slice): 75 | _raise_slice_error() 76 | ret = slice_.dims 77 | elif isinstance(slice_, ast.Tuple): 78 | for elt in slice_.elts: 79 | if isinstance(elt, ast.Slice): 80 | _raise_slice_error() 81 | ret = slice_ 82 | elif not isinstance(slice_, ast.Slice): 83 | ret = slice_ 84 | else: 85 | _raise_slice_error() 86 | 87 | if force_list: 88 | if isinstance(ret, ast.Tuple): 89 | ret = ret.elts 90 | 91 | if not isinstance(ret, (tuple, list)): 92 | ret = [ret] 93 | 94 | return ret 95 | 96 | 97 | def ast_from_source(source, keyword: str): 98 | """ 99 | Return the AST representation of `source`. `source` might be a function or 100 | string of source code. 101 | 102 | If `source` is a function, `keyword` is used to find the very start of its 103 | source code. 104 | """ 105 | if inspect.isfunction(source): 106 | lines, lnum = inspect.findsource(source.__code__) 107 | 108 | # Append line-ending if necessary 109 | for idx in range(len(lines)): 110 | if not lines[idx].endswith("\n"): 111 | lines[idx] += "\n" 112 | 113 | # Lines starting from `lnum` may contain enclosing tokens of previous expression 114 | # We use `keyword` to locate the true start point of source 115 | while True: 116 | first_keyword_loc = lines[lnum].find(keyword) 117 | if first_keyword_loc >= 0: 118 | break 119 | lnum -= 1 120 | 121 | lines = [lines[lnum][first_keyword_loc:]] + lines[lnum + 1 :] 122 | # Prepend the lines with newlines, so that parsed AST will have correct lineno 123 | source_lines = ["\n"] * lnum + inspect.getblock(lines) 124 | source = "".join(source_lines) 125 | 126 | # Some garbage may still remain at the end, we alternatively try compiling 127 | # and popping the last character until the source is valid. 128 | exc = None 129 | original_source = source 130 | while source: 131 | try: 132 | return ast.parse(source).body[0] 133 | except SyntaxError as e: 134 | source = source[:-1] 135 | exc = e 136 | else: 137 | # This should never happen 138 | raise RuntimeError( 139 | "cannot parse the snippet into AST:\n{}".format(original_source) 140 | ) from exc 141 | 142 | 143 | def recursively_set_attr(node: ast.AST, attrname: str, value): 144 | """ 145 | Recursively set attribute `attrname` to `value` on node and its children, 146 | if the field exists. 147 | """ 148 | for n in ast.walk(node): 149 | if attrname in n._fields: 150 | setattr(n, attrname, value) 151 | 152 | return node 153 | 154 | 155 | def copy_lineinfo(src: ast.AST, dst: ast.AST): 156 | """ 157 | Copy metadata of lineno and column offset from `src` to `dst`. 158 | """ 159 | for field in ("lineno", "col_offset", "end_lineno", "end_col_offset"): 160 | setattr(dst, field, getattr(src, field, None)) 161 | 162 | return dst 163 | 164 | 165 | def is_lvalue(node: ast.AST) -> bool: 166 | """ 167 | Check whether `node` can be L-value. 168 | """ 169 | from collections import deque 170 | 171 | todo = deque([node]) 172 | while todo: 173 | n = todo.popleft() 174 | if "ctx" not in n._fields: 175 | return False, n 176 | if isinstance(n, (ast.List, ast.Tuple, ast.Starred)): 177 | todo.extend( 178 | cn 179 | for cn in ast.iter_child_nodes(n) 180 | if not isinstance(cn, ast.expr_context) 181 | ) 182 | 183 | return True, None 184 | 185 | 186 | def cast_to_ctx(node: ast.AST, ctx=ast.Store()): 187 | """ 188 | Recursively set `ctx` to `Store()` on `node` and its children. This 189 | function assumes that `is_lvalue()` check has passed. 190 | 191 | The behavior ony propagates down to children with type `ast.List`, 192 | `ast.Tuple` and `ast.Starred`. e.g. name `attr` in `a[attr]` will 193 | not be set. 194 | """ 195 | from collections import deque 196 | 197 | todo = deque([node]) 198 | while todo: 199 | n = todo.popleft() 200 | n.ctx = ctx 201 | if isinstance(n, (ast.List, ast.Tuple, ast.Starred)): 202 | todo.extend(ast.iter_child_nodes(n)) 203 | 204 | return node 205 | 206 | 207 | def _expected_syntax_repr(type_, num): 208 | assert type_ in COMPARATORS_S2A 209 | op = COMPARATORS_S2A[type_] 210 | 211 | if num is None: 212 | return "... {op} ... [{op} ...]".format(op=op) 213 | else: 214 | return " {op} ".format(op=op).join(["..."] * num) 215 | 216 | 217 | def check_compare(ctx, node: ast.Compare, expected_type, expected_num=None): 218 | """ 219 | Check that `node.ops` are all with type `expected_type`. If 220 | `expected_num` given, also check that `node` has `expected_num` 221 | operands. 222 | 223 | Return a tuple of all operands of `node`. 224 | """ 225 | syntax_repr = _expected_syntax_repr(expected_type, expected_num) 226 | op_repr = COMPARATORS_S2A[expected_type] 227 | ctx.assert_is_instance(node, ast.Compare, "expect {!r}".format(syntax_repr)) 228 | 229 | for idx, op in enumerate(node.ops): 230 | ctx.assert_( 231 | isinstance(op, expected_type), 232 | "expect {!r} before".format(op_repr), 233 | node.comparators[idx], 234 | ) 235 | 236 | if expected_num is not None: 237 | assert expected_num == 2 238 | ctx.assert_( 239 | expected_num == len(node.ops) + 1, 240 | "too many operands", 241 | lambda: node.comparators[expected_num - 1], 242 | ) 243 | 244 | return (node.left, *node.comparators) 245 | 246 | 247 | def check_as(ctx, node: ast.expr, as_op, *, rhs_is_identifier=False): 248 | """ 249 | Check that `node` has pattern `lhs > rhs`. Return `(lhs, rhs)` 250 | if matched, otherwise `(any, None)`. 251 | 252 | If `rhs_is_identifier` is `True`, `rhs` will be converted to L-value. 253 | """ 254 | if not isinstance(node, ast.Compare): 255 | return node, None 256 | 257 | lhs, rhs = check_compare(ctx, node, as_op, 2) 258 | 259 | if rhs_is_identifier: 260 | ctx.assert_is_instance(rhs, ast.Name, "expect identifier") 261 | return lhs, rhs.id 262 | else: 263 | ctx.assert_lvalue(rhs) 264 | return lhs, cast_to_ctx(rhs) 265 | 266 | 267 | def is_coroutine_ast(x): 268 | """ 269 | Check if `x` is coroutine AST node or AST type. 270 | """ 271 | if isinstance(x, ast.AST): 272 | x = type(x) 273 | return x in (ast.AsyncFunctionDef, ast.AsyncWith, ast.AsyncFor, ast.Await) 274 | 275 | 276 | empty_arguments = ast.arguments( 277 | posonlyargs=[], 278 | args=[], 279 | vararg=None, 280 | kwonlyargs=[], 281 | kw_defaults=[], 282 | kwarg=None, 283 | defaults=[], 284 | ) 285 | 286 | None_node = ast.parse("None", "", mode="eval").body 287 | -------------------------------------------------------------------------------- /lambdex/utils/compat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import enum 3 | import types 4 | 5 | if sys.version_info < (3, 6): 6 | ModuleNotFoundError = ImportError 7 | else: 8 | ModuleNotFoundError = ModuleNotFoundError 9 | 10 | if sys.version_info < (3, 8): 11 | _code_fields = [ 12 | "argcount", 13 | "kwonlyargcount", 14 | "nlocals", 15 | "stacksize", 16 | "flags", 17 | "codestring", 18 | "consts", 19 | "names", 20 | "varnames", 21 | "filename", 22 | "name", 23 | "firstlineno", 24 | "lnotab", 25 | "freevars", 26 | "cellvars", 27 | ] 28 | 29 | _code_a2f = {n: "co_" + n for n in _code_fields} 30 | _code_a2f["codestring"] = "co_code" 31 | _code_f2a = {v: k for k, v in _code_a2f.items()} 32 | 33 | def code_replace(code_obj, **kwargs): 34 | arguments = {} 35 | for field in _code_fields: 36 | arguments[field] = getattr(code_obj, _code_a2f[field]) 37 | 38 | for key, value in kwargs.items(): 39 | assert key.startswith("co_") 40 | arguments[_code_f2a[key]] = value 41 | 42 | arguments = [arguments[n] for n in _code_fields] 43 | 44 | return types.CodeType(*arguments) 45 | 46 | 47 | else: 48 | code_replace = types.CodeType.replace 49 | 50 | if sys.version_info < (3, 6): 51 | 52 | def enum_auto(): 53 | _counter = 0 54 | 55 | def _auto(): 56 | nonlocal _counter 57 | _counter += 1 58 | return _counter 59 | 60 | return _auto 61 | 62 | 63 | else: 64 | 65 | def enum_auto(): 66 | return enum.auto 67 | -------------------------------------------------------------------------------- /lambdex/utils/ops.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | # Mapping from operator string to AST node type 4 | COMPARATORS = { 5 | "<": ast.Lt, 6 | "<=": ast.LtE, 7 | ">": ast.Gt, 8 | ">=": ast.GtE, 9 | "in": ast.In, 10 | } 11 | 12 | COMPARATORS_S2A = {v: k for k, v in COMPARATORS.items()} 13 | -------------------------------------------------------------------------------- /lambdex/utils/registry.py: -------------------------------------------------------------------------------- 1 | __all__ = ["FunctionRegistry"] 2 | 3 | 4 | class FunctionRegistry: 5 | """ 6 | A registry with values being functions. 7 | """ 8 | 9 | _empty = object() 10 | 11 | def __init__(self, name: str): 12 | self.name = name 13 | self.mapping = {} 14 | self.__default = self._empty 15 | 16 | def register(self, key, value=_empty): 17 | """ 18 | If `value` provided, register it with key `key`. 19 | Otherwise return a decorator. 20 | """ 21 | if value is not self._empty: 22 | if not callable(value): 23 | value = lambda: value 24 | self.mapping[key] = value 25 | return 26 | 27 | def _decorator(f): 28 | self.mapping[key] = f 29 | return f 30 | 31 | return _decorator 32 | 33 | def get(self, key, default=_empty): 34 | """ 35 | Return corresponding value of key `key`. 36 | 37 | If not found and `default` provided, `default` is returned. 38 | Otherwise, if default value of `self` set, the value is returned. 39 | Otherwise, raise a KeyError. 40 | """ 41 | value = self.mapping.get(key, self._empty) 42 | if value is not self._empty: 43 | return value 44 | if default is not self._empty: 45 | return default 46 | if self.__default is not self._empty: 47 | return self.__default 48 | raise KeyError(key) 49 | 50 | def set_default(self, value): 51 | """ 52 | Set `value` as the default value of the registry. 53 | """ 54 | self.__default = value 55 | return self 56 | -------------------------------------------------------------------------------- /lambdex/utils/repl_compat/__init__.py: -------------------------------------------------------------------------------- 1 | from . import checks 2 | 3 | 4 | def _noop(): 5 | pass 6 | 7 | 8 | if checks.is_idle: 9 | # Patch Python IDLE 10 | from .idle import patch 11 | elif checks.is_ipython: 12 | # Check IPython before builtin, since `is_ipython is True` implies `is_builtin is True` 13 | 14 | # IPython requires no patching 15 | patch = _noop 16 | elif checks.is_builtin: 17 | # Patch Python builtin REPL 18 | from .builtin import patch 19 | else: 20 | # Otherwise, do nothing 21 | patch = _noop 22 | 23 | del _noop 24 | -------------------------------------------------------------------------------- /lambdex/utils/repl_compat/builtin.py: -------------------------------------------------------------------------------- 1 | """ 2 | REPL patch for builtin Python REPL. 3 | 4 | Since the builtin interactive loop is implemented in C and there is no 5 | approach to hack the input, we decide to start a custom interactive 6 | console that takes over the loop. 7 | """ 8 | 9 | import sys 10 | import inspect 11 | import linecache 12 | import code as blcode 13 | import __main__ 14 | 15 | _lines_cache = {} 16 | 17 | from ... import _exports 18 | 19 | 20 | class InteractiveConsole(blcode.InteractiveConsole): 21 | def __init__(self): 22 | locals = __main__.__dict__ 23 | # Add keywords to globals dict 24 | # This ensures def_ is available after `from lambdex.repl import *`, 25 | # since the statements will never return 26 | locals.update({k: getattr(_exports, k) for k in _exports.__all__}) 27 | 28 | super(InteractiveConsole, self).__init__(locals, "") 29 | self._counter = 1 30 | 31 | def runsource(self, source, _): 32 | """ 33 | Overwrite to cache input lines and setting filename. 34 | """ 35 | filename = self.filename.format(self._counter) 36 | _lines_cache[filename] = source.splitlines() 37 | 38 | more = super(InteractiveConsole, self).runsource(source, filename) 39 | if not more: 40 | # The counter will not increase until the source is complete 41 | self._counter += 1 42 | 43 | return more 44 | 45 | def interact(self): 46 | """ 47 | Overwrite to directly exit the process after interact() ends. 48 | """ 49 | kwargs = dict(banner="") 50 | if sys.version_info > (3, 5, float("inf")): 51 | kwargs["exitmsg"] = "" 52 | super(InteractiveConsole, self).interact(**kwargs) 53 | sys.exit(0) 54 | 55 | 56 | def _extended_linecache_getlines( 57 | filename, 58 | globals_dict=None, 59 | orig_getlines=linecache.getlines, # Store the old function here 60 | ): 61 | """ 62 | Patch `linecache.getlines` to obtain lines from `_lines_cache`. 63 | """ 64 | if filename.startswith(" 2 else 2: [ 60 | pass_ 61 | ]) 62 | 63 | self.assert_has_n_lambdex(f, 1) 64 | 65 | def test_arg_default_logic_folded(self): 66 | def f(): 67 | def_(lambda a=1 or 2, b=1 and 2, c=1 or 2 and 3: [ 68 | pass_ 69 | ]) 70 | 71 | self.assert_has_n_lambdex(f, 1) 72 | 73 | def test_arg_default_logic_unfolded(self): 74 | def f(): 75 | def_(lambda a=1 + 1 > 2 or 2, b=1 + 1 > 2 and 2, c=a or b or c: [ 76 | pass_ 77 | ]) 78 | 79 | self.assert_has_n_lambdex(f, 1) 80 | 81 | def test_arg_default_lambda(self): 82 | def f(): 83 | def_(lambda a=lambda: 1: [ 84 | pass_ 85 | ]) 86 | 87 | self.assert_has_n_lambdex(f, 1) 88 | 89 | def test_arg_default_lambda_call(self): 90 | def f(): 91 | def_(lambda a=a(lambda: 1)(): [ 92 | pass_ 93 | ]) 94 | 95 | self.assert_has_n_lambdex(f, 1) 96 | 97 | def test_arg_default_lambdex(self): 98 | def f(): 99 | def_(lambda a=def_(lambda: [ 100 | pass_ 101 | ]): [ 102 | pass_ 103 | ]) 104 | 105 | self.assert_has_n_lambdex(f, 2) 106 | 107 | def test_arg_default_lambdex_x2(self): 108 | def f(): 109 | def_(lambda a=def_(lambda: [ 110 | pass_ 111 | ]), b=def_(lambda: [ 112 | pass_ 113 | ]): [ 114 | pass_ 115 | ]) 116 | 117 | self.assert_has_n_lambdex(f, 3) 118 | 119 | def test_arg_default_lambdex_arg_default_lambdex(self): 120 | def f(): 121 | def_(lambda a=def_(lambda a=def_(lambda: [ 122 | pass_ 123 | ]): [ 124 | pass_ 125 | ]): [ 126 | pass_ 127 | ]) 128 | 129 | self.assert_has_n_lambdex(f, 3) 130 | 131 | def test_complex(self): 132 | def f(): 133 | def_(lambda a=def_(lambda a=def_(lambda: [ 134 | pass_ 135 | ]), b=def_(lambda: [ 136 | pass_ 137 | ]), c=a if b else c or d and e or f: [ 138 | pass_ 139 | ]): [ 140 | pass_ 141 | ]) 142 | 143 | self.assert_has_n_lambdex(f, 4) 144 | -------------------------------------------------------------------------------- /tests/asm/test_frontend.py: -------------------------------------------------------------------------------- 1 | import os.path as osp 2 | import sys 3 | import dis 4 | import unittest 5 | import importlib 6 | 7 | import lambdex 8 | from lambdex import asmopt 9 | from lambdex.compiler.asm import frontend 10 | 11 | 12 | class TestFrontend(unittest.TestCase): 13 | def setUp(self): 14 | self.old_path = sys.path 15 | sys.path = [osp.dirname(__file__)] + sys.path 16 | 17 | self.old_def_ = lambdex.def_ 18 | frontend._done_queue = frontend.SimpleQueue() 19 | 20 | def tearDown(self): 21 | sys.path = self.old_path 22 | lambdex.def_ = self.old_def_ 23 | frontend._done_queue = None 24 | 25 | def test_module(self): 26 | 27 | lambdex.def_ = None 28 | with self.assertRaises(TypeError): 29 | import sample 30 | 31 | lambdex.def_ = self.old_def_ 32 | import sample 33 | self.assertEqual(sample.s, 4950) 34 | 35 | self.assertEqual(frontend._done_queue.get(), (sample.__file__, 'ok')) 36 | 37 | lambdex.def_ = None 38 | sample = importlib.reload(sample) 39 | self.assertEqual(sample.s, 4950) 40 | 41 | def test_asmopt(self): 42 | var = 1 43 | 44 | @asmopt 45 | def f(): 46 | var2 = 3 47 | a = def_.a(lambda: [ 48 | nonlocal_[var], 49 | var < var + var2, 50 | return_[callee_] 51 | ]) 52 | b = def_.b(lambda: [ 53 | nonlocal_[var], 54 | var < var * var2, 55 | return_[callee_] 56 | ]) 57 | c = def_.c(lambda i: [ 58 | nonlocal_[var], 59 | var < var + i, 60 | return_[callee_] 61 | ]) 62 | return a, b, c 63 | 64 | # import dis 65 | # dis.dis(f) 66 | a, b, c = f() 67 | ca = a() 68 | self.assertEqual(var, 4) 69 | self.assertIs(a, ca) 70 | cb = b() 71 | self.assertEqual(var, 12) 72 | self.assertIs(b, cb) 73 | cc = c(1) 74 | self.assertEqual(var, 13) 75 | self.assertIs(c, cc) 76 | -------------------------------------------------------------------------------- /tests/asm/test_transpile.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import dis 3 | import unittest 4 | import traceback 5 | import linecache 6 | from pathlib import Path 7 | from textwrap import dedent 8 | from lambdex.compiler.asm.core import transpile 9 | 10 | from lambdex import def_ 11 | 12 | 13 | class TestTranspile(unittest.TestCase): 14 | def setUp(self): 15 | self.orig_getlines = linecache.getlines 16 | 17 | def _mock_getlines(filename, g=None): 18 | if filename == '': 19 | return self.lines 20 | return self.orig_getlines(filename, g) 21 | 22 | linecache.getlines = _mock_getlines 23 | 24 | def tearDown(self): 25 | linecache.getlines = self.orig_getlines 26 | 27 | def makecode(self, source: str): 28 | source = dedent(source).strip() 29 | self.lines = source.splitlines(keepends=True) 30 | 31 | code1 = compile(source, '', 'exec') 32 | code2 = transpile(code1, ismod=True) 33 | 34 | return code1, code2 35 | 36 | def test_case1(self): 37 | source = ''' 38 | a = 1 39 | b = 2 40 | f = def_(lambda a=1 if 1 + 1 > 2 else 2, b=def_.a(lambda: [return_[callee_]]), c=a+b and b-a-a or a if a else b: [ 41 | return_[a, b, c, callee_] 42 | ]) 43 | ''' 44 | 45 | code1, code2 = self.makecode(source) 46 | 47 | g = {'def_': def_} 48 | exec(code1, g) 49 | g = {} 50 | exec(code2, g) 51 | f = g['f'] 52 | 53 | two, b, one, f_callee = f() 54 | self.assertEqual(one, 1) 55 | self.assertEqual(two, 2) 56 | self.assertIs(f, f_callee) 57 | self.assertEqual(b.__name__, 'a') 58 | self.assertIs(b, b()) 59 | 60 | def test_case2(self): 61 | source = ''' 62 | def func(): 63 | return 1 64 | 65 | def gen_f(): 66 | a = 1 67 | b = 2 68 | return def_(lambda a=1 if 1 + 1 > 2 else 2, b=def_.a(lambda: [return_[callee_]]), c=a+b and b-a-a or a if a else b: [ 69 | return_[a, b, c, callee_] 70 | ]) 71 | ''' 72 | 73 | code1, code2 = self.makecode(source) 74 | 75 | g = {'def_': def_} 76 | exec(code1, g) 77 | g = {} 78 | exec(code2, g) 79 | gen_f = g['gen_f'] 80 | func = g['func'] 81 | 82 | self.assertEqual(func(), 1) 83 | f = gen_f() 84 | two, b, one, f_callee = f() 85 | self.assertEqual(one, 1) 86 | self.assertEqual(two, 2) 87 | self.assertIs(f, f_callee) 88 | self.assertEqual(b.__name__, 'a') 89 | self.assertIs(b, b()) 90 | 91 | def test_runtime_error(self): 92 | source = ''' 93 | def_(lambda b=1/0 if a==0 else 1: [ 94 | if_[a == 1] [1/0 95 | ].elif_[a==2] [ 96 | 1/0 97 | ].elif_[a==3] [ 98 | 99 | 1/0] 100 | ])() 101 | 102 | if a == 4: 103 | 1/0 104 | 105 | f=def_(lambda 106 | b=1/0 if a == 5 else 0: [pass_]) 107 | 108 | f() 109 | 110 | {stmt_bundles} 111 | 112 | if a == 6: 113 | 1/0 114 | '''.format(stmt_bundles='var=1;' * 1000) 115 | 116 | def assert_error_at(a, lineno): 117 | try: 118 | self.assertRaises(NonexistantError, exec, code2, {'a': a}) 119 | except ZeroDivisionError: 120 | _, _, tb = sys.exc_info() 121 | tb = traceback.extract_tb(tb) 122 | self.assertEqual(tb[-1].lineno, lineno, 'a={}'.format(a)) 123 | 124 | _, code2 = self.makecode(source) 125 | for a, lineno in enumerate([1, 2, 4, 7, 11, 14, 21]): 126 | assert_error_at(a, lineno) 127 | 128 | 129 | class NonexistantError(Exception): 130 | pass 131 | -------------------------------------------------------------------------------- /tests/compiler/aliases/.lambdex.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | 3 | def_ = Def 4 | if_ = If 5 | else_ = Else 6 | Assignment = <= -------------------------------------------------------------------------------- /tests/compiler/aliases/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/aliases/__init__.py -------------------------------------------------------------------------------- /tests/compiler/aliases/test_aliases.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import importlib 4 | import unittest 5 | 6 | from astcheck import assert_ast_like 7 | 8 | Def = None 9 | 10 | 11 | def _reload_lambdex(): 12 | for modname in sorted(filter(lambda x: "lambdex" in x, sys.modules)): 13 | del sys.modules[modname] 14 | 15 | 16 | class TestAliases(unittest.TestCase): 17 | def setUp(self): 18 | os.environ["LXALIAS"] = "1" 19 | _reload_lambdex() 20 | 21 | from lambdex import Def as d 22 | from lambdex.compiler import core 23 | 24 | global Def 25 | Def = d 26 | 27 | core.__DEBUG__ = True 28 | 29 | def tearDown(self): 30 | from lambdex.compiler import core 31 | 32 | core.__DEBUG__ = False 33 | 34 | del os.environ["LXALIAS"] 35 | _reload_lambdex() 36 | 37 | def assert_ast_like(self, f, target): 38 | from lambdex.utils.ast import ( 39 | ast_from_source, 40 | pformat, 41 | pprint, 42 | recursively_set_attr, 43 | ) 44 | 45 | ast_f = f.__ast__ 46 | recursively_set_attr(ast_f, "type_comment", None) 47 | ast_target = ast_from_source(target, "def") 48 | ast_target.name = ast_f.name 49 | 50 | try: 51 | assert_ast_like(ast_f, ast_target) 52 | except AssertionError as cause: 53 | msg = "\n".join( 54 | [ 55 | "", 56 | "===> Compiled:", 57 | pformat(ast_f), 58 | "===> Target:", 59 | pformat(ast_target), 60 | ] 61 | ) 62 | raise AssertionError(msg) from cause 63 | 64 | def test_aliases(self): 65 | f = Def(lambda: [ 66 | If[True] [ 67 | a <= 1 68 | ].Else[ 69 | a <= 2 70 | ] 71 | ]) 72 | 73 | def target(): 74 | if True: 75 | a = 1 76 | else: 77 | a = 2 78 | 79 | self.assert_ast_like(f, target) 80 | -------------------------------------------------------------------------------- /tests/compiler/await_attribute/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/await_attribute/__init__.py -------------------------------------------------------------------------------- /tests/compiler/await_attribute/disabled/.lambdex.cfg: -------------------------------------------------------------------------------- 1 | [features] 2 | await_attribute = off -------------------------------------------------------------------------------- /tests/compiler/await_attribute/disabled/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/await_attribute/disabled/__init__.py -------------------------------------------------------------------------------- /tests/compiler/await_attribute/disabled/test_await_attribute.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import importlib 3 | import unittest 4 | 5 | from astcheck import assert_ast_like 6 | 7 | 8 | def _reload_lambdex(): 9 | for modname in sorted(filter(lambda x: "lambdex" in x, sys.modules)): 10 | del sys.modules[modname] 11 | 12 | 13 | async_def_ = None 14 | 15 | 16 | class TestAwaitAttribute(unittest.TestCase): 17 | def setUp(self): 18 | _reload_lambdex() 19 | 20 | from lambdex import async_def_ as d 21 | from lambdex.compiler import core 22 | 23 | global async_def_ 24 | async_def_ = d 25 | 26 | core.__DEBUG__ = True 27 | 28 | def tearDown(self): 29 | from lambdex.compiler import core 30 | 31 | core.__DEBUG__ = False 32 | 33 | _reload_lambdex() 34 | 35 | def assert_ast_like(self, f, target): 36 | from lambdex.utils.ast import ( 37 | ast_from_source, 38 | pformat, 39 | pprint, 40 | recursively_set_attr, 41 | ) 42 | 43 | ast_f = f.__ast__ 44 | recursively_set_attr(ast_f, "type_comment", None) 45 | ast_target = ast_from_source(target, "async def") 46 | ast_target.name = ast_f.name 47 | 48 | try: 49 | assert_ast_like(ast_f, ast_target) 50 | except AssertionError as cause: 51 | msg = "\n".join( 52 | [ 53 | "", 54 | "===> Compiled:", 55 | pformat(ast_f), 56 | "===> Target:", 57 | pformat(ast_target), 58 | ] 59 | ) 60 | raise AssertionError(msg) from cause 61 | 62 | def test_await_attribute(self): 63 | f = async_def_(lambda: [ 64 | a.await_.b.c.await_.d 65 | ]) 66 | 67 | async def target(): 68 | a.await_.b.c.await_.d 69 | 70 | self.assert_ast_like(f, target) 71 | -------------------------------------------------------------------------------- /tests/compiler/await_attribute/enabled/.lambdex.cfg: -------------------------------------------------------------------------------- 1 | [features] 2 | await_attribute = on -------------------------------------------------------------------------------- /tests/compiler/await_attribute/enabled/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/await_attribute/enabled/__init__.py -------------------------------------------------------------------------------- /tests/compiler/await_attribute/enabled/test_await_attribute.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import importlib 3 | import unittest 4 | 5 | from astcheck import assert_ast_like 6 | 7 | 8 | def _reload_lambdex(): 9 | for modname in sorted(filter(lambda x: "lambdex" in x, sys.modules)): 10 | del sys.modules[modname] 11 | 12 | 13 | async_def_ = None 14 | 15 | 16 | class TestAwaitAttribute(unittest.TestCase): 17 | def setUp(self): 18 | _reload_lambdex() 19 | 20 | from lambdex import async_def_ as d 21 | from lambdex.compiler import core 22 | 23 | global async_def_ 24 | async_def_ = d 25 | 26 | core.__DEBUG__ = True 27 | 28 | def tearDown(self): 29 | from lambdex.compiler import core 30 | 31 | core.__DEBUG__ = False 32 | 33 | _reload_lambdex() 34 | 35 | def assert_ast_like(self, f, target): 36 | from lambdex.utils.ast import ( 37 | ast_from_source, 38 | pformat, 39 | pprint, 40 | recursively_set_attr, 41 | ) 42 | 43 | ast_f = f.__ast__ 44 | recursively_set_attr(ast_f, "type_comment", None) 45 | ast_target = ast_from_source(target, "async def") 46 | ast_target.name = ast_f.name 47 | 48 | try: 49 | assert_ast_like(ast_f, ast_target) 50 | except AssertionError as cause: 51 | msg = "\n".join( 52 | [ 53 | "", 54 | "===> Compiled:", 55 | pformat(ast_f), 56 | "===> Target:", 57 | pformat(ast_target), 58 | ] 59 | ) 60 | raise AssertionError(msg) from cause 61 | 62 | def test_await_attribute(self): 63 | f = async_def_(lambda: [ 64 | a.await_.b.c.await_.d 65 | ]) 66 | 67 | async def target(): 68 | (await (await a).b.c).d 69 | 70 | self.assert_ast_like(f, target) 71 | -------------------------------------------------------------------------------- /tests/compiler/implicit_return/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/implicit_return/__init__.py -------------------------------------------------------------------------------- /tests/compiler/implicit_return/disabled/.lambdex.cfg: -------------------------------------------------------------------------------- 1 | [features] 2 | implicit_return = off -------------------------------------------------------------------------------- /tests/compiler/implicit_return/disabled/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/implicit_return/disabled/__init__.py -------------------------------------------------------------------------------- /tests/compiler/implicit_return/disabled/test_implicit_return.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import importlib 3 | import unittest 4 | 5 | from astcheck import assert_ast_like 6 | 7 | 8 | def _reload_lambdex(): 9 | for modname in sorted(filter(lambda x: "lambdex" in x, sys.modules)): 10 | del sys.modules[modname] 11 | 12 | 13 | async_def_ = None 14 | 15 | 16 | class TestImplicitReturn(unittest.TestCase): 17 | def setUp(self): 18 | _reload_lambdex() 19 | 20 | from lambdex import async_def_ as d 21 | from lambdex.compiler import core 22 | 23 | global async_def_ 24 | async_def_ = d 25 | 26 | core.__DEBUG__ = True 27 | 28 | def tearDown(self): 29 | from lambdex.compiler import core 30 | 31 | core.__DEBUG__ = False 32 | 33 | _reload_lambdex() 34 | 35 | def assert_ast_like(self, f, target): 36 | from lambdex.utils.ast import ( 37 | ast_from_source, 38 | pformat, 39 | pprint, 40 | recursively_set_attr, 41 | ) 42 | 43 | ast_f = f.__ast__ 44 | recursively_set_attr(ast_f, "type_comment", None) 45 | ast_target = ast_from_source(target, "async def") 46 | ast_target.name = ast_f.name 47 | 48 | try: 49 | assert_ast_like(ast_f, ast_target) 50 | except AssertionError as cause: 51 | msg = "\n".join( 52 | [ 53 | "", 54 | "===> Compiled:", 55 | pformat(ast_f), 56 | "===> Target:", 57 | pformat(ast_target), 58 | ] 59 | ) 60 | raise AssertionError(msg) from cause 61 | 62 | def test_implicit_return(self): 63 | f = async_def_(lambda: [ 64 | 1 65 | ]) 66 | 67 | async def target(): 68 | 1 69 | 70 | self.assert_ast_like(f, target) 71 | -------------------------------------------------------------------------------- /tests/compiler/implicit_return/enabled/.lambdex.cfg: -------------------------------------------------------------------------------- 1 | [features] 2 | implicit_return = on -------------------------------------------------------------------------------- /tests/compiler/implicit_return/enabled/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hsfzxjy/lambdex/64fae676063d16af4f861441e825740c5696001d/tests/compiler/implicit_return/enabled/__init__.py -------------------------------------------------------------------------------- /tests/compiler/implicit_return/enabled/test_implicit_return.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import importlib 3 | import unittest 4 | 5 | from astcheck import assert_ast_like 6 | 7 | 8 | def _reload_lambdex(): 9 | for modname in sorted(filter(lambda x: "lambdex" in x, sys.modules)): 10 | del sys.modules[modname] 11 | 12 | 13 | async_def_ = None 14 | 15 | 16 | class TestImplicitReturn(unittest.TestCase): 17 | def setUp(self): 18 | _reload_lambdex() 19 | 20 | from lambdex import async_def_ as d 21 | from lambdex.compiler import core 22 | 23 | global async_def_ 24 | async_def_ = d 25 | 26 | core.__DEBUG__ = True 27 | 28 | def tearDown(self): 29 | from lambdex.compiler import core 30 | 31 | core.__DEBUG__ = False 32 | 33 | _reload_lambdex() 34 | 35 | def assert_ast_like(self, f, target): 36 | from lambdex.utils.ast import ( 37 | ast_from_source, 38 | pformat, 39 | pprint, 40 | recursively_set_attr, 41 | ) 42 | 43 | ast_f = f.__ast__ 44 | recursively_set_attr(ast_f, "type_comment", None) 45 | ast_target = ast_from_source(target, "async def") 46 | ast_target.name = ast_f.name 47 | 48 | try: 49 | assert_ast_like(ast_f, ast_target) 50 | except AssertionError as cause: 51 | msg = "\n".join( 52 | [ 53 | "", 54 | "===> Compiled:", 55 | pformat(ast_f), 56 | "===> Target:", 57 | pformat(ast_target), 58 | ] 59 | ) 60 | raise AssertionError(msg) from cause 61 | 62 | def test_implicit_return_last_expr(self): 63 | f = async_def_(lambda: [ 64 | 1 65 | ]) 66 | 67 | async def target(): 68 | return 1 69 | 70 | self.assert_ast_like(f, target) 71 | 72 | def test_implicit_return_last_stmt(self): 73 | f = async_def_(lambda: [ 74 | for_[a in b] [ 75 | c 76 | ] 77 | ]) 78 | 79 | async def target(): 80 | for a in b: 81 | c 82 | 83 | self.assert_ast_like(f, target) 84 | 85 | def test_implicit_return_last_assignment(self): 86 | f = async_def_(lambda: [ 87 | a < 1 88 | ]) 89 | 90 | async def target(): 91 | a = 1 92 | 93 | self.assert_ast_like(f, target) 94 | -------------------------------------------------------------------------------- /tests/compiler/test_cache.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lambdex.keywords import def_ 4 | from lambdex.compiler import cache, core 5 | 6 | 7 | class TestCacheEnableDisable(unittest.TestCase): 8 | def setUp(self): 9 | core.__DEBUG__ = True 10 | 11 | def tearDown(self): 12 | cache.set_enabled(True) 13 | core.__DEBUG__ = False 14 | 15 | def test_code_and_ast_is_same_when_cache_enabled(self): 16 | def f(): 17 | return def_(lambda: [ 18 | return_[1 + 1], 19 | ]) 20 | 21 | cache.set_enabled(True) 22 | f1 = f() 23 | f2 = f() 24 | self.assertEqual(f1.__code__.co_name, f2.__code__.co_name) 25 | self.assertIs(f1.__ast__, f2.__ast__) 26 | 27 | def test_code_and_ast_is_different_when_cache_disabled(self): 28 | def f(): 29 | return def_(lambda: [ 30 | return_[1 + 1], 31 | ]) 32 | 33 | cache.set_enabled(False) 34 | f1 = f() 35 | f2 = f() 36 | self.assertNotEqual(f1.__code__.co_name, f2.__code__.co_name) 37 | self.assertIsNot(f1.__ast__, f2.__ast__) 38 | 39 | 40 | class TestEdgeCase(unittest.TestCase): 41 | def setUp(self): 42 | core.__DEBUG__ = True 43 | 44 | def tearDown(self): 45 | core.__DEBUG__ = False 46 | 47 | def test_lambdex_on_the_same_line_should_be_different(self): 48 | def f(): 49 | # lxfmt: off 50 | return def_.a(lambda: [return_[1 + 1]]), def_(lambda: [return_[1 + 2]]) 51 | # lxfmt: on 52 | 53 | f1, f2 = f() 54 | self.assertNotEqual(f1.__code__.co_name, f2.__code__.co_name) 55 | self.assertIsNot(f1.__ast__, f2.__ast__) 56 | -------------------------------------------------------------------------------- /tests/compiler/test_renames.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lambdex import def_ 4 | 5 | 6 | class TestRename(unittest.TestCase): 7 | def test_outer_rename(self): 8 | f = def_.myfunc(lambda: [ 9 | pass_ 10 | ]) 11 | 12 | self.assertEqual(f.__code__.co_name, "myfunc") 13 | 14 | def test_inner_rename(self): 15 | f = def_.myfunc(lambda: [ 16 | return_[def_.myfunc_inner(lambda: [ 17 | pass_ 18 | ])] 19 | ]) 20 | 21 | self.assertEqual(f().__code__.co_name, "myfunc_inner") 22 | 23 | def test_inner_rename_not_expose(self): 24 | f = def_.myfunc(lambda: [ 25 | a < def_.myfunc_inner(lambda: [ 26 | pass_ 27 | ]), 28 | myfunc_inner, 29 | ]) 30 | 31 | with self.assertRaises(NameError) as cm: 32 | f() 33 | -------------------------------------------------------------------------------- /tests/compiler/test_scoping.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lambdex.keywords import def_ 4 | 5 | VAR = 1 6 | 7 | 8 | class TestScoping(unittest.TestCase): 9 | def setUp(self): 10 | global VAR 11 | VAR = 1 12 | 13 | def test_load_global_without_keyword(self): 14 | f = def_(lambda: [ 15 | return_[VAR], 16 | ]) 17 | 18 | self.assertEqual(f(), 1) 19 | 20 | def test_load_global_with_keyword(self): 21 | f = def_(lambda: [ 22 | global_[VAR], 23 | return_[VAR], 24 | ]) 25 | 26 | self.assertEqual(f(), 1) 27 | 28 | def test_set_global(self): 29 | f = def_(lambda: [ 30 | global_[VAR], 31 | VAR < 2, 32 | return_[VAR], 33 | ]) 34 | 35 | self.assertEqual(f(), 2) 36 | self.assertEqual(VAR, 2) 37 | 38 | def test_set_global_fail_of_no_global_keyword(self): 39 | f = def_(lambda: [ 40 | VAR < 2, 41 | return_[VAR], 42 | ]) 43 | 44 | self.assertEqual(f(), 2) 45 | self.assertEqual(VAR, 1) 46 | 47 | def test_load_nonlocal_without_keyword(self): 48 | VAR = 2 49 | f = def_(lambda: [ 50 | return_[VAR], 51 | ]) 52 | 53 | self.assertEqual(f(), 2) 54 | 55 | def test_load_nonlocal_with_keyword(self): 56 | VAR = 2 57 | f = def_(lambda: [ 58 | nonlocal_[VAR], 59 | return_[VAR, callee_], 60 | ]) 61 | 62 | var, fc = f() 63 | self.assertEqual(var, 2) 64 | self.assertIs(fc, f) 65 | 66 | def test_set_nonlocal(self): 67 | VAR = 2 68 | f = def_(lambda: [ 69 | nonlocal_[VAR], 70 | VAR < VAR + 2, 71 | return_[VAR], 72 | ]) 73 | 74 | def _assert_global_VAR_is(x): 75 | global VAR 76 | self.assertEqual(VAR, x) 77 | 78 | self.assertEqual(f(), 4) 79 | self.assertEqual(VAR, 4) 80 | _assert_global_VAR_is(1) 81 | 82 | def test_set_nonlocal_fail_of_no_nonlocal_keyword(self): 83 | VAR = 2 84 | f = def_(lambda: [ 85 | VAR < 4, 86 | return_[VAR], 87 | ]) 88 | 89 | self.assertEqual(f(), 4) 90 | self.assertEqual(VAR, 2) 91 | 92 | def test_set_global_while_same_name_variable_exists_in_parent_scope(self): 93 | VAR = 2 94 | f = def_(lambda: [ 95 | global_[VAR], 96 | VAR < 4, 97 | return_[VAR], 98 | ]) 99 | 100 | def _assert_global_VAR_is(x): 101 | global VAR 102 | self.assertEqual(VAR, x) 103 | 104 | self.assertEqual(f(), 4) 105 | self.assertEqual(VAR, 2) 106 | _assert_global_VAR_is(4) 107 | 108 | 109 | class TestNested(unittest.TestCase): 110 | def test_load_nonlocal(self): 111 | f = def_(lambda VAR: [ 112 | return_[ 113 | def_(lambda: [ 114 | return_[VAR], 115 | ]) 116 | ], 117 | ]) 118 | 119 | self.assertEqual(f(4)(), 4) 120 | self.assertEqual(VAR, 1) 121 | 122 | def test_IIFE(self): 123 | import lambdex.compiler.core as core 124 | from lambdex.utils.ast import pprint 125 | 126 | core.__DEBUG__ = True 127 | f = def_(lambda: [ 128 | ret < [], 129 | for_[i in range(10)] [ 130 | def_(lambda i: [ 131 | ret.append( 132 | def_(lambda: [ 133 | return_[i], 134 | ]) 135 | ), 136 | ])(i), 137 | ], 138 | return_[ret], 139 | ]) 140 | 141 | ret = f() 142 | stored_variables = [x() for x in ret] 143 | self.assertEqual(stored_variables, list(range(10))) 144 | core.__DEBUG__ = False 145 | -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/aliases_1/.lambdex.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | def_ = Def 3 | except_ = Except 4 | if_ = If 5 | else_ = Else 6 | try_ = Try 7 | return_ = Return 8 | -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/aliases_1/test_demo.dst.py: -------------------------------------------------------------------------------- 1 | from lambdex import Def 2 | 3 | def f(): 4 | return Def.myfunc(lambda a, b: [ # comment1 5 | # comment2 6 | If[condition] [ 7 | f2 < Def(lambda: [ 8 | a+b 9 | ]), 10 | Return[f2], 11 | ], 12 | Try[ # comment3 13 | body, 14 | ].Except[Exception > e] [ 15 | Excepthandler 16 | ], # comment4 17 | # comment5 18 | ]) -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/aliases_1/test_demo.src.py: -------------------------------------------------------------------------------- 1 | from lambdex import Def 2 | 3 | def f(): 4 | return Def.myfunc( # comment1 5 | lambda a, b: [# comment2 6 | If[condition] [ 7 | f2 < Def(lambda:[a+b]), 8 | Return[f2], 9 | ],Try[# comment3 10 | body, 11 | ].Except[Exception > e] [ 12 | Excepthandler 13 | ] # comment4 14 | , ],# comment5 15 | ) -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/test_augassign.dst.py: -------------------------------------------------------------------------------- 1 | def_(lambda: [ 2 | a +_< 1, 3 | a -_< 1, 4 | a *_< 1, 5 | a /_< 1, 6 | a //_< 1, 7 | a @_< 1, 8 | a %_< 1, 9 | a <<_< 1, 10 | a >>_< 1, 11 | a **_< 1, 12 | a &_< 1, 13 | a |_< 1, 14 | a ^_< 1, 15 | ]) -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/test_augassign.src.py: -------------------------------------------------------------------------------- 1 | def_ (lambda: [ 2 | a + _ < 1, 3 | a - _ < 1, 4 | a * _ < 1, 5 | a / _ < 1, 6 | a // _ < 1, 7 | a @ _ < 1, 8 | a % _ < 1, 9 | a << _ < 1, 10 | a >> _ < 1, 11 | a ** _ < 1, 12 | a & _ < 1, 13 | a | _ < 1, 14 | a ^ _ < 1, 15 | ]) -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/test_demo.dst.py: -------------------------------------------------------------------------------- 1 | from lambdex import def_ 2 | 3 | def f(): 4 | return def_.myfunc(lambda a, b: [ # comment1 5 | # comment2 6 | if_[condition] [ 7 | f2 < def_(lambda: [ 8 | a+b 9 | ]), 10 | return_[f2], 11 | ].else_[ 12 | c 13 | ], 14 | try_[ # comment3 15 | body, 16 | ].except_[Exception > e] [ 17 | except_handler 18 | ], # comment4 19 | # comment5 20 | ]) 21 | -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/test_demo.src.py: -------------------------------------------------------------------------------- 1 | from lambdex import def_ 2 | 3 | def f(): 4 | return def_.myfunc( # comment1 5 | lambda a, b: [# comment2 6 | if_[condition] [ 7 | f2 < def_(lambda:[a+b]), 8 | return_[f2], 9 | ].else_[c],try_[# comment3 10 | body, 11 | ].except_[Exception > e] [ 12 | except_handler 13 | ] # comment4 14 | , ],# comment5 15 | ) -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/test_lxfmt_directive.dst.py: -------------------------------------------------------------------------------- 1 | from lambdex import def_ 2 | 3 | # lxfmt: off 4 | def_(lambda: [ pass_ 5 | ]) 6 | # lxfmt: on 7 | def_(lambda: [ 8 | pass_ 9 | ]) 10 | -------------------------------------------------------------------------------- /tests/fmt/fmt_samples/test_lxfmt_directive.src.py: -------------------------------------------------------------------------------- 1 | from lambdex import def_ 2 | 3 | # lxfmt: off 4 | def_(lambda: [ pass_ 5 | ]) 6 | # lxfmt: on 7 | def_(lambda: [pass_ 8 | ]) 9 | -------------------------------------------------------------------------------- /tests/fmt/test_fmt_result.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pathlib 4 | import difflib 5 | import unittest 6 | import subprocess 7 | 8 | TEST_DIR = pathlib.Path(__file__).parent 9 | 10 | 11 | def _pad(string): 12 | if not string.endswith("\n"): 13 | string += "\n" 14 | return string 15 | 16 | 17 | def _test_cases(): 18 | fmt_samples_dir = TEST_DIR / "fmt_samples" 19 | for src in fmt_samples_dir.rglob("*.src.py"): 20 | dst = src.parent / src.name.replace(".src", ".dst") 21 | yield (src.name.replace(".", "_"), src, dst) 22 | 23 | 24 | def _spawn_fmt(args): 25 | return subprocess.Popen( 26 | args, 27 | stdout=subprocess.PIPE, 28 | stderr=subprocess.PIPE, 29 | cwd=str(TEST_DIR.parent.parent), 30 | env=dict(LXALIAS="1", **os.environ), 31 | ) 32 | 33 | 34 | def _build_test_func(src, dst): 35 | def _func(self): 36 | self.maxDiff = None 37 | p = _spawn_fmt([sys.executable, "-m", "lambdex.fmt", str(src.absolute())]) 38 | stdout, stderr = p.communicate() 39 | output = stdout.decode() 40 | desired_output = _pad(dst.read_text()) 41 | self.assertEqual(p.returncode, 0, msg="STDERR:\n" + stderr.decode()) 42 | self.assertEqual(output, desired_output, output) 43 | 44 | return _func 45 | 46 | 47 | class TestFmtResult(unittest.TestCase): 48 | for name, src, dst in _test_cases(): 49 | locals()[name] = _build_test_func(src, dst) 50 | 51 | def test_multi_files(self): 52 | self.maxDiff = None 53 | 54 | srcs = [] 55 | dsts = [] 56 | for _, src, dst in _test_cases(): 57 | srcs.append(src) 58 | dsts.append(dst) 59 | 60 | p = _spawn_fmt( 61 | [sys.executable, "-m", "lambdex.fmt"] + [str(x.absolute()) for x in srcs] 62 | ) 63 | stdout, stderr = p.communicate() 64 | output = stdout.decode() 65 | desired_output = "".join(map(_pad, map(pathlib.Path.read_text, dsts))) 66 | self.assertEqual(p.returncode, 0, msg="STDERR:\n" + stderr.decode()) 67 | self.assertEqual(output, desired_output, output) 68 | -------------------------------------------------------------------------------- /tests/repl/_cases.py: -------------------------------------------------------------------------------- 1 | import io 2 | import re 3 | import textwrap 4 | 5 | import pexpect 6 | 7 | 8 | def _remove_ANSI_escape( 9 | string: str, 10 | regexp=re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])"), 11 | ) -> str: 12 | """ 13 | Remove ANSI escape sequence in a string. 14 | 15 | See https://stackoverflow.com/questions/14693701/ 16 | """ 17 | return regexp.sub("", string) 18 | 19 | 20 | def get_output(inputs, *cmds, timeout=1): 21 | """ 22 | Spawn a REPL, feed the inputs and get the outputs. 23 | """ 24 | inputs = textwrap.dedent(inputs).strip() 25 | 26 | p = pexpect.spawn(*cmds) 27 | # Turn off ECHO so that they won't mess up the outputs 28 | p.setecho(False) 29 | 30 | # Save the stdout and stderr in `log` 31 | log = io.BytesIO() 32 | p.logfile_read = log 33 | 34 | for line in inputs.split("\n"): 35 | p.sendline(line.encode()) 36 | 37 | try: 38 | p.expect(pexpect.EOF, timeout=timeout) 39 | except pexpect.TIMEOUT as exc: 40 | raise RuntimeError("OUTPUT:\n" + log.getvalue().decode()) from exc 41 | else: 42 | p.wait() 43 | 44 | return _remove_ANSI_escape(log.getvalue().decode()) 45 | 46 | 47 | class _Cases: 48 | def test_runtime_error(self): 49 | inputs = """ 50 | from lambdex.repl import * 51 | f = def_(lambda: [ 52 | 1 / 0, ]) 53 | 54 | f() 55 | exit() 56 | """ 57 | 58 | outputs = """ 59 | 1 / 0, ]) 60 | ZeroDivisionError: division by zero 61 | >>> """ 62 | 63 | self._test(inputs, outputs) 64 | 65 | def test_compiletime_error(self): 66 | inputs = """ 67 | from lambdex.repl import * 68 | f = def_(lambda: [ 69 | if_[a], ]) 70 | 71 | exit() 72 | """ 73 | 74 | outputs = """ 75 | if_[a], ]) 76 | ^ 77 | SyntaxError: expect another group of '[]' 78 | >>> >>> """ 79 | 80 | self._test(inputs, outputs) 81 | -------------------------------------------------------------------------------- /tests/repl/test_builtin.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import sys 4 | import time 5 | import unittest 6 | import textwrap 7 | 8 | import pexpect 9 | 10 | sys.path.append(os.path.dirname(__file__)) 11 | from _cases import _Cases, get_output 12 | 13 | 14 | class TestBuiltinREPL(unittest.TestCase, _Cases): 15 | def _test(self, inputs, outputs): 16 | self.maxDiff = None 17 | outputs = textwrap.dedent(outputs) 18 | outputs = outputs.lstrip("\r\n").replace("\n", "\r\n") 19 | stdout = get_output(inputs, sys.executable, ["-i"]) 20 | self.assertEqual(stdout[-len(outputs) :], outputs, stdout) 21 | -------------------------------------------------------------------------------- /tests/repl/test_idle.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import unittest 5 | import textwrap 6 | import threading 7 | import multiprocessing 8 | 9 | try: 10 | import idlelib 11 | 12 | del idlelib 13 | except ImportError: 14 | IDLE_AVAILABLE = False 15 | else: 16 | IDLE_AVAILABLE = True 17 | 18 | sys.path.append(os.path.dirname(__file__)) 19 | from _cases import _Cases 20 | 21 | EOF = -1 22 | 23 | 24 | def _subprocess(Q_in: multiprocessing.Queue, Q_out: multiprocessing.Queue): 25 | """ 26 | The entry point of IDLE process. We hack everything here to avoid 27 | pollution in the host process. 28 | """ 29 | # Reset the argv so that IDLE will open a clean console 30 | sys.argv[1:] = [] 31 | 32 | if sys.version_info < (3, 6): 33 | from idlelib import PyShell as pyshell 34 | else: 35 | from idlelib import pyshell 36 | 37 | outputs = [] 38 | 39 | class _PyShell(pyshell.PyShell): 40 | """ 41 | Hack `pyshell.PyShell` to: 42 | 43 | - Record output string from the execution backend; 44 | - Remove the prompt at exiting. 45 | """ 46 | 47 | def write(self, s, tags=()): 48 | outputs.append(s) 49 | return super(_PyShell, self).write(s, tags) 50 | 51 | def close(self): 52 | self.stop_readline() 53 | self.canceled = True 54 | self.closing = True 55 | return pyshell.EditorWindow.close(self) 56 | 57 | pyshell.PyShell = _PyShell 58 | 59 | # Start a thread for IDLE main looping 60 | th = threading.Thread(target=pyshell.main) 61 | th.start() 62 | # Wait until `pyshell.flish.pyshell` is ready 63 | while not hasattr(pyshell, "flist") or pyshell.flist.pyshell is None: 64 | time.sleep(0.1) 65 | 66 | pyshell_object = pyshell.flist.pyshell 67 | while True: 68 | string = Q_in.get() 69 | if string == EOF: 70 | break 71 | 72 | # Set the string onto the editor 73 | pyshell_object.text.insert("insert", string) 74 | # Trigger logic in ENTER event 75 | pyshell_object.enter_callback("") 76 | 77 | # Wait until the editor is ready to get more inputs 78 | while pyshell_object.executing and not pyshell_object.reading: 79 | time.sleep(0.1) 80 | 81 | th.join() 82 | # Send out the outputs 83 | Q_out.put(outputs) 84 | 85 | 86 | def get_output_from_idle(inputs) -> str: 87 | """ 88 | Feed the inputs into an IDLE process, and returns the output as 89 | string. 90 | """ 91 | input_lines = inputs = textwrap.dedent(inputs).strip().split("\n") 92 | 93 | Q_in = multiprocessing.Queue() 94 | Q_out = multiprocessing.Queue() 95 | p = multiprocessing.Process(target=_subprocess, args=(Q_in, Q_out)) 96 | p.start() 97 | 98 | for line in input_lines: 99 | Q_in.put(line) 100 | Q_in.put(EOF) 101 | 102 | outputs = None 103 | while outputs is None: 104 | try: 105 | outputs = Q_out.get(block=False, timeout=1) 106 | except multiprocessing.queues.Empty: 107 | if not p.is_alive(): 108 | raise RuntimeError("IDLE process gone") 109 | time.sleep(0.5) 110 | else: 111 | break 112 | p.join() 113 | return "".join(outputs).replace(" \t", "") 114 | 115 | 116 | @unittest.skipIf(not IDLE_AVAILABLE, "IDLE not available") 117 | class TestIDLE(unittest.TestCase, _Cases): 118 | def _test(self, inputs, outputs): 119 | self.maxDiff = None 120 | stdout = get_output_from_idle(inputs) 121 | outputs = textwrap.dedent(outputs).lstrip("\r\n") 122 | self.assertEqual(stdout[-len(outputs) :], outputs, stdout) 123 | -------------------------------------------------------------------------------- /tests/repl/test_ipython.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import textwrap 4 | import unittest 5 | 6 | sys.path.append(os.path.dirname(__file__)) 7 | from _cases import _Cases, get_output 8 | 9 | 10 | class TestIPython(unittest.TestCase, _Cases): 11 | def _test(self, inputs, outputs): 12 | self.maxDiff = None 13 | outputs = textwrap.dedent(outputs) 14 | outputs = outputs.lstrip("\r\n") 15 | stdout = get_output( 16 | inputs, 17 | sys.executable, 18 | ["-m", "IPython", "--quiet", "--classic", "--quick", "--no-confirm-exit"], 19 | timeout=10, 20 | ) 21 | for line in outputs.splitlines(keepends=False): 22 | if not line or ">>>" in line: 23 | continue 24 | self.assertTrue( 25 | line in stdout, "\n{!r} not in\n{}".format(repr(line), stdout) 26 | ) 27 | --------------------------------------------------------------------------------