├── .github └── workflows │ ├── publish.yml │ └── python.yml ├── .gitignore ├── .readthedocs.yml ├── LICENSE ├── README.md ├── docs ├── Makefile ├── conf.py ├── index.rst └── requirements.txt ├── pyproject.toml ├── synr ├── __init__.py ├── ast.py ├── compiler.py ├── diagnostic_context.py └── transformer.py └── tests ├── __init__.py └── test_synr.py /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Synr 2 | 3 | on: 4 | release: 5 | types: [published, edited] 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set up Python 13 | uses: actions/setup-python@v2 14 | with: 15 | python-version: 3.8 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install poetry 19 | poetry install 20 | - name: Publish 21 | run: | 22 | poetry publish --build --username __token__ --password ${{ secrets.PYPI_API_KEY }} 23 | -------------------------------------------------------------------------------- /.github/workflows/python.yml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] 16 | os: [ubuntu-latest, macos-latest] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v2 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install poetry 27 | poetry install 28 | - name: Formatting 29 | run: | 30 | poetry run black . --check 31 | - name: Type checking 32 | run: | 33 | poetry run mypy synr 34 | - name: Testing 35 | run: | 36 | poetry run pytest 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | *.vscode 132 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/conf.py 11 | 12 | # Optionally set the version of Python and requirements required to build your docs 13 | python: 14 | version: 3.8 15 | install: 16 | - requirements: docs/requirements.txt 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Synr 2 | 3 | Synr is a library that provides a stable Abstract Syntax Tree for Python. 4 | 5 | ## Features 6 | 7 | - The Synr AST does not change between Python versions. 8 | - Every AST node contains line and column information. 9 | - There is a single AST node for assignments (compared to three in Python's ast module). 10 | - Support for returning multiple errors at once. 11 | - Support for custom error reporting. 12 | 13 | ## Usage 14 | 15 | ```python 16 | import synr 17 | 18 | def test_program(x: int): 19 | return x + 2 20 | 21 | # Parse a Python function into an AST 22 | ast = synr.to_ast(test_program, synr.PrinterDiagnosticContext()) 23 | ``` 24 | 25 | ## Documentation 26 | 27 | Please see [https://synr.readthedocs.io/en/latest/](https://synr.readthedocs.io/en/latest/) for documentation. 28 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("..")) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = "Synr" 22 | copyright = "2020, OctoML, inc" 23 | author = "Jared Roesch, Tristan Konolige" 24 | 25 | # The full version, including alpha/beta/rc tags 26 | release = "0.2.0" 27 | 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ["_templates"] 38 | 39 | # List of patterns, relative to source directory, that match files and 40 | # directories to ignore when looking for source files. 41 | # This pattern also affects html_static_path and html_extra_path. 42 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 43 | 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | 47 | # The theme to use for HTML and HTML Help pages. See the documentation for 48 | # a list of builtin themes. 49 | # 50 | html_theme = "sphinx_rtd_theme" 51 | 52 | # Add any paths that contain custom static files (such as style sheets) here, 53 | # relative to this directory. They are copied after the builtin static files, 54 | # so a file named "default.css" will overwrite the builtin "default.css". 55 | html_static_path = ["_static"] 56 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Synr: A stable AST for python 2 | ============================= 3 | 4 | Converting from the Python AST 5 | ------------------------------ 6 | 7 | .. autofunction:: synr.to_ast 8 | 9 | The Synr AST 10 | ------------ 11 | 12 | .. automodule:: synr.ast 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | 17 | Error Handling 18 | -------------- 19 | 20 | Synr uses a :py:class:`DiagnosticContext` to accumulate errors that occurred 21 | during parsing. Multiple errors can be accumulated and they are all return 22 | after parsing. :py:class:`DiagnosticContext` can be subclassed to customize 23 | error handling behavior. We also provide a 24 | :py:class:`PrinterDiagnosticContext`, which prints out errors as they occur. 25 | 26 | .. autoclass:: synr.DiagnosticContext 27 | :members: 28 | :undoc-members: 29 | 30 | .. autoclass:: synr.PrinterDiagnosticContext 31 | :show-inheritance: 32 | 33 | Transforming the Synr AST to a Different Representation 34 | ------------------------------------------------------- 35 | 36 | :py:func:`to_ast` allows you to transform a Synr AST into your desired 37 | representation while using the existing Synr error handling infrastructure. 38 | First implement a class that inherits from :py:class:`Transformer`, then 39 | pass an instance of this class to :py:func:`to_ast`. :py:func:`to_ast` will 40 | either return your converted class or an errors that occurred. 41 | 42 | .. autoclass:: synr.Transformer 43 | :members: 44 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | attrs 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "synr" 3 | version = "0.6.0" 4 | description = "A consistent AST for Python" 5 | authors = ["Jared Roesch ", "Tristan Konolige "] 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.6.2" 10 | attrs = "*" 11 | 12 | [tool.poetry.dev-dependencies] 13 | pytest = "^6.2.4" 14 | black = { version = "^21.10b0", allow-prereleases = true } 15 | mypy = "^0.782" 16 | 17 | [build-system] 18 | requires = ["poetry>=0.12"] 19 | build-backend = "poetry.masonry.api" 20 | -------------------------------------------------------------------------------- /synr/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The goal of synr is to provide a syntactic representation 3 | for the core of the Python programming language for building 4 | embedded domain specific langauges. 5 | 6 | Synr provides 7 | - A python version agnostic syntax tree. 8 | - Line and column information for every node in the AST. 9 | """ 10 | 11 | __version__ = "0.6.0" 12 | 13 | from .compiler import to_ast 14 | from .diagnostic_context import PrinterDiagnosticContext, DiagnosticContext 15 | from .transformer import Transformer 16 | -------------------------------------------------------------------------------- /synr/ast.py: -------------------------------------------------------------------------------- 1 | """ 2 | `synr.ast` contains definitions of all nodes in the synr AST. This AST is 3 | independent of python version: different versions of python will give the same 4 | AST. In addition, all nodes contain span information. 5 | """ 6 | import ast as py_ast 7 | import sys 8 | 9 | import attr 10 | from enum import Enum, auto 11 | import typing 12 | from typing import Optional, Any, List, Dict, Union, Sequence 13 | 14 | NoneType = type(None) 15 | 16 | 17 | @attr.s(auto_attribs=True, frozen=True) 18 | class Span: 19 | """An contiguous interval in a source file from a starting line and column 20 | to an ending line and column. 21 | 22 | Notes 23 | ----- 24 | Line and column numbers are one indexed. The interval spanned is inclusive 25 | of the start and exclusive of the end. 26 | """ 27 | 28 | filename: str 29 | start_line: int 30 | start_column: int 31 | end_line: int 32 | end_column: int 33 | 34 | @staticmethod 35 | def from_ast(filename: str, node: py_ast.AST) -> "Span": 36 | """Extract the span of a python AST node""" 37 | lineno = node.lineno 38 | # A workaround for function def lineno before Python 3.8 39 | if isinstance(node, py_ast.FunctionDef) and sys.version_info < (3, 8): 40 | lineno += len(node.decorator_list) 41 | 42 | if hasattr(node, "end_lineno") and node.end_lineno is not None: # type: ignore 43 | end_lineno = node.end_lineno # type: ignore 44 | else: 45 | end_lineno = lineno 46 | if hasattr(node, "end_col_offset") and node.end_col_offset is not None: # type: ignore 47 | end_col_offset = node.end_col_offset + 1 # type: ignore 48 | else: 49 | end_col_offset = node.col_offset + 2 50 | return Span(filename, lineno, node.col_offset + 1, end_lineno, end_col_offset) 51 | 52 | def merge(self, span: "Span") -> "Span": 53 | """Return the span starting from the beginning of the first span and 54 | ending at the end of the second span. 55 | 56 | Notes 57 | ----- 58 | Gaps between the end of the first span and the start of the second span 59 | are contained in the merged span. The spans also do not need to be in 60 | order. 61 | 62 | Example 63 | ------- 64 | 65 | >>> Span("", 1, 2, 1, 4).merge(Span("", 2, 1, 2, 3)) 66 | Span(filename="", start_line=1, start_column=2, end_line=2, end_column=3) 67 | >>> Span("", 2, 3, 2, 1).merge(Span("", 1, 2, 1, 4)) 68 | Span(filename="", start_line=1, start_column=2, end_line=2, end_column=3) 69 | 70 | """ 71 | assert self.filename == span.filename, "Spans must be from the same file" 72 | self_start = (self.start_line, self.start_column) 73 | span_start = (span.start_line, span.start_column) 74 | start_line, start_col = min(self_start, span_start) 75 | 76 | self_end = (self.end_line, self.end_column) 77 | span_end = (span.end_line, span.end_column) 78 | end_line, end_col = max(self_end, span_end) 79 | 80 | return Span(self.filename, start_line, start_col, end_line, end_col) 81 | 82 | @staticmethod 83 | def union(spans: Sequence["Span"]) -> "Span": 84 | """A span containing all the given spans with no gaps. 85 | 86 | This function is the equivalent to merge with more than two spans. 87 | 88 | Example 89 | ------- 90 | 91 | >>> Span.union(Span("", 1, 2, 1, 4), Span("", 2, 1, 2, 3)) 92 | Span(filename="", start_line=1, start_column=2, end_line=2, end_column=3) 93 | 94 | """ 95 | if len(spans) == 0: 96 | return Span.invalid() 97 | span = spans[0] 98 | for s in spans[1:]: 99 | span = span.merge(s) 100 | return span 101 | 102 | def between(self, span: "Span") -> "Span": 103 | """The span starting after the end of the first span and ending before 104 | the start of the second span. 105 | 106 | Example 107 | -------- 108 | 109 | >>> Span("", 1, 2, 1, 4).between(Span("", 2, 1, 2, 3)) 110 | Span(filename="", start_line=1, start_column=4, end_line=2, end_column=1) 111 | 112 | """ 113 | assert self.filename == span.filename, "Spans must be from the same file" 114 | return Span( 115 | self.filename, 116 | self.end_line, 117 | self.end_column, 118 | span.start_line, 119 | span.start_column, 120 | ) 121 | 122 | def subtract(self, span: "Span") -> "Span": 123 | """The span from the start of the first span to the start of the second span. 124 | 125 | Example 126 | -------- 127 | 128 | >>> Span("", 1, 2, 2, 4).subtract(Span("", 2, 1, 2, 3)) 129 | Span(filename="", start_line=1, start_column=2, end_line=2, end_column=1) 130 | 131 | """ 132 | assert self.filename == span.filename, "Spans must be from the same file" 133 | return Span( 134 | self.filename, 135 | self.start_line, 136 | self.start_column, 137 | span.start_line, 138 | span.start_column, 139 | ) 140 | 141 | @staticmethod 142 | def invalid() -> "Span": 143 | """An invalid span""" 144 | return Span("", -1, -1, -1, -1) 145 | 146 | 147 | Name = str # TODO: add span to this 148 | 149 | 150 | @attr.s(auto_attribs=True, frozen=True) 151 | class Node: 152 | """Base class of any AST node. 153 | 154 | All AST nodes must have a span. 155 | """ 156 | 157 | span: Span 158 | 159 | 160 | @attr.s(auto_attribs=True, frozen=True) 161 | class Id(Node): 162 | name: str 163 | 164 | @staticmethod 165 | def invalid() -> "Id": 166 | return Id(Span.invalid(), "") 167 | 168 | 169 | class Type(Node): 170 | """Base class of a type in the AST.""" 171 | 172 | pass 173 | 174 | 175 | class Stmt(Node): 176 | """Base class of a statement in the AST.""" 177 | 178 | pass 179 | 180 | 181 | class Expr(Node): 182 | """Base class of a expression in the AST.""" 183 | 184 | pass 185 | 186 | 187 | @attr.s(auto_attribs=True, frozen=True) 188 | class Parameter(Node): 189 | """Parameter in a function declaration. 190 | 191 | Example 192 | ------- 193 | :code:`x: str` in :code:`def my_function(x: str)`. 194 | """ 195 | 196 | name: Name 197 | ty: Optional[Type] 198 | 199 | 200 | @attr.s(auto_attribs=True, frozen=True) 201 | class Var(Expr): 202 | """A variable in an expression. 203 | 204 | Examples 205 | -------- 206 | :code:`x` and :code:`y` in :code:`x = y`. 207 | :code:`x` in :code:`x + 2`. 208 | """ 209 | 210 | id: Id 211 | 212 | @staticmethod 213 | def invalid() -> "Var": 214 | return Var(Span.invalid(), Id.invalid()) 215 | 216 | 217 | Pattern = List[Var] 218 | 219 | 220 | @attr.s(auto_attribs=True, frozen=True) 221 | class Attr(Expr): 222 | """Field access on variable or structure or module. 223 | 224 | Examples 225 | -------- 226 | In :code:`x.y`, :code:`x` is the :code:`object` and :code:`y` is the 227 | :code:`field`. For multiple field accesses in a row, they are grouped left 228 | to right. For example, :code:`x.y.z` becomes 229 | :code:`Attr(Attr(object='x', field='y'), field='z')`. 230 | """ 231 | 232 | object: Expr 233 | field: Id 234 | 235 | 236 | @attr.s(auto_attribs=True, frozen=True) 237 | class TypeVar(Type): 238 | """Type variable in a type expression. 239 | 240 | This is equivalent to :code:`Var`, but for types. 241 | 242 | Example 243 | ------- 244 | :code:`X` in :code:`y: X = 2`. 245 | """ 246 | 247 | id: Id 248 | 249 | 250 | @attr.s(auto_attribs=True, frozen=True) 251 | class TypeAttr(Type): 252 | """Field access in a type expression. 253 | 254 | This is equivalent to :code:`Attr`, but for types. 255 | 256 | Example 257 | ------- 258 | In :code:`y: X.Z = 2`, :code:`object` is :code:`X` and :code:`field` is 259 | :code:`Z`. 260 | """ 261 | 262 | object: Type 263 | field: Id 264 | 265 | 266 | class BuiltinOp(Enum): 267 | """Various built in operators including binary operators, unary operators, 268 | and array access. 269 | 270 | Examples 271 | -------- 272 | :code:`+` in :code:`2 + 3`. 273 | :code:`[]` in :code:`x[2]`. 274 | :code:`or` in :code:`true or false`. 275 | :code:`>` in :code:`3 > 2`. 276 | """ 277 | 278 | Add = auto() 279 | Sub = auto() 280 | Mul = auto() 281 | Div = auto() 282 | FloorDiv = auto() 283 | Mod = auto() 284 | Subscript = auto() 285 | SubscriptAssign = auto() 286 | And = auto() 287 | Or = auto() 288 | Eq = auto() 289 | NotEq = auto() 290 | GT = auto() 291 | GE = auto() 292 | LT = auto() 293 | LE = auto() 294 | Not = auto() 295 | BitOr = auto() 296 | BitAnd = auto() 297 | BitXor = auto() 298 | USub = auto() 299 | UAdd = auto() 300 | Invert = auto() 301 | Invalid = auto() # placeholder op if op failed to parse 302 | 303 | 304 | @attr.s(auto_attribs=True, frozen=True) 305 | class TypeCall(Type): 306 | """A function call in type expression. 307 | 308 | This is equivalent to :code:`Call`, but for type expressions. 309 | 310 | Example 311 | ------- 312 | In :code:`x : List[type(None)]`, :code:`type(None)` is a :code:`TypeCall`. 313 | :code:`type` is the :code:`func_name` and :code:`None` is 314 | :code:`params[0]`. 315 | """ 316 | 317 | func_name: Union[Type, BuiltinOp] 318 | params: List[Type] 319 | keyword_params: Dict[Type, Type] 320 | 321 | 322 | @attr.s(auto_attribs=True, frozen=True) 323 | class TypeApply(Type): 324 | """Type application. 325 | 326 | Example 327 | ------- 328 | In :code:`x: List[str]`, :code:`List[str]` is a :code:`TypeCall`. In this 329 | case, :code:`List` is the :code:`func_name`, and :code:`str` is 330 | :code:`params[0]`. 331 | """ 332 | 333 | func_name: Type 334 | params: Sequence[Type] 335 | 336 | 337 | @attr.s(auto_attribs=True, frozen=True) 338 | class TypeTuple(Type): 339 | """A tuple in a type expression. 340 | 341 | Example 342 | ------- 343 | :code:`(str, int)` in :code:`x: (str, int)`. 344 | """ 345 | 346 | values: Sequence[Expr] 347 | 348 | 349 | @attr.s(auto_attribs=True, frozen=True) 350 | class TypeConstant(Type): 351 | """A literal value in a type expression. 352 | 353 | This is equivalent to :code:`Constant`, but for type expressions. 354 | 355 | Examples 356 | -------- 357 | :code:`1` in :code:`x: Array[1]`. 358 | :code:`None` in :code:`def my_function() -> None:`. 359 | """ 360 | 361 | value: Union[str, NoneType, bool, complex, float, int] 362 | 363 | 364 | @attr.s(auto_attribs=True, frozen=True) 365 | class TypeSlice(Type): 366 | """A slice in a type expression. 367 | 368 | This is equivalent to :code:`Slice`, but for type expressions. 369 | 370 | Example 371 | ------- 372 | :code:`1:2` in :code:`x: Array[1:2]`. 373 | """ 374 | 375 | start: Type 376 | step: Type 377 | end: Type 378 | 379 | 380 | @attr.s(auto_attribs=True, frozen=True) 381 | class Tuple(Expr): 382 | """A tuple in an expression. 383 | 384 | Example 385 | ------- 386 | :code:`(1, 2)` in :code:`x = (1, 2)`. 387 | """ 388 | 389 | values: Sequence[Expr] 390 | 391 | 392 | @attr.s(auto_attribs=True, frozen=True) 393 | class DictLiteral(Expr): 394 | """A sictionary literal in an expression. 395 | 396 | Example 397 | ------- 398 | :code:`{"x": 2}` in :code:`x = {"x": 2}`. 399 | """ 400 | 401 | keys: Sequence[Expr] 402 | values: Sequence[Expr] 403 | 404 | 405 | @attr.s(auto_attribs=True, frozen=True) 406 | class ArrayLiteral(Expr): 407 | """An array literal in an expression. 408 | 409 | Example 410 | ------- 411 | :code:`[1, 2]` in :code:`x = [1, 2]`. 412 | """ 413 | 414 | values: Sequence[Expr] 415 | 416 | 417 | @attr.s(auto_attribs=True, frozen=True) 418 | class Op(Node): 419 | """A builtin operator. 420 | 421 | See :code:`BuiltinOp` for supported operators. 422 | 423 | Example 424 | ------- 425 | :code:`+` in :code:`x = 1 + 2`. 426 | """ 427 | 428 | name: BuiltinOp 429 | 430 | 431 | @attr.s(auto_attribs=True, frozen=True) 432 | class Constant(Expr): 433 | """A literal value in an expression. 434 | 435 | Example 436 | ------- 437 | :code:`1` in :code:`x = 1`. 438 | """ 439 | 440 | value: Union[str, NoneType, bool, complex, float, int] 441 | 442 | 443 | @attr.s(auto_attribs=True, frozen=True) 444 | class Call(Expr): 445 | """A function call. 446 | 447 | Function calls can be: 448 | 449 | - A regular function call of the form :code:`x.y(1, 2, z=3)`. In this case, 450 | :code:`func_name` will contain the function name (`x.y`), :code:`params` 451 | will contain the arguments (`1, 2`), and :code:`keywords_params` will 452 | contain any keyword arguments (`z=3`). 453 | - A binary operation like :code:`x + 2`. In this case, :code:`func_name` 454 | will be the binary operation from :code:`BuiltinOp`, :code:`params[0]` 455 | will be the left hand side (`x`), and :code:`params[1]` will be the right 456 | hand side :code:`2`. 457 | - A unary operation like :code:`not x`. In this case, :code:`func_name` 458 | will be the unary operation from :code:`BuiltinOp`, and :code:`params[0]` 459 | will be the operand (`x`). 460 | - An array access like :code:`x[2, y]`. In this case, :code:`func_name` 461 | will be :code:`BuiltinOp.Subscript`, :code:`params[0]` will be the 462 | operand (`x`), :code:`params[1]` will be a :code:`Tuple` containing the 463 | indices (`2, y`). 464 | - An array assignment like :code:`x[2, 3] = y`. In this case, 465 | :code:`func_name` will be :code:`BuiltinOp.SubscriptAssign`, 466 | :code:`params[0]` will be the operand (`x`), :code:`params[1]` will be a 467 | :code:`Tuple` containing the indices (`2, 3`), and :code:`params[2]` will 468 | contain the right hand side of the assignment (`y`).""" 469 | 470 | func_name: Union[Expr, Op] 471 | params: List[Expr] 472 | keyword_params: Dict[Expr, Expr] 473 | 474 | 475 | @attr.s(auto_attribs=True, frozen=True) 476 | class Slice(Expr): 477 | """A slice in an expression. 478 | 479 | A slice in a range from [start,end) with step size step. 480 | 481 | Notes 482 | ----- 483 | If not defined, start is 0, end is -1 and step is 1. 484 | 485 | Example 486 | ------- 487 | :code:`1:2` in :code:`x = y[1:2]`. 488 | """ 489 | 490 | start: Expr 491 | step: Expr 492 | end: Expr 493 | 494 | 495 | @attr.s(auto_attribs=True, frozen=True) 496 | class Lambda(Expr): 497 | """A lambda expression 498 | 499 | Example 500 | ------- 501 | In :code:`lambda x, y: x + y`, :code:`x, y` are :code:`params`, 502 | :code:`x + y` is :code:`body`. 503 | """ 504 | 505 | params: List[Parameter] 506 | body: Expr 507 | 508 | 509 | @attr.s(auto_attribs=True, frozen=True) 510 | class Return(Stmt): 511 | """A return statement. 512 | 513 | Example 514 | ------- 515 | :code:`return x`. 516 | """ 517 | 518 | value: Optional[Expr] 519 | 520 | 521 | @attr.s(auto_attribs=True, frozen=True) 522 | class Assign(Stmt): 523 | """An assignment statement. 524 | 525 | Notes 526 | ----- 527 | Augmented assignment statements like :code:`x += 2` are translated into an 528 | operator call and an assignment (i.e. `x = x + 2`). 529 | 530 | Example 531 | ------- 532 | In :code:`x: int = 2`, :code:`x` is :code:`lhs`, :code:`int` is :code:`ty`, 533 | and :code:`2` is :code:`rhs`. 534 | """ 535 | 536 | lhs: Pattern 537 | ty: Optional[Type] 538 | rhs: Expr 539 | 540 | 541 | @attr.s(auto_attribs=True, frozen=True) 542 | class UnassignedCall(Stmt): 543 | """A standalone function call. 544 | 545 | Example 546 | ------- 547 | .. code-block:: python 548 | 549 | def my_function(): 550 | test_call() 551 | return 2 552 | 553 | Here :code:`test_call()` is an :code:`UnassignedCall`. 554 | """ 555 | 556 | call: Call 557 | 558 | 559 | @attr.s(auto_attribs=True, frozen=True) 560 | class Nonlocal(Stmt): 561 | """A nonlocal statement. 562 | 563 | Example 564 | ------- 565 | .. code-block:: python 566 | x, y = 1, 2 567 | def foo(): 568 | nonlocal x, y 569 | return x 570 | 571 | In :code:`nonlocal x, y`, :code:`vars` is :code`[x, y]`. 572 | """ 573 | 574 | vars: List[Var] 575 | 576 | 577 | @attr.s(auto_attribs=True, frozen=True) 578 | class Global(Stmt): 579 | """A global statement. 580 | 581 | Example 582 | ------- 583 | .. code-block:: python 584 | x, y = 1, 2 585 | def foo(): 586 | global x, y 587 | return x 588 | 589 | In :code:`global x, y`, :code:`vars` is :code`[x, y]`. 590 | """ 591 | 592 | vars: List[Var] 593 | 594 | 595 | @attr.s(auto_attribs=True, frozen=True) 596 | class Block(Node): 597 | """A sequence of statements. 598 | 599 | Examples 600 | -------- 601 | .. code-block:: python 602 | 603 | def my_function(): 604 | test_call() 605 | return 2 606 | 607 | 608 | Here :code:`test_call()` and :code:`return 2` forms a :code:`Block`. 609 | 610 | .. code-block:: python 611 | 612 | if x > 2: 613 | y = 2 614 | 615 | Here :code:`y = 2` is a :code:`Block`. 616 | """ 617 | 618 | stmts: List[Stmt] 619 | 620 | 621 | @attr.s(auto_attribs=True, frozen=True) 622 | class Function(Stmt): 623 | """A function declaration. 624 | 625 | Example 626 | ------- 627 | .. code-block:: python 628 | 629 | @F 630 | def my_function(x: int): 631 | return x + 2 632 | 633 | Here :code:`name` is :code:`my_function`, :code:`x: int` is 634 | :code:`params[0]`, :code:`body` is :code:`return x + 2`, and 635 | :code:`decorators` is :code:`[F]`. 636 | """ 637 | 638 | name: Name 639 | params: List[Parameter] 640 | ret_type: Optional[Type] 641 | body: Block 642 | decorators: List[Expr] 643 | 644 | 645 | @attr.s(auto_attribs=True, frozen=True) 646 | class For(Stmt): 647 | """A for statement. 648 | 649 | Example 650 | ------- 651 | .. code-block:: python 652 | 653 | for x in range(2): 654 | pass 655 | 656 | Here :code:`lhs` will be :code:`x`, :code:`rhs` will be :code:`range(2)`, 657 | and :code:`body` will be :code:`pass`. 658 | """ 659 | 660 | lhs: Pattern 661 | rhs: Expr 662 | body: Block 663 | 664 | 665 | @attr.s(auto_attribs=True, frozen=True) 666 | class While(Stmt): 667 | """An while statement. 668 | 669 | Examples 670 | -------- 671 | .. code-block:: python 672 | 673 | while x <= 2: 674 | pass 675 | 676 | Here :code:`condition` is :code:`x <= 2`, and :code:`body` will be :code:`pass`. 677 | """ 678 | 679 | condition: Expr 680 | body: Block 681 | 682 | 683 | @attr.s(auto_attribs=True, frozen=True) 684 | class With(Stmt): 685 | """A with statement. 686 | 687 | Example 688 | ------- 689 | .. code-block:: python 690 | 691 | with open(x) as f: 692 | pass 693 | 694 | Here :code:`lhs` will be :code:`f`, :code:`rhs` will be :code:`open(x)`, 695 | and :code:`body` will be :code:`pass`. 696 | """ 697 | 698 | lhs: Pattern 699 | rhs: Expr 700 | body: Block 701 | 702 | 703 | @attr.s(auto_attribs=True, frozen=True) 704 | class Assert(Stmt): 705 | """An assert statement. 706 | 707 | Example 708 | ------- 709 | In :code:`assert 2 == 2, "Oh no!"`, :code:`condition` is :code:`2 == 2`, 710 | and :code:`msg` is :code:`"Oh no!"`. 711 | """ 712 | 713 | condition: Expr 714 | msg: Optional[Expr] 715 | 716 | 717 | @attr.s(auto_attribs=True, frozen=True) 718 | class If(Stmt): 719 | """An if statement. 720 | 721 | Notes 722 | ----- 723 | An if statement with :code:`elif` branches becomes multiple nested if 724 | statements. 725 | 726 | Examples 727 | -------- 728 | .. code-block:: python 729 | 730 | if x == 2: 731 | return x 732 | else: 733 | return 3 734 | 735 | Here :code:`condition` is :code:`x == 2`, :code:`true` is :code:`return x`, 736 | and :code:`false` is :code:`return 3`. 737 | 738 | Multiple :code:`elif` statements become nested ifs. For example, 739 | 740 | .. code-block:: python 741 | 742 | if x == 2: 743 | return x 744 | elif x == 3: 745 | return "hi" 746 | else: 747 | return 3 748 | 749 | becomes :code:`If('x == 2', 'return x', If('x == 3', 'return "hi"', 'return 3'))`. 750 | """ 751 | 752 | condition: Expr 753 | true: Block 754 | false: Block 755 | 756 | 757 | @attr.s(auto_attribs=True, frozen=True) 758 | class Class(Node): 759 | """A class definition. 760 | 761 | Example 762 | ------- 763 | .. code-block:: python 764 | 765 | class MyClass: 766 | x = 2 767 | def return_x(): 768 | return x 769 | 770 | Here :code:`name` is :code:`MyClass`, :code:`funcs["return_x"]` is 771 | :code:`def return_x():\n return x`, and :code:`assignments[0]` is 772 | :code:`x = 2`. 773 | """ 774 | 775 | name: Name 776 | funcs: Dict[Name, Function] 777 | assignments: List[Assign] 778 | 779 | 780 | @attr.s(auto_attribs=True, frozen=True) 781 | class Module(Node): 782 | """A collection of classes and functions.""" 783 | 784 | funcs: Dict[Name, Union[Class, Function]] 785 | -------------------------------------------------------------------------------- /synr/compiler.py: -------------------------------------------------------------------------------- 1 | """This module contains the main compiler from the python AST to the synr AST.""" 2 | import ast as py_ast 3 | import inspect 4 | from typing import Optional, Any, List, Union, Sequence 5 | import sys 6 | 7 | from .ast import * 8 | from .diagnostic_context import DiagnosticContext 9 | from .transformer import Transformer 10 | 11 | 12 | class Compiler: 13 | filename: str 14 | start_line: int 15 | start_column: int 16 | transformer: Optional[Transformer] 17 | diagnostic_ctx: DiagnosticContext 18 | 19 | _builtin_ops = { 20 | py_ast.Add: BuiltinOp.Add, 21 | py_ast.Sub: BuiltinOp.Sub, 22 | py_ast.Mult: BuiltinOp.Mul, 23 | py_ast.Div: BuiltinOp.Div, 24 | py_ast.FloorDiv: BuiltinOp.FloorDiv, 25 | py_ast.Mod: BuiltinOp.Mod, 26 | py_ast.Eq: BuiltinOp.Eq, 27 | py_ast.NotEq: BuiltinOp.NotEq, 28 | py_ast.GtE: BuiltinOp.GE, 29 | py_ast.LtE: BuiltinOp.LE, 30 | py_ast.Gt: BuiltinOp.GT, 31 | py_ast.Lt: BuiltinOp.LT, 32 | py_ast.Not: BuiltinOp.Not, 33 | py_ast.Or: BuiltinOp.Or, 34 | py_ast.And: BuiltinOp.And, 35 | py_ast.BitOr: BuiltinOp.BitOr, 36 | py_ast.BitAnd: BuiltinOp.BitAnd, 37 | py_ast.BitXor: BuiltinOp.BitXor, 38 | py_ast.USub: BuiltinOp.USub, 39 | py_ast.UAdd: BuiltinOp.UAdd, 40 | py_ast.Invert: BuiltinOp.Invert, 41 | } 42 | 43 | def __init__( 44 | self, 45 | filename: str, 46 | start_line: int, 47 | start_column: int, 48 | transformer: Optional[Transformer], 49 | diagnostic_ctx: DiagnosticContext, 50 | ): 51 | self.filename = filename 52 | self.start_line = start_line - 1 53 | self.start_column = start_column 54 | self.transformer = transformer 55 | self.diagnostic_ctx = diagnostic_ctx 56 | 57 | def error(self, message, span): 58 | self.diagnostic_ctx.emit("error", message, span) 59 | 60 | def span_from_ast(self, node: py_ast.AST) -> Span: 61 | if isinstance(node, py_ast.withitem): 62 | span = Span.from_ast(self.filename, node.context_expr) 63 | if node.optional_vars: 64 | end_span = Span.from_ast(self.filename, node.optional_vars) 65 | span = span.merge(end_span) 66 | elif isinstance(node, py_ast.Slice): 67 | spans = [] 68 | if node.lower is not None: 69 | spans.append(Span.from_ast(self.filename, node.lower)) 70 | if node.upper is not None: 71 | spans.append(Span.from_ast(self.filename, node.upper)) 72 | if node.step is not None: 73 | spans.append(Span.from_ast(self.filename, node.step)) 74 | span = Span.union(spans) 75 | elif isinstance(node, py_ast.Index): 76 | span = Span.from_ast(self.filename, node.value) # type: ignore[attr-defined] 77 | elif isinstance(node, py_ast.keyword): 78 | # unfortunately this span does not include the keyword name itself 79 | span = Span.from_ast(self.filename, node.value) 80 | else: 81 | span = Span.from_ast(self.filename, node) 82 | return Span( 83 | span.filename, 84 | span.start_line + self.start_line, 85 | span.start_column + self.start_column, 86 | span.end_line + self.start_line, 87 | span.end_column + self.start_column, 88 | ) 89 | 90 | def span_from_asts(self, nodes: Sequence[py_ast.AST]) -> Span: 91 | return Span.union([self.span_from_ast(node) for node in nodes]) 92 | 93 | def compile_module(self, program: py_ast.Module) -> Module: 94 | funcs: Dict[Name, Union[Function, Class]] = {} 95 | # Merge later 96 | span = self.span_from_ast(program.body[0]) 97 | for stmt in program.body: 98 | # need to build module 99 | if isinstance(stmt, py_ast.FunctionDef): 100 | new_func = self.compile_def(stmt) 101 | assert isinstance(new_func, Function) 102 | funcs[stmt.name] = new_func 103 | elif isinstance(stmt, py_ast.ClassDef): 104 | new_class = self.compile_class(stmt) 105 | assert isinstance(new_class, Class) 106 | funcs[stmt.name] = new_class 107 | else: 108 | self.error( 109 | "can only transform top-level functions, other statements are invalid", 110 | span, 111 | ) 112 | return Module(span, funcs) 113 | 114 | def compile_block(self, stmts: List[py_ast.stmt]) -> Block: 115 | assert ( 116 | len(stmts) != 0 117 | ), "Internal Error: python ast forbids empty function bodies." 118 | return Block( 119 | self.span_from_asts(stmts), 120 | [self.compile_stmt(st) for st in stmts], 121 | ) 122 | 123 | def compile_args_to_params(self, args: py_ast.arguments) -> List[Parameter]: 124 | # TODO: arguments object doesn't have line column so its either the 125 | # functions span needs to be passed or we need to reconstruct span 126 | # information from arg objects. 127 | # 128 | # The below solution is temporary hack. 129 | 130 | if hasattr(args, "posonlyargs") and len(args.posonlyargs): # type: ignore 131 | self.error( 132 | "currently synr only supports non-position only arguments", 133 | self.span_from_asts(args.posonlyargs), # type: ignore 134 | ) 135 | 136 | if args.vararg: 137 | self.error( 138 | "currently synr does not support varargs", 139 | self.span_from_ast(args.vararg), 140 | ) 141 | 142 | if len(args.kw_defaults): 143 | self.error( 144 | "currently synr does not support kw_defaults", 145 | self.span_from_asts(args.kw_defaults), 146 | ) 147 | 148 | if args.kwarg: 149 | self.error( 150 | "currently synr does not support kwarg", self.span_from_ast(args.kwarg) 151 | ) 152 | 153 | if len(args.defaults) > 0: 154 | self.error( 155 | "currently synr does not support defaults", 156 | self.span_from_asts(args.defaults), 157 | ) 158 | 159 | params = [] 160 | for arg in args.args: 161 | span = self.span_from_ast(arg) 162 | ty: Optional[Type] = None 163 | if arg.annotation is not None: 164 | ty = self.compile_type(arg.annotation) 165 | params.append(Parameter(span, arg.arg, ty)) 166 | 167 | return params 168 | 169 | def compile_def(self, stmt: py_ast.stmt) -> Function: 170 | stmt_span = self.span_from_ast(stmt) 171 | if isinstance(stmt, py_ast.FunctionDef): 172 | name = stmt.name 173 | args = stmt.args 174 | params = self.compile_args_to_params(args) 175 | body = self.compile_block(stmt.body) 176 | ty: Optional[Type] = None 177 | if stmt.returns is not None: 178 | ty = self.compile_type(stmt.returns) 179 | decorators = [self.compile_expr(dec) for dec in stmt.decorator_list] 180 | return Function(stmt_span, name, params, ty, body, decorators) 181 | else: 182 | self.error( 183 | f"Unexpected {type(stmt)} when looking for a function definition", 184 | stmt_span, 185 | ) 186 | return Function( 187 | Span.invalid(), 188 | "", 189 | [], 190 | Type(Span.invalid()), 191 | Block(Span.invalid(), []), 192 | [], 193 | ) 194 | 195 | def compile_stmt(self, stmt: py_ast.stmt) -> Stmt: 196 | stmt_span = self.span_from_ast(stmt) 197 | if isinstance(stmt, py_ast.Return): 198 | if stmt.value: 199 | value = self.compile_expr(stmt.value) 200 | return Return(stmt_span, value) 201 | else: 202 | return Return(stmt_span, None) 203 | 204 | elif ( 205 | isinstance(stmt, py_ast.Assign) 206 | or isinstance(stmt, py_ast.AugAssign) 207 | or isinstance(stmt, py_ast.AnnAssign) 208 | ): 209 | if isinstance(stmt, py_ast.Assign): 210 | if len(stmt.targets) > 1: 211 | self.error("Multiple assignment is not supported", stmt_span) 212 | lhs = self.compile_expr(stmt.targets[0]) 213 | else: 214 | lhs = self.compile_expr(stmt.target) 215 | 216 | if stmt.value is None: 217 | self.error("Empty type assignment not supported", stmt_span) 218 | rhs = Expr(stmt_span) 219 | else: 220 | rhs = self.compile_expr(stmt.value) 221 | 222 | if isinstance(stmt, py_ast.AugAssign): 223 | op = self._builtin_ops.get(type(stmt.op)) 224 | op_span = lhs.span.between(rhs.span) 225 | if op is None: 226 | self.error( 227 | "Unsupported op {type(op)} in assignment", 228 | op_span, 229 | ) 230 | op = BuiltinOp.Invalid 231 | rhs = Call(stmt_span, Op(op_span, op), [lhs, rhs], {}) 232 | 233 | # if the lhs is a subscript, we replace the whole expression with a SubscriptAssign 234 | if ( 235 | isinstance(lhs, Call) 236 | and isinstance(lhs.func_name, Op) 237 | and lhs.func_name.name == BuiltinOp.Subscript 238 | ): 239 | return UnassignedCall( 240 | stmt_span, 241 | Call( 242 | stmt_span, 243 | Op(lhs.func_name.span, BuiltinOp.SubscriptAssign), 244 | [ 245 | lhs.params[0], 246 | Tuple( 247 | Span.union([x.span for x in lhs.params[1].values]), # type: ignore 248 | lhs.params[1].values, # type: ignore 249 | ), 250 | rhs, 251 | ], 252 | {}, 253 | ), 254 | ) 255 | elif isinstance(lhs, Var): 256 | lhs_vars = [lhs] 257 | elif isinstance(lhs, ArrayLiteral) or isinstance(lhs, Tuple): 258 | lhs_vars = [] 259 | for x in lhs.values: 260 | if isinstance(x, Var): 261 | lhs_vars.append(x) 262 | else: 263 | self.error( 264 | "Left hand side of assignment (x in `x = y`) must be a variable or a list of variables (`x, z = y`), but it is " 265 | + str(type(x)), 266 | x.span, 267 | ) 268 | else: 269 | self.error( 270 | "Left hand side of assignment (x in `x = y`) must be a variable or a list of variables", 271 | lhs.span, 272 | ) 273 | lhs_vars = [Var.invalid()] 274 | ty: Optional[Type] = None 275 | if isinstance(stmt, py_ast.AnnAssign): 276 | ty = self.compile_type(stmt.annotation) 277 | return Assign(stmt_span, lhs_vars, ty, rhs) 278 | 279 | elif isinstance(stmt, py_ast.For): 280 | l = self.compile_expr(stmt.target) 281 | if isinstance(l, Var): 282 | lhs_vars = [l] 283 | elif isinstance(l, Tuple): 284 | lhs_vars = [] 285 | for x in l.values: 286 | if isinstance(x, Var): 287 | lhs_vars.append(x) 288 | else: 289 | self.error( 290 | "Left hand side of for loop (the x in `for x in range(...)`) must be one or more variables, but it is " 291 | + str(type(x)), 292 | x.span, 293 | ) 294 | else: 295 | self.error( 296 | "Left hand side of for loop (the x in `for x in range(...)`) must be one or more variables", 297 | self.span_from_ast(stmt.target), 298 | ) 299 | lhs_vars = [Var(Span.invalid(), Id.invalid())] 300 | rhs = self.compile_expr(stmt.iter) 301 | body = self.compile_block(stmt.body) 302 | return For(self.span_from_ast(stmt), lhs_vars, rhs, body) 303 | 304 | elif isinstance(stmt, py_ast.While): 305 | condition = self.compile_expr(stmt.test) 306 | body = self.compile_block(stmt.body) 307 | return While(stmt_span, condition, body) 308 | 309 | elif isinstance(stmt, py_ast.With): 310 | if len(stmt.items) != 1: 311 | self.error( 312 | "Only one `x as y` statement allowed in with statements", 313 | self.span_from_asts(stmt.items), 314 | ) 315 | wth = stmt.items[0] 316 | if wth.optional_vars: 317 | l = self.compile_expr(wth.optional_vars) 318 | if isinstance(l, Var): 319 | lhs_vars = [l] 320 | elif isinstance(l, ArrayLiteral) or isinstance(l, Tuple): 321 | lhs_vars = [] 322 | for x in l.values: 323 | if isinstance(x, Var): 324 | lhs_vars.append(x) 325 | else: 326 | self.error( 327 | "Right hand side of with statement (y in `with x as y:`) must be a variable, list of var or tuple of var, but it is " 328 | + str(type(x)), 329 | x.span, 330 | ) 331 | else: 332 | self.error( 333 | "Right hand side of with statement (y in `with x as y:`) must be a variable, list of var or tuple of var", 334 | self.span_from_ast(wth.optional_vars), 335 | ) 336 | lhs_vars = [Var.invalid()] 337 | else: 338 | lhs_vars = [] 339 | rhs = self.compile_expr(wth.context_expr) 340 | body = self.compile_block(stmt.body) 341 | return With(self.span_from_ast(stmt), lhs_vars, rhs, body) 342 | 343 | elif isinstance(stmt, py_ast.If): 344 | true = self.compile_block(stmt.body) 345 | if len(stmt.orelse) > 0: 346 | false = self.compile_block(stmt.orelse) 347 | else: 348 | false = Block(stmt_span, []) # TODO: this span isn't correct 349 | condition = self.compile_expr(stmt.test) 350 | return If(stmt_span, condition, true, false) 351 | 352 | elif isinstance(stmt, py_ast.Expr): 353 | expr = self.compile_expr(stmt.value) 354 | if isinstance(expr, Call): 355 | return UnassignedCall(expr.span, expr) 356 | else: 357 | self.error( 358 | f"Found unexpected expression of type {type(expr)} when looking for a statement", 359 | expr.span, 360 | ) 361 | return Stmt(Span.invalid()) 362 | 363 | elif isinstance(stmt, py_ast.FunctionDef): 364 | return self.compile_def(stmt) 365 | 366 | elif isinstance(stmt, py_ast.Assert): 367 | return Assert( 368 | stmt_span, 369 | self.compile_expr(stmt.test), 370 | None if stmt.msg is None else self.compile_expr(stmt.msg), 371 | ) 372 | 373 | elif isinstance(stmt, py_ast.Nonlocal): 374 | # TODO: the variable spans here are incorrect as the Python AST stores each identifier 375 | # as a raw string (with no span information), so we just use the statement's span 376 | return Nonlocal( 377 | stmt_span, [Var(stmt_span, Id(stmt_span, name)) for name in stmt.names] 378 | ) 379 | 380 | elif isinstance(stmt, py_ast.Global): 381 | # TODO: the variable spans here are incorrect as the Python AST stores each identifier 382 | # as a raw string (with no span information), so we just use the statement's span 383 | return Global( 384 | stmt_span, [Var(stmt_span, Id(stmt_span, name)) for name in stmt.names] 385 | ) 386 | 387 | else: 388 | self.error(f"Found unexpected {type(stmt)} when compiling stmt", stmt_span) 389 | return Stmt(Span.invalid()) 390 | 391 | def compile_var(self, expr: py_ast.expr) -> Union[Var, Attr]: 392 | expr_span = self.span_from_ast(expr) 393 | if isinstance(expr, py_ast.Name): 394 | return Var(expr_span, Id(expr_span, expr.id)) 395 | if isinstance(expr, py_ast.Attribute): 396 | sub_var = self.compile_expr(expr.value) 397 | # The name of the field we are accessing comes at the end of the 398 | # span, we can just infer the span for it from counting from the 399 | # end. 400 | attr_span = Span( 401 | expr_span.filename, 402 | expr_span.end_line, 403 | expr_span.end_column - len(expr.attr), 404 | expr_span.end_line, 405 | expr_span.end_column, 406 | ) 407 | return Attr(expr_span, sub_var, Id(attr_span, expr.attr)) 408 | self.error("Expected a variable name of the form a.b.c", expr_span) 409 | return Var.invalid() 410 | 411 | def compile_expr(self, expr: py_ast.expr) -> Expr: 412 | expr_span = self.span_from_ast(expr) 413 | if isinstance(expr, py_ast.Name) or isinstance(expr, py_ast.Attribute): 414 | return self.compile_var(expr) 415 | if isinstance(expr, py_ast.Constant): 416 | if ( 417 | isinstance(expr.value, float) 418 | or isinstance(expr.value, complex) 419 | or isinstance(expr.value, int) 420 | or isinstance(expr.value, str) 421 | or isinstance(expr.value, type(None)) 422 | or isinstance(expr.value, bool) 423 | ): 424 | return Constant(expr_span, expr.value) 425 | self.error( 426 | "Only float, complex, int, str, bool, and None constants are allowed. " 427 | f"{type(expr.value)} was provided.", 428 | expr_span, 429 | ) 430 | return Constant(expr_span, float("nan")) 431 | if isinstance(expr, py_ast.Num): 432 | return Constant(expr_span, expr.n) 433 | if isinstance(expr, py_ast.Str): 434 | return Constant(expr_span, expr.s) 435 | if isinstance(expr, py_ast.NameConstant): 436 | return Constant(expr_span, expr.value) 437 | if isinstance(expr, py_ast.Call): 438 | return self.compile_call(expr) 439 | if isinstance(expr, py_ast.BinOp): 440 | lhs = self.compile_expr(expr.left) 441 | rhs = self.compile_expr(expr.right) 442 | op_span = lhs.span.between(rhs.span) 443 | ty = type(expr.op) 444 | op = self._builtin_ops.get(ty) 445 | if op is None: 446 | self.error( 447 | f"Binary operator {ty} is not supported", 448 | op_span, 449 | ) 450 | op = BuiltinOp.Invalid 451 | return Call(expr_span, Op(op_span, op), [lhs, rhs], {}) 452 | if isinstance(expr, py_ast.Compare): 453 | lhs = self.compile_expr(expr.left) 454 | if ( 455 | len(expr.ops) != 1 456 | or len(expr.comparators) != 1 457 | or len(expr.comparators) != len(expr.ops) 458 | ): 459 | self.error( 460 | "Only one comparison operator is allowed", 461 | self.span_from_asts(expr.comparators), 462 | ) 463 | rhs = self.compile_expr(expr.comparators[0]) 464 | op_span = lhs.span.between(rhs.span) 465 | ty2 = type(expr.ops[0]) 466 | op = self._builtin_ops.get(ty2) 467 | if op is None: 468 | self.error( 469 | f"Comparison operator {ty2} is not supported", 470 | op_span, 471 | ) 472 | op = BuiltinOp.Invalid 473 | return Call(expr_span, Op(op_span, op), [lhs, rhs], {}) 474 | if isinstance(expr, py_ast.UnaryOp): 475 | lhs = self.compile_expr(expr.operand) 476 | op_span = expr_span.subtract(lhs.span) 477 | ty3 = type(expr.op) 478 | op = self._builtin_ops.get(ty3) 479 | if op is None: 480 | self.error( 481 | f"Unary operator {ty3} is not supported", 482 | op_span, 483 | ) 484 | op = BuiltinOp.Invalid 485 | return Call(expr_span, Op(op_span, op), [lhs], {}) 486 | if isinstance(expr, py_ast.BoolOp): 487 | lhs = self.compile_expr(expr.values[0]) 488 | rhs = self.compile_expr(expr.values[1]) 489 | op_span = lhs.span.between(rhs.span) 490 | ty4 = type(expr.op) 491 | op = self._builtin_ops.get(ty4) 492 | if op is None: 493 | self.error( 494 | f"Binary operator {ty4} is not supported", 495 | op_span, 496 | ) 497 | op = BuiltinOp.Invalid 498 | call = Call(lhs.span.merge(rhs.span), Op(op_span, op), [lhs, rhs], {}) 499 | for arg in expr.values[2:]: 500 | rhs = self.compile_expr(arg) 501 | call = Call( 502 | call.span.merge(rhs.span), 503 | Op(call.span.between(rhs.span), op), 504 | [call, rhs], 505 | {}, 506 | ) 507 | return call 508 | if isinstance(expr, py_ast.Subscript): 509 | lhs = self.compile_expr(expr.value) 510 | rhs = self.compile_subscript_slice(expr.slice) 511 | return Call(expr_span, Op(rhs.span, BuiltinOp.Subscript), [lhs, rhs], {}) 512 | if isinstance(expr, py_ast.Tuple): 513 | return Tuple(expr_span, [self.compile_expr(x) for x in expr.elts]) 514 | if isinstance(expr, py_ast.Dict): 515 | return DictLiteral( 516 | expr_span, 517 | [self.compile_expr(x) for x in expr.keys], # type: ignore 518 | [self.compile_expr(x) for x in expr.values], 519 | ) 520 | if isinstance(expr, py_ast.List): 521 | return ArrayLiteral(expr_span, [self.compile_expr(x) for x in expr.elts]) 522 | if isinstance(expr, py_ast.Slice): 523 | return self.compile_slice(expr) 524 | if isinstance(expr, py_ast.Lambda): 525 | params = self.compile_args_to_params(expr.args) 526 | body = self.compile_expr(expr.body) 527 | return Lambda(expr_span, params, body) 528 | 529 | self.error(f"Unexpected expression {type(expr)}", expr_span) 530 | return Expr(Span.invalid()) 531 | 532 | def _compile_slice(self, slice: py_ast.slice) -> Expr: 533 | if isinstance(slice, py_ast.Slice): 534 | return self.compile_slice(slice) 535 | # x[1:2, z] is an ExtSlice in the python ast 536 | if isinstance(slice, py_ast.ExtSlice): 537 | slices = [self._compile_slice(d) for d in slice.dims] # type: ignore[attr-defined] 538 | return Tuple(Span.union([s.span for s in slices]), slices) 539 | if isinstance(slice, py_ast.Index): 540 | return self.compile_expr(slice.value) # type: ignore[attr-defined] 541 | self.error(f"Unexpected slice type {type(slice)}", self.span_from_ast(slice)) 542 | return Expr(Span.invalid()) 543 | 544 | def compile_subscript_slice(self, slice: Union[py_ast.slice, py_ast.expr]) -> Tuple: 545 | # In 3.9, multiple slices are just tuples 546 | if sys.version_info.major == 3 and sys.version_info.minor >= 9: 547 | s = self.compile_expr(slice) # type: ignore 548 | else: 549 | s = self._compile_slice(slice) # type: ignore 550 | # We ensure that slices are always a tuple 551 | if isinstance(s, Tuple): 552 | return s 553 | return Tuple(s.span, [s]) 554 | 555 | def compile_slice(self, slice: py_ast.Slice) -> Slice: 556 | # TODO: handle spans for slice without start and end 557 | if slice.lower is None: 558 | start: Expr = Constant(Span.invalid(), 0) 559 | else: 560 | start = self.compile_expr(slice.lower) 561 | if slice.upper is None: 562 | end: Expr = Constant(Span.invalid(), -1) 563 | else: 564 | end = self.compile_expr(slice.upper) 565 | if slice.step is None: 566 | step: Expr = Constant(Span.invalid(), 1) 567 | else: 568 | step = self.compile_expr(slice.step) 569 | span = self.span_from_ast(slice) 570 | return Slice(span, start, step, end) 571 | 572 | def compile_call(self, call: py_ast.Call) -> Call: 573 | kws: Dict[Expr, Expr] = dict() 574 | for x in call.keywords: 575 | if x.arg in kws: 576 | self.error( 577 | f"Duplicate keyword argument {x.arg}", self.span_from_ast(x.value) 578 | ) 579 | kws[Constant(self.span_from_ast(x), x.arg)] = self.compile_expr(x.value) 580 | 581 | func = self.compile_expr(call.func) 582 | 583 | args = [] 584 | for arg in call.args: 585 | args.append(self.compile_expr(arg)) 586 | 587 | return Call(self.span_from_ast(call), func, args, kws) 588 | 589 | def _expr2type(self, expr: Expr) -> Type: 590 | if isinstance(expr, Var): 591 | return TypeVar(expr.span, expr.id) 592 | if isinstance(expr, Constant): 593 | return TypeConstant(expr.span, expr.value) 594 | if isinstance(expr, Tuple): 595 | return TypeTuple(expr.span, expr.values) 596 | if isinstance(expr, Slice): 597 | return TypeSlice( 598 | expr.span, 599 | self._expr2type(expr.start), 600 | self._expr2type(expr.step), 601 | self._expr2type(expr.end), 602 | ) 603 | if isinstance(expr, Call): 604 | if isinstance(expr.func_name, Op): 605 | if expr.func_name.name == BuiltinOp.Subscript: # type: ignore 606 | assert isinstance( 607 | expr.params[1], Tuple 608 | ), f"Expected subscript call to have rhs of Tuple, but it is {type(expr.params[1])}" 609 | return TypeApply( 610 | expr.span, 611 | self._expr2type(expr.params[0]), 612 | [self._expr2type(x) for x in expr.params[1].values], 613 | ) 614 | return TypeCall( 615 | expr.span, 616 | expr.func_name.name, 617 | [self._expr2type(x) for x in expr.params], 618 | {}, 619 | ) 620 | elif isinstance(expr.func_name, Expr): 621 | return TypeCall( 622 | expr.span, 623 | self._expr2type(expr.func_name), 624 | [self._expr2type(x) for x in expr.params], 625 | { 626 | self._expr2type(key): self._expr2type(value) 627 | for key, value in expr.keyword_params.items() 628 | }, 629 | ) 630 | 631 | if isinstance(expr, Attr): 632 | return TypeAttr(expr.span, self._expr2type(expr.object), expr.field) 633 | self.error(f"Found unknown kind (type of type) {type(expr)}", expr.span) 634 | return Type(Span.invalid()) 635 | 636 | def compile_type(self, ty: py_ast.expr) -> Type: 637 | return self._expr2type(self.compile_expr(ty)) 638 | 639 | def compile_class(self, cls: py_ast.ClassDef) -> Class: 640 | span = self.span_from_ast(cls) 641 | funcs: Dict[Name, Function] = {} 642 | stmts: List[Assign] = [] 643 | for stmt in cls.body: 644 | stmt_span = self.span_from_ast(stmt) 645 | if isinstance(stmt, py_ast.FunctionDef): 646 | f = self.compile_def(stmt) 647 | funcs[f.name] = f 648 | elif isinstance(stmt, (py_ast.AnnAssign, py_ast.Assign)): 649 | assign = self.compile_stmt(stmt) 650 | if isinstance(assign, Assign): 651 | stmts.append(assign) 652 | else: 653 | self.error( 654 | f"Unexpected {type(assign)} in class definition. Only function definitions and assignments are allowed", 655 | stmt_span, 656 | ) 657 | else: 658 | self.error( 659 | "Only functions definitions and assignments are allowed within a class", 660 | stmt_span, 661 | ) 662 | return Class(span, cls.name, funcs, stmts) 663 | 664 | 665 | def _get_full_source(program: Any, source: str) -> str: 666 | """Get full source code of the program 667 | 668 | Parameters 669 | ---------- 670 | program : Any 671 | The python function or class. 672 | 673 | source : str 674 | The source code of the program without other codes in the same file. 675 | 676 | Returns 677 | ------- 678 | str 679 | The full source code 680 | """ 681 | try: 682 | # It will cause a problem when running in Jupyter Notebook. 683 | # `mod` will be , which is a built-in module 684 | # and `getsource` will throw a TypeError 685 | mod = inspect.getmodule(program) 686 | if mod is not None: 687 | return inspect.getsource(mod) 688 | else: 689 | return source 690 | except TypeError: 691 | # It's a work around for Jupyter problem. 692 | # Since `findsource` is an internal API of inspect, we just use it 693 | # as a fallback method. 694 | full_source, _ = inspect.findsource(program) 695 | return "".join(full_source) 696 | 697 | 698 | def to_ast( 699 | program: Union[Any, str], 700 | diagnostic_ctx: DiagnosticContext, 701 | transformer: Optional[Transformer] = None, 702 | ) -> Any: 703 | """Parse an abstract syntax tree from a Python program. 704 | 705 | Examples 706 | -------- 707 | 708 | :py:func:`to_ast` can be used with a given python function or class: 709 | 710 | .. code-block:: python 711 | 712 | import synr 713 | 714 | def my_function(x): 715 | return x + 2 716 | 717 | synr.to_ast(my_function, synr.PrinterDiagnosticContext()) 718 | 719 | 720 | :py:func:`to_ast` can also be used with a string containing python code: 721 | 722 | .. code-block:: python 723 | 724 | import synr 725 | f = "def my_function(x):\\\\n return x + 2" 726 | synr.to_ast(f, synr.PrinterDiagnosticContext()) 727 | 728 | 729 | 730 | Parameters 731 | ---------- 732 | program : Union[Any, str] 733 | A string containing a python program or a python function or class. If 734 | a python function or class is used, then line numbers will be localized 735 | to the file that the function or class came from. 736 | diagnostic_ctx : DiagnosticContext 737 | A diagnostic context to handle reporting of errors. 738 | transformer : Optional[Transformer] 739 | An optional transformer to apply to the AST after parsing. The 740 | transformer allows for converting from synr's AST to a user specified 741 | AST using the same diagnostic context and error handling. 742 | 743 | Returns 744 | ------- 745 | Union[synr.ast.Module, Any] 746 | A synr AST if no transformer was specified and not errors occured. Or 747 | an error if one occured. Or the result of applying the transformer to 748 | the synr AST. 749 | """ 750 | if isinstance(program, str): 751 | source_name = "" 752 | source = program 753 | full_source = source 754 | start_line = 1 755 | start_column = 0 756 | else: 757 | source_name = inspect.getsourcefile(program) # type: ignore 758 | assert source_name, "source name must be valid" 759 | lines, start_line = inspect.getsourcelines(program) 760 | start_column = 0 761 | if len(lines) > 0: 762 | start_column = len(lines[0]) - len(lines[0].lstrip()) 763 | if start_column == 0: 764 | source = "".join(lines) 765 | else: 766 | # make sure to preserve blank lines for correct spans 767 | source = "\n".join([l[start_column:].rstrip() for l in lines]) 768 | full_source = _get_full_source(program, source) 769 | 770 | diagnostic_ctx.add_source(source_name, full_source) 771 | program_ast = py_ast.parse(source) 772 | compiler = Compiler( 773 | source_name, start_line, start_column, transformer, diagnostic_ctx 774 | ) 775 | assert isinstance(program_ast, py_ast.Module), "Synr only supports module inputs" 776 | prog = compiler.compile_module(program_ast) 777 | err = diagnostic_ctx.render() 778 | if err is not None: 779 | return err 780 | if transformer is not None: 781 | transformed = transformer.do_transform(prog, diagnostic_ctx) 782 | err = diagnostic_ctx.render() 783 | if err is not None: 784 | return err 785 | return transformed 786 | else: 787 | return prog 788 | -------------------------------------------------------------------------------- /synr/diagnostic_context.py: -------------------------------------------------------------------------------- 1 | """Error handling in Synr is done through a `DiagnosticContext`. This context 2 | accumulates errors and renders them together after parsing has finished. 3 | """ 4 | from .ast import Span 5 | from typing import Optional, Sequence, List, Tuple, Any, Dict 6 | import attr 7 | 8 | 9 | class DiagnosticContext: 10 | def add_source(self, name: str, source: str) -> None: 11 | """Add a file with source code to the context. This will be called 12 | before any call to :py:func:`emit` that contains a span in this 13 | file. 14 | """ 15 | raise NotImplementedError("You must subclass DiagnosticContext") 16 | 17 | def emit(self, level: str, message: str, span: Span) -> None: 18 | """Called when an error has occured.""" 19 | raise NotImplementedError("You must subclass DiagnosticContext") 20 | 21 | def render(self) -> Optional[Any]: 22 | """Render out all error messages. Can either return a value or raise 23 | and execption. 24 | """ 25 | raise NotImplementedError("You must subclass DiagnosticContext") 26 | 27 | 28 | @attr.s(auto_attribs=True) 29 | class PrinterDiagnosticContext(DiagnosticContext): 30 | """Simple diagnostic context that prints the error and underlines its 31 | location in the source. Raises a RuntimeError on the first error hit. 32 | """ 33 | 34 | sources: Dict[str, Sequence[str]] = attr.ib(default=attr.Factory(dict)) 35 | errors: List[Tuple[str, str, Span]] = attr.ib(default=attr.Factory(list)) 36 | 37 | def add_source(self, name: str, source: str) -> None: 38 | self.sources[name] = source.split("\n") 39 | 40 | def emit(self, level: str, message: str, span: Span): 41 | self.errors.append((level, message, span)) 42 | raise RuntimeError(self.render()) 43 | 44 | def render(self) -> Optional[str]: 45 | msgs = [] 46 | for level, message, span in self.errors: 47 | msg = f"Parse error on line {span.filename}:{span.start_line}:{span.start_column}:\n" 48 | msg += "\n".join( 49 | self.sources[span.filename][span.start_line - 1 : span.end_line] 50 | ) 51 | msg += "\n" 52 | msg += ( 53 | " " * (span.start_column - 1) 54 | + "^" * (span.end_column - span.start_column) 55 | + "\n" 56 | ) 57 | msg += message 58 | msgs.append(msg) 59 | if len(msgs) == 0: 60 | return None 61 | else: 62 | return "\n\n".join(msgs) 63 | -------------------------------------------------------------------------------- /synr/transformer.py: -------------------------------------------------------------------------------- 1 | """This module handles converting a synr AST into the users desired 2 | representation. We provide a visitor class that the user can inherit from to 3 | write their conversion. 4 | """ 5 | from typing import TypeVar, Generic, Union 6 | 7 | from . import ast 8 | from .diagnostic_context import DiagnosticContext 9 | 10 | M = TypeVar("M") 11 | F = TypeVar("F") 12 | S = TypeVar("S") 13 | E = TypeVar("E") 14 | T = TypeVar("T") 15 | B = TypeVar("B") 16 | P = TypeVar("P") 17 | 18 | 19 | class Transformer(Generic[M, F, S, E, B, T]): 20 | """A visitor to handle user specified transformations on the AST.""" 21 | 22 | def do_transform( 23 | self, node: ast.Node, diag: "DiagnosticContext" 24 | ) -> Union[M, F, S, E, B, P, T, None]: 25 | """Entry point for the transformation. 26 | 27 | This is called with the synr AST and the diagnostic context used in parsing the python AST. 28 | """ 29 | self._diagnostic_context = diag 30 | return self.transform(node) 31 | 32 | def error(self, message, span): 33 | """Report an error on a given span.""" 34 | self._diagnostic_context.emit("error", message, span) 35 | 36 | def transform(self, node: ast.Node) -> Union[M, F, S, E, B, P, T, None]: 37 | """Visitor function. 38 | 39 | Call this to recurse into child nodes. 40 | """ 41 | if isinstance(node, ast.Module): 42 | return self.transform_module(node) 43 | if isinstance(node, ast.Function): 44 | return self.transform_function(node) 45 | if isinstance(node, ast.Stmt): 46 | return self.transform_stmt(node) 47 | if isinstance(node, ast.Expr): 48 | return self.transform_expr(node) 49 | if isinstance(node, ast.Type): 50 | return self.transform_type(node) 51 | if isinstance(node, ast.Block): 52 | return self.transform_block(node) 53 | if isinstance(node, ast.Parameter): 54 | return self.transform_parameter(node) 55 | self.error(f"Unexpected synr ast type {type(node)}", node.span) 56 | return None 57 | 58 | def transform_module(self, mod: ast.Module) -> M: 59 | pass 60 | 61 | def transform_function(self, func: ast.Function) -> F: 62 | pass 63 | 64 | def transform_stmt(self, stmt: ast.Stmt) -> S: 65 | pass 66 | 67 | def transform_expr(self, expr: ast.Expr) -> E: 68 | pass 69 | 70 | def transform_block(self, expr: ast.Block) -> B: 71 | pass 72 | 73 | def transform_parameter(self, expr: ast.Parameter) -> P: 74 | pass 75 | 76 | def transform_type(self, ty: ast.Type) -> T: 77 | pass 78 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/octoml/synr/6f8ba7fdbd359fe7972456dd69c18a9c49a50c0d/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_synr.py: -------------------------------------------------------------------------------- 1 | import synr 2 | from synr import __version__ 3 | from typing import Any 4 | import inspect 5 | import sys 6 | 7 | 8 | def test_version(): 9 | assert __version__ == "0.6.0" 10 | 11 | 12 | def to_ast(program: Any) -> Any: 13 | diag_ctx = synr.PrinterDiagnosticContext() 14 | transformer = None 15 | res = synr.to_ast(program, diag_ctx, transformer) 16 | if isinstance(res, str): 17 | raise (RuntimeError(res)) 18 | return res 19 | 20 | 21 | def assert_one_fn(module, name, no_params=None): 22 | func = module.funcs.get(name) 23 | assert func, "the function `%s` was not found" % name 24 | if no_params: 25 | assert len(func.params) == no_params, "the parameters do not match" 26 | assert isinstance(func.body, synr.ast.Block) 27 | return func 28 | 29 | 30 | def identity(x): 31 | return x 32 | 33 | 34 | def test_id_function(): 35 | module = to_ast(identity) 36 | ast_fn = assert_one_fn(module, "identity", no_params=1) 37 | return_var = ast_fn.body.stmts[-1].value 38 | assert isinstance(return_var, synr.ast.Var) 39 | assert return_var.id.name == "x" 40 | 41 | 42 | class ExampleClass: 43 | def func(): 44 | return 3 45 | 46 | 47 | def test_class(): 48 | module = to_ast(ExampleClass) 49 | cls = module.funcs.get("ExampleClass") 50 | assert cls, "ExampleClass not found" 51 | assert isinstance(cls, synr.ast.Class), "ExampleClass was not parsed as a Class" 52 | assert len(cls.funcs) == 1, "func not found" 53 | fn = cls.funcs["func"] 54 | assert isinstance(fn, synr.ast.Function), "func not found" 55 | assert fn.name == "func", "func not found" 56 | return_var = fn.body.stmts[-1].value 57 | assert isinstance(return_var, synr.ast.Constant) 58 | assert return_var.value == 3 59 | 60 | 61 | def func_for(): 62 | for x in range(3): 63 | return x 64 | 65 | for x, y in grid(5, 6): 66 | return x 67 | 68 | 69 | def test_for(): 70 | module = to_ast(func_for) 71 | fn = assert_one_fn(module, "func_for", no_params=0) 72 | 73 | fr = fn.body.stmts[0] 74 | assert isinstance(fr, synr.ast.For), "Did not find for loop" 75 | assert fr.lhs[0].id.name == "x", "For lhs is incorrect" 76 | assert isinstance(fr.rhs, synr.ast.Call) 77 | assert fr.rhs.func_name.id.name == "range" 78 | assert fr.rhs.params[0].value == 3 79 | assert isinstance(fr.body.stmts[0], synr.ast.Return) 80 | assert fr.body.stmts[0].value.id.name == "x" 81 | 82 | fr = fn.body.stmts[1] 83 | assert isinstance(fr, synr.ast.For), "Did not find for loop" 84 | assert len(fr.lhs) == 2 85 | assert fr.lhs[0].id.name == "x", "For lhs is incorrect" 86 | assert fr.lhs[1].id.name == "y", "For lhs is incorrect" 87 | assert isinstance(fr.rhs, synr.ast.Call) 88 | assert fr.rhs.func_name.id.name == "grid" 89 | assert fr.rhs.params[0].value == 5 90 | assert fr.rhs.params[1].value == 6 91 | assert isinstance(fr.body.stmts[0], synr.ast.Return) 92 | assert fr.body.stmts[0].value.id.name == "x" 93 | 94 | 95 | def func_while(): 96 | while x < 10: 97 | return x 98 | 99 | 100 | def test_while(): 101 | module = to_ast(func_while) 102 | fn = assert_one_fn(module, "func_while", no_params=0) 103 | 104 | while_stmt = fn.body.stmts[0] 105 | assert isinstance(while_stmt, synr.ast.While) 106 | assert isinstance(while_stmt.body.stmts[0], synr.ast.Return) 107 | assert while_stmt.body.stmts[0].value.id.name == "x" 108 | cond = while_stmt.condition 109 | assert isinstance(cond, synr.ast.Call) 110 | assert cond.func_name.name == synr.ast.BuiltinOp.LT 111 | assert cond.params[0].id.name == "x" 112 | assert cond.params[1].value == 10 113 | 114 | 115 | def func_with(): 116 | with x as y: 117 | return x 118 | 119 | with block() as [x, y]: 120 | return x 121 | 122 | with block() as (): 123 | return True 124 | 125 | with block(): 126 | return True 127 | 128 | 129 | def test_with(): 130 | module = to_ast(func_with) 131 | fn = assert_one_fn(module, "func_with", no_params=0) 132 | wth = fn.body.stmts[0] 133 | assert isinstance( 134 | wth, synr.ast.With 135 | ), "Did not find With statement, found %s" % type(wth) 136 | assert wth.rhs.id.name == "x" 137 | assert wth.lhs[0].id.name == "y" 138 | assert isinstance(wth.body.stmts[0], synr.ast.Return) 139 | assert wth.body.stmts[0].value.id.name == "x" 140 | 141 | wth = fn.body.stmts[1] 142 | assert isinstance( 143 | wth, synr.ast.With 144 | ), "Did not find With statement, found %s" % type(wth) 145 | assert isinstance(wth.rhs, synr.ast.Call) 146 | assert wth.rhs.func_name.id.name == "block" 147 | assert len(wth.lhs) == 2 148 | assert wth.lhs[0].id.name == "x" 149 | assert wth.lhs[1].id.name == "y" 150 | assert isinstance(wth.body.stmts[0], synr.ast.Return) 151 | assert wth.body.stmts[0].value.id.name == "x" 152 | 153 | wth = fn.body.stmts[2] 154 | assert isinstance( 155 | wth, synr.ast.With 156 | ), "Did not find With statement, found %s" % type(wth) 157 | assert isinstance(wth.rhs, synr.ast.Call) 158 | assert wth.rhs.func_name.id.name == "block" 159 | assert len(wth.lhs) == 0 160 | 161 | 162 | def func_block(): 163 | y = x 164 | z = y 165 | return z 166 | 167 | 168 | def test_block(): 169 | module = to_ast(func_block) 170 | fn = assert_one_fn(module, "func_block", no_params=0) 171 | block = fn.body 172 | assert isinstance(block, synr.ast.Block) 173 | assert len(block.stmts) == 3 174 | assert isinstance(block.stmts[0], synr.ast.Assign) 175 | assert isinstance(block.stmts[1], synr.ast.Assign) 176 | assert isinstance(block.stmts[2], synr.ast.Return) 177 | 178 | 179 | def func_assign(): 180 | y = 2 181 | x, y = 2, 2 182 | (x, y) = 2, 2 183 | [x, y] = 2, 2 184 | 185 | 186 | def test_assign(): 187 | module = to_ast(func_assign) 188 | fn = assert_one_fn(module, "func_assign", no_params=0) 189 | assign = fn.body.stmts[0] 190 | assert isinstance(assign, synr.ast.Assign) 191 | assert isinstance(assign.lhs[0], synr.ast.Var) 192 | assert assign.lhs[0].id.name == "y" 193 | assert isinstance(assign.rhs, synr.ast.Constant) 194 | assert assign.rhs.value == 2 195 | 196 | def _check_multi_assign(assign): 197 | assert isinstance(assign, synr.ast.Assign) 198 | 199 | assert len(assign.lhs) == 2 200 | assert isinstance(assign.lhs[0], synr.ast.Var) 201 | assert assign.lhs[0].id.name == "x" 202 | assert isinstance(assign.lhs[1], synr.ast.Var) 203 | assert assign.lhs[1].id.name == "y" 204 | 205 | _check_multi_assign(fn.body.stmts[1]) 206 | _check_multi_assign(fn.body.stmts[2]) 207 | _check_multi_assign(fn.body.stmts[3]) 208 | 209 | 210 | def func_var(): 211 | return x.y.z 212 | 213 | 214 | def test_var(): 215 | module = to_ast(func_var) 216 | fn = assert_one_fn(module, "func_var", no_params=0) 217 | ret = fn.body.stmts[0] 218 | assert ret.value.field.name == "z" 219 | assert ret.value.object.field.name == "y" 220 | assert ret.value.object.object.id.name == "x" 221 | 222 | 223 | def func_binop(): 224 | x = 1 + 2 225 | x = 1 - 2 226 | x = 1 * 2 227 | x = 1 / 2 228 | x = 1 // 2 229 | x = 1 % 2 230 | x = 1 == 2 231 | x = 1 != 2 232 | x = 1 >= 2 233 | x = 1 <= 2 234 | x = 1 < 2 235 | x = 1 > 2 236 | x = not True 237 | x = True and False 238 | x = True or False 239 | x += 1 240 | x -= 1 241 | x /= 1 242 | x *= 1 243 | x //= 1 244 | x %= 1 245 | x = (1 + 3) / (4 % 2) 246 | x = -1 247 | x = +1 248 | x = ~1 249 | 250 | 251 | def test_binop(): 252 | module = to_ast(func_binop) 253 | fn = assert_one_fn(module, "func_binop", no_params=0) 254 | stmts = fn.body.stmts 255 | 256 | def verify(stmt, op, vals): 257 | assert isinstance(stmt, synr.ast.Call) 258 | assert stmt.func_name.name == op, f"Expect {op.name}, got {stmt.func_name}" 259 | assert len(vals) == len(stmt.params) 260 | for i in range(len(vals)): 261 | assert stmt.params[i].value == vals[i] 262 | 263 | verify(stmts[0].rhs, synr.ast.BuiltinOp.Add, [1, 2]) 264 | verify(stmts[1].rhs, synr.ast.BuiltinOp.Sub, [1, 2]) 265 | verify(stmts[2].rhs, synr.ast.BuiltinOp.Mul, [1, 2]) 266 | verify(stmts[3].rhs, synr.ast.BuiltinOp.Div, [1, 2]) 267 | verify(stmts[4].rhs, synr.ast.BuiltinOp.FloorDiv, [1, 2]) 268 | verify(stmts[5].rhs, synr.ast.BuiltinOp.Mod, [1, 2]) 269 | verify(stmts[6].rhs, synr.ast.BuiltinOp.Eq, [1, 2]) 270 | verify(stmts[7].rhs, synr.ast.BuiltinOp.NotEq, [1, 2]) 271 | verify(stmts[8].rhs, synr.ast.BuiltinOp.GE, [1, 2]) 272 | verify(stmts[9].rhs, synr.ast.BuiltinOp.LE, [1, 2]) 273 | verify(stmts[10].rhs, synr.ast.BuiltinOp.LT, [1, 2]) 274 | verify(stmts[11].rhs, synr.ast.BuiltinOp.GT, [1, 2]) 275 | verify(stmts[12].rhs, synr.ast.BuiltinOp.Not, [True]) 276 | verify(stmts[13].rhs, synr.ast.BuiltinOp.And, [True, False]) 277 | verify(stmts[14].rhs, synr.ast.BuiltinOp.Or, [True, False]) 278 | 279 | def verify_assign(stmt, op, vals): 280 | assert isinstance(stmt.rhs, synr.ast.Call) 281 | assert stmt.rhs.func_name.name == op, f"Expect {op.name}, got {stmt.id.name}" 282 | assert len(vals) + 1 == len(stmt.rhs.params) 283 | assert stmt.lhs[0].id.name == stmt.rhs.params[0].id.name 284 | for i in range(len(vals)): 285 | assert stmt.rhs.params[i + 1].value == vals[i] 286 | 287 | verify_assign(stmts[15], synr.ast.BuiltinOp.Add, [1]) 288 | verify_assign(stmts[16], synr.ast.BuiltinOp.Sub, [1]) 289 | verify_assign(stmts[17], synr.ast.BuiltinOp.Div, [1]) 290 | verify_assign(stmts[18], synr.ast.BuiltinOp.Mul, [1]) 291 | verify_assign(stmts[19], synr.ast.BuiltinOp.FloorDiv, [1]) 292 | verify_assign(stmts[20], synr.ast.BuiltinOp.Mod, [1]) 293 | verify(stmts[22].rhs, synr.ast.BuiltinOp.USub, [1]) 294 | verify(stmts[23].rhs, synr.ast.BuiltinOp.UAdd, [1]) 295 | verify(stmts[24].rhs, synr.ast.BuiltinOp.Invert, [1]) 296 | 297 | 298 | def func_if(): 299 | if 1 and 2 and 3 or 4: 300 | return 1 301 | elif 1: 302 | return 2 303 | else: 304 | return 3 305 | 306 | 307 | def test_if(): 308 | module = to_ast(func_if) 309 | fn = assert_one_fn(module, "func_if", no_params=0) 310 | 311 | if_stmt = fn.body.stmts[0] 312 | assert isinstance(if_stmt, synr.ast.If) 313 | assert isinstance(if_stmt.true.stmts[0], synr.ast.Return) 314 | assert if_stmt.true.stmts[0].value.value == 1 315 | cond = if_stmt.condition 316 | assert isinstance(cond, synr.ast.Call) 317 | assert cond.func_name.name == synr.ast.BuiltinOp.Or 318 | assert cond.params[1].value == 4 319 | elif_stmt = if_stmt.false.stmts[0] 320 | assert isinstance(elif_stmt.true.stmts[0], synr.ast.Return) 321 | assert elif_stmt.true.stmts[0].value.value == 2 322 | assert elif_stmt.condition.value == 1 323 | assert isinstance(elif_stmt.false.stmts[0], synr.ast.Return) 324 | assert elif_stmt.false.stmts[0].value.value == 3 325 | 326 | 327 | def func_subscript(): 328 | z = x[1:2, y] 329 | z = x[1.0:3.0:2] 330 | x[1:2] = 3 331 | z = x[y, z] 332 | return x[:1] 333 | 334 | 335 | def test_subscript(): 336 | module = to_ast(func_subscript) 337 | fn = assert_one_fn(module, "func_subscript", no_params=0) 338 | 339 | sub = fn.body.stmts[0].rhs 340 | assert isinstance(sub, synr.ast.Call) 341 | assert sub.func_name.name == synr.ast.BuiltinOp.Subscript 342 | assert sub.params[0].id.name == "x" 343 | assert sub.params[1].values[0].start.value == 1 344 | assert sub.params[1].values[0].step.value == 1 345 | assert sub.params[1].values[0].end.value == 2 346 | assert sub.params[1].values[1].id.name == "y" 347 | 348 | sub2 = fn.body.stmts[1].rhs 349 | assert sub2.params[1].values[0].step.value == 2 350 | 351 | sub3 = fn.body.stmts[2] 352 | assert isinstance(sub3, synr.ast.UnassignedCall) 353 | assert isinstance(sub3.call, synr.ast.Call) 354 | assert sub3.call.func_name.name == synr.ast.BuiltinOp.SubscriptAssign 355 | assert sub3.call.params[0].id.name == "x" 356 | assert isinstance(sub3.call.params[1], synr.ast.Tuple) 357 | assert isinstance(sub3.call.params[1].values[0], synr.ast.Slice) 358 | assert sub3.call.params[1].values[0].start.value == 1 359 | assert sub3.call.params[1].values[0].end.value == 2 360 | assert sub3.call.params[2].value == 3 361 | 362 | sub4 = fn.body.stmts[3].rhs 363 | assert sub4.params[1].values[0].id.name == "y" 364 | assert sub4.params[1].values[1].id.name == "z" 365 | 366 | 367 | def func_literals(): 368 | x = 1 369 | x = 2.0 370 | x = (1, 2.0) 371 | 372 | 373 | def test_literals(): 374 | module = to_ast(func_literals) 375 | fn = assert_one_fn(module, "func_literals", no_params=0) 376 | 377 | assert fn.body.stmts[0].rhs.value == 1 378 | assert isinstance(fn.body.stmts[0].rhs.value, int) 379 | 380 | assert fn.body.stmts[1].rhs.value == 2.0 381 | assert isinstance(fn.body.stmts[1].rhs.value, float) 382 | 383 | assert fn.body.stmts[2].rhs.values[0].value == 1 384 | assert fn.body.stmts[2].rhs.values[1].value == 2.0 385 | assert isinstance(fn.body.stmts[2].rhs, synr.ast.Tuple) 386 | 387 | 388 | class X: 389 | pass 390 | 391 | 392 | class Y: 393 | pass 394 | 395 | 396 | def func_type(x: X) -> Y: 397 | x: test.X = 1 398 | x: X[Y] = 1 399 | x: X[X, Y] = 1 400 | x: X[X:Y] = 1 401 | x: X[1] = 1 402 | x: test.X[Y] = 1 403 | x: test.X(Y) = 1 404 | x: X + Y = 1 405 | x: test.X(Y_TYPE=Y) = 1 406 | 407 | 408 | def test_type(): 409 | module = to_ast(func_type) 410 | fn = assert_one_fn(module, "func_type", no_params=0) 411 | 412 | assert isinstance(fn.ret_type, synr.ast.TypeVar) 413 | assert isinstance(fn.params[0].ty, synr.ast.TypeVar), fn.params[0].ty 414 | assert fn.params[0].ty.id.name == "X" 415 | 416 | stmts = fn.body.stmts 417 | assert stmts[0].ty.object.id.name == "test" 418 | assert stmts[0].ty.field.name == "X" 419 | 420 | assert isinstance(stmts[1].ty, synr.ast.TypeApply) 421 | assert stmts[1].ty.func_name.id.name == "X" 422 | assert stmts[1].ty.params[0].id.name == "Y" 423 | 424 | assert isinstance(stmts[2].ty, synr.ast.TypeApply) 425 | assert stmts[2].ty.func_name.id.name == "X" 426 | assert stmts[2].ty.params[0].id.name == "X" 427 | assert stmts[2].ty.params[1].id.name == "Y" 428 | 429 | assert isinstance(stmts[5].ty, synr.ast.TypeApply) 430 | assert isinstance(stmts[5].ty.func_name, synr.ast.TypeAttr) 431 | assert stmts[5].ty.func_name.object.id.name == "test" 432 | assert stmts[5].ty.func_name.field.name == "X" 433 | assert stmts[5].ty.params[0].id.name == "Y" 434 | 435 | assert isinstance(stmts[6].ty, synr.ast.TypeCall) 436 | assert isinstance(stmts[6].ty.func_name, synr.ast.TypeAttr) 437 | assert stmts[6].ty.func_name.object.id.name == "test" 438 | assert stmts[6].ty.func_name.field.name == "X" 439 | assert stmts[6].ty.params[0].id.name == "Y" 440 | 441 | assert isinstance(stmts[7].ty, synr.ast.TypeCall) 442 | assert stmts[7].ty.func_name == synr.ast.BuiltinOp.Add 443 | assert stmts[7].ty.params[0].id.name == "X" 444 | assert stmts[7].ty.params[1].id.name == "Y" 445 | 446 | # test TypeCall with kwargs 447 | assert isinstance(stmts[8].ty, synr.ast.TypeCall) 448 | assert isinstance(stmts[8].ty.func_name, synr.ast.TypeAttr) 449 | assert stmts[8].ty.func_name.object.id.name == "test" 450 | assert stmts[8].ty.func_name.field.name == "X" 451 | for k, v in stmts[8].ty.keyword_params.items(): 452 | assert k.value == "Y_TYPE" 453 | assert v.id.name == "Y" 454 | 455 | 456 | def func_call(): 457 | test() 458 | 459 | 460 | def test_call(): 461 | module = to_ast(func_call) 462 | fn = assert_one_fn(module, "func_call", no_params=0) 463 | 464 | assert isinstance(fn.body.stmts[0], synr.ast.UnassignedCall) 465 | assert fn.body.stmts[0].call.func_name.id.name == "test" 466 | 467 | 468 | def func_constants(): 469 | x = {"test": 1, "another": 3j} 470 | y = ["an", "array", 2.0, None, True, False] 471 | z = ("hi",) 472 | 473 | 474 | def test_constants(): 475 | module = to_ast(func_constants) 476 | fn = assert_one_fn(module, "func_constants", no_params=0) 477 | 478 | d = fn.body.stmts[0].rhs 479 | assert isinstance(d, synr.ast.DictLiteral) 480 | k = [x.value for x in d.keys] 481 | v = [x.value for x in d.values] 482 | assert dict(zip(k, v)) == {"test": 1, "another": 3j} 483 | 484 | ary = fn.body.stmts[1].rhs 485 | assert isinstance(ary, synr.ast.ArrayLiteral) 486 | assert [x.value for x in ary.values] == ["an", "array", 2.0, None, True, False] 487 | 488 | t = fn.body.stmts[2].rhs 489 | assert isinstance(t, synr.ast.Tuple) 490 | assert [x.value for x in t.values] == ["hi"] 491 | 492 | 493 | class ErrorAccumulator: 494 | def __init__(self): 495 | self.errors = {} 496 | self.sources = {} 497 | 498 | def add_source(self, name, source): 499 | self.sources[name] = source 500 | 501 | def emit(self, level, message, span): 502 | if span.start_line in self.errors: 503 | self.errors[span.start_line].append((level, message, span)) 504 | else: 505 | self.errors[span.start_line] = [(level, message, span)] 506 | 507 | def render(self): 508 | return self.errors 509 | 510 | 511 | def to_ast_err(program: Any) -> Any: 512 | diag_ctx = ErrorAccumulator() 513 | transformer = None 514 | return synr.to_ast(program, diag_ctx, transformer) 515 | 516 | 517 | def func_err(x=2, *args, **kwargs): 518 | x: X 519 | 520 | 521 | def test_err_msg(): 522 | _, start = inspect.getsourcelines(func_err) 523 | errs = to_ast_err(func_err) 524 | def_errs = sorted( 525 | [(x[1], x[2]) for x in errs[start]], key=lambda x: x[1].start_column 526 | ) 527 | 528 | def check_err(err, msg, filename, start_line, start_column): 529 | assert ( 530 | err[0] == msg 531 | ), f"Error message `{err[0]}` does not match expected message `{msg}`" 532 | span = err[1] 533 | assert span.filename.endswith( 534 | filename 535 | ), f"File name `{span.filename}` does not end with `{filename}`" 536 | assert ( 537 | span.start_line == start_line 538 | ), f"Starting line of error does not match expected: {span.start_line} vs {start_line}" 539 | assert ( 540 | span.start_column == start_column 541 | ), f"Starting column of error does not match expected: {span.start_column} vs {start_column}" 542 | 543 | check_err( 544 | def_errs[0], 545 | "currently synr does not support defaults", 546 | "test_synr.py", 547 | start, 548 | 16, 549 | ) 550 | check_err( 551 | def_errs[1], 552 | "currently synr does not support varargs", 553 | "test_synr.py", 554 | start, 555 | 20, 556 | ) 557 | check_err( 558 | def_errs[2], 559 | "currently synr does not support kwarg", 560 | "test_synr.py", 561 | start, 562 | 28, 563 | ) 564 | 565 | assert errs[start + 1][0][1] == "Empty type assignment not supported" 566 | 567 | 568 | def test_scoped_func(): 569 | global_var = 0 570 | 571 | def func(): 572 | return global_var 573 | 574 | module = to_ast(func) 575 | fn = assert_one_fn(module, "func", no_params=0) 576 | stmts = fn.body.stmts 577 | assert isinstance(stmts[0], synr.ast.Return) 578 | assert stmts[0].value.id.name == "global_var" 579 | _, start_line = inspect.getsourcelines(func) 580 | assert stmts[0].span.start_line == start_line + 1 581 | assert stmts[0].span.start_column == 9 582 | 583 | 584 | def test_local_func(): 585 | def foo(): 586 | def bar(): 587 | return 1 588 | 589 | return bar() 590 | 591 | module = to_ast(foo) 592 | fn = assert_one_fn(module, "foo") 593 | stmts = fn.body.stmts 594 | assert isinstance(stmts[0], synr.ast.Function) 595 | assert stmts[0].name == "bar" 596 | assert len(stmts[0].params) == 0 597 | _, start_line = inspect.getsourcelines(foo) 598 | assert stmts[0].span.start_line == start_line + 1 599 | assert stmts[0].span.start_column == 9 600 | 601 | 602 | def test_decorators(): 603 | def A(f): 604 | return f 605 | 606 | @A 607 | def foo(): 608 | @B 609 | @C 610 | def bar(): 611 | return 1 612 | 613 | return bar() 614 | 615 | module = to_ast(foo) 616 | fn = assert_one_fn(module, "foo") 617 | _, start_line = inspect.getsourcelines(foo) 618 | assert fn.span.start_line == start_line + 1 619 | 620 | assert len(fn.decorators) == 1 621 | assert isinstance(fn.decorators[0], synr.ast.Var) 622 | assert fn.decorators[0].id.name == "A" 623 | assert fn.decorators[0].span.start_line == start_line 624 | 625 | # end_lineno was added in Python 3.8 so we check it here 626 | if sys.version_info >= (3, 8): 627 | assert fn.span.end_line == start_line + 7 628 | 629 | bar = fn.body.stmts[0] 630 | assert bar.span.start_line == start_line + 4 631 | 632 | assert len(bar.decorators) == 2 633 | 634 | assert isinstance(bar.decorators[0], synr.ast.Var) 635 | assert bar.decorators[0].id.name == "B" 636 | assert bar.decorators[0].span.start_line == start_line + 2 637 | 638 | assert isinstance(bar.decorators[1], synr.ast.Var) 639 | assert bar.decorators[1].id.name == "C" 640 | assert bar.decorators[1].span.start_line == start_line + 3 641 | 642 | 643 | def test_nonlocal(): 644 | x, y = 1, 2 645 | 646 | def foo(): 647 | nonlocal x, y 648 | return x + y 649 | 650 | module = to_ast(foo) 651 | fn = assert_one_fn(module, "foo") 652 | nl = fn.body.stmts[0] 653 | assert isinstance(nl, synr.ast.Nonlocal) 654 | assert len(nl.vars) == 2 655 | x, y = nl.vars 656 | assert isinstance(x, synr.ast.Var) and x.id.name == "x" 657 | assert isinstance(y, synr.ast.Var) and y.id.name == "y" 658 | 659 | _, start_line = inspect.getsourcelines(foo) 660 | assert nl.span.start_line == start_line + 1 661 | # NOTE: variable spans are a bit hacky so we don't check them here 662 | 663 | 664 | def test_global(): 665 | def foo(): 666 | global x, y 667 | return x + y 668 | 669 | module = to_ast(foo) 670 | fn = assert_one_fn(module, "foo") 671 | gl = fn.body.stmts[0] 672 | assert isinstance(gl, synr.ast.Global) 673 | assert len(gl.vars) == 2 674 | x, y = gl.vars 675 | assert isinstance(x, synr.ast.Var) and x.id.name == "x" 676 | assert isinstance(y, synr.ast.Var) and y.id.name == "y" 677 | 678 | _, start_line = inspect.getsourcelines(foo) 679 | assert gl.span.start_line == start_line + 1 680 | 681 | 682 | def test_lambda(): 683 | def foo(): 684 | return lambda x, y: x + y 685 | 686 | module = to_ast(foo) 687 | fn = assert_one_fn(module, "foo") 688 | 689 | assert isinstance(fn.body.stmts[0], synr.ast.Return) 690 | assert isinstance(fn.body.stmts[0].value, synr.ast.Lambda) 691 | node = fn.body.stmts[0].value 692 | assert len(node.params) == 2 693 | assert node.params[0].name == "x" 694 | assert node.params[0].ty == None 695 | assert node.params[1].name == "y" 696 | assert node.params[1].ty == None 697 | 698 | assert isinstance(node.body, synr.ast.Call) 699 | assert node.body.func_name.name == synr.ast.BuiltinOp.Add 700 | assert node.body.params[0].id.name == "x" 701 | assert node.body.params[1].id.name == "y" 702 | 703 | _, start_line = inspect.getsourcelines(foo) 704 | assert node.span.start_line == start_line + 1 705 | 706 | 707 | if __name__ == "__main__": 708 | test_id_function() 709 | test_class() 710 | test_for() 711 | test_while() 712 | test_with() 713 | test_block() 714 | test_assign() 715 | test_var() 716 | test_binop() 717 | test_if() 718 | test_subscript() 719 | test_literals() 720 | test_type() 721 | test_call() 722 | test_constants() 723 | test_err_msg() 724 | test_scoped_func() 725 | test_local_func() 726 | test_decorators() 727 | test_nonlocal() 728 | test_global() 729 | test_lambda() 730 | --------------------------------------------------------------------------------