├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── astutils ├── __init__.py ├── ast.py ├── ply.py └── py.typed ├── examples ├── README.md ├── foo │ ├── __init__.py │ ├── ast.py │ ├── backend.py │ └── lexyacc.py └── setup.py ├── setup.py └── tests ├── ast_test.py └── ply_test.py /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # configuration for GitHub Actions 3 | name: astutils tests 4 | on: 5 | push: 6 | pull_request: 7 | schedule: 8 | - cron: '43 5 4 * *' 9 | jobs: 10 | build: 11 | name: Build 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: [ 16 | '3.10', 17 | '3.11', 18 | '3.12', 19 | ] 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v4 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Update Python environment 27 | run: | 28 | pip install --upgrade pip setuptools 29 | - name: Install `astutils` 30 | run: | 31 | python setup.py sdist 32 | pip install dist/astutils-*.tar.gz 33 | - name: Install test dependencies 34 | run: | 35 | pip install pytest 36 | - name: Run `astutils` tests 37 | run: | 38 | set -o posix 39 | echo "Exported environment variables:" 40 | export -p 41 | # run tests 42 | cd tests/ 43 | pytest -v --continue-on-collection-errors . 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /astutils/_version.py 2 | .pytest_cache/ 3 | /build/* 4 | /tests/foo.py 5 | *.egg-info/ 6 | *.egg 7 | __pycache__/ 8 | *.pyc 9 | *.swp 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2022 California Institute of Technology 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the California Institute of Technology nor 16 | the names of its contributors may be used to endorse or promote 17 | products derived from this software without specific prior 18 | written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CALTECH OR THE 24 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 25 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 26 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 27 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 28 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 29 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include astutils/py.typed 4 | include examples/*.py 5 | include examples/README.md 6 | include examples/foo/*.py 7 | include tests/*_test.py 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status][build_img]][ci] 2 | 3 | 4 | About 5 | ===== 6 | 7 | Bare essentials for building abstract syntax trees (AST) and Python 8 | `lex`-`yacc` ([PLY](https://github.com/dabeaz/ply)) parsers. 9 | The package includes: 10 | 11 | - two classes for tree nodes: `Terminal`, `Operator` 12 | - a `Lexer` and `Parser` class, and a helper function to erase and 13 | rewrite the table files. 14 | 15 | The examples under `examples/` demonstrate how to use these classes to create 16 | a richer AST, a parser, and different backends that use the same parser. 17 | 18 | These classes provide the boilerplate for parsing with PLY, and are based on 19 | code that was developed in [`tulip`]( 20 | https://github.com/tulip-control/tulip-control) 21 | and [`promela`](https://github.com/johnyf/promela). 22 | 23 | 24 | License 25 | ======= 26 | [BSD-3](https://opensource.org/licenses/BSD-3-Clause), see file `LICENSE`. 27 | 28 | 29 | [build_img]: https://github.com/johnyf/astutils/actions/workflows/main.yml/badge.svg?branch=main 30 | [ci]: https://github.com/johnyf/astutils/actions 31 | -------------------------------------------------------------------------------- /astutils/__init__.py: -------------------------------------------------------------------------------- 1 | """Utilities for abstract syntax trees and parsing with PLY.""" 2 | # Copyright 2014-2022 by California Institute of Technology 3 | # All rights reserved. Licensed under 3-clause BSD. 4 | # 5 | from astutils.ast import Terminal, Operator 6 | from astutils.ply import Lexer, Parser, rewrite_tables 7 | try: 8 | import astutils._version as _version 9 | __version__ = _version.version 10 | except ImportError: 11 | __version__ = None 12 | -------------------------------------------------------------------------------- /astutils/ast.py: -------------------------------------------------------------------------------- 1 | """Abstract syntax tree nodes.""" 2 | # Copyright 2014-2022 by California Institute of Technology 3 | # All rights reserved. Licensed under 3-clause BSD. 4 | # 5 | 6 | 7 | class Terminal: 8 | """Nullary symbol.""" 9 | 10 | def __init__( 11 | self, 12 | value: 13 | str, 14 | dtype: 15 | str='terminal' 16 | ) -> None: 17 | try: 18 | value + 's' 19 | except TypeError: 20 | raise TypeError( 21 | 'value must be a string, ' 22 | f'got: {value}') 23 | self.type = dtype 24 | self.value = value 25 | 26 | def __hash__( 27 | self 28 | ) -> int: 29 | return id(self) 30 | 31 | def __repr__( 32 | self 33 | ) -> str: 34 | class_name = type(self).__name__ 35 | return ( 36 | f'{class_name}(' 37 | f'{self.value!r}, ' 38 | f'{self.type!r})') 39 | 40 | def __str__( 41 | self, 42 | *arg, 43 | **kw 44 | ) -> str: 45 | return self.value 46 | 47 | def __len__( 48 | self 49 | ) -> int: 50 | return 1 51 | 52 | def __eq__( 53 | self, 54 | other 55 | ) -> bool: 56 | return ( 57 | hasattr(other, 'type') and 58 | hasattr(other, 'value') and 59 | self.type == other.type and 60 | self.value == other.value) 61 | 62 | def flatten( 63 | self, 64 | *arg, 65 | **kw): 66 | return self.value 67 | 68 | 69 | class Operator: 70 | """Operator with arity > 0.""" 71 | 72 | def __init__( 73 | self, 74 | operator: 75 | str, 76 | *operands 77 | ) -> None: 78 | try: 79 | operator + 'a' 80 | except TypeError: 81 | raise TypeError( 82 | 'operator must be string, ' 83 | f'got: {operator}') 84 | self.type = 'operator' 85 | self.operator = operator 86 | self.operands = list(operands) 87 | 88 | def __repr__( 89 | self 90 | ) -> str: 91 | class_name = type(self).__name__ 92 | xyz = ', '.join(map(repr, self.operands)) 93 | return ( 94 | f'{class_name}(' 95 | f'{self.operator!r}, ' 96 | f'{xyz})') 97 | 98 | def __str__( 99 | self 100 | ) -> str: 101 | xyz = ' '.join(map(str, self.operands)) 102 | return f'({self.operator} {xyz})' 103 | 104 | def __len__( 105 | self 106 | ) -> int: 107 | return 1 + sum(map(len, self.operands)) 108 | 109 | def flatten( 110 | self, 111 | *arg, 112 | **kw): 113 | csv = ', '.join( 114 | x.flatten(*arg, **kw) 115 | for x in self.operands) 116 | return f'( {self.operator} {csv} )' 117 | -------------------------------------------------------------------------------- /astutils/ply.py: -------------------------------------------------------------------------------- 1 | """Utilities for Python lex-yacc (PLY).""" 2 | # Copyright 2014-2022 by California Institute of Technology 3 | # All rights reserved. Licensed under 3-clause BSD. 4 | # 5 | import logging 6 | import os 7 | import textwrap as _tw 8 | import typing as _ty 9 | import warnings 10 | 11 | import ply.lex 12 | import ply.yacc 13 | 14 | import astutils.ast as _ast 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class Lexer: 21 | """Init and build methods.""" 22 | 23 | def __init__( 24 | self, 25 | debug: 26 | bool=False 27 | ) -> None: 28 | self.reserved = getattr( 29 | self, 'reserved', dict()) 30 | self.delimiters = getattr( 31 | self, 'delimiters', list()) 32 | self.operators = getattr( 33 | self, 'operators', list()) 34 | self.misc = getattr( 35 | self, 'misc', list()) 36 | self.logger = getattr( 37 | self, 'logger', logger) 38 | self.tokens = ( 39 | self.delimiters + 40 | self.operators + 41 | self.misc + 42 | sorted(set(self.reserved.values()))) 43 | self.build(debug=debug) 44 | 45 | def t_error( 46 | self, 47 | t 48 | ) -> _ty.NoReturn: 49 | raise RuntimeError( 50 | f'Illegal character "{t.value[0]}"') 51 | 52 | def build( 53 | self, 54 | debug: 55 | bool=False, 56 | debuglog=None, 57 | **kwargs 58 | ) -> None: 59 | """Create a lexer.""" 60 | if debug and debuglog is None: 61 | debuglog = self.logger 62 | self.lexer = ply.lex.lex( 63 | module=self, 64 | debug=debug, 65 | debuglog=debuglog, 66 | **kwargs) 67 | 68 | 69 | class Parser: 70 | """Init, build and parse methods. 71 | 72 | To subclass, overwrite the class attributes 73 | defined below, and add production rules. 74 | """ 75 | 76 | def __init__( 77 | self, 78 | nodes=None, 79 | lexer=None 80 | ) -> None: 81 | self.tabmodule = getattr( 82 | self, 'tabmodule', None) 83 | self.start = getattr( 84 | self, 'start', 'expr') 85 | # low to high 86 | self.precedence = getattr( 87 | self, 'precedence', tuple()) 88 | self.nodes = getattr( 89 | self, 'nodes', _ast) 90 | self.logger = getattr( 91 | self, 'logger', logger) 92 | if nodes is not None: 93 | self.nodes = nodes 94 | if lexer is not None: 95 | self._lexer = lexer 96 | elif hasattr(self, 'Lexer'): 97 | warnings.warn(_tw.dedent(f''' 98 | The parser attribute `Lexer` 99 | has been deprecated. Instead, 100 | pass argument `lexer`, for example: 101 | 102 | ```py 103 | lexer = Lexer() 104 | super().__init__(lexer=lexer) 105 | ``` 106 | '''), 107 | DeprecationWarning) 108 | self._lexer = self.Lexer() 109 | else: 110 | raise ValueError( 111 | 'pass argument `lexer` to ' 112 | '`Parser.__init__()`') 113 | self.tokens = self._lexer.tokens 114 | self.parser = None 115 | 116 | def build( 117 | self, 118 | tabmodule: 119 | str | 120 | None=None, 121 | outputdir: 122 | str='', 123 | write_tables: 124 | bool=False, 125 | debug: 126 | bool=False, 127 | debuglog=None 128 | ) -> None: 129 | """Build parser using `ply.yacc`.""" 130 | if tabmodule is None: 131 | tabmodule = self.tabmodule 132 | if debug and debuglog is None: 133 | debuglog = self.logger 134 | self.parser = ply.yacc.yacc( 135 | method='LALR', 136 | module=self, 137 | start=self.start, 138 | tabmodule=tabmodule, 139 | outputdir=outputdir, 140 | write_tables=write_tables, 141 | debug=debug, 142 | debuglog=debuglog) 143 | 144 | def parse( 145 | self, 146 | formula: 147 | str, 148 | debuglog=None 149 | ) -> _ty.Any: 150 | """Parse string `formula`. 151 | 152 | @param formula: 153 | input to the parser 154 | @return: 155 | what the parser returns 156 | (many parsers return a syntax tree) 157 | """ 158 | if self.parser is None: 159 | self.build() 160 | root = self.parser.parse( 161 | input=formula, 162 | lexer=self._lexer.lexer, 163 | debug=debuglog) 164 | self._clear_lr_stack() 165 | if root is not None: 166 | return root 167 | raise RuntimeError( 168 | f'failed to parse:\n\t{formula}') 169 | 170 | def _clear_lr_stack( 171 | self 172 | ) -> None: 173 | """Ensure no references remain. 174 | 175 | Otherwise, references can prevent `gc` from 176 | collecting objects that are expected to 177 | have become unreachable. 178 | """ 179 | has_lr_stack = ( 180 | self.parser is not None and 181 | hasattr(self.parser, 'statestack') and 182 | hasattr(self.parser, 'symstack')) 183 | if not has_lr_stack: 184 | return 185 | self.parser.restart() 186 | 187 | def p_error( 188 | self, 189 | p 190 | ) -> _ty.NoReturn: 191 | s = list() 192 | while True: 193 | tok = self.parser.token() 194 | if tok is None: 195 | break 196 | s.append(tok.value) 197 | s = ' '.join(s) 198 | raise RuntimeError( 199 | f'Syntax error at "{p.value}"\n' 200 | f'remaining input:\n{s}\n') 201 | 202 | 203 | def rewrite_tables( 204 | parser_class: 205 | type[Parser], 206 | tabmodule: 207 | str, 208 | outputdir: 209 | str 210 | ) -> None: 211 | """Write the parser table file. 212 | 213 | Overwrites any preexisting parser file. 214 | 215 | The module name (after last dot) in `tabmodule` 216 | is appended to `outputdir` to form the path. 217 | 218 | Example use: 219 | 220 | ```python 221 | _TABMODULE = 'packagename.modulename_parsetab' 222 | 223 | 224 | class Parser(...): 225 | ... 226 | 227 | 228 | if __name__ == '__main__': 229 | outputdir = './' 230 | rewrite_tables(Parser, _TABMODULE, outputdir) 231 | ``` 232 | 233 | @param parser_class: 234 | PLY production rules 235 | @param tabmodule: 236 | module name for table file 237 | @param outputdir: 238 | dump parser file 239 | in this directory 240 | """ 241 | if outputdir is None: 242 | raise ValueError( 243 | '`outputdir` must be `str`.') 244 | *_, table = tabmodule.rpartition('.') 245 | for ext in ('.py', '.pyc'): 246 | path = outputdir + table + ext 247 | if os.path.isfile(path): 248 | logger.info(f'found file `{path}`') 249 | os.remove(path) 250 | logger.info(f'removed file `{path}`') 251 | parser = parser_class() 252 | debugfile = ply.yacc.debug_file 253 | path = os.path.join(outputdir, debugfile) 254 | with open(path, 'w') as debuglog_file: 255 | debuglog = ply.yacc.PlyLogger(debuglog_file) 256 | parser.build( 257 | write_tables=True, 258 | outputdir=outputdir, 259 | tabmodule=table, 260 | debug=True, 261 | debuglog=debuglog) 262 | -------------------------------------------------------------------------------- /astutils/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnyf/astutils/4f0a3baa87d245925c7094edeeec9db70a8f523c/astutils/py.typed -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | This example shows how `astutils` can be used to write a package that 2 | includes a parser, whose tables are created and copied during installation. 3 | 4 | Other examples of parsers that use `astutils` are contained in the modules: 5 | 6 | - [`dd._parser`](https://github.com/johnyf/dd/blob/main/dd/_parser.py) 7 | - [`omega.logic.ast`]( 8 | https://github.com/tulip-control/omega/blob/main/omega/logic/ast.py) 9 | - [`omega.logic.lexyacc`]( 10 | https://github.com/tulip-control/omega/blob/main/omega/logic/lexyacc.py) 11 | - [`omega.symbolic.bdd`]( 12 | https://github.com/tulip-control/omega/blob/main/omega/symbolic/bdd.py) 13 | -------------------------------------------------------------------------------- /examples/foo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/johnyf/astutils/4f0a3baa87d245925c7094edeeec9db70a8f523c/examples/foo/__init__.py -------------------------------------------------------------------------------- /examples/foo/ast.py: -------------------------------------------------------------------------------- 1 | """Examples of subclassing the basic AST node classes. 2 | 3 | The main purpose of subclassing is to create 4 | different flattening methods. By creating a larger 5 | variety of AST node classes, the same parser can 6 | be used with different backends. Each backend is 7 | created by subclassing the below classes, then 8 | overriding the `flatten` method. 9 | """ 10 | import astutils 11 | 12 | 13 | class Nodes: 14 | """Namespace of AST node classes.""" 15 | 16 | Terminal = astutils.Terminal 17 | Operator = astutils.Operator 18 | 19 | class Var(astutils.Terminal): 20 | """Variable identifier.""" 21 | 22 | def __init__(self, value, dtype='var'): 23 | super(Nodes.Var, self).__init__(value, dtype) 24 | 25 | class Bool(astutils.Terminal): 26 | """Boolean constant.""" 27 | 28 | def __init__(self, value, dtype='bool'): 29 | super(Nodes.Bool, self).__init__(value, dtype) 30 | 31 | class Num(astutils.Terminal): 32 | """Numerical costant.""" 33 | 34 | def __init__(self, value, dtype='num'): 35 | super(Nodes.Num, self).__init__(value, dtype) 36 | 37 | class Unary(astutils.Operator): 38 | """Unary operator.""" 39 | 40 | class Binary(astutils.Operator): 41 | """Binary operator.""" 42 | 43 | def flatten(self, *arg, **kw): 44 | return ' '.join([ 45 | '(', 46 | self.operands[0].flatten(*arg, **kw), 47 | self.operator, 48 | self.operands[1].flatten(*arg, **kw), 49 | ')']) 50 | -------------------------------------------------------------------------------- /examples/foo/backend.py: -------------------------------------------------------------------------------- 1 | """Example of subclassing the derived AST to change the flatteners. 2 | 3 | This allows changing backend, without touching the 4 | frontend (parser) in `lexyacc`. 5 | """ 6 | import foo.ast as _ast 7 | import foo.lexyacc as _lexyacc 8 | 9 | 10 | class Nodes(_ast.Nodes): 11 | """Further subclassing that changes selected flatteners.""" 12 | 13 | class Var(_ast.Nodes.Var): 14 | def flatten(self, varmap=None): 15 | """Rename variable, if found in `varmap`.""" 16 | if varmap is None: 17 | return self.value 18 | else: 19 | return varmap.get(self.value, self.value) 20 | 21 | class Binary(_ast.Nodes.Binary): 22 | def flatten(self, *arg, **kw): 23 | """Produce postfix syntax.""" 24 | return ' '.join([ 25 | '(', 26 | self.operands[0].flatten(*arg, **kw), 27 | self.operands[1].flatten(*arg, **kw), 28 | self.operator, 29 | ')']) 30 | 31 | 32 | parser = _lexyacc.Parser(nodes=Nodes()) 33 | -------------------------------------------------------------------------------- /examples/foo/lexyacc.py: -------------------------------------------------------------------------------- 1 | """Example usage of `astutils` classes. 2 | 3 | This file is `foo.lexyacc`, so that 4 | `setup.py` can `from foo import lexyacc`. 5 | """ 6 | import astutils 7 | import foo.ast as _ast 8 | 9 | 10 | _TABMODULE = 'foo.calc_parsetab' 11 | 12 | 13 | class Lexer(astutils.Lexer): 14 | """Lexer for Boolean formulae.""" 15 | 16 | def __init__(self, **kw): 17 | self.reserved = { 18 | 'False': 'FALSE', 19 | 'True': 'TRUE'} 20 | self.delimiters = [ 21 | 'LPAREN', 'RPAREN', 'COMMA'] 22 | self.operators = [ 23 | 'NOT', 'AND', 'OR', 'XOR', 24 | 'IMP', 'BIMP', 25 | 'EQUALS', 'NEQUALS'] 26 | self.misc = ['NAME', 'NUMBER'] 27 | super(Lexer, self).__init__(**kw) 28 | 29 | def t_NAME(self, t): 30 | r""" 31 | [A-Za-z_] 32 | [A-Za-z0-9]* 33 | """ 34 | t.type = self.reserved.get( 35 | t.value, 'NAME') 36 | return t 37 | 38 | def t_AND(self, t): 39 | r' \& \& ' 40 | t.value = '&' 41 | return t 42 | 43 | def t_OR(self, t): 44 | r' \| \| ' 45 | t.value = '|' 46 | return t 47 | 48 | t_NOT = r' \! ' 49 | t_XOR = r' \^ ' 50 | t_LPAREN = r' \( ' 51 | t_RPAREN = r' \) ' 52 | t_NUMBER = r' \d+ ' 53 | t_IMP = ' -> ' 54 | t_BIMP = r' \< -> ' 55 | t_ignore = ''.join(['\x20', '\t']) 56 | 57 | def t_comment(self, t): 58 | r' \#.* ' 59 | return 60 | 61 | def t_newline(self, t): 62 | r' \n+ ' 63 | t.lexer.lineno += t.value.count('\n') 64 | 65 | 66 | class Parser(astutils.Parser): 67 | """Parser for Boolean formulae.""" 68 | 69 | def __init__(self, **kw): 70 | self.tabmodule = _TABMODULE 71 | self.start = 'expr' 72 | # low to high 73 | self.precedence = ( 74 | ('left', 'BIMP'), 75 | ('left', 'IMP'), 76 | ('left', 'XOR'), 77 | ('left', 'OR'), 78 | ('left', 'AND'), 79 | ('left', 'EQUALS', 'NEQUALS'), 80 | ('right', 'NOT')) 81 | self.nodes = _ast.Nodes 82 | kw.setdefault('lexer', Lexer()) 83 | super(Parser, self).__init__(**kw) 84 | 85 | def p_bool(self, p): 86 | """expr : TRUE 87 | | FALSE 88 | """ 89 | p[0] = self.nodes.Bool(p[1]) 90 | 91 | def p_number(self, p): 92 | """expr : NUMBER""" 93 | p[0] = self.nodes.Num(p[1]) 94 | 95 | def p_var(self, p): 96 | """expr : NAME""" 97 | p[0] = self.nodes.Var(p[1]) 98 | 99 | def p_unary(self, p): 100 | """expr : NOT expr""" 101 | p[0] = self.nodes.Unary(p[1], p[2]) 102 | 103 | def p_binary(self, p): 104 | """expr : expr AND expr 105 | | expr OR expr 106 | | expr XOR expr 107 | | expr IMP expr 108 | | expr BIMP expr 109 | | expr EQUALS expr 110 | | expr NEQUALS expr 111 | """ 112 | p[0] = self.nodes.Binary(p[2], p[1], p[3]) 113 | 114 | def p_paren(self, p): 115 | """expr : LPAREN expr RPAREN""" 116 | p[0] = p[2] 117 | 118 | 119 | def _rewrite_tables(outputdir='./'): 120 | astutils.rewrite_tables( 121 | Parser, _TABMODULE, outputdir) 122 | 123 | 124 | # this is a convenience to regenerate the tables 125 | # during development 126 | if __name__ == '__main__': 127 | _rewrite_tables() 128 | -------------------------------------------------------------------------------- /examples/setup.py: -------------------------------------------------------------------------------- 1 | """Example of a `setup.py` that builds PLY tables. 2 | 3 | It uses `pip` to install `ply`, 4 | then it calls `_rewrite_tables` to (re)write the table files, 5 | and finally uses `setuptools` to install the package. 6 | 7 | If the package contains multiple parsers, 8 | then each parsing module can provide a `_rewrite_tables` 9 | function that hides the details (table file name, etc). 10 | 11 | If some parsing module has extra dependenies, 12 | then these can be installed using `pip`. 13 | 14 | Although `pip` installs dependencies in `install_requires` first, 15 | the below works also if one runs `python setup.py install`, 16 | assuming that `pip` is available. 17 | """ 18 | import subprocess 19 | import sys 20 | 21 | import setuptools 22 | 23 | 24 | PACKAGE_NAME = 'foo' 25 | DESCRIPTION = 'foo is very useful.' 26 | PACKAGE_URL = f'https://example.org/{PACKAGE_NAME}' 27 | README = 'README.md' 28 | VERSION_FILE = f'{PACKAGE_NAME}/_version.py' 29 | VERSION = '0.0.1' 30 | VERSION_FILE_TEXT = ( 31 | '# This file was generated from setup.py\n' 32 | f"version = '{VERSION}'\n") 33 | PYTHON_REQUIRES = '>=3.10' 34 | PLY_REQUIRED = 'ply >= 3.4, <= 3.10' 35 | INSTALL_REQUIRES = [PLY_REQUIRED] 36 | 37 | 38 | def run_setup(): 39 | """Install.""" 40 | with open(VERSION_FILE, 'w') as f: 41 | f.write(VERSION_FILE_TEXT) 42 | # first install PLY, then build the tables 43 | _install_ply() 44 | _build_parser() 45 | # so that they will be copied to `site-packages` 46 | with open(README) as f: 47 | long_description = f.read() 48 | setuptools.setup( 49 | name=PACKAGE_NAME, 50 | version=VERSION, 51 | description=DESCRIPTION, 52 | long_description=long_description, 53 | author='Name', 54 | author_email='name@example.org', 55 | url=PACKAGE_URL, 56 | license='BSD', 57 | python_requires=PYTHON_REQUIRES, 58 | install_requires=INSTALL_REQUIRES, 59 | packages=[PACKAGE_NAME], 60 | package_dir={PACKAGE_NAME: PACKAGE_NAME}, 61 | keywords=['parsing', 'setup']) 62 | 63 | 64 | def _install_ply(): 65 | cmd = [ 66 | sys.executable, 67 | '-m', 'pip', 'install', 68 | PLY_REQUIRED] 69 | subprocess.check_call(cmd) 70 | 71 | 72 | def _build_parser(): 73 | from foo import lexyacc 74 | lexyacc._rewrite_tables( 75 | outputdir=PACKAGE_NAME) 76 | 77 | 78 | if __name__ == '__main__': 79 | run_setup() 80 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Installation script.""" 2 | import setuptools 3 | 4 | 5 | PACKAGE_NAME = 'astutils' 6 | DESCRIPTION = 'Utilities for abstract syntax trees and parsing with PLY.' 7 | PACKAGE_URL = f'https://github.com/johnyf/{PACKAGE_NAME}' 8 | README = 'README.md' 9 | VERSION_FILE = f'{PACKAGE_NAME}/_version.py' 10 | VERSION = '0.0.7' 11 | VERSION_FILE_TEXT = ( 12 | '# This file was generated from setup.py\n' 13 | f"version = '{VERSION}'\n") 14 | PYTHON_REQUIRES = '>=3.10' 15 | TESTS_REQUIRE = ['pytest >= 4.6.11'] 16 | KEYWORDS = [ 17 | 'lexing', 'parsing', 'syntax tree', 'abstract syntax tree', 18 | 'AST', 'PLY', 'lex', 'yacc'] 19 | CLASSIFIERS = [ 20 | 'Development Status :: 2 - Pre-Alpha', 21 | 'Intended Audience :: Developers', 22 | 'License :: OSI Approved :: BSD License', 23 | 'Operating System :: OS Independent', 24 | 'Programming Language :: Python :: 3 :: Only', 25 | 'Topic :: Software Development'] 26 | 27 | 28 | def run_setup(): 29 | """Install.""" 30 | with open(VERSION_FILE, 'w') as f: 31 | f.write(VERSION_FILE_TEXT) 32 | with open(README) as f: 33 | long_description = f.read() 34 | setuptools.setup( 35 | name=PACKAGE_NAME, 36 | version=VERSION, 37 | description=DESCRIPTION, 38 | long_description=long_description, 39 | long_description_content_type='text/markdown', 40 | author='Ioannis Filippidis', 41 | author_email='jfilippidis@gmail.com', 42 | url=PACKAGE_URL, 43 | license='BSD', 44 | python_requires=PYTHON_REQUIRES, 45 | install_requires=['ply >= 3.4, <= 3.10'], 46 | packages=[PACKAGE_NAME], 47 | package_dir={PACKAGE_NAME: PACKAGE_NAME}, 48 | classifiers=CLASSIFIERS, 49 | keywords=KEYWORDS) 50 | 51 | 52 | if __name__ == '__main__': 53 | run_setup() 54 | -------------------------------------------------------------------------------- /tests/ast_test.py: -------------------------------------------------------------------------------- 1 | import astutils.ast as _ast 2 | 3 | 4 | def test_terminal(): 5 | value = 'a' 6 | t = _ast.Terminal(value) 7 | r = repr(t) 8 | assert r == "Terminal('a', 'terminal')", r 9 | r = str(t) 10 | assert r == 'a', r 11 | r = len(t) 12 | assert r == 1, r 13 | r = t.flatten() 14 | assert r == value, r 15 | 16 | 17 | def test_hash(): 18 | # different AST node instances should 19 | # have different hash 20 | # 21 | # terminals 22 | value = 'foo' 23 | a = _ast.Terminal(value) 24 | b = _ast.Terminal(value) 25 | assert hash(a) != hash(b) 26 | # operators 27 | op = 'bar' 28 | a = _ast.Operator(op) 29 | b = _ast.Operator(op) 30 | assert hash(a) != hash(b) 31 | 32 | 33 | def test_eq(): 34 | value = 'a' 35 | t = _ast.Terminal(value) 36 | p = _ast.Terminal(value) 37 | assert t == p, (t, p) 38 | p = _ast.Terminal('b') 39 | assert t != p, (t, p) 40 | p = _ast.Terminal(value, 'number') 41 | assert t != p, (t, p) 42 | p = 54 43 | assert t != p, (t, p) 44 | 45 | 46 | def test_operator(): 47 | a = _ast.Terminal('a') 48 | b = _ast.Terminal('b') 49 | op = '+' 50 | operands = [a, b] # 'a', 'b' fail due to `str` 51 | t = _ast.Operator(op, *operands) 52 | r = repr(t) 53 | r_ = ( 54 | "Operator('+', " 55 | "Terminal('a', 'terminal'), " 56 | "Terminal('b', 'terminal'))") 57 | assert r == r_, r 58 | r = str(t) 59 | assert r == '(+ a b)', r 60 | r = len(t) 61 | assert r == 3, r 62 | r = t.flatten() 63 | assert r == '( + a, b )', r 64 | -------------------------------------------------------------------------------- /tests/ply_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import astutils.ply 4 | import pytest 5 | 6 | 7 | class Lexer(astutils.ply.Lexer): 8 | 9 | def __init__(self, **kw): 10 | self.operators = ['NOT', 'AND'] 11 | self.misc = ['NAME'] 12 | super(Lexer, self).__init__(**kw) 13 | 14 | t_NAME = r'[A-Za-z_][A-za-z0-9]*' 15 | t_NOT = r'~' 16 | t_AND = r'/\\' 17 | t_ignore = ' \t' 18 | 19 | 20 | class Parser(astutils.ply.Parser): 21 | 22 | def __init__(self, **kw): 23 | self.tabmodule = 'testing_parsetab' 24 | self.start = 'expr' 25 | self.precedence = ( 26 | ('left', 'AND'), 27 | ('right', 'NOT')) 28 | kw.setdefault('lexer', Lexer()) 29 | super(Parser, self).__init__(**kw) 30 | 31 | def p_not(self, p): 32 | """expr : NOT expr""" 33 | p[0] = not p[2] 34 | 35 | def p_and(self, p): 36 | """expr : expr AND expr""" 37 | p[0] = p[1] and p[3] 38 | 39 | def p_name(self, p): 40 | """expr : NAME""" 41 | s = p[1] 42 | p[0] = self.names[s] 43 | 44 | 45 | def test_parser(): 46 | parser = Parser() 47 | parser.names = {'True': True, 'False': False} 48 | s = 'True' 49 | r = parser.parse(s) 50 | assert r is True, r 51 | s = r'True /\ True' 52 | r = parser.parse(s) 53 | assert r is True, r 54 | s = r'False /\ True' 55 | r = parser.parse(s) 56 | assert r is False, r 57 | s = r'~ False /\ ~ True' 58 | r = parser.parse(s) 59 | assert r is False, r 60 | s = r'~ False /\ True' 61 | r = parser.parse(s) 62 | assert r is True, r 63 | 64 | 65 | def test_illegal_character(): 66 | parser = Parser() 67 | parser.names = {'True': True} 68 | s = '( True' 69 | with pytest.raises(RuntimeError): 70 | parser.parse(s) 71 | 72 | 73 | def test_syntax_error(): 74 | parser = Parser() 75 | parser.names = {'True': True} 76 | s = 'True True' 77 | with pytest.raises(RuntimeError): 78 | parser.parse(s) 79 | 80 | 81 | def test_rewrite_tables(): 82 | prefix = 'foo' 83 | outputdir = './' 84 | for ext in ('.py', '.pyc'): 85 | try: 86 | os.remove(prefix + ext) 87 | except: 88 | pass 89 | f = prefix + '.py' 90 | assert not os.path.isfile(f) 91 | astutils.ply.rewrite_tables( 92 | Parser, prefix, outputdir) 93 | assert os.path.isfile(f) 94 | --------------------------------------------------------------------------------