├── .gitignore
├── 01-Hello
├── 01-Hello.iml
├── Hello.g4
├── input.txt
└── test_hello.py
├── 03-Array
├── 03-Array.iml
├── ArrayInit.g4
├── README.md
├── input.txt
├── rewriter.py
└── test_array.py
├── 04-Calc
├── 04-Calc.iml
├── LabeledExpr.g4
├── MyVisitor.py
├── README.md
├── calc.py
└── t.expr
├── 04-Expr
├── 04-Expr.iml
├── Expr.g4
├── MyListener.py
├── README.md
├── t.expr
└── test_Expr.py
├── 04-ExtractInterface
├── 04-ExtractInterface.iml
├── Demo.java
├── ExtractInterfaceListener.py
├── Java.g4
├── README.md
└── test_extract.py
├── 04-Rows-visitor
├── 04-Rows-visitor.iml
├── README.md
├── Rows.g4
├── col.py
└── t.rows
├── 04-modechange
├── 04-modechange.iml
├── README.md
├── XMLLexer.g4
├── t.xml
└── test.py
├── 04-sempred
├── 04-sempred.iml
├── Data.g4
├── Readme.md
├── t.data
└── test_data.py
├── 07-LableRule
├── 07-LableRule.iml
├── LExpr.g4
├── README.md
├── t.expr
├── test_EvalContext.py
├── test_EvalListener.py
└── test_EvalVistor.py
├── 07-Property
├── 07-Property.iml
├── PropertyFile.g4
├── README.md
├── t.properties
├── test_as_loader.py
├── test_as_print.py
├── test_listener.py
└── test_visitor.py
├── 08-CSV
├── 08-CSV.iml
├── CSV.g4
├── CSV_Loader.py
├── README.md
└── t.csv
├── 08-CallGraph
├── 08-CallGraph.iml
├── CallGraph.py
├── CheckSymbols.py
├── Cymbol.g4
├── Readme.md
├── SymbolScope.py
├── output.dot
├── t.cymbol
├── vars.cymbol
└── vars2.cymbol
├── 08-JSON
├── 08-JSON.iml
├── JSON.g4
├── README.md
├── json2xml.py
└── t.json
├── 09-simple
├── README.md
├── Simple.g4
├── TestE_Listener.py
├── in1.txt
├── in2.txt
├── in3.txt
├── in4.txt
├── in5.txt
├── in6.txt
└── in7.txt
├── 10-CSV
├── 10-CSV.iml
├── CSV.g4
├── README.md
├── test_csv.py
└── users.csv
├── 10-Keyword
├── 10-Keyword.iml
├── Keywords.g4
├── input1.txt
├── input2.txt
├── input3.txt
└── test_Keyword.py
├── 10-calc
├── 10-calc.iml
├── Expr.g4
├── README.md
└── calc.py
├── 11-CppState
├── 11-CppState.iml
├── CppStat.g4
├── PredCppStat.g4
├── input.cpp
└── input2.cpp
├── 11-predicate_lexer
├── 11-predicate_lexer.iml
├── Enum2.g4
├── README.md
├── Temp.java
└── test_EnumLexer.py
├── 11-predicate_parser
├── 11-predicate_parser.iml
├── Enum.g4
├── README.md
└── Temp.java
├── 12-channel
├── 12-channel.iml
├── Cymbol.g4
├── README.md
├── shift_var_comments.py
└── t.cym
├── 12-id_as_keyword
├── IDKeyword.g4
├── PredKeyword.g4
├── input.txt
└── run.bat
├── 12-sea_of_text
├── Mode.txt
├── ModeTagsLexer.g4
├── ModeTagsParser.g4
└── Tags.g4
├── README.md
├── antlr.jar
├── antlr4module
├── bin
├── antlr-4.7.2-complete.jar
├── antlr4env.bat
└── pygrun
├── img
└── IDEA.png
└── requirements.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | __pycache__/
3 | *.tokens
4 | *.swp
5 |
6 | # Byte-compiled / optimized / DLL files
7 | __pycache__/
8 | *.py[cod]
9 | *$py.class
10 |
11 | # C extensions
12 | *.so
13 |
14 | # Distribution / packaging
15 | .Python
16 | build/
17 | develop-eggs/
18 | dist/
19 | downloads/
20 | eggs/
21 | .eggs/
22 | lib/
23 | lib64/
24 | parts/
25 | sdist/
26 | var/
27 | wheels/
28 | pip-wheel-metadata/
29 | share/python-wheels/
30 | *.egg-info/
31 | .installed.cfg
32 | *.egg
33 | MANIFEST
34 |
35 | # PyInstaller
36 | # Usually these files are written by a python script from a template
37 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
38 | *.manifest
39 | *.spec
40 |
41 | # Installer logs
42 | pip-log.txt
43 | pip-delete-this-directory.txt
44 |
45 | # Unit test / coverage reports
46 | htmlcov/
47 | .tox/
48 | .nox/
49 | .coverage
50 | .coverage.*
51 | .cache
52 | nosetests.xml
53 | coverage.xml
54 | *.cover
55 | *.py,cover
56 | .hypothesis/
57 | .pytest_cache/
58 | cover/
59 |
60 | # Translations
61 | *.mo
62 | *.pot
63 |
64 | # Django stuff:
65 | *.log
66 | local_settings.py
67 | db.sqlite3
68 | db.sqlite3-journal
69 |
70 | # Flask stuff:
71 | instance/
72 | .webassets-cache
73 |
74 | # Scrapy stuff:
75 | .scrapy
76 |
77 | # Sphinx documentation
78 | docs/_build/
79 |
80 | # PyBuilder
81 | .pybuilder/
82 | target/
83 |
84 | # Jupyter Notebook
85 | .ipynb_checkpoints
86 |
87 | # IPython
88 | profile_default/
89 | ipython_config.py
90 |
91 | # pyenv
92 | # For a library or package, you might want to ignore these files since the code is
93 | # intended to run in multiple environments; otherwise, check them in:
94 | # .python-version
95 |
96 | # pipenv
97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
100 | # install all needed dependencies.
101 | #Pipfile.lock
102 |
103 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
104 | __pypackages__/
105 |
106 | # Celery stuff
107 | celerybeat-schedule
108 | celerybeat.pid
109 |
110 | # SageMath parsed files
111 | *.sage.py
112 |
113 | # Environments
114 | .env
115 | .venv
116 | env/
117 | venv/
118 | ENV/
119 | env.bak/
120 | venv.bak/
121 |
122 | # Spyder project settings
123 | .spyderproject
124 | .spyproject
125 |
126 | # Rope project settings
127 | .ropeproject
128 |
129 | # mkdocs documentation
130 | /site
131 |
132 | # mypy
133 | .mypy_cache/
134 | .dmypy.json
135 | dmypy.json
136 |
137 | # Pyre type checker
138 | .pyre/
139 |
140 | # pytype static type analyzer
141 | .pytype/
142 |
143 | # Cython debug symbols
144 | cython_debug/
145 |
146 | # static files generated from Django application using `collectstatic`
147 | media
148 | static
149 |
--------------------------------------------------------------------------------
/01-Hello/01-Hello.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/01-Hello/Hello.g4:
--------------------------------------------------------------------------------
1 | grammar Hello; // Define a grammar called Hello
2 | r : 'hello' ID ; // match keyword hello followed by an identifier
3 | ID : [a-z]+ ; // match lower-case identifiers
4 | WS : [ \t\r\n]+ -> skip ; // skip spaces, tabs, newlines, \r (Windows)
5 |
--------------------------------------------------------------------------------
/01-Hello/input.txt:
--------------------------------------------------------------------------------
1 | hello world
--------------------------------------------------------------------------------
/01-Hello/test_hello.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from HelloLexer import HelloLexer
4 | from HelloParser import HelloParser
5 |
6 |
7 | def main(argv):
8 | input = FileStream(argv[1])
9 | lexer = HelloLexer(input)
10 | stream = CommonTokenStream(lexer)
11 | parser = HelloParser(stream)
12 | tree = parser.r()
13 | print(tree.toStringTree(recog=parser))
14 |
15 | if __name__ == '__main__':
16 | main(sys.argv)
17 |
--------------------------------------------------------------------------------
/03-Array/03-Array.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/03-Array/ArrayInit.g4:
--------------------------------------------------------------------------------
1 | /** Grammars always start with a grammar header. This grammar is called
2 | * ArrayInit and must match the filename: ArrayInit.g4
3 | */
4 | grammar ArrayInit;
5 |
6 | /** A rule called init that matches comma-separated values between {...}. */
7 | init : '{' value (',' value)* '}' ; // must match at least one value
8 |
9 | /** A value can be either a nested array/struct or a simple integer (INT) */
10 | value : init
11 | | INT
12 | ;
13 |
14 | // parser rules start with lowercase letters, lexer rules with uppercase
15 | INT : [0-9]+ ; // Define token INT as one or more digits
16 | WS : [ \t\r\n]+ -> skip ; // Define whitespace rule, toss it out
17 |
--------------------------------------------------------------------------------
/03-Array/README.md:
--------------------------------------------------------------------------------
1 | # Introduction
2 | This example shows how to use listener to do simple translation of input array.
3 |
4 | # How to run
5 | ```
6 | % antlr4py3 ArrayInit.g4
7 | % pygrun ArrayInit init --tree input.txt
8 | [@0,0:0='{',<1>,1:0]
9 | [@1,1:2='99',<4>,1:1]
10 | [@2,3:3=',',<2>,1:3]
11 | [@3,5:5='3',<4>,1:5]
12 | [@4,6:6=',',<2>,1:6]
13 | [@5,8:10='451',<4>,1:8]
14 | [@6,11:11='}',<3>,1:11]
15 | [@7,12:11='',<-1>,1:12]
16 |
17 | % pygrun ArrayInit init --tokens input.txt
18 | (init {
19 | (value 99) ,
20 | (value 3) ,
21 | (value 451) })
22 |
23 | % python test_array.py input.txt
24 | (init { (value 99) , (value 3) , (value 451) })
25 | "\u0063\u0003\u01c3"
26 |
27 | ```
--------------------------------------------------------------------------------
/03-Array/input.txt:
--------------------------------------------------------------------------------
1 | {99, 3, 451}
--------------------------------------------------------------------------------
/03-Array/rewriter.py:
--------------------------------------------------------------------------------
1 | from ArrayInitListener import ArrayInitListener
2 |
3 |
4 | class RewriteListener(ArrayInitListener):
5 | # Enter a parse tree produced by ArrayInitParser#init.
6 | def enterInit(self, ctx):
7 | print("\"", end='')
8 |
9 | # Exit a parse tree produced by ArrayInitParser#init.
10 | def exitInit(self, ctx):
11 | print("\"", end='')
12 |
13 | # Enter a parse tree produced by ArrayInitParser#value.
14 | def enterValue(self, ctx):
15 | pass
16 |
17 | # Exit a parse tree produced by ArrayInitParser#value.
18 | def exitValue(self, ctx):
19 | data = ctx.INT().getText()
20 | print('\\u%04x' % int(data), end='')
--------------------------------------------------------------------------------
/03-Array/test_array.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from ArrayInitLexer import ArrayInitLexer
4 | from ArrayInitParser import ArrayInitParser
5 | from rewriter import RewriteListener
6 |
7 | def main(argv):
8 | istream = FileStream(argv[1])
9 | lexer = ArrayInitLexer(istream)
10 | stream = CommonTokenStream(lexer)
11 | parser = ArrayInitParser(stream)
12 | tree = parser.init()
13 | print(tree.toStringTree(recog=parser))
14 |
15 | walker = ParseTreeWalker()
16 | walker.walk(RewriteListener(), tree)
17 | print()
18 |
19 | if __name__ == '__main__':
20 | main(sys.argv)
21 |
--------------------------------------------------------------------------------
/04-Calc/04-Calc.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/04-Calc/LabeledExpr.g4:
--------------------------------------------------------------------------------
1 | grammar LabeledExpr; // rename to distinguish from Expr.g4
2 |
3 | prog: stat+ ;
4 |
5 | stat: expr NEWLINE # printExpr
6 | | ID '=' expr NEWLINE # assign
7 | | NEWLINE # blank
8 | ;
9 |
10 | expr: expr op=('*'|'/') expr # MulDiv
11 | | expr op=('+'|'-') expr # AddSub
12 | | INT # int
13 | | ID # id
14 | | '(' expr ')' # parens
15 | ;
16 |
17 | MUL : '*' ; // assigns token name to '*' used above in grammar
18 | DIV : '/' ;
19 | ADD : '+' ;
20 | SUB : '-' ;
21 | ID : [a-zA-Z]+ ; // match identifiers
22 | INT : [0-9]+ ; // match integers
23 | NEWLINE:'\r'? '\n' ; // return newlines to parser (is end-statement signal)
24 | WS : [ \t]+ -> skip ; // toss out whitespace
25 |
--------------------------------------------------------------------------------
/04-Calc/MyVisitor.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | from LabeledExprVisitor import LabeledExprVisitor
4 | from LabeledExprParser import LabeledExprParser
5 |
6 |
7 | class MyVisitor(LabeledExprVisitor):
8 | def __init__(self):
9 | self.memory = {}
10 |
11 | def visitAssign(self, ctx):
12 | name = ctx.ID().getText()
13 | value = self.visit(ctx.expr())
14 | self.memory[name] = value
15 | return value
16 |
17 | def visitPrintExpr(self, ctx):
18 | value = self.visit(ctx.expr())
19 | print(value)
20 | return 0
21 |
22 | def visitInt(self, ctx):
23 | return ctx.INT().getText()
24 |
25 | def visitId(self, ctx):
26 | name = ctx.ID().getText()
27 | if name in self.memory:
28 | return self.memory[name]
29 | return 0
30 |
31 | def visitMulDiv(self, ctx):
32 | left = int(self.visit(ctx.expr(0)))
33 | right = int(self.visit(ctx.expr(1)))
34 | if ctx.op.type == LabeledExprParser.MUL:
35 | return left * right
36 | return left / right
37 |
38 | def visitAddSub(self, ctx):
39 | left = int(self.visit(ctx.expr(0)))
40 | right = int(self.visit(ctx.expr(1)))
41 | if ctx.op.type == LabeledExprParser.ADD:
42 | return left + right
43 | return left - right
44 |
45 | def visitParens(self, ctx):
46 | return self.visit(ctx.expr())
47 |
--------------------------------------------------------------------------------
/04-Calc/README.md:
--------------------------------------------------------------------------------
1 | # Calculator
2 | This example shows
3 | - Visitor Pattern
4 | - Use of Alternative Label
5 |
6 | # How to run
7 | ```
8 | % antlr4py3 -visitor -no-listener LabeledExpr.g4
9 | % type t.expr
10 | 193
11 | a = 5
12 | b = 6
13 | a+b*2
14 | (1+2)*3
15 |
16 | % python calc.py t.expr
17 | 193
18 | 17
19 | 9
20 | ```
--------------------------------------------------------------------------------
/04-Calc/calc.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4.InputStream import InputStream
6 | from LabeledExprLexer import LabeledExprLexer
7 | from LabeledExprParser import LabeledExprParser
8 | from MyVisitor import MyVisitor
9 |
10 | if __name__ == '__main__':
11 | if len(sys.argv) > 1:
12 | input_stream = FileStream(sys.argv[1])
13 | else:
14 | input_stream = InputStream(sys.stdin.readline())
15 |
16 | lexer = LabeledExprLexer(input_stream)
17 | token_stream = CommonTokenStream(lexer)
18 | parser = LabeledExprParser(token_stream)
19 | tree = parser.prog()
20 |
21 | #lisp_tree_str = tree.toStringTree(recog=parser)
22 | #print(lisp_tree_str)
23 |
24 | visitor = MyVisitor()
25 | visitor.visit(tree)
26 |
--------------------------------------------------------------------------------
/04-Calc/t.expr:
--------------------------------------------------------------------------------
1 | 193
2 | a = 5
3 | b = 6
4 | a+b*2
5 | (1+2)*3
6 |
--------------------------------------------------------------------------------
/04-Expr/04-Expr.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/04-Expr/Expr.g4:
--------------------------------------------------------------------------------
1 | grammar Expr;
2 |
3 | /** The start rule; begin parsing here. */
4 | prog: stat+ ;
5 |
6 | stat: expr NEWLINE
7 | | ID '=' expr NEWLINE
8 | | NEWLINE
9 | ;
10 |
11 | expr: expr ('*'|'/') expr
12 | | expr ('+'|'-') expr
13 | | INT
14 | | ID
15 | | '(' expr ')'
16 | ;
17 |
18 | ID : [a-zA-Z]+ ; // match identifiers
19 | INT : [0-9]+ ; // match integers
20 | NEWLINE:'\r'? '\n' ; // return newlines to parser (is end-statement signal)
21 | WS : [ \t]+ -> skip ; // toss out whitespace
22 |
--------------------------------------------------------------------------------
/04-Expr/MyListener.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | from ExprListener import ExprListener
4 |
5 |
6 | class MyListener(ExprListener):
7 | # Enter a parse tree produced by ExprParser#prog.
8 | def enterProg(self, ctx):
9 | print('EnterProg')
10 |
11 | # Exit a parse tree produced by ExprParser#prog.
12 | def exitProg(self, ctx):
13 | print('ExitProg')
14 |
15 |
16 | # Enter a parse tree produced by ExprParser#stat.
17 | def enterStat(self, ctx):
18 | print('EnterStat')
19 |
20 | # Exit a parse tree produced by ExprParser#stat.
21 | def exitStat(self, ctx):
22 | print('ExitStat')
23 |
24 |
25 | # Enter a parse tree produced by ExprParser#expr.
26 | def enterExpr(self, ctx):
27 | print('EnterExpr')
28 |
29 | # Exit a parse tree produced by ExprParser#expr.
30 | def exitExpr(self, ctx):
31 | print('ExitExpr')
32 |
33 |
34 |
--------------------------------------------------------------------------------
/04-Expr/README.md:
--------------------------------------------------------------------------------
1 | # How to run
2 | ```
3 | % pygrun Expr prog --tree t.expr
4 | (prog
5 | (stat
6 | (expr 193) \n)
7 | (stat a =
8 | (expr 5) \n)
9 | (stat b =
10 | (expr 6) \n)
11 | (stat
12 | (expr
13 | (expr a) +
14 | (expr
15 | (expr b) *
16 | (expr 2))) \n)
17 | (stat
18 | (expr
19 | (expr (
20 | (expr
21 | (expr 1) +
22 | (expr 2)) )) *
23 | (expr 3)) \n))
24 |
25 | % python test_Expr.py t.expr
26 | (prog
27 | (stat (expr 193) \n)
28 | (stat a = (expr 5) \n)
29 | (stat b = (expr 6) \n)
30 | (stat (expr (expr a) + (expr (expr b) * (expr 2))) \n)
31 | (stat (expr (expr ( (expr (expr 1) + (expr 2)) )) * (expr 3)) \n)
32 | )
33 | ```
--------------------------------------------------------------------------------
/04-Expr/t.expr:
--------------------------------------------------------------------------------
1 | 193
2 | a = 5
3 | b = 6
4 | a+b*2
5 | (1+2)*3
6 |
--------------------------------------------------------------------------------
/04-Expr/test_Expr.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4.InputStream import InputStream
6 | from ExprLexer import ExprLexer
7 | from ExprParser import ExprParser
8 |
9 | if __name__ == '__main__':
10 | if len(sys.argv) > 1:
11 | input_stream = FileStream(sys.argv[1])
12 | else:
13 | input_stream = InputStream(sys.stdin.readline())
14 |
15 | lexer = ExprLexer(input_stream)
16 | token_stream = CommonTokenStream(lexer)
17 | parser = ExprParser(token_stream)
18 | tree = parser.prog()
19 |
20 | lisp_tree_str = tree.toStringTree(recog=parser)
21 | print(lisp_tree_str)
22 |
23 |
--------------------------------------------------------------------------------
/04-ExtractInterface/04-ExtractInterface.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/04-ExtractInterface/Demo.java:
--------------------------------------------------------------------------------
1 | /***
2 | * Excerpted from "The Definitive ANTLR 4 Reference",
3 | * published by The Pragmatic Bookshelf.
4 | * Copyrights apply to this code. It may not be used to create training material,
5 | * courses, books, articles, and the like. Contact us if you are in doubt.
6 | * We make no guarantees that this code is fit for any purpose.
7 | * Visit http://www.pragmaticprogrammer.com/titles/tpantlr2 for more book information.
8 | ***/
9 | import java.util.List;
10 | import java.util.Map;
11 | public class Demo {
12 | int f(int x, String y) { }
13 | int[ ] g(/*no args*/) { return null; }
14 | List>[] h() { return null; }
15 | }
16 |
--------------------------------------------------------------------------------
/04-ExtractInterface/ExtractInterfaceListener.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | from antlr4 import *
4 | from JavaListener import JavaListener
5 | from JavaParser import JavaParser
6 |
7 |
8 | class ExtractInterfaceListener(JavaListener):
9 |
10 | # need parser to extract token stream
11 | def __init__(self, parser: JavaParser):
12 | self.parser = parser
13 |
14 | # Enter a parse tree produced by JavaParser#classDeclaration.
15 | def enterClassDeclaration(self, ctx):
16 | print('interface I', ctx.Identifier(), ' {', sep='')
17 |
18 | # Exit a parse tree produced by JavaParser#classDeclaration.
19 | def exitClassDeclaration(self, ctx):
20 | print('}')
21 |
22 | # should the function becomes function of Rule Context?
23 | def getAllText(self, ctx): # include hidden channel
24 | token_stream = ctx.parser.getTokenStream()
25 | lexer = token_stream.tokenSource
26 | input_stream = lexer.inputStream
27 | start = ctx.start.start
28 | stop = ctx.stop.stop
29 | return input_stream.getText(start, stop)
30 |
31 | def enterMethodDeclaration(self, ctx: JavaParser.MethodDeclarationContext):
32 |
33 | dt = 'void' # extract data type string, None means void
34 | dt_ctx = ctx.datatype()
35 | if dt_ctx is not None:
36 | dt = self.getAllText(dt_ctx)
37 | args = self.getAllText(ctx.formalParameters())
38 | print("\t", dt, ' ', ctx.Identifier(), args, ';', sep='')
--------------------------------------------------------------------------------
/04-ExtractInterface/Java.g4:
--------------------------------------------------------------------------------
1 | /*
2 | [The "BSD licence"]
3 | Copyright (c) 2013 Terence Parr, Sam Harwell
4 | All rights reserved.
5 |
6 | Redistribution and use in source and binary forms, with or without
7 | modification, are permitted provided that the following conditions
8 | are met:
9 | 1. Redistributions of source code must retain the above copyright
10 | notice, this list of conditions and the following disclaimer.
11 | 2. Redistributions in binary form must reproduce the above copyright
12 | notice, this list of conditions and the following disclaimer in the
13 | documentation and/or other materials provided with the distribution.
14 | 3. The name of the author may not be used to endorse or promote products
15 | derived from this software without specific prior written permission.
16 |
17 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 | IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 | NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 | */
28 |
29 | /** A Java 1.7 grammar for ANTLR v4 derived from ANTLR v3 Java grammar.
30 | * Uses ANTLR v4's left-recursive expression notation.
31 | * It parses ECJ, Netbeans, JDK etc...
32 | *
33 | * Sam Harwell cleaned this up significantly and updated to 1.7!
34 | *
35 | * You can test with
36 | *
37 | * $ antlr4 Java.g4
38 | * $ javac *.java
39 | * $ grun Java compilationUnit *.java
40 | */
41 | grammar Java;
42 |
43 | // starting point for parsing a java file
44 | compilationUnit
45 | : packageDeclaration? importDeclaration* typeDeclaration* EOF
46 | ;
47 |
48 | packageDeclaration
49 | : annotation* 'package' qualifiedName ';'
50 | ;
51 |
52 | importDeclaration
53 | : 'import' 'static'? qualifiedName ('.' '*')? ';'
54 | ;
55 |
56 | typeDeclaration
57 | : classOrInterfaceModifier* classDeclaration
58 | | classOrInterfaceModifier* enumDeclaration
59 | | classOrInterfaceModifier* interfaceDeclaration
60 | | classOrInterfaceModifier* annotationTypeDeclaration
61 | | ';'
62 | ;
63 |
64 | modifier
65 | : classOrInterfaceModifier
66 | | ( 'native'
67 | | 'synchronized'
68 | | 'transient'
69 | | 'volatile'
70 | )
71 | ;
72 |
73 | classOrInterfaceModifier
74 | : annotation // class or interface
75 | | ( 'public' // class or interface
76 | | 'protected' // class or interface
77 | | 'private' // class or interface
78 | | 'static' // class or interface
79 | | 'abstract' // class or interface
80 | | 'final' // class only -- does not apply to interfaces
81 | | 'strictfp' // class or interface
82 | )
83 | ;
84 |
85 | variableModifier
86 | : 'final'
87 | | annotation
88 | ;
89 |
90 | classDeclaration
91 | : 'class' Identifier typeParameters?
92 | ('extends' datatype)?
93 | ('implements' typeList)?
94 | classBody
95 | ;
96 |
97 | typeParameters
98 | : '<' typeParameter (',' typeParameter)* '>'
99 | ;
100 |
101 | typeParameter
102 | : Identifier ('extends' typeBound)?
103 | ;
104 |
105 | typeBound
106 | : datatype ('&' datatype)*
107 | ;
108 |
109 | enumDeclaration
110 | : ENUM Identifier ('implements' typeList)?
111 | '{' enumConstants? ','? enumBodyDeclarations? '}'
112 | ;
113 |
114 | enumConstants
115 | : enumConstant (',' enumConstant)*
116 | ;
117 |
118 | enumConstant
119 | : annotation* Identifier arguments? classBody?
120 | ;
121 |
122 | enumBodyDeclarations
123 | : ';' classBodyDeclaration*
124 | ;
125 |
126 | interfaceDeclaration
127 | : 'interface' Identifier typeParameters? ('extends' typeList)? interfaceBody
128 | ;
129 |
130 | typeList
131 | : datatype (',' datatype)*
132 | ;
133 |
134 | classBody
135 | : '{' classBodyDeclaration* '}'
136 | ;
137 |
138 | interfaceBody
139 | : '{' interfaceBodyDeclaration* '}'
140 | ;
141 |
142 | classBodyDeclaration
143 | : ';'
144 | | 'static'? block
145 | | modifier* memberDeclaration
146 | ;
147 |
148 | memberDeclaration
149 | : methodDeclaration
150 | | genericMethodDeclaration
151 | | fieldDeclaration
152 | | constructorDeclaration
153 | | genericConstructorDeclaration
154 | | interfaceDeclaration
155 | | annotationTypeDeclaration
156 | | classDeclaration
157 | | enumDeclaration
158 | ;
159 |
160 | /* We use rule this even for void methods which cannot have [] after parameters.
161 | This simplifies grammar and we can consider void to be a type, which
162 | renders the [] matching as a context-sensitive issue or a semantic check
163 | for invalid return type after parsing.
164 | */
165 | methodDeclaration
166 | : (datatype|'void') Identifier formalParameters ('[' ']')*
167 | ('throws' qualifiedNameList)?
168 | ( methodBody
169 | | ';'
170 | )
171 | ;
172 |
173 | genericMethodDeclaration
174 | : typeParameters methodDeclaration
175 | ;
176 |
177 | constructorDeclaration
178 | : Identifier formalParameters ('throws' qualifiedNameList)?
179 | constructorBody
180 | ;
181 |
182 | genericConstructorDeclaration
183 | : typeParameters constructorDeclaration
184 | ;
185 |
186 | fieldDeclaration
187 | : datatype variableDeclarators ';'
188 | ;
189 |
190 | interfaceBodyDeclaration
191 | : modifier* interfaceMemberDeclaration
192 | | ';'
193 | ;
194 |
195 | interfaceMemberDeclaration
196 | : constDeclaration
197 | | interfaceMethodDeclaration
198 | | genericInterfaceMethodDeclaration
199 | | interfaceDeclaration
200 | | annotationTypeDeclaration
201 | | classDeclaration
202 | | enumDeclaration
203 | ;
204 |
205 | constDeclaration
206 | : datatype constantDeclarator (',' constantDeclarator)* ';'
207 | ;
208 |
209 | constantDeclarator
210 | : Identifier ('[' ']')* '=' variableInitializer
211 | ;
212 |
213 | // see matching of [] comment in methodDeclaratorRest
214 | interfaceMethodDeclaration
215 | : (datatype|'void') Identifier formalParameters ('[' ']')*
216 | ('throws' qualifiedNameList)?
217 | ';'
218 | ;
219 |
220 | genericInterfaceMethodDeclaration
221 | : typeParameters interfaceMethodDeclaration
222 | ;
223 |
224 | variableDeclarators
225 | : variableDeclarator (',' variableDeclarator)*
226 | ;
227 |
228 | variableDeclarator
229 | : variableDeclaratorId ('=' variableInitializer)?
230 | ;
231 |
232 | variableDeclaratorId
233 | : Identifier ('[' ']')*
234 | ;
235 |
236 | variableInitializer
237 | : arrayInitializer
238 | | expression
239 | ;
240 |
241 | arrayInitializer
242 | : '{' (variableInitializer (',' variableInitializer)* (',')? )? '}'
243 | ;
244 |
245 | enumConstantName
246 | : Identifier
247 | ;
248 |
249 | datatype
250 | : classOrInterfaceType ('[' ']')*
251 | | primitiveType ('[' ']')*
252 | ;
253 |
254 | classOrInterfaceType
255 | : Identifier typeArguments? ('.' Identifier typeArguments? )*
256 | ;
257 |
258 | primitiveType
259 | : 'boolean'
260 | | 'char'
261 | | 'byte'
262 | | 'short'
263 | | 'int'
264 | | 'long'
265 | | 'float'
266 | | 'double'
267 | ;
268 |
269 | typeArguments
270 | : '<' typeArgument (',' typeArgument)* '>'
271 | ;
272 |
273 | typeArgument
274 | : datatype
275 | | '?' (('extends' | 'super') datatype)?
276 | ;
277 |
278 | qualifiedNameList
279 | : qualifiedName (',' qualifiedName)*
280 | ;
281 |
282 | formalParameters
283 | : '(' formalParameterList? ')'
284 | ;
285 |
286 | formalParameterList
287 | : formalParameter (',' formalParameter)* (',' lastFormalParameter)?
288 | | lastFormalParameter
289 | ;
290 |
291 | formalParameter
292 | : variableModifier* datatype variableDeclaratorId
293 | ;
294 |
295 | lastFormalParameter
296 | : variableModifier* datatype '...' variableDeclaratorId
297 | ;
298 |
299 | methodBody
300 | : block
301 | ;
302 |
303 | constructorBody
304 | : block
305 | ;
306 |
307 | qualifiedName
308 | : Identifier ('.' Identifier)*
309 | ;
310 |
311 | literal
312 | : IntegerLiteral
313 | | FloatingPointLiteral
314 | | CharacterLiteral
315 | | StringLiteral
316 | | BooleanLiteral
317 | | 'null'
318 | ;
319 |
320 | // ANNOTATIONS
321 |
322 | annotation
323 | : '@' annotationName ( '(' ( elementValuePairs | elementValue )? ')' )?
324 | ;
325 |
326 | annotationName : qualifiedName ;
327 |
328 | elementValuePairs
329 | : elementValuePair (',' elementValuePair)*
330 | ;
331 |
332 | elementValuePair
333 | : Identifier '=' elementValue
334 | ;
335 |
336 | elementValue
337 | : expression
338 | | annotation
339 | | elementValueArrayInitializer
340 | ;
341 |
342 | elementValueArrayInitializer
343 | : '{' (elementValue (',' elementValue)*)? (',')? '}'
344 | ;
345 |
346 | annotationTypeDeclaration
347 | : '@' 'interface' Identifier annotationTypeBody
348 | ;
349 |
350 | annotationTypeBody
351 | : '{' (annotationTypeElementDeclaration)* '}'
352 | ;
353 |
354 | annotationTypeElementDeclaration
355 | : modifier* annotationTypeElementRest
356 | | ';' // this is not allowed by the grammar, but apparently allowed by the actual compiler
357 | ;
358 |
359 | annotationTypeElementRest
360 | : datatype annotationMethodOrConstantRest ';'
361 | | classDeclaration ';'?
362 | | interfaceDeclaration ';'?
363 | | enumDeclaration ';'?
364 | | annotationTypeDeclaration ';'?
365 | ;
366 |
367 | annotationMethodOrConstantRest
368 | : annotationMethodRest
369 | | annotationConstantRest
370 | ;
371 |
372 | annotationMethodRest
373 | : Identifier '(' ')' defaultValue?
374 | ;
375 |
376 | annotationConstantRest
377 | : variableDeclarators
378 | ;
379 |
380 | defaultValue
381 | : 'default' elementValue
382 | ;
383 |
384 | // STATEMENTS / BLOCKS
385 |
386 | block
387 | : '{' blockStatement* '}'
388 | ;
389 |
390 | blockStatement
391 | : localVariableDeclarationStatement
392 | | statement
393 | | typeDeclaration
394 | ;
395 |
396 | localVariableDeclarationStatement
397 | : localVariableDeclaration ';'
398 | ;
399 |
400 | localVariableDeclaration
401 | : variableModifier* datatype variableDeclarators
402 | ;
403 |
404 | statement
405 | : block
406 | | ASSERT expression (':' expression)? ';'
407 | | 'if' parExpression statement ('else' statement)?
408 | | 'for' '(' forControl ')' statement
409 | | 'while' parExpression statement
410 | | 'do' statement 'while' parExpression ';'
411 | | 'try' block (catchClause+ finallyBlock? | finallyBlock)
412 | | 'try' resourceSpecification block catchClause* finallyBlock?
413 | | 'switch' parExpression '{' switchBlockStatementGroup* switchLabel* '}'
414 | | 'synchronized' parExpression block
415 | | 'return' expression? ';'
416 | | 'throw' expression ';'
417 | | 'break' Identifier? ';'
418 | | 'continue' Identifier? ';'
419 | | ';'
420 | | statementExpression ';'
421 | | Identifier ':' statement
422 | ;
423 |
424 | catchClause
425 | : 'catch' '(' variableModifier* catchType Identifier ')' block
426 | ;
427 |
428 | catchType
429 | : qualifiedName ('|' qualifiedName)*
430 | ;
431 |
432 | finallyBlock
433 | : 'finally' block
434 | ;
435 |
436 | resourceSpecification
437 | : '(' resources ';'? ')'
438 | ;
439 |
440 | resources
441 | : resource (';' resource)*
442 | ;
443 |
444 | resource
445 | : variableModifier* classOrInterfaceType variableDeclaratorId '=' expression
446 | ;
447 |
448 | /** Matches cases then statements, both of which are mandatory.
449 | * To handle empty cases at the end, we add switchLabel* to statement.
450 | */
451 | switchBlockStatementGroup
452 | : switchLabel+ blockStatement+
453 | ;
454 |
455 | switchLabel
456 | : 'case' constantExpression ':'
457 | | 'case' enumConstantName ':'
458 | | 'default' ':'
459 | ;
460 |
461 | forControl
462 | : enhancedForControl
463 | | forInit? ';' expression? ';' forUpdate?
464 | ;
465 |
466 | forInit
467 | : localVariableDeclaration
468 | | expressionList
469 | ;
470 |
471 | enhancedForControl
472 | : variableModifier* datatype variableDeclaratorId ':' expression
473 | ;
474 |
475 | forUpdate
476 | : expressionList
477 | ;
478 |
479 | // EXPRESSIONS
480 |
481 | parExpression
482 | : '(' expression ')'
483 | ;
484 |
485 | expressionList
486 | : expression (',' expression)*
487 | ;
488 |
489 | statementExpression
490 | : expression
491 | ;
492 |
493 | constantExpression
494 | : expression
495 | ;
496 |
497 | expression
498 | : primary
499 | | expression '.' Identifier
500 | | expression '.' 'this'
501 | | expression '.' 'new' nonWildcardTypeArguments? innerCreator
502 | | expression '.' 'super' superSuffix
503 | | expression '.' explicitGenericInvocation
504 | | expression '[' expression ']'
505 | | expression '(' expressionList? ')'
506 | | 'new' creator
507 | | '(' datatype ')' expression
508 | | expression ('++' | '--')
509 | | ('+'|'-'|'++'|'--') expression
510 | | ('~'|'!') expression
511 | | expression ('*'|'/'|'%') expression
512 | | expression ('+'|'-') expression
513 | | expression ('<' '<' | '>' '>' '>' | '>' '>') expression
514 | | expression ('<=' | '>=' | '>' | '<') expression
515 | | expression 'instanceof' datatype
516 | | expression ('==' | '!=') expression
517 | | expression '&' expression
518 | | expression '^' expression
519 | | expression '|' expression
520 | | expression '&&' expression
521 | | expression '||' expression
522 | | expression '?' expression ':' expression
523 | | expression
524 | ( '='
525 | | '+='
526 | | '-='
527 | | '*='
528 | | '/='
529 | | '&='
530 | | '|='
531 | | '^='
532 | | '>>='
533 | | '>>>='
534 | | '<<='
535 | | '%='
536 | )
537 | expression
538 | ;
539 |
540 | primary
541 | : '(' expression ')'
542 | | 'this'
543 | | 'super'
544 | | literal
545 | | Identifier
546 | | datatype '.' 'class'
547 | | 'void' '.' 'class'
548 | | nonWildcardTypeArguments (explicitGenericInvocationSuffix | 'this' arguments)
549 | ;
550 |
551 | creator
552 | : nonWildcardTypeArguments createdName classCreatorRest
553 | | createdName (arrayCreatorRest | classCreatorRest)
554 | ;
555 |
556 | createdName
557 | : Identifier typeArgumentsOrDiamond? ('.' Identifier typeArgumentsOrDiamond?)*
558 | | primitiveType
559 | ;
560 |
561 | innerCreator
562 | : Identifier nonWildcardTypeArgumentsOrDiamond? classCreatorRest
563 | ;
564 |
565 | arrayCreatorRest
566 | : '['
567 | ( ']' ('[' ']')* arrayInitializer
568 | | expression ']' ('[' expression ']')* ('[' ']')*
569 | )
570 | ;
571 |
572 | classCreatorRest
573 | : arguments classBody?
574 | ;
575 |
576 | explicitGenericInvocation
577 | : nonWildcardTypeArguments explicitGenericInvocationSuffix
578 | ;
579 |
580 | nonWildcardTypeArguments
581 | : '<' typeList '>'
582 | ;
583 |
584 | typeArgumentsOrDiamond
585 | : '<' '>'
586 | | typeArguments
587 | ;
588 |
589 | nonWildcardTypeArgumentsOrDiamond
590 | : '<' '>'
591 | | nonWildcardTypeArguments
592 | ;
593 |
594 | superSuffix
595 | : arguments
596 | | '.' Identifier arguments?
597 | ;
598 |
599 | explicitGenericInvocationSuffix
600 | : 'super' superSuffix
601 | | Identifier arguments
602 | ;
603 |
604 | arguments
605 | : '(' expressionList? ')'
606 | ;
607 |
608 | // LEXER
609 |
610 | // §3.9 Keywords
611 |
612 | ABSTRACT : 'abstract';
613 | ASSERT : 'assert';
614 | BOOLEAN : 'boolean';
615 | BREAK : 'break';
616 | BYTE : 'byte';
617 | CASE : 'case';
618 | CATCH : 'catch';
619 | CHAR : 'char';
620 | CLASS : 'class';
621 | CONST : 'const';
622 | CONTINUE : 'continue';
623 | DEFAULT : 'default';
624 | DO : 'do';
625 | DOUBLE : 'double';
626 | ELSE : 'else';
627 | ENUM : 'enum';
628 | EXTENDS : 'extends';
629 | FINAL : 'final';
630 | FINALLY : 'finally';
631 | FLOAT : 'float';
632 | FOR : 'for';
633 | IF : 'if';
634 | GOTO : 'goto';
635 | IMPLEMENTS : 'implements';
636 | IMPORT : 'import';
637 | INSTANCEOF : 'instanceof';
638 | INT : 'int';
639 | INTERFACE : 'interface';
640 | LONG : 'long';
641 | NATIVE : 'native';
642 | NEW : 'new';
643 | PACKAGE : 'package';
644 | PRIVATE : 'private';
645 | PROTECTED : 'protected';
646 | PUBLIC : 'public';
647 | RETURN : 'return';
648 | SHORT : 'short';
649 | STATIC : 'static';
650 | STRICTFP : 'strictfp';
651 | SUPER : 'super';
652 | SWITCH : 'switch';
653 | SYNCHRONIZED : 'synchronized';
654 | THIS : 'this';
655 | THROW : 'throw';
656 | THROWS : 'throws';
657 | TRANSIENT : 'transient';
658 | TRY : 'try';
659 | VOID : 'void';
660 | VOLATILE : 'volatile';
661 | WHILE : 'while';
662 |
663 | // §3.10.1 Integer Literals
664 |
665 | IntegerLiteral
666 | : DecimalIntegerLiteral
667 | | HexIntegerLiteral
668 | | OctalIntegerLiteral
669 | | BinaryIntegerLiteral
670 | ;
671 |
672 | fragment
673 | DecimalIntegerLiteral
674 | : DecimalNumeral IntegerTypeSuffix?
675 | ;
676 |
677 | fragment
678 | HexIntegerLiteral
679 | : HexNumeral IntegerTypeSuffix?
680 | ;
681 |
682 | fragment
683 | OctalIntegerLiteral
684 | : OctalNumeral IntegerTypeSuffix?
685 | ;
686 |
687 | fragment
688 | BinaryIntegerLiteral
689 | : BinaryNumeral IntegerTypeSuffix?
690 | ;
691 |
692 | fragment
693 | IntegerTypeSuffix
694 | : [lL]
695 | ;
696 |
697 | fragment
698 | DecimalNumeral
699 | : '0'
700 | | NonZeroDigit (Digits? | Underscores Digits)
701 | ;
702 |
703 | fragment
704 | Digits
705 | : Digit (DigitOrUnderscore* Digit)?
706 | ;
707 |
708 | fragment
709 | Digit
710 | : '0'
711 | | NonZeroDigit
712 | ;
713 |
714 | fragment
715 | NonZeroDigit
716 | : [1-9]
717 | ;
718 |
719 | fragment
720 | DigitOrUnderscore
721 | : Digit
722 | | '_'
723 | ;
724 |
725 | fragment
726 | Underscores
727 | : '_'+
728 | ;
729 |
730 | fragment
731 | HexNumeral
732 | : '0' [xX] HexDigits
733 | ;
734 |
735 | fragment
736 | HexDigits
737 | : HexDigit (HexDigitOrUnderscore* HexDigit)?
738 | ;
739 |
740 | fragment
741 | HexDigit
742 | : [0-9a-fA-F]
743 | ;
744 |
745 | fragment
746 | HexDigitOrUnderscore
747 | : HexDigit
748 | | '_'
749 | ;
750 |
751 | fragment
752 | OctalNumeral
753 | : '0' Underscores? OctalDigits
754 | ;
755 |
756 | fragment
757 | OctalDigits
758 | : OctalDigit (OctalDigitOrUnderscore* OctalDigit)?
759 | ;
760 |
761 | fragment
762 | OctalDigit
763 | : [0-7]
764 | ;
765 |
766 | fragment
767 | OctalDigitOrUnderscore
768 | : OctalDigit
769 | | '_'
770 | ;
771 |
772 | fragment
773 | BinaryNumeral
774 | : '0' [bB] BinaryDigits
775 | ;
776 |
777 | fragment
778 | BinaryDigits
779 | : BinaryDigit (BinaryDigitOrUnderscore* BinaryDigit)?
780 | ;
781 |
782 | fragment
783 | BinaryDigit
784 | : [01]
785 | ;
786 |
787 | fragment
788 | BinaryDigitOrUnderscore
789 | : BinaryDigit
790 | | '_'
791 | ;
792 |
793 | // §3.10.2 Floating-Point Literals
794 |
795 | FloatingPointLiteral
796 | : DecimalFloatingPointLiteral
797 | | HexadecimalFloatingPointLiteral
798 | ;
799 |
800 | fragment
801 | DecimalFloatingPointLiteral
802 | : Digits '.' Digits? ExponentPart? FloatTypeSuffix?
803 | | '.' Digits ExponentPart? FloatTypeSuffix?
804 | | Digits ExponentPart FloatTypeSuffix?
805 | | Digits FloatTypeSuffix
806 | ;
807 |
808 | fragment
809 | ExponentPart
810 | : ExponentIndicator SignedInteger
811 | ;
812 |
813 | fragment
814 | ExponentIndicator
815 | : [eE]
816 | ;
817 |
818 | fragment
819 | SignedInteger
820 | : Sign? Digits
821 | ;
822 |
823 | fragment
824 | Sign
825 | : [+-]
826 | ;
827 |
828 | fragment
829 | FloatTypeSuffix
830 | : [fFdD]
831 | ;
832 |
833 | fragment
834 | HexadecimalFloatingPointLiteral
835 | : HexSignificand BinaryExponent FloatTypeSuffix?
836 | ;
837 |
838 | fragment
839 | HexSignificand
840 | : HexNumeral '.'?
841 | | '0' [xX] HexDigits? '.' HexDigits
842 | ;
843 |
844 | fragment
845 | BinaryExponent
846 | : BinaryExponentIndicator SignedInteger
847 | ;
848 |
849 | fragment
850 | BinaryExponentIndicator
851 | : [pP]
852 | ;
853 |
854 | // §3.10.3 Boolean Literals
855 |
856 | BooleanLiteral
857 | : 'true'
858 | | 'false'
859 | ;
860 |
861 | // §3.10.4 Character Literals
862 |
863 | CharacterLiteral
864 | : '\'' SingleCharacter '\''
865 | | '\'' EscapeSequence '\''
866 | ;
867 |
868 | fragment
869 | SingleCharacter
870 | : ~['\\]
871 | ;
872 |
873 | // §3.10.5 String Literals
874 |
875 | StringLiteral
876 | : '"' StringCharacters? '"'
877 | ;
878 |
879 | fragment
880 | StringCharacters
881 | : StringCharacter+
882 | ;
883 |
884 | fragment
885 | StringCharacter
886 | : ~["\\]
887 | | EscapeSequence
888 | ;
889 |
890 | // §3.10.6 Escape Sequences for Character and String Literals
891 |
892 | fragment
893 | EscapeSequence
894 | : '\\' [btnfr"'\\]
895 | | OctalEscape
896 | | UnicodeEscape
897 | ;
898 |
899 | fragment
900 | OctalEscape
901 | : '\\' OctalDigit
902 | | '\\' OctalDigit OctalDigit
903 | | '\\' ZeroToThree OctalDigit OctalDigit
904 | ;
905 |
906 | fragment
907 | UnicodeEscape
908 | : '\\' 'u' HexDigit HexDigit HexDigit HexDigit
909 | ;
910 |
911 | fragment
912 | ZeroToThree
913 | : [0-3]
914 | ;
915 |
916 | // §3.10.7 The Null Literal
917 |
918 | NullLiteral
919 | : 'null'
920 | ;
921 |
922 | // §3.11 Separators
923 |
924 | LPAREN : '(';
925 | RPAREN : ')';
926 | LBRACE : '{';
927 | RBRACE : '}';
928 | LBRACK : '[';
929 | RBRACK : ']';
930 | SEMI : ';';
931 | COMMA : ',';
932 | DOT : '.';
933 |
934 | // §3.12 Operators
935 |
936 | ASSIGN : '=';
937 | GT : '>';
938 | LT : '<';
939 | BANG : '!';
940 | TILDE : '~';
941 | QUESTION : '?';
942 | COLON : ':';
943 | EQUAL : '==';
944 | LE : '<=';
945 | GE : '>=';
946 | NOTEQUAL : '!=';
947 | AND : '&&';
948 | OR : '||';
949 | INC : '++';
950 | DEC : '--';
951 | ADD : '+';
952 | SUB : '-';
953 | MUL : '*';
954 | DIV : '/';
955 | BITAND : '&';
956 | BITOR : '|';
957 | CARET : '^';
958 | MOD : '%';
959 |
960 | ADD_ASSIGN : '+=';
961 | SUB_ASSIGN : '-=';
962 | MUL_ASSIGN : '*=';
963 | DIV_ASSIGN : '/=';
964 | AND_ASSIGN : '&=';
965 | OR_ASSIGN : '|=';
966 | XOR_ASSIGN : '^=';
967 | MOD_ASSIGN : '%=';
968 | LSHIFT_ASSIGN : '<<=';
969 | RSHIFT_ASSIGN : '>>=';
970 | URSHIFT_ASSIGN : '>>>=';
971 |
972 | // §3.8 Identifiers (must appear after all keywords in the grammar)
973 |
974 | Identifier
975 | : JavaLetter JavaLetterOrDigit*
976 | ;
977 |
978 | fragment
979 | JavaLetter
980 | : [a-zA-Z$_] // these are the "java letters" below 0xFF
981 | // | // covers all characters above 0xFF which are not a surrogate
982 | // ~[\u0000-\u00FF\uD800-\uDBFF]
983 | // {Character.isJavaIdentifierStart(_input.LA(-1))}?
984 | // | // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
985 | // [\uD800-\uDBFF] [\uDC00-\uDFFF]
986 | // {Character.isJavaIdentifierStart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
987 | ;
988 |
989 | fragment
990 | JavaLetterOrDigit
991 | : [a-zA-Z0-9$_] // these are the "java letters or digits" below 0xFF
992 | // | // covers all characters above 0xFF which are not a surrogate
993 | // ~[\u0000-\u00FF\uD800-\uDBFF]
994 | // {Character.isJavaIdentifierPart(_input.LA(-1))}?
995 | // | // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
996 | // [\uD800-\uDBFF] [\uDC00-\uDFFF]
997 | // {Character.isJavaIdentifierPart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
998 | ;
999 |
1000 | //
1001 | // Additional symbols not defined in the lexical specification
1002 | //
1003 |
1004 | AT : '@';
1005 | ELLIPSIS : '...';
1006 |
1007 | //
1008 | // Whitespace and comments
1009 | //
1010 |
1011 | WS : [ \t\r\n\u000C]+ -> skip
1012 | ;
1013 |
1014 | COMMENT
1015 | : '/*' .*? '*/' -> skip
1016 | ;
1017 |
1018 | LINE_COMMENT
1019 | : '//' ~[\r\n]* -> skip
1020 | ;
1021 |
--------------------------------------------------------------------------------
/04-ExtractInterface/README.md:
--------------------------------------------------------------------------------
1 | # How to Run
2 | ```
3 | % python test_extract.py Demo.java
4 | interface IDemo {
5 | int f(int x, String y);
6 | int[ ] g(/*no args*/);
7 | List>[] h();
8 | }
9 | ```
10 |
11 | # to make this test pass for python
12 | - use the latest JAVA g4 file from ANTLR4 grammar depot
13 | - comment out java action part for unicode
14 | ```antlr4
15 | fragment
16 | JavaLetter
17 | : [a-zA-Z$_] // these are the "java letters" below 0xFF
18 | // | // covers all characters above 0xFF which are not a surrogate
19 | // ~[\u0000-\u00FF\uD800-\uDBFF]
20 | // {Character.isJavaIdentifierStart(_input.LA(-1))}?
21 | // | // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
22 | // [\uD800-\uDBFF] [\uDC00-\uDFFF]
23 | // {Character.isJavaIdentifierStart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
24 | ;
25 |
26 | fragment
27 | JavaLetterOrDigit
28 | : [a-zA-Z0-9$_] // these are the "java letters or digits" below 0xFF
29 | // | // covers all characters above 0xFF which are not a surrogate
30 | // ~[\u0000-\u00FF\uD800-\uDBFF]
31 | // {Character.isJavaIdentifierPart(_input.LA(-1))}?
32 | // | // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
33 | // [\uD800-\uDBFF] [\uDC00-\uDFFF]
34 | // {Character.isJavaIdentifierPart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
35 | ;
36 | - change type to datatype to avoid conflict with python keyword.
37 | ```
38 | - add function to print the original text (including hidden channel)
39 | ```python
40 | def getAllText(self, ctx): # include hidden channel
41 | token_stream = ctx.parser.getTokenStream()
42 | lexer = token_stream.tokenSource
43 | input_stream = lexer.inputStream
44 | start = ctx.start.start
45 | stop = ctx.stop.stop
46 | return input_stream.getText(start, stop)
47 | ```
--------------------------------------------------------------------------------
/04-ExtractInterface/test_extract.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4.InputStream import InputStream
6 |
7 | from JavaLexer import JavaLexer
8 | from JavaParser import JavaParser
9 |
10 | from ExtractInterfaceListener import ExtractInterfaceListener
11 |
12 | if __name__ == '__main__':
13 | if len(sys.argv) > 1:
14 | input_stream = FileStream(sys.argv[1])
15 | else:
16 | input_stream = InputStream(sys.stdin.readline())
17 |
18 | lexer = JavaLexer(input_stream)
19 | token_stream = CommonTokenStream(lexer)
20 | parser = JavaParser(token_stream)
21 | tree = parser.compilationUnit()
22 |
23 | listener = ExtractInterfaceListener(parser)
24 | walker = ParseTreeWalker()
25 | walker.walk(listener, tree)
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/04-Rows-visitor/04-Rows-visitor.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/04-Rows-visitor/README.md:
--------------------------------------------------------------------------------
1 | # What's in this example
2 | - With embedded python actions
3 | - local variable and initialization
4 | - get column from command line and set in parser
5 | - Do not build tree
6 |
7 | # How to Run
8 | ```
9 | % python col.py 1 t.rows
10 | parrt
11 | tombu
12 | bke
13 |
14 | % python col.py 2 t.rows
15 | Terence Parr
16 | Tom Burns
17 | Kevin Edgar
18 |
19 | % python col.py 3 t.rows
20 | 101
21 | 020
22 | 008
23 | ```
--------------------------------------------------------------------------------
/04-Rows-visitor/Rows.g4:
--------------------------------------------------------------------------------
1 | grammar Rows;
2 |
3 | @parser::members {
4 | @property
5 | def column(self):
6 | return self._col
7 |
8 | @column.setter
9 | def column(self, value):
10 | self._col = value
11 |
12 | }
13 |
14 | rows: (row NL)+ ;
15 |
16 | row
17 | locals [i = 0]
18 | : ( STUFF
19 | {
20 | $i = $i + 1
21 | if $i == self.column:
22 | print($STUFF.text)
23 | }
24 | )+
25 | ;
26 |
27 | TAB : '\t' -> skip ; // match but don't pass to the parser
28 | NL : '\r'? '\n' ; // match and pass to the parser
29 | STUFF: ~[\t\r\n]+ ; // match any chars except tab, newline
30 |
--------------------------------------------------------------------------------
/04-Rows-visitor/col.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4.InputStream import InputStream
6 |
7 | from RowsLexer import RowsLexer
8 | from RowsParser import RowsParser
9 |
10 | if __name__ == '__main__':
11 | if len(sys.argv) > 1:
12 | col_num = int(sys.argv[1])
13 | input_stream = FileStream(sys.argv[2])
14 | else:
15 | print('Usage: python col.py #col file')
16 | exit(1)
17 |
18 | lexer = RowsLexer(input_stream)
19 | token_stream = CommonTokenStream(lexer)
20 | parser = RowsParser(token_stream)
21 | parser.column = col_num
22 | parser.buildParseTrees = False
23 | tree = parser.rows()
--------------------------------------------------------------------------------
/04-Rows-visitor/t.rows:
--------------------------------------------------------------------------------
1 | parrt Terence Parr 101
2 | tombu Tom Burns 020
3 | bke Kevin Edgar 008
4 |
--------------------------------------------------------------------------------
/04-modechange/04-modechange.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/04-modechange/README.md:
--------------------------------------------------------------------------------
1 | # How to run
2 | ```
3 | % python test.py t.xml
4 | 0 8 3 413 413 4 \n
5 | 0 9 0 414 414 1 <
6 | 0 9 1 415 419 10 tools
7 | 0 9 6 420 420 5 >
8 | 0 9 7 421 422 4 \n\t
9 | 0 10 1 423 423 1 <
10 | 0 10 2 424 427 10 tool
11 | 0 10 7 429 432 10 name
12 | 0 10 11 433 433 7 =
13 | 0 10 12 434 440 8 "ANTLR"
14 | 0 10 19 441 441 5 >
15 | 0 10 20 442 459 4 A parser generator
16 | 0 10 38 460 460 1 <
17 | 0 10 39 461 465 9 /tool
18 | 0 10 44 466 466 5 >
19 | 0 10 45 467 467 4 \n
20 | 0 11 0 468 468 1 <
21 | 0 11 1 469 474 9 /tools
22 | 0 11 7 475 475 5 >
23 | 0 11 8 476 476 4 \n
24 | ```
25 |
26 | # What's in this test
27 | - Lexer Only, call nextToken() method till end of file (Token.EOF)
28 | - Switch Mode
29 |
--------------------------------------------------------------------------------
/04-modechange/XMLLexer.g4:
--------------------------------------------------------------------------------
1 | lexer grammar XMLLexer;
2 |
3 | // Default "mode": Everything OUTSIDE of a tag
4 | OPEN : '<' -> pushMode(INSIDE) ;
5 | COMMENT : '' -> skip ;
6 | EntityRef : '&' [a-z]+ ';' ;
7 | TEXT : ~('<'|'&')+ ; // match any 16 bit char minus < and &
8 |
9 | // ----------------- Everything INSIDE of a tag ---------------------
10 | mode INSIDE;
11 |
12 | CLOSE : '>' -> popMode ; // back to default mode
13 | SLASH_CLOSE : '/>' -> popMode ;
14 | EQUALS : '=' ;
15 | STRING : '"' .*? '"' ;
16 | SlashName : '/' Name ;
17 | Name : ALPHA (ALPHA|DIGIT)* ;
18 | S : [ \t\r\n] -> skip ;
19 |
20 | fragment
21 | ALPHA : [a-zA-Z] ;
22 |
23 | fragment
24 | DIGIT : [0-9] ;
25 |
--------------------------------------------------------------------------------
/04-modechange/t.xml:
--------------------------------------------------------------------------------
1 |
9 |
10 | A parser generator
11 |
12 |
--------------------------------------------------------------------------------
/04-modechange/test.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 |
5 | from antlr4 import *
6 | from antlr4.InputStream import InputStream
7 | from XMLLexer import XMLLexer
8 |
9 | if __name__ == '__main__':
10 | if len(sys.argv) > 1:
11 | input_stream = FileStream(sys.argv[1])
12 | else:
13 | input_stream = InputStream(sys.stdin.read())
14 |
15 | lexer = XMLLexer(input_stream)
16 |
17 | t = lexer.nextToken()
18 | while t.type != Token.EOF:
19 | txt = t.text
20 | if txt is not None:
21 | txt = txt.replace("\n","\\n")
22 | txt = txt.replace("\r","\\r")
23 | txt = txt.replace("\t","\\t")
24 | else:
25 | txt = ""
26 | print(t.channel, t.line, t.column, t.start, t.stop, t.type, txt)
27 | t = lexer.nextToken()
28 |
--------------------------------------------------------------------------------
/04-sempred/04-sempred.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/04-sempred/Data.g4:
--------------------------------------------------------------------------------
1 | grammar Data;
2 |
3 | top : group+ ;
4 |
5 | group: INT sequence[$INT.int] ;
6 |
7 | sequence[n]
8 | locals [i=0]
9 | : ( {$i<$n}? INT {$i=$i+1;} )* // match n integers
10 | ;
11 |
12 | INT : [0-9]+ ; // match integers
13 | WS : [ \t\n\r]+ -> skip ; // toss out all whitespace
14 |
--------------------------------------------------------------------------------
/04-sempred/Readme.md:
--------------------------------------------------------------------------------
1 | # What's in This Example
2 | - pass parameter to rule
3 | - local variable
4 | - semantic predication: 1st data decide the number of data followed
5 |
6 | # How to Run
7 | ```
8 | % type t.data
9 | 2 9 10 3 1 2 3
10 | % python test_data.py t.data
11 | (top
12 | (group 2 (sequence 9 10))
13 | (group 3 (sequence 1 2 3))
14 | )
15 | ```
--------------------------------------------------------------------------------
/04-sempred/t.data:
--------------------------------------------------------------------------------
1 | 2 9 10 3 1 2 3
2 |
--------------------------------------------------------------------------------
/04-sempred/test_data.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 | import sys
3 | from antlr4 import *
4 | from antlr4.InputStream import InputStream
5 | from DataLexer import DataLexer
6 | from DataParser import DataParser
7 |
8 | if __name__ == '__main__':
9 | if len(sys.argv) > 1:
10 | input_stream = FileStream(sys.argv[1])
11 | else:
12 | input_stream = InputStream(sys.stdin.read())
13 |
14 | lexer = DataLexer(input_stream)
15 | token_stream = CommonTokenStream(lexer)
16 | parser = DataParser(token_stream)
17 | tree = parser.top()
18 |
19 | lisp_tree_str = tree.toStringTree(recog=parser)
20 | print(lisp_tree_str)
--------------------------------------------------------------------------------
/07-LableRule/07-LableRule.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/07-LableRule/LExpr.g4:
--------------------------------------------------------------------------------
1 | grammar LExpr;
2 |
3 | s : e ;
4 |
5 | e : e MULT e # Mult
6 | | e ADD e # Add
7 | | INT # Int
8 | ;
9 |
10 | MULT: '*' ;
11 | ADD : '+' ;
12 | INT : [0-9]+ ;
13 | WS : [ \t\r\n]+ -> skip ;
14 |
--------------------------------------------------------------------------------
/07-LableRule/README.md:
--------------------------------------------------------------------------------
1 | # Label Rule Alternative
2 |
3 | # How to Run
4 | ```
5 | antlr4py3 -visitor LExpr.g4
6 | python test_EvalVistor.py t.expr
7 | python test_EvalListener.py t.expr
8 | python test_EvalContext.py t.expr
9 | ```
--------------------------------------------------------------------------------
/07-LableRule/t.expr:
--------------------------------------------------------------------------------
1 | 1+2*3
2 |
--------------------------------------------------------------------------------
/07-LableRule/test_EvalContext.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 | """
3 | To Store value with the context associated to the tree
4 | """
5 |
6 |
7 | import sys
8 | from antlr4 import *
9 | from antlr4.InputStream import InputStream
10 |
11 |
12 | from LExprLexer import LExprLexer
13 | from LExprParser import LExprParser
14 | from LExprListener import LExprListener
15 |
16 |
17 | class ContextMap(LExprListener):
18 | def __init__(self):
19 | self.tree_property = {}
20 |
21 | def getValue(self, node):
22 | return self.tree_property[node]
23 |
24 | def setValue(self, node, value):
25 | self.tree_property[node] = value
26 |
27 | def exitInt(self, ctx):
28 | self.setValue(ctx, int(ctx.INT().getText()))
29 |
30 | def exitAdd(self, ctx):
31 | left = self.getValue(ctx.e(0))
32 | right= self.getValue(ctx.e(1))
33 | self.setValue(ctx, left+right)
34 |
35 | def exitMult(self, ctx):
36 | left = self.getValue(ctx.e(0))
37 | right= self.getValue(ctx.e(1))
38 | self.setValue(ctx, left*right)
39 |
40 | def exitS(self, ctx):
41 | self.setValue(ctx, self.getValue(ctx.e()))
42 |
43 |
44 | if __name__ == '__main__':
45 | if len(sys.argv) > 1:
46 | input_stream = FileStream(sys.argv[1])
47 | else:
48 | input_stream = InputStream(sys.stdin.read())
49 |
50 | lexer = LExprLexer(input_stream)
51 | token_stream = CommonTokenStream(lexer)
52 | parser = LExprParser(token_stream)
53 | tree = parser.s()
54 |
55 | lisp_tree_str = tree.toStringTree(recog=parser)
56 | print(lisp_tree_str)
57 |
58 | # listener
59 | print("Start Walking...")
60 | listener = ContextMap()
61 | walker = ParseTreeWalker()
62 | walker.walk(listener, tree)
63 | print('result_at_top =', listener.getValue(tree))
64 |
--------------------------------------------------------------------------------
/07-LableRule/test_EvalListener.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 | """
3 | Using Listener, store data in stack. result at the top of the stack
4 | """
5 |
6 | import sys
7 | from antlr4 import *
8 | from antlr4.InputStream import InputStream
9 |
10 |
11 | from LExprLexer import LExprLexer
12 | from LExprParser import LExprParser
13 | from LExprListener import LExprListener
14 |
15 |
16 | class EvalListener(LExprListener):
17 | def __init__(self):
18 | self.stack = []
19 |
20 | def exitMult(self, ctx):
21 | right = self.stack.pop()
22 | left = self.stack.pop()
23 | self.stack.append(right * left)
24 |
25 | def exitAdd(self, ctx):
26 | right = self.stack.pop()
27 | left = self.stack.pop()
28 | self.stack.append(right + left)
29 |
30 | def exitInt(self, ctx):
31 | self.stack.append(int(ctx.INT().getText()))
32 |
33 |
34 | if __name__ == '__main__':
35 | if len(sys.argv) > 1:
36 | input_stream = FileStream(sys.argv[1])
37 | else:
38 | input_stream = InputStream(sys.stdin.read())
39 |
40 | lexer = LExprLexer(input_stream)
41 | token_stream = CommonTokenStream(lexer)
42 | parser = LExprParser(token_stream)
43 | tree = parser.s()
44 |
45 | lisp_tree_str = tree.toStringTree(recog=parser)
46 | print(lisp_tree_str)
47 |
48 | # listener
49 | print("Start Walking...")
50 | listener = EvalListener()
51 | walker = ParseTreeWalker()
52 | walker.walk(listener, tree)
53 | print('result_stack=', listener.stack)
54 |
--------------------------------------------------------------------------------
/07-LableRule/test_EvalVistor.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 | """
3 | Using Visitor. Customized visitor walking down the tree and do calculation at
4 | certain tree stage and eventually return result from the entry call.
5 | """
6 | import sys
7 | from antlr4 import *
8 | from antlr4.InputStream import InputStream
9 |
10 |
11 | from LExprLexer import LExprLexer
12 | from LExprParser import LExprParser
13 | from LExprVisitor import LExprVisitor
14 |
15 |
16 | class EvalVisitor(LExprVisitor):
17 | def visitMult(self, ctx):
18 | return self.visit(ctx.e(0)) * self.visit(ctx.e(1))
19 |
20 | def visitAdd(self, ctx):
21 | return self.visit(ctx.e(0)) + self.visit(ctx.e(1))
22 |
23 | def visitInt(self, ctx):
24 | return int(ctx.INT().getText())
25 |
26 |
27 | if __name__ == '__main__':
28 | if len(sys.argv) > 1:
29 | input_stream = FileStream(sys.argv[1])
30 | else:
31 | input_stream = InputStream(sys.stdin.read())
32 |
33 | lexer = LExprLexer(input_stream)
34 | token_stream = CommonTokenStream(lexer)
35 | parser = LExprParser(token_stream)
36 | tree = parser.s()
37 |
38 | lisp_tree_str = tree.toStringTree(recog=parser)
39 | print(lisp_tree_str)
40 |
41 | visitor = EvalVisitor()
42 | result = visitor.visit(tree)
43 | print("result=", result)
44 |
45 |
--------------------------------------------------------------------------------
/07-Property/07-Property.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/07-Property/PropertyFile.g4:
--------------------------------------------------------------------------------
1 | grammar PropertyFile;
2 |
3 | @members {
4 | def startFile(self):
5 | pass
6 | def finishFile(self):
7 | pass
8 | def defineProperty(self, name, value):
9 | pass
10 | }
11 |
12 | top : {self.startFile()} prop+ {self.finishFile()};
13 | prop : ID '=' STRING '\n' {self.defineProperty($ID, $STRING)};
14 | ID : [a-z]+ ;
15 | STRING : '"' .*? '"' ;
16 |
--------------------------------------------------------------------------------
/07-Property/README.md:
--------------------------------------------------------------------------------
1 | # Property Parsing
2 | - Using Listener/Visitor
3 | - Call back inserted in action
4 |
5 | # How to run
6 | ```
7 | % antlr4py3 -visitor PropertyFile.g4
8 |
9 | % python test_listener.py t.properties
10 | user => "parrt"
11 | machine => "maniac"
12 |
13 | % python test_visitor.py t.properties
14 | machine => "maniac"
15 | user => "parrt"
16 |
17 | % python test_as_print.py t.properties
18 | user = "parrt"
19 | machine = "maniac"
20 |
21 | % python test_as_loader.py t.properties
22 | machine => "maniac"
23 | user => "parrt"
24 | ```
--------------------------------------------------------------------------------
/07-Property/t.properties:
--------------------------------------------------------------------------------
1 | user="parrt"
2 | machine="maniac"
3 |
--------------------------------------------------------------------------------
/07-Property/test_as_loader.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from antlr4.InputStream import InputStream
4 |
5 | from PropertyFileLexer import PropertyFileLexer
6 | from PropertyFileParser import PropertyFileParser
7 |
8 |
9 | class PropertyFileLoader(PropertyFileParser):
10 | def __init__(self, input:TokenStream):
11 | super().__init__(input)
12 | self.props = {}
13 |
14 | def defineProperty(self, name, value):
15 | self.props[name.text] = value.text
16 |
17 | def show(self):
18 | for (key, value) in self.props.items():
19 | print(key, '=>', value)
20 |
21 | if __name__ == '__main__':
22 | if len(sys.argv) > 1:
23 | input_stream = FileStream(sys.argv[1])
24 | else:
25 | input_stream = InputStream(sys.stdin.read())
26 |
27 | lexer = PropertyFileLexer(input_stream)
28 | token_stream = CommonTokenStream(lexer)
29 | parser = PropertyFileLoader(token_stream)
30 | parser.top()
31 | parser.show()
--------------------------------------------------------------------------------
/07-Property/test_as_print.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from antlr4.InputStream import InputStream
4 |
5 | from PropertyFileLexer import PropertyFileLexer
6 | from PropertyFileParser import PropertyFileParser
7 |
8 |
9 | class PropertyFilePrinter(PropertyFileParser):
10 | def defineProperty(self, name, value):
11 | print(name.text, ' = ', value.text)
12 |
13 |
14 | if __name__ == '__main__':
15 | if len(sys.argv) > 1:
16 | input_stream = FileStream(sys.argv[1])
17 | else:
18 | input_stream = InputStream(sys.stdin.read())
19 |
20 | lexer = PropertyFileLexer(input_stream)
21 | token_stream = CommonTokenStream(lexer)
22 | parser = PropertyFilePrinter(token_stream)
23 | parser.top()
24 |
--------------------------------------------------------------------------------
/07-Property/test_listener.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from antlr4.InputStream import InputStream
4 |
5 | from PropertyFileLexer import PropertyFileLexer
6 | from PropertyFileParser import PropertyFileParser
7 | from PropertyFileListener import PropertyFileListener
8 |
9 |
10 | class PropertyFileLoader(PropertyFileListener):
11 | def __init__(self):
12 | super().__init__()
13 | self.props = {}
14 |
15 | def exitProp(self, ctx):
16 | self.props[ctx.ID().getText()] = ctx.STRING().getText()
17 |
18 | def show(self):
19 | for (key, value) in self.props.items():
20 | print(key, '=>', value)
21 |
22 | if __name__ == '__main__':
23 | if len(sys.argv) > 1:
24 | input_stream = FileStream(sys.argv[1])
25 | else:
26 | input_stream = InputStream(sys.stdin.read())
27 |
28 | lexer = PropertyFileLexer(input_stream)
29 | token_stream = CommonTokenStream(lexer)
30 | parser = PropertyFileParser(token_stream)
31 | tree = parser.top()
32 |
33 | walker = ParseTreeWalker()
34 | listener = PropertyFileLoader()
35 | walker.walk(listener, tree)
36 |
37 | listener.show()
--------------------------------------------------------------------------------
/07-Property/test_visitor.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from antlr4.InputStream import InputStream
4 |
5 | from PropertyFileLexer import PropertyFileLexer
6 | from PropertyFileParser import PropertyFileParser
7 | from PropertyFileVisitor import PropertyFileVisitor
8 |
9 |
10 | class PropertyFileLoader(PropertyFileVisitor):
11 | def __init__(self):
12 | super().__init__()
13 | self.props = {}
14 |
15 | def visitProp(self, ctx):
16 | self.props[ctx.ID().getText()] = ctx.STRING().getText()
17 |
18 | def show(self):
19 | for (key, value) in self.props.items():
20 | print(key, '=>', value)
21 |
22 | if __name__ == '__main__':
23 | if len(sys.argv) > 1:
24 | input_stream = FileStream(sys.argv[1])
25 | else:
26 | input_stream = InputStream(sys.stdin.read())
27 |
28 | lexer = PropertyFileLexer(input_stream)
29 | token_stream = CommonTokenStream(lexer)
30 | parser = PropertyFileParser(token_stream)
31 | tree = parser.top()
32 |
33 | visitor = PropertyFileLoader()
34 | visitor.visit(tree)
35 |
36 | visitor.show()
--------------------------------------------------------------------------------
/08-CSV/08-CSV.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/08-CSV/CSV.g4:
--------------------------------------------------------------------------------
1 | grammar CSV;
2 |
3 | top : hdr row+ ;
4 | hdr : row ;
5 |
6 | row : field (',' field)* '\r'? '\n' ;
7 |
8 | field
9 | : TEXT # text
10 | | STRING # string
11 | | # empty
12 | ;
13 |
14 | TEXT : ~[,\n\r"]+ ;
15 | STRING : '"' ('""'|~'"')* '"' ;
16 |
--------------------------------------------------------------------------------
/08-CSV/CSV_Loader.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 |
4 | import sys
5 | from antlr4 import *
6 | from antlr4.InputStream import InputStream
7 |
8 | from CSVLexer import CSVLexer
9 | from CSVParser import CSVParser
10 | from CSVListener import CSVListener
11 |
12 |
13 | class CsvLoader(CSVListener):
14 | def __init__(self):
15 | self.rows = []
16 | self.header = []
17 | self.current = []
18 |
19 | def exitString(self, ctx):
20 | self.current.append(ctx.STRING().getText())
21 |
22 | def exitText(self, ctx):
23 | self.current.append(ctx.TEXT().getText())
24 |
25 | def exitEmpty(self, ctx):
26 | self.current.append('')
27 |
28 | def exitHdr(self, ctx):
29 | self.header = self.current
30 |
31 | def enterRow(self, ctx):
32 | self.current = []
33 |
34 | def exitRow(self, ctx):
35 | # getParent() method does not exist, use 'parentCtx' field here.
36 | if ctx.parentCtx.getRuleIndex() == CSVParser.RULE_hdr:
37 | return
38 | m = dict(zip(self.header, self.current))
39 | self.rows.append(m)
40 |
41 |
42 | if __name__ == '__main__':
43 | if len(sys.argv) > 1:
44 | input_stream = FileStream(sys.argv[1])
45 | else:
46 | input_stream = InputStream(sys.stdin.read())
47 |
48 | lexer = CSVLexer(input_stream)
49 | token_stream = CommonTokenStream(lexer)
50 | parser = CSVParser(token_stream)
51 | tree = parser.top()
52 |
53 | lisp_tree_str = tree.toStringTree(recog=parser)
54 | print(lisp_tree_str)
55 |
56 | # listener
57 | print("Start Walking...")
58 | listener = CsvLoader()
59 | walker = ParseTreeWalker()
60 | walker.walk(listener, tree)
61 | print('result =', listener.rows)
62 |
--------------------------------------------------------------------------------
/08-CSV/README.md:
--------------------------------------------------------------------------------
1 | # Parse CSV file
2 |
3 | ```
4 | % type t.csv
5 | Details,Month,Amount
6 | Mid Bonus,June,"$2,000"
7 | ,January,"""zippo"""
8 | Total Bonuses,"","$5,000"
9 |
10 | % python CSV_Loader.py t.csv
11 | (top
12 | (hdr
13 | (row
14 | (field Details) ,
15 | (field Month) ,
16 | (field Amount) \r \n))
17 | (row (field Mid Bonus) , (field June) , (field "$2,000") \r \n)
18 | (row field , (field January) , (field """zippo""") \r \n)
19 | (row (field Total Bonuses) , (field "") , (field "$5,000") \r \n))
20 |
21 | Start Walking...
22 | result = [
23 | {'Month': 'June','Details': 'Mid Bonus', 'Amount': '"$2,000"'},
24 | {'Month': 'January', 'Details': '', 'Amount': '"""zippo"""'},
25 | {'Month': '""', 'Details': 'Total Bonuses', 'Amount': '"$5,000"'}
26 | ]
27 | ```
28 |
29 |
--------------------------------------------------------------------------------
/08-CSV/t.csv:
--------------------------------------------------------------------------------
1 | Details,Month,Amount
2 | Mid Bonus,June,"$2,000"
3 | ,January,"""zippo"""
4 | Total Bonuses,"","$5,000"
5 |
--------------------------------------------------------------------------------
/08-CallGraph/08-CallGraph.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/08-CallGraph/CallGraph.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from collections import OrderedDict
5 | from string import Template
6 |
7 | from antlr4 import *
8 | from antlr4.InputStream import InputStream
9 |
10 | from CymbolLexer import CymbolLexer
11 | from CymbolParser import CymbolParser
12 | from CymbolListener import CymbolListener
13 |
14 |
15 | class Graph:
16 | def __init__(self):
17 | self.nodes = OrderedDict()
18 | self.edges = OrderedDict()
19 |
20 | def add_edge(self, src, dst):
21 | if src in self.edges:
22 | self.edges[src].append(dst)
23 | else:
24 | self.edges[src] = [dst]
25 |
26 | def add_node(self, function_name):
27 | self.nodes[function_name] = True
28 |
29 | def __str__(self):
30 | return "edges : " + self.edges.__str__() + ", functions : " + list(self.nodes.keys()).__str__()
31 |
32 | def toDOT(self):
33 | # funcs = ';'.join(self.nodes.keys())
34 | funcs = ""
35 | for f in self.nodes.keys():
36 | funcs += f + ';'
37 | edges = ""
38 | for (key, value) in self.edges.items():
39 | for dst in value:
40 | edges += " " + key + " -> " + dst + ";\n"
41 |
42 | tpl_str = """
43 | digraph G {
44 | ranksep=.25;
45 | edge [arrowsize=.5]
46 | node [shape=circle, fontname="ArialNarrow",
47 | fontsize=12, fixedsize=true, height=.45];
48 |
49 | $func_list
50 | $edge_list
51 | }
52 | """
53 | tpl = Template(tpl_str)
54 | return tpl.substitute(func_list=funcs, edge_list=edges)
55 |
56 |
57 | class FunctionListener(CymbolListener):
58 | def __init__(self):
59 | self.graph = Graph()
60 | self.current_function_name = None
61 |
62 | def enterFunctionDecl(self, ctx: CymbolParser.FunctionDeclContext):
63 | name = ctx.ID().getText()
64 | self.current_function_name = name
65 | self.graph.add_node(name)
66 |
67 | def exitCall(self, ctx):
68 | name = ctx.ID().getText()
69 | self.graph.add_edge(self.current_function_name, name)
70 |
71 |
72 | if __name__ == '__main__':
73 | if len(sys.argv) > 1:
74 | input_stream = FileStream(sys.argv[1])
75 | else:
76 | input_stream = InputStream(sys.stdin.read())
77 |
78 | lexer = CymbolLexer(input_stream)
79 | token_stream = CommonTokenStream(lexer)
80 | parser = CymbolParser(token_stream)
81 | tree = parser.top()
82 |
83 | # lisp_tree_str = tree.toStringTree(recog=parser)
84 | # print(lisp_tree_str)
85 |
86 | walker = ParseTreeWalker()
87 | collector = FunctionListener()
88 | walker.walk(collector, tree)
89 | # print(collector.graph)
90 | print(collector.graph.toDOT())
91 |
92 |
--------------------------------------------------------------------------------
/08-CallGraph/CheckSymbols.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 |
5 | from antlr4 import *
6 | from antlr4.InputStream import InputStream
7 | from SymbolScope import *
8 |
9 | from CymbolLexer import CymbolLexer
10 | from CymbolParser import CymbolParser
11 | from CymbolListener import CymbolListener
12 |
13 |
14 | def get_type(tokenType):
15 | if tokenType == CymbolParser.K_FLOAT:
16 | return Symbol.TypeEnum.FLOAT
17 | elif tokenType == CymbolParser.K_INT:
18 | return Symbol.TypeEnum.INT
19 | elif tokenType == CymbolParser.K_VOID:
20 | return Symbol.TypeEnum.VOID
21 | else:
22 | return Symbol.TypeEnum.INVALID
23 |
24 |
25 | def error(token, msg):
26 | print('[Error] line %d:%d %s' % (token.line, token.column, msg))
27 |
28 |
29 | class DefPhase(CymbolListener):
30 | def __init__(self):
31 | self.globals = None
32 | self.currentScope = None
33 | self.scopes = {}
34 | pass
35 |
36 | def enterTop(self, ctx):
37 | self.globals = GlobalScope(None)
38 | self.currentScope = self.globals
39 |
40 | def exitTop(self, ctx):
41 | print(self.globals)
42 |
43 | def enterFunctionDecl(self, ctx: CymbolParser.FunctionDeclContext):
44 | name = ctx.ID().getText()
45 | stoken_type = ctx.primtype().start.type
46 | stype = get_type(stoken_type)
47 | #
48 | function = FunctionSymbol(name, stype, self.currentScope)
49 | self.currentScope.define(function)
50 | self.scopes[ctx] = function
51 | self.currentScope = function
52 |
53 | def exitFunctionDecl(self, ctx):
54 | print(self.currentScope)
55 | self.currentScope = self.currentScope.getEnclosingScope()
56 |
57 | def enterBlock(self, ctx):
58 | self.currentScope = LocalScope(self.currentScope)
59 | self.scopes[ctx] = self.currentScope
60 |
61 | def exitBlock(self, ctx):
62 | print(self.currentScope)
63 | self.currentScope = self.currentScope.getEnclosingScope()
64 |
65 | def exitFormalParameter(self, ctx: CymbolParser.FormalParameterContext):
66 | stoken_type = ctx.primtype().start.type
67 | stype =get_type(stoken_type)
68 | var = VariableSymbol(ctx.ID().getText(), stype)
69 | self.currentScope.define(var)
70 |
71 | def exitVarDecl(self, ctx):
72 | stoken_type = ctx.primtype().start.type
73 | stype =get_type(stoken_type)
74 | var = VariableSymbol(ctx.ID().getText(), stype)
75 | self.currentScope.define(var)
76 |
77 |
78 | class RefPhase(CymbolListener):
79 | def __init__(self, glbs, scopes):
80 | self.globals = glbs
81 | self.scopes = scopes
82 | self.currentScope = None
83 |
84 | def enterTop(self, ctx):
85 | self.currentScope = self.globals
86 |
87 | def enterFunctionDecl(self, ctx):
88 | self.currentScope = self.scopes[ctx]
89 |
90 | def exitFunctionDecl(self, ctx):
91 | self.currentScope = self.currentScope.getEnclosingScope()
92 |
93 | def enterBlock(self, ctx):
94 | self.currentScope = self.scopes[ctx]
95 |
96 | def exitBlock(self, ctx):
97 | self.currentScope = self.currentScope.getEnclosingScope()
98 |
99 | def exitVar(self, ctx: CymbolParser.VarContext):
100 | name = ctx.ID().getText()
101 | var = self.currentScope.resolve(name)
102 | if var is None:
103 | error(ctx.ID().getSymbol(), "no such variable: "+name)
104 | if isinstance(var, FunctionSymbol):
105 | error(ctx.ID().getSymbol(), name+" is not a variable")
106 |
107 | def exitCall(self, ctx: CymbolParser.CallContext):
108 | funcname = ctx.ID().getText()
109 | meth = self.currentScope.resolve(funcname)
110 | if meth is None:
111 | error(ctx.ID().getSymbol(), "no such function: "+funcname)
112 | if isinstance(meth, VariableSymbol):
113 | error(ctx.ID().getSymbol(), funcname+" is not a function")
114 |
115 |
116 | if __name__ == '__main__':
117 | if len(sys.argv) > 1:
118 | input_stream = FileStream(sys.argv[1])
119 | else:
120 | input_stream = InputStream(sys.stdin.read())
121 |
122 | lexer = CymbolLexer(input_stream)
123 | token_stream = CommonTokenStream(lexer)
124 | parser = CymbolParser(token_stream)
125 | tree = parser.top()
126 |
127 | # lisp_tree_str = tree.toStringTree(recog=parser)
128 | # print(lisp_tree_str)
129 |
130 | walker = ParseTreeWalker()
131 |
132 | # definition phase, collect data
133 | print('*** Scan Definitions ***')
134 | def_phase = DefPhase()
135 | walker.walk(def_phase, tree)
136 |
137 | print()
138 |
139 | # reference phase, check error
140 | print('*** Check errors ***')
141 | ref_phase = RefPhase(def_phase.globals, def_phase.scopes)
142 | walker.walk(ref_phase, tree)
--------------------------------------------------------------------------------
/08-CallGraph/Cymbol.g4:
--------------------------------------------------------------------------------
1 | /** Simple statically-typed programming language with functions and variables
2 | * taken from "Language Implementation Patterns" book.
3 | */
4 | grammar Cymbol;
5 |
6 | top: (functionDecl | varDecl)+ ;
7 |
8 | varDecl
9 | : primtype ID ('=' expr)? ';'
10 | ;
11 | primtype: 'float' | 'int' | 'void' ; // user-defined types
12 |
13 | functionDecl
14 | : primtype ID '(' formalParameters? ')' block // "void f(int x) {...}"
15 | ;
16 |
17 | formalParameters
18 | : formalParameter (',' formalParameter)*
19 | ;
20 | formalParameter
21 | : primtype ID
22 | ;
23 |
24 | block: '{' stat* '}' ; // possibly empty statement block
25 |
26 | stat: block
27 | | varDecl
28 | | 'if' expr 'then' stat ('else' stat)?
29 | | 'return' expr? ';'
30 | | expr '=' expr ';' // assignment
31 | | expr ';' // func call
32 | ;
33 |
34 | expr: ID '(' exprList? ')' # Call
35 | | expr '[' expr ']' # Index
36 | | '-' expr # Negate
37 | | '!' expr # Not
38 | | expr '*' expr # Mult
39 | | expr ('+'|'-') expr # AddSub
40 | | expr '==' expr # Equal
41 | | ID # Var
42 | | INT # Int
43 | | '(' expr ')' # Parens
44 | ;
45 |
46 | exprList : expr (',' expr)* ; // arg list
47 |
48 | K_FLOAT : 'float';
49 | K_INT : 'int';
50 | K_VOID : 'void';
51 | ID : LETTER (LETTER | [0-9])* ;
52 |
53 | fragment
54 | LETTER : [a-zA-Z] ;
55 |
56 | INT : [0-9]+ ;
57 |
58 | WS : [ \t\n\r]+ -> skip ;
59 |
60 | SL_COMMENT
61 | : '//' .*? '\n' -> skip
62 | ;
63 |
--------------------------------------------------------------------------------
/08-CallGraph/Readme.md:
--------------------------------------------------------------------------------
1 | # Cymbol
2 | Two example here
3 | - Generate call graph
4 | - Check variable
5 |
6 | # How to run
7 | ```
8 | % antlr4py3 Cymbol.g4
9 | % python CallGraph.py t.cymbol
10 |
11 | digraph G {
12 | ranksep=.25;
13 | edge [arrowsize=.5]
14 | node [shape=circle, fontname="ArialNarrow",
15 | fontsize=12, fixedsize=true, height=.45];
16 |
17 | main;fact;a;b;c;d;e;
18 | main -> fact;
19 | main -> a;
20 | fact -> print;
21 | fact -> fact;
22 | a -> b;
23 | a -> c;
24 | a -> d;
25 | b -> c;
26 | c -> b;
27 |
28 | }
29 |
30 | % python CheckSymbols.py vars.cymbol
31 | *** Scan Definitions ***
32 | locals : []
33 | function : ;;
34 | locals : ['y', 'x']
35 | function :
36 | globals : ['g', 'f']
37 |
38 | *** Check errors ***
39 | [Error] line 3:4 no such variable: i
40 | [Error] line 4:4 g is not a variable
41 | [Error] line 13:4 no such function: z
42 | [Error] line 14:4 y is not a function
43 | [Error] line 15:8 f is not a variable
44 |
45 | % python CheckSymbols.py vars2.cymbol
46 | *** Scan Definitions ***
47 | locals : ['y']
48 | locals : ['x']
49 | function :
50 | locals : []
51 | function : ;
52 | globals : ['b', 'y', 'x', 'a']
53 |
54 | *** Check errors ***
55 |
56 | ```
--------------------------------------------------------------------------------
/08-CallGraph/SymbolScope.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 | """
3 | This is helper classes and interfaces for symbol scope
4 | recording and identification
5 | """
6 |
7 | from enum import Enum
8 |
9 |
10 | class Scope:
11 | def getScopeName(self):
12 | pass
13 |
14 | def getEnclosingScope(self):
15 | pass
16 |
17 | def define(self, sym):
18 | pass
19 |
20 | def resolve(self, name):
21 | pass
22 |
23 |
24 | class Symbol:
25 | class TypeEnum(Enum):
26 | INVALID = 1
27 | VOID = 2
28 | INT = 3
29 | FLOAT = 4
30 |
31 | def __init__(self, name='', stype=TypeEnum.INVALID):
32 | self.name = name
33 | self.type = stype
34 | self.scope = None
35 |
36 | def __str__(self):
37 | if self.type != Symbol.TypeEnum.INVALID:
38 | return '<'+self.name+':'+self.type.name+'>'
39 | else:
40 | return self.name
41 |
42 |
43 | class BaseScope(Scope):
44 | def __init__(self, scope: Scope):
45 | self.enclosingScope = scope
46 | self.symbols = {}
47 |
48 | def resolve(self, name):
49 | s = self.symbols.get(name)
50 | if s is not None:
51 | return s
52 | if self.enclosingScope is not None:
53 | return self.enclosingScope.resolve(name)
54 | return None
55 |
56 | def define(self, sym: Symbol):
57 | self.symbols[sym.name] = sym
58 |
59 | def getEnclosingScope(self):
60 | return self.enclosingScope
61 |
62 | def __str__(self):
63 | buf = self.getScopeName() + ' : ' + list(self.symbols.keys()).__str__()
64 | return buf
65 |
66 |
67 | class GlobalScope(BaseScope):
68 | def __init__(self, scope):
69 | super().__init__(scope)
70 |
71 | def getScopeName(self):
72 | return 'globals'
73 |
74 |
75 | class LocalScope(BaseScope):
76 | def __init__(self, parent):
77 | super().__init__(parent)
78 |
79 | def getScopeName(self):
80 | return 'locals'
81 |
82 |
83 | class FunctionSymbol(Symbol, Scope):
84 | def __init__(self, name='', stype=Symbol.TypeEnum.INVALID, scope=None):
85 | super().__init__(name, stype)
86 | self.enclosingScope = scope
87 | self.arguments = {}
88 |
89 | def resolve(self, name):
90 | s = self.arguments.get(name)
91 | if s is not None:
92 | return s
93 | if self.enclosingScope is not None:
94 | return self.enclosingScope.resolve(name)
95 | return None
96 |
97 | def define(self, sym: Symbol):
98 | self.arguments[sym.name] = sym
99 | sym.scope = self
100 |
101 | def getEnclosingScope(self):
102 | return self.enclosingScope
103 |
104 | def getScopeName(self):
105 | return self.name
106 |
107 | def __str__(self):
108 | buf = "function "
109 | buf += super().__str__()
110 | buf += " : "
111 | for par in self.arguments.values():
112 | buf += par.__str__() + ';'
113 | return buf
114 |
115 |
116 | class VariableSymbol(Symbol):
117 | def __init__(self, name, stype):
118 | super().__init__(name, stype)
--------------------------------------------------------------------------------
/08-CallGraph/output.dot:
--------------------------------------------------------------------------------
1 | digraph G {
2 | ranksep=.25;
3 | edge [arrowsize=.5]
4 | node [shape=circle, fontname="ArialNarrow",
5 | fontsize=12, fixedsize=true, height=.45];
6 |
7 | main;fact;a;b;c;d;e;
8 | main -> fact;
9 | main -> a;
10 | fact -> print;
11 | fact -> fact;
12 | a -> b;
13 | a -> c;
14 | a -> d;
15 | b -> c;
16 | c -> b;
17 |
18 | }
19 |
20 |
--------------------------------------------------------------------------------
/08-CallGraph/t.cymbol:
--------------------------------------------------------------------------------
1 | int main() { fact(); a(); }
2 |
3 | float fact(int n) {
4 | print(n);
5 |
6 | if ( n==0 ) then return 1;
7 | return n * fact(n-1);
8 | }
9 |
10 | void a() { int x = b(); if false then {c(); d();} }
11 | void b() { c(); }
12 | void c() { b(); }
13 | void d() { }
14 | void e() { }
15 |
16 |
--------------------------------------------------------------------------------
/08-CallGraph/vars.cymbol:
--------------------------------------------------------------------------------
1 | int f(int x, float y) {
2 | g(); // forward reference is ok
3 | i = 3; // no declaration for i (error)
4 | g = 4; // g is not variable (error)
5 | return x + y; // x, y are defined, so no problem
6 | }
7 |
8 | void g() {
9 | int x = 0;
10 | float y;
11 | y = 9; // y is defined
12 | f(); // backward reference is ok
13 | z(); // no such function (error)
14 | y(); // y is not function (error)
15 | x = f; // f is not a variable (error)
16 | }
17 |
--------------------------------------------------------------------------------
/08-CallGraph/vars2.cymbol:
--------------------------------------------------------------------------------
1 | int x; //(1)
2 | int y;
3 | void a() //(2)
4 | { //(3)
5 | int x;
6 | x = 1; // x resolves to current scope, not x in global scope
7 | y = 2; // y is not found in current scope, but resolves in global
8 | { int y = x; } //(4)
9 | }
10 | void b(int z) //(5)
11 | { } //(6)
12 |
--------------------------------------------------------------------------------
/08-JSON/08-JSON.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/08-JSON/JSON.g4:
--------------------------------------------------------------------------------
1 | // Derived from http://json.org
2 | grammar JSON;
3 |
4 | json: obj
5 | | array
6 | ;
7 |
8 | obj
9 | : '{' pair (',' pair)* '}' # AnObject
10 | | '{' '}' # EmptyObject
11 | ;
12 |
13 | array
14 | : '[' value (',' value)* ']' # ArrayOfValues
15 | | '[' ']' # EmptyArray
16 | ;
17 |
18 | pair: STRING ':' value ;
19 |
20 | value
21 | : STRING # String
22 | | NUMBER # Atom
23 | | obj # ObjectValue
24 | | array # ArrayValue
25 | | 'true' # Atom
26 | | 'false' # Atom
27 | | 'null' # Atom
28 | ;
29 |
30 | LCURLY : '{' ;
31 | LBRACK : '[' ;
32 | STRING : '"' (ESC | ~["\\])* '"' ;
33 |
34 | fragment ESC : '\\' (["\\/bfnrt] | UNICODE) ;
35 | fragment UNICODE : 'u' HEX HEX HEX HEX ;
36 | fragment HEX : [0-9a-fA-F] ;
37 |
38 | NUMBER
39 | : '-'? INT '.' INT EXP? // 1.35, 1.35E-9, 0.3, -4.5
40 | | '-'? INT EXP // 1e10 -3e4
41 | | '-'? INT // -3, 45
42 | ;
43 | fragment INT : '0' | '1'..'9' '0'..'9'* ; // no leading zeros
44 | fragment EXP : [Ee] [+\-]? INT ; // \- since - means "range" inside [...]
45 |
46 | WS : [ \t\n\r]+ -> skip ;
47 |
--------------------------------------------------------------------------------
/08-JSON/README.md:
--------------------------------------------------------------------------------
1 | # What's New
2 | - Label multiple alternative with the same name and process with same function.
3 |
4 | # How to Run
5 | ```
6 | % type t.json
7 | {
8 | "description" : "An imaginary server config file",
9 | "logs" : {"level":"verbose", "dir":"/var/log"},
10 | "host" : "antlr.org",
11 | "admin": ["parrt", "tombu"],
12 | "aliases": []
13 | }
14 |
15 | % python json2xml.py t.json
16 |
17 | An imaginary server config file
18 |
19 | verbose
20 | /var/log
21 |
22 | antlr.org
23 |
24 | parrt
25 | tombu
26 |
27 |
28 |
29 | ```
--------------------------------------------------------------------------------
/08-JSON/json2xml.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4.InputStream import InputStream
6 |
7 | from JSONLexer import JSONLexer
8 | from JSONParser import JSONParser
9 | from JSONListener import JSONListener
10 |
11 |
12 | class XmlEmitter(JSONListener):
13 | def __init__(self):
14 | self.xml = {}
15 |
16 | def getXML(self, ctx):
17 | return self.xml[ctx]
18 |
19 | def setXML(self, ctx, value):
20 | self.xml[ctx] = value
21 |
22 | def exitAtom(self, ctx):
23 | self.setXML(ctx, ctx.getText())
24 |
25 | def exitString(self, ctx):
26 | self.setXML(ctx, ctx.getText().strip('"'))
27 |
28 | def exitObjectValue(self, ctx: JSONParser.ObjectValueContext):
29 | self.setXML(ctx, self.getXML(ctx.obj()))
30 |
31 | def exitPair(self, ctx: JSONParser.PairContext):
32 | tag = ctx.STRING().getText().strip('"')
33 | val = self.getXML(ctx.value())
34 | x = '<%s>%s\n' % (tag, val, tag)
35 | self.setXML(ctx, x)
36 |
37 | def exitAnObject(self, ctx: JSONParser.AnObjectContext):
38 | buf = "\n"
39 | for pctx in ctx.pair():
40 | buf += self.getXML(pctx)
41 | self.setXML(ctx, buf)
42 |
43 | def exitArrayOfValues(self, ctx: JSONParser.ArrayOfValuesContext):
44 | buf = "\n"
45 | for vctx in ctx.value():
46 | buf += ""
47 | buf += self.getXML(vctx)
48 | buf += "\n"
49 | self.setXML(ctx, buf)
50 |
51 | def exitArrayValue(self, ctx: JSONParser.ArrayValueContext):
52 | self.setXML(ctx, self.getXML(ctx.array()))
53 |
54 | def exitEmptyArray(self, ctx: JSONParser.EmptyArrayContext):
55 | self.setXML(ctx, "")
56 |
57 | def exitJson(self, ctx: JSONParser.JsonContext):
58 | self.setXML(ctx, self.getXML(ctx.getChild(0)))
59 |
60 |
61 | if __name__ == '__main__':
62 | if len(sys.argv) > 1:
63 | input_stream = FileStream(sys.argv[1])
64 | else:
65 | input_stream = InputStream(sys.stdin.read())
66 |
67 | lexer = JSONLexer(input_stream)
68 | token_stream = CommonTokenStream(lexer)
69 | parser = JSONParser(token_stream)
70 | tree = parser.json()
71 |
72 | lisp_tree_str = tree.toStringTree(recog=parser)
73 | print(lisp_tree_str)
74 |
75 | # listener
76 | print("Start Walking...")
77 | listener = XmlEmitter()
78 | walker = ParseTreeWalker()
79 | walker.walk(listener, tree)
80 | print(listener.getXML(tree))
81 |
82 |
--------------------------------------------------------------------------------
/08-JSON/t.json:
--------------------------------------------------------------------------------
1 | {
2 | "description" : "An imaginary server config file",
3 | "logs" : {"level":"verbose", "dir":"/var/log"},
4 | "host" : "antlr.org",
5 | "admin": ["parrt", "tombu"],
6 | "aliases": []
7 | }
8 |
--------------------------------------------------------------------------------
/09-simple/README.md:
--------------------------------------------------------------------------------
1 | # The build-in Error report
2 |
3 | ```
4 | % antlr4py3 Simple.g4
5 | % pygrun Simple prog in1.txt
6 | var i
7 | class T
8 |
9 | % pygrun Simple prog in2.txt
10 | line 2:19 mismatched input '4' expecting ';'
11 | method: f
12 | class T
13 |
14 | % pygrun Simple prog in3.txt
15 | line 1:7 extraneous input ';' expecting '{'
16 | var i
17 | class T
18 |
19 | % pygrun Simple prog in4.txt
20 | found assign: a=3;
21 | method: f
22 | line 2:21 missing '}' at ''
23 | class T
24 |
25 | % pygrun Simple prog in5.txt
26 | line 1:14 no viable alternative at input 'int;'
27 | class T
28 |
29 | % pygrun Simple prog in6.txt
30 | line 1:6 token recognition error at: '#'
31 | line 1:8 missing ID at '{'
32 | var i
33 | class
34 | ```
35 |
36 | # Verbose Listener
37 |
38 | ```
39 | % python TestE_Listener.py in7.txt
40 | rule stack: ['prog', 'classDef']
41 | line 1 : 8 at [@2,8:8='T',<10>,1:8] : extraneous input 'T' expecting '{'
42 | rule stack: ['prog', 'classDef', 'member']
43 | line 2 : 6 at [@5,19:19=';',<5>,2:6] : no viable alternative at input 'int;'
44 | class T
45 | ```
--------------------------------------------------------------------------------
/09-simple/Simple.g4:
--------------------------------------------------------------------------------
1 | grammar Simple;
2 |
3 | prog: classDef+ ; // match one or more class definitions
4 |
5 | classDef
6 | : 'class' ID '{' member+ '}' // a class has one or more members
7 | {print("class "+$ID.text);}
8 | ;
9 |
10 | member
11 | : 'int' ID ';' // field definition
12 | {print("var "+$ID.text);}
13 | | 'int' f=ID '(' ID ')' '{' stat '}' // method definition
14 | {print("method: "+$f.text);}
15 | ;
16 |
17 | stat: expr ';'
18 | {print("found expr: " + $text);}
19 | | ID '=' expr ';'
20 | {print("found assign: " + $text);}
21 | ;
22 |
23 | expr: INT
24 | | ID '(' INT ')'
25 | ;
26 |
27 | INT : [0-9]+ ;
28 | ID : [a-zA-Z]+ ;
29 | WS : [ \t\r\n]+ -> skip ;
30 |
31 |
--------------------------------------------------------------------------------
/09-simple/TestE_Listener.py:
--------------------------------------------------------------------------------
1 | import sys
2 | from antlr4 import *
3 | from antlr4.error.ErrorListener import *
4 | from pprint import pprint
5 |
6 | from SimpleLexer import SimpleLexer
7 | from SimpleParser import SimpleParser
8 |
9 |
10 |
11 | class VerboseListener(ErrorListener) :
12 | def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e):
13 | stack = recognizer.getRuleInvocationStack()
14 | stack.reverse()
15 | print("rule stack: ", str(stack))
16 | print("line", line, ":", column, "at", offendingSymbol, ":", msg)
17 |
18 |
19 | def main(argv):
20 | istream = FileStream(argv[1])
21 | lexer = SimpleLexer(istream)
22 | stream = CommonTokenStream(lexer)
23 | parser = SimpleParser(stream)
24 | parser.removeErrorListeners()
25 | parser.addErrorListener(VerboseListener())
26 | parser.prog()
27 |
28 |
29 | if __name__ == '__main__':
30 | main(sys.argv)
31 |
--------------------------------------------------------------------------------
/09-simple/in1.txt:
--------------------------------------------------------------------------------
1 | class T { int i; }
2 |
3 |
--------------------------------------------------------------------------------
/09-simple/in2.txt:
--------------------------------------------------------------------------------
1 | class T {
2 | int f(x) { a = 3 4 5; }
3 | }
--------------------------------------------------------------------------------
/09-simple/in3.txt:
--------------------------------------------------------------------------------
1 | class T; { int i; }
2 |
3 |
--------------------------------------------------------------------------------
/09-simple/in4.txt:
--------------------------------------------------------------------------------
1 | class T {
2 | int f(x) { a = 3; }
--------------------------------------------------------------------------------
/09-simple/in5.txt:
--------------------------------------------------------------------------------
1 | class T { int ; }
--------------------------------------------------------------------------------
/09-simple/in6.txt:
--------------------------------------------------------------------------------
1 | class # { int i; }
--------------------------------------------------------------------------------
/09-simple/in7.txt:
--------------------------------------------------------------------------------
1 | class T T {
2 | int ;
3 | }
--------------------------------------------------------------------------------
/10-CSV/10-CSV.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/10-CSV/CSV.g4:
--------------------------------------------------------------------------------
1 | grammar CSV;
2 |
3 | @header {
4 | from pprint import pprint
5 |
6 | }
7 |
8 | /** Derived from rule "start : hdr row+ ;" */
9 | start
10 | locals [i=0]
11 | : hdr ( rows+=row[$hdr.text.split(',')] {$i+=1} )+
12 | {
13 | print(str($i)+" rows");
14 | for r in $rows:
15 | print("row token interval: "+str(r.getSourceInterval()))
16 | }
17 | ;
18 |
19 | hdr : row[None] {print("header: '"+$text.strip()+"'")} ;
20 |
21 | row[columns] returns [values]
22 | locals [col=0]
23 | @init {
24 | $values = {}
25 | }
26 | @after {
27 | if ($values != None) or (len($values) > 0):
28 | pprint($values)
29 | }
30 | : field
31 | {
32 | if ($columns!=None) :
33 | $values[$columns[$col].strip()] = $field.text.strip()
34 | $col += 1
35 | }
36 | ( ',' field
37 | {
38 | if ($columns!=None) :
39 | $values[$columns[$col].strip()] = $field.text.strip()
40 | $col += 1
41 | }
42 | )* '\r'? '\n'
43 | ;
44 |
45 | field
46 | : TEXT
47 | | STRING
48 | |
49 | ;
50 |
51 | TEXT : ~[,\n\r"]+ ;
52 | STRING : '"' ('""'|~'"')* '"' ; // quote-quote is an escaped quote
53 |
--------------------------------------------------------------------------------
/10-CSV/README.md:
--------------------------------------------------------------------------------
1 | Rule with parameter and return.
2 |
3 | ```
4 | % antlr4py3 CSV.g4
5 |
6 | % python test_csv.py users.csv
7 | {}
8 | header: 'User, Name, Dept'
9 | {'Dept': '101', 'Name': 'Terence', 'User': 'parrt'}
10 | {'Dept': '020', 'Name': 'Tom', 'User': 'tombu'}
11 | {'Dept': '008', 'Name': 'Kevin', 'User': 'bke'}
12 | 3 rows
13 | row token interval: (6, 11)
14 | row token interval: (12, 17)
15 | row token interval: (18, 23)
16 | ```
--------------------------------------------------------------------------------
/10-CSV/test_csv.py:
--------------------------------------------------------------------------------
1 |
2 | import sys
3 | from antlr4 import *
4 | from antlr4 import InputStream
5 |
6 | from CSVLexer import CSVLexer
7 | from CSVParser import CSVParser
8 |
9 | if __name__ == '__main__':
10 | if len(sys.argv) > 1:
11 | input_stream = FileStream(sys.argv[1])
12 | else:
13 | input_stream = InputStream(sys.stdin.read())
14 |
15 | lexer = CSVLexer(input_stream)
16 | token_stream = CommonTokenStream(lexer)
17 | parser = CSVParser(token_stream)
18 | parser.buildParseTrees = False
19 | parser.start()
--------------------------------------------------------------------------------
/10-CSV/users.csv:
--------------------------------------------------------------------------------
1 | User, Name, Dept
2 | parrt, Terence, 101
3 | tombu, Tom, 020
4 | bke, Kevin, 008
5 |
--------------------------------------------------------------------------------
/10-Keyword/10-Keyword.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/10-Keyword/Keywords.g4:
--------------------------------------------------------------------------------
1 | grammar Keywords;
2 | // @lexer::header {
3 | // }
4 |
5 | // explicitly define keyword token types to avoid implicit def warnings
6 | tokens { BEGIN, END, IF, THEN, WHILE }
7 |
8 | // @lexer::members {
9 | // }
10 |
11 | stat: BEGIN stat* END
12 | | IF expr THEN stat
13 | | WHILE expr stat
14 | | ID '=' expr ';'
15 | ;
16 |
17 | expr: INT | CHAR ;
18 |
19 | ID : [a-zA-Z]+
20 | {
21 | if self.text in self.keywords:
22 | self.type = self.keywords.get(self.text)
23 | }
24 | ;
25 |
26 | /** Convert 3-char 'x' input sequence to string x */
27 | CHAR: '\'' . '\'' {self.text = self.text[1]} ;
28 |
29 | INT : [0-9]+ ;
30 |
31 | WS : [ \t\n\r]+ -> skip ;
32 |
--------------------------------------------------------------------------------
/10-Keyword/input1.txt:
--------------------------------------------------------------------------------
1 | if = 34;
--------------------------------------------------------------------------------
/10-Keyword/input2.txt:
--------------------------------------------------------------------------------
1 | if 1 then i=4;
2 |
--------------------------------------------------------------------------------
/10-Keyword/input3.txt:
--------------------------------------------------------------------------------
1 | x = 34;
2 |
--------------------------------------------------------------------------------
/10-Keyword/test_Keyword.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4 import InputStream
6 | from KeywordsLexer import KeywordsLexer
7 | from KeywordsParser import KeywordsParser
8 |
9 | if __name__ == '__main__':
10 | if len(sys.argv) > 1:
11 | input_stream = FileStream(sys.argv[1])
12 | else:
13 | input_stream = InputStream(sys.stdin.read())
14 |
15 | lexer = KeywordsLexer(input_stream)
16 | # there is no way to insert code in the __init__ function
17 | # from ANTLR file. just hack here, or could just create new class
18 | setattr(lexer, 'keywords', {
19 | 'begin': KeywordsParser.BEGIN,
20 | 'if': KeywordsParser.IF,
21 | 'then': KeywordsParser.THEN,
22 | 'while': KeywordsParser.WHILE
23 | })
24 |
25 | # tk = lexer.nextToken()
26 | # while tk.type != Token.EOF:
27 | # print(tk)
28 | # tk = lexer.nextToken()
29 |
30 | token_stream = CommonTokenStream(lexer)
31 | parser = KeywordsParser(token_stream)
32 | parser.stat()
33 |
--------------------------------------------------------------------------------
/10-calc/10-calc.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/10-calc/Expr.g4:
--------------------------------------------------------------------------------
1 | /** Grammar from tour chapter augmented with actions */
2 | grammar Expr;
3 |
4 | @header {
5 | }
6 |
7 | @parser::members {
8 | @property
9 | def memory(self):
10 | if not hasattr(self, '_map'):
11 | setattr(self, '_map', {})
12 | return self._map
13 |
14 | @memory.setter
15 | def memory_setter(self, value):
16 | if not hasattr(self, '_map'):
17 | setattr(self, '_map', {})
18 | self._map = value
19 |
20 | def eval(self, left, op, right):
21 | if ExprParser.MUL == op.type:
22 | return left * right
23 | elif ExprParser.DIV == op.type:
24 | return left / right
25 | elif ExprParser.ADD == op.type:
26 | return left + right
27 | elif ExprParser.SUB == op.type:
28 | return left - right
29 | else:
30 | return 0
31 | }
32 |
33 | stat: e NEWLINE {print($e.v);}
34 | | ID '=' e NEWLINE {self.memory[$ID.text] = $e.v}
35 | | NEWLINE
36 | ;
37 |
38 | e returns [int v]
39 | : a=e op=('*'|'/') b=e {$v = self.eval($a.v, $op, $b.v)}
40 | | a=e op=('+'|'-') b=e {$v = self.eval($a.v, $op, $b.v)}
41 | | INT {$v = $INT.int}
42 | | ID
43 | {
44 | id = $ID.text
45 | $v = self.memory.get(id, 0)
46 | }
47 | | '(' e ')' {$v = $e.v}
48 | ;
49 |
50 | MUL : '*' ;
51 | DIV : '/' ;
52 | ADD : '+' ;
53 | SUB : '-' ;
54 |
55 | ID : [a-zA-Z]+ ; // match identifiers
56 | INT : [0-9]+ ; // match integers
57 | NEWLINE:'\r'? '\n' ; // return newlines to parser (is end-statement signal)
58 | WS : [ \t]+ -> skip ; // toss out whitespace
59 |
--------------------------------------------------------------------------------
/10-calc/README.md:
--------------------------------------------------------------------------------
1 | # What's in this example
2 | - Interactive
3 | - Create new input stream every line.
4 | - Do not build AST tree
5 |
6 | # How to run
7 | ```
8 | % python calc.py
9 | 1+2*3
10 | 7
11 | var=10
12 | var*100
13 | 1000
14 | ^Z
15 | ```
--------------------------------------------------------------------------------
/10-calc/calc.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4.InputStream import InputStream
6 |
7 | from ExprLexer import ExprLexer
8 | from ExprParser import ExprParser
9 |
10 | if __name__ == '__main__':
11 | parser = ExprParser(None)
12 | parser.buildParseTrees = False
13 |
14 | line = sys.stdin.readline()
15 | lineno = 1
16 |
17 | while line != '':
18 | line = line.strip()
19 | #print(lineno, line)
20 |
21 | istream = InputStream(line + "\n")
22 | lexer = ExprLexer(istream)
23 | lexer.line = lineno
24 | lexer.column = 0
25 | token_stream = CommonTokenStream(lexer)
26 | parser.setInputStream(token_stream)
27 | parser.stat()
28 |
29 | line = sys.stdin.readline()
30 | lineno += 1
31 |
--------------------------------------------------------------------------------
/11-CppState/11-CppState.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/11-CppState/CppStat.g4:
--------------------------------------------------------------------------------
1 | grammar CppStat;
2 |
3 | stat: decl ';' {print("decl "+$decl.text)}
4 | | expr ';' {print("expr "+$expr.text)}
5 | ;
6 |
7 | decl: ID ID // E.g., "Point p"
8 | | ID '(' ID ')' // E.g., "Point (p)", same as ID ID
9 | ;
10 |
11 | expr: INT // integer literal
12 | | ID // identifier
13 | | ID '(' expr ')' // function call
14 | ;
15 |
16 | ID : [a-zA-Z]+ ;
17 | INT : [0-9]+ ;
18 | WS : [ \t\n\r]+ -> skip ;
19 |
--------------------------------------------------------------------------------
/11-CppState/PredCppStat.g4:
--------------------------------------------------------------------------------
1 | grammar PredCppStat;
2 |
3 | @parser::header {
4 | }
5 |
6 | @parser::members {
7 | @property
8 | def types_table(self):
9 | try:
10 | return self._types_table
11 | except AttributeError:
12 | self._types_table = ['T']
13 | return self._types_table
14 |
15 | def istype(self):
16 | if self.getCurrentToken().text in self.types_table:
17 | return True
18 | else:
19 | return False
20 | }
21 |
22 | stat: decl ';' {print("decl "+$decl.text);}
23 | | expr ';' {print("expr "+$expr.text);}
24 | ;
25 |
26 | decl: ID ID // E.g., "Point p"
27 | | {self.istype()}? ID '(' ID ')' // E.g., "Point (p)", same as ID ID
28 | ;
29 |
30 | expr: INT // integer literal
31 | | ID // identifier
32 | | {not self.istype()}? ID '(' expr ')' // function call
33 | ;
34 |
35 | ID : [a-zA-Z]+ ;
36 | INT : [0-9]+ ;
37 | WS : [ \t\n\r]+ -> skip ;
38 |
--------------------------------------------------------------------------------
/11-CppState/input.cpp:
--------------------------------------------------------------------------------
1 | f(i);
2 | T(i);
3 |
--------------------------------------------------------------------------------
/11-CppState/input2.cpp:
--------------------------------------------------------------------------------
1 | T(i);
2 |
--------------------------------------------------------------------------------
/11-predicate_lexer/11-predicate_lexer.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/11-predicate_lexer/Enum2.g4:
--------------------------------------------------------------------------------
1 | grammar Enum2;
2 | @lexer::members {_java5 = False}
3 |
4 | prog: ( stat
5 | | enumDecl
6 | )+
7 | ;
8 |
9 | stat: ID '=' expr ';' {print($ID.text+"="+$expr.text)} ;
10 |
11 | expr: ID
12 | | INT
13 | ;
14 |
15 | // No predicate needed here because 'enum' token undefined if !java5
16 | enumDecl
17 | : 'enum' name=ID '{' ID (',' ID)* '}'
18 | {print("enum "+$name.text)}
19 | ;
20 |
21 | ENUM: 'enum' {self._java5}? ; // must be before ID
22 | ID : [a-zA-Z]+ ;
23 |
24 |
25 | INT : [0-9]+ ;
26 | WS : [ \t\r\n]+ -> skip ;
27 |
--------------------------------------------------------------------------------
/11-predicate_lexer/README.md:
--------------------------------------------------------------------------------
1 | Predicate in Lexer
2 |
3 | ```
4 | When you set Enum2Lexer._java5 = True
5 | % python test_EnumLexer.py Temp.java
6 | enum Temp
7 |
8 | When you set Enum2Lexer._java5 = False
9 | % python test_EnumLexer.py Temp.java
10 | line 1:5 missing '=' at 'Temp'
11 | line 1:15 mismatched input ',' expecting '='
12 | line 1:22 mismatched input '}' expecting '='
13 | ```
--------------------------------------------------------------------------------
/11-predicate_lexer/Temp.java:
--------------------------------------------------------------------------------
1 | enum Temp { HOT, COLD }
2 |
--------------------------------------------------------------------------------
/11-predicate_lexer/test_EnumLexer.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from Enum2Lexer import Enum2Lexer
6 | from Enum2Parser import Enum2Parser
7 |
8 | if __name__ == '__main__':
9 | Enum2Lexer._java5 = True
10 |
11 | if len(sys.argv) > 1:
12 | input_stream = FileStream(sys.argv[1])
13 | else:
14 | input_stream = InputStream(sys.stdin.read())
15 |
16 | lexer = Enum2Lexer(input_stream)
17 | token_stream = CommonTokenStream(lexer)
18 | parser = Enum2Parser(token_stream)
19 | parser.buildParseTrees = False
20 | parser.prog()
--------------------------------------------------------------------------------
/11-predicate_parser/11-predicate_parser.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/11-predicate_parser/Enum.g4:
--------------------------------------------------------------------------------
1 | grammar Enum;
2 | @parser::init {}
3 | @parser::members {_java5 = False}
4 |
5 | prog: ( stat
6 | | enumDecl
7 | )+
8 | ;
9 |
10 | stat: identifier '=' expr ';' {print($identifier.text+"="+$expr.text)} ;
11 |
12 | expr
13 | : identifier
14 | | INT
15 | ;
16 |
17 | enumDecl
18 | : {self._java5}? 'enum' name=identifier '{' identifier (',' identifier)* '}'
19 | {print("enum "+$name.text)}
20 | ;
21 |
22 | identifier : ID
23 | | {not self._java5}? 'enum'
24 | ;
25 |
26 | ID : [a-zA-Z]+ ;
27 | INT : [0-9]+ ;
28 | WS : [ \t\r\n]+ -> skip ;
29 |
--------------------------------------------------------------------------------
/11-predicate_parser/README.md:
--------------------------------------------------------------------------------
1 | Predicate in Parser
2 |
3 | ```
4 | % antlr4py3 Enum.g4
5 |
6 | Set EnumParser._java5 = True
7 | % python test_EnumParser.py Temp.java
8 | enum Temp
9 |
10 | Set EnumParser._java5 = False
11 | % python test_EnumParser.py Temp.java
12 | line 1:0 no viable alternative at input 'enum'
13 | ```
14 |
--------------------------------------------------------------------------------
/11-predicate_parser/Temp.java:
--------------------------------------------------------------------------------
1 | enum Temp { HOT, COLD }
2 |
--------------------------------------------------------------------------------
/12-channel/12-channel.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/12-channel/Cymbol.g4:
--------------------------------------------------------------------------------
1 | /** Simple statically-typed programming language with functions and variables
2 | * taken from "Language Implementation Patterns" book.
3 | */
4 | grammar Cymbol;
5 |
6 | @lexer::members {
7 | WHITESPACE = 1
8 | COMMENTS = 2
9 | }
10 |
11 | startrule: (functionDecl | varDecl)+ ;
12 |
13 | varDecl
14 | : datatype ID ('=' expr)? ';'
15 | ;
16 | datatype: 'float' | 'int' | 'void' ; // user-defined types
17 |
18 | functionDecl
19 | : datatype ID '(' formalParameters? ')' block // "void f(int x) {...}"
20 | ;
21 |
22 | formalParameters
23 | : formalParameter (',' formalParameter)*
24 | ;
25 | formalParameter
26 | : datatype ID
27 | ;
28 |
29 | block: '{' stat* '}' ; // possibly empty statement block
30 |
31 | stat: block
32 | | varDecl
33 | | 'if' expr 'then' stat ('else' stat)?
34 | | 'return' expr? ';'
35 | | expr '=' expr ';' // assignment
36 | | expr ';' // func call
37 | ;
38 |
39 | expr: ID '(' exprList? ')' // func call like f(), f(x), f(1,2)
40 | | expr '[' expr ']' // array index like a[i], a[i][j]
41 | | '-' expr // unary minus
42 | | '!' expr // boolean not
43 | | expr '*' expr
44 | | expr ('+'|'-') expr
45 | | expr '==' expr // equality comparison (lowest priority op)
46 | | ID // variable reference
47 | | INT
48 | | '(' expr ')'
49 | ;
50 |
51 | exprList : expr (',' expr)* ; // arg list
52 |
53 | ID : LETTER (LETTER | [0-9])* ;
54 | fragment
55 | LETTER : [a-zA-Z] ;
56 |
57 | INT : [0-9]+ ;
58 |
59 | WS : [ \t\n\r]+ -> channel(1) ; // channel(1)
60 |
61 | SL_COMMENT
62 | : '//' .*? '\n' -> channel(2) // channel(2)
63 | ;
64 |
--------------------------------------------------------------------------------
/12-channel/README.md:
--------------------------------------------------------------------------------
1 | # Shift Comment to top
2 | There is no TokenStreamRewriter in the python runtime. It does not have to. if you dive into the source code, the CommonTokenStream is derived from BufferedTokenStream which has a list of CommonToken. It is re-writable.
3 |
4 | in the listener
5 | ```python
6 | class CommentShifter(CymbolListener):
7 | def __init__(self, tokens:CommonTokenStream):
8 | super().__init__()
9 | self.tokens = tokens # record the token stream from parser
10 |
11 | def exitVarDecl(self, ctx:CymbolParser.VarDeclContext):
12 | startIndex = ctx.start.tokenIndex
13 | stopIndex = ctx.stop.tokenIndex
14 | cmtChannel = self.tokens.getHiddenTokensToRight(stopIndex, CymbolLexer.COMMENTS)
15 | if cmtChannel != None:
16 | tok = cmtChannel[0]
17 | if tok != None: # find out a comment followed, it is one token
18 | # self.tokens is CommonTokenStream
19 | # self.tokens.tokens is the list of CommonToken
20 | token_array = self.tokens.tokens
21 | token_array.insert(startIndex, tok.clone())
22 | tok.text = "\n"
23 | ```
24 |
25 | # How to run
26 | ```
27 | % python shift_var_comments.py t.cym
28 | (startrule
29 | (varDecl (datatype int) n = (expr 0) ;)
30 | (varDecl (datatype int) i = (expr 9) ;)
31 | )
32 | // define a counter
33 | int n = 0;
34 | int i = 9;
35 | ```
--------------------------------------------------------------------------------
/12-channel/shift_var_comments.py:
--------------------------------------------------------------------------------
1 | __author__ = 'jszheng'
2 |
3 | import sys
4 | from antlr4 import *
5 | from antlr4 import InputStream
6 |
7 | from CymbolLexer import CymbolLexer
8 | from CymbolParser import CymbolParser
9 | from CymbolListener import CymbolListener
10 |
11 |
12 | class CommentShifter(CymbolListener):
13 | def __init__(self, tokens:CommonTokenStream):
14 | super().__init__()
15 | self.tokens = tokens
16 |
17 | def exitVarDecl(self, ctx:CymbolParser.VarDeclContext):
18 | startIndex = ctx.start.tokenIndex
19 | stopIndex = ctx.stop.tokenIndex
20 | cmtChannel = self.tokens.getHiddenTokensToRight(stopIndex, CymbolLexer.COMMENTS)
21 | if cmtChannel != None:
22 | tok = cmtChannel[0]
23 | if tok != None:
24 | token_array = self.tokens.tokens
25 | token_array.insert(startIndex, tok.clone())
26 | tok.text = "\n"
27 |
28 |
29 | if __name__ == '__main__':
30 | if len(sys.argv) > 1:
31 | input_stream = FileStream(sys.argv[1])
32 | else:
33 | input_stream = InputStream(sys.stdin.read())
34 |
35 | lexer = CymbolLexer(input_stream)
36 | token_stream = CommonTokenStream(lexer)
37 | parser = CymbolParser(token_stream)
38 | tree = parser.startrule()
39 |
40 | lisp_tree_str = tree.toStringTree(recog=parser)
41 | print(lisp_tree_str)
42 |
43 | walker = ParseTreeWalker()
44 | collector = CommentShifter(token_stream)
45 | walker.walk(collector, tree)
46 | print(token_stream.getText())
47 |
--------------------------------------------------------------------------------
/12-channel/t.cym:
--------------------------------------------------------------------------------
1 | int n = 0; // define a counter
2 | int i = 9;
3 |
--------------------------------------------------------------------------------
/12-id_as_keyword/IDKeyword.g4:
--------------------------------------------------------------------------------
1 | grammar IDKeyword;
2 |
3 | prog: stat+ ;
4 |
5 | stat: 'if' expr 'then' stat
6 | | 'call' id_rule ';'
7 | | ';'
8 | ;
9 |
10 | expr: id_rule ;
11 |
12 | id_rule : 'if' | 'call' | 'then' | ID ;
13 |
14 | ID : [a-z]+ ;
15 | WS : [ \r\n]+ -> skip ;
16 |
--------------------------------------------------------------------------------
/12-id_as_keyword/PredKeyword.g4:
--------------------------------------------------------------------------------
1 | grammar PredKeyword;
2 |
3 | prog: stat+ ;
4 |
5 | stat: keyIF expr 'then' stat
6 | | keyCALL ID ';'
7 | | ';'
8 | ;
9 |
10 | expr: ID
11 | ;
12 |
13 | keyIF : {self._input.LT(1).text == "if"}? ID ;
14 |
15 | keyCALL : {self._input.LT(1).text == "call"}? ID ;
16 |
17 | ID : 'a'..'z'+ ;
18 | WS : (' '|'\n')+ -> skip;
19 |
--------------------------------------------------------------------------------
/12-id_as_keyword/input.txt:
--------------------------------------------------------------------------------
1 | if if then call call;
--------------------------------------------------------------------------------
/12-id_as_keyword/run.bat:
--------------------------------------------------------------------------------
1 | pygrun IDKeyword prog input.txt --tree
2 | pygrun PredKeyword prog input.txt --trace --diagnostics
--------------------------------------------------------------------------------
/12-sea_of_text/Mode.txt:
--------------------------------------------------------------------------------
1 | Hello John
2 |
--------------------------------------------------------------------------------
/12-sea_of_text/ModeTagsLexer.g4:
--------------------------------------------------------------------------------
1 | lexer grammar ModeTagsLexer;
2 |
3 | // Default mode rules (the SEA)
4 | OPEN : '<' -> mode(ISLAND) ; // switch to ISLAND mode
5 | TEXT : ~'<'+ ; // clump all text together
6 |
7 | mode ISLAND;
8 | CLOSE : '>' -> mode(DEFAULT_MODE) ; // back to SEA mode
9 | SLASH : '/' ;
10 | ID : [a-zA-Z]+ ; // match/send ID in tag to parser
11 |
--------------------------------------------------------------------------------
/12-sea_of_text/ModeTagsParser.g4:
--------------------------------------------------------------------------------
1 | parser grammar ModeTagsParser;
2 |
3 | options { tokenVocab=ModeTagsLexer; } // use tokens from ModeTagsLexer.g4
4 |
5 | entry: (tag | TEXT)* ;
6 |
7 | tag : '<' ID '>'
8 | | '<' '/' ID '>'
9 | ;
10 |
--------------------------------------------------------------------------------
/12-sea_of_text/Tags.g4:
--------------------------------------------------------------------------------
1 | grammar Tags;
2 | entry : (TAG|ENTITY|TEXT|CDATA)* ;
3 |
4 | COMMENT : '' -> skip ;
5 | CDATA : '' ;
6 | TAG : '<' .*? '>' ; // must come after other tag-like structures
7 | ENTITY : '&' .*? ';' ;
8 | TEXT : ~[<&]+ ; // any sequence of chars except < and & chars
9 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # py3antlr4book
2 | Covert ANTLR4 book source code to Python 3 version.
3 |
4 | # How to use
5 |
6 | ## Windows
7 |
8 | - Install Python
9 | - Install antlr4 python3 runtime
10 |
11 | ```bash
12 | pip install antlr4-python3-runtime
13 | ```
14 |
15 | - open command and run bin/antlr4.bat
16 |
17 | You may need to compile the latest ANTLR4 and put it into bin, modify the bat file
18 |
19 | You may also need to use the latest runtime, just copy the source to [PythonDir]\Lib\site-packages\antlr4
20 |
21 | ## Ubuntu
22 |
23 | ```bash
24 | sudo apt install environment-modules
25 | sudo -H pip3 install antlr4-python3-runtime
26 | git clone https://github.com/jszheng/py3antlr4book.git
27 | cd py3antlr4book
28 | # goto https://www.antlr.org/download.html
29 | # download the latest ver
30 | wget https://www.antlr.org/download/antlr-4.7.2-complete.jar
31 | ln -s antlr-4.7.2-complete.jar antlr.jar
32 | module load ./antlr4module
33 | ```
34 |
35 |
36 |
37 | # run first example
38 |
39 | ```
40 | % antlr4py3 Hello.g4
41 | % pygrun Hello r --tokens input.txt
42 | [@0,0:4='hello',<1>,1:0]
43 | [@1,6:10='world',<2>,1:6]
44 | [@2,11:10='',<-1>,1:11]
45 |
46 | % pygrun Hello r --tree input.txt
47 | (r hello world)
48 |
49 | % python test_hello.py input.txt
50 | (r hello world)
51 | ```
52 |
53 | # IDE
54 | You can install the best python IDE in the world (no 'one of') from www.jetbrains.com. They provide two community version free of charge and just enough for the experiment here.
55 |
56 | The only problem is the tool take a large amount of memory just like all JAVA based IDE, you may not want to open IDEA and PyCharm at the same time. I only install IDEA and install the python community plug-in and also ANTLR4 plug-in to debug the .g4 file.
57 | 
58 |
59 |
60 |
--------------------------------------------------------------------------------
/antlr.jar:
--------------------------------------------------------------------------------
1 | bin/antlr-4.7.2-complete.jar
--------------------------------------------------------------------------------
/antlr4module:
--------------------------------------------------------------------------------
1 | #%Module1.0
2 |
3 | conflict antlr4
4 | set installdir [pwd]
5 |
6 | prepend-path CLASSPATH $installdir/antlr.jar
7 |
8 | set-alias antrl4 "java org.antlr.v4.Tool"
9 | set-alias grun "java org.antlr.v4.runtime.misc.TestRig"
10 | set-alias antlr4py3 "java org.antlr.v4.Tool -Dlanguage=Python3"
11 | set-alias antlr4py2 "java org.antlr.v4.Tool -Dlanguage=Python2"
12 | set-alias antlr4vpy3 "java org.antlr.v4.Tool -Dlanguage=Python3 -no-listener -visitor"
13 | set-alias antlr4vpy2 "java org.antlr.v4.Tool -Dlanguage=Python2 -no-listener -visitor"
14 | set-alias pygrun "python3 $installdir/bin/pygrun"
15 |
16 |
--------------------------------------------------------------------------------
/bin/antlr-4.7.2-complete.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jszheng/py3antlr4book/669f1f76d9d8ab54dfc79cf76ede2e08565abf98/bin/antlr-4.7.2-complete.jar
--------------------------------------------------------------------------------
/bin/antlr4env.bat:
--------------------------------------------------------------------------------
1 | SET CLASSPATH=%~dp0antlr-4.5.3-complete.jar;.
2 | doskey antlr4=java org.antlr.v4.Tool $*
3 | doskey grun=java org.antlr.v4.runtime.misc.TestRig $*
4 | doskey antlr4py3=java org.antlr.v4.Tool -Dlanguage=Python3 $*
5 | doskey antlr4py2=java org.antlr.v4.Tool -Dlanguage=Python2 $*
6 | doskey antlr4vpy3=java org.antlr.v4.Tool -Dlanguage=Python3 -no-listener -visitor $*
7 | doskey antlr4vpy2=java org.antlr.v4.Tool -Dlanguage=Python2 -no-listener -visitor $*
8 | doskey pygrun=python %~dp0pygrun $*
9 |
10 |
--------------------------------------------------------------------------------
/bin/pygrun:
--------------------------------------------------------------------------------
1 | #!/bin/env python
2 | __author__ = 'jszheng'
3 | import optparse
4 | import os
5 | import sys
6 | import re
7 |
8 | # for GUI
9 | from PySide2.QtCore import *
10 | from PySide2.QtGui import *
11 | from PySide2.QtWidgets import QDialog
12 |
13 | # put antlr4 runtime at the same directory and uncomment these line.
14 | # tied run-time with antlr generate file to avoid unexpected error due to version change.
15 | # antlr_runtime_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '.'))
16 | # sys.path.insert(0, antlr_runtime_path)
17 | from antlr4 import *
18 |
19 |
20 | # this is a python version of TestRig
21 | def beautify_lisp_string(in_string):
22 | indent_size = 3
23 | add_indent = ' ' * indent_size
24 | out_string = in_string[0] # no indent for 1st (
25 | indent = ''
26 | for i in range(1, len(in_string)):
27 | if in_string[i] == '(' and in_string[i + 1] != ' ':
28 | indent += add_indent
29 | out_string += "\n" + indent + '('
30 | elif in_string[i] == ')':
31 | out_string += ')'
32 | if len(indent) > 0:
33 | indent = indent.replace(add_indent, '', 1)
34 | else:
35 | out_string += in_string[i]
36 | return out_string
37 |
38 |
39 | class AntlrParserTreeViewer(QDialog):
40 | def __init__(self, parent=None):
41 | super(AntlrParserTreeViewer, self).__init__(parent)
42 | self.re_context = re.compile(r'([\w\d_]+)Context\'\>')
43 |
44 | self.tree = QTreeWidget()
45 | self.tree.setColumnCount(4)
46 | self.tree.setColumnWidth(0, 350)
47 | self.tree.setColumnWidth(1, 400)
48 | self.tree.setColumnWidth(2, 200)
49 | self.tree.setColumnWidth(3, 200)
50 | self.tree.setHeaderLabels(['name', 'Text', 'start', 'stop'])
51 |
52 | self.layout = QVBoxLayout()
53 | self.layout.addWidget(self.tree)
54 | self.setLayout(self.layout)
55 |
56 | self.setWindowTitle('AST Tree')
57 | self.resize(1200, 800)
58 |
59 | def set_data(self, py_obj):
60 | self.tree.clear()
61 | self.add_sub_tree(py_obj, self.tree)
62 |
63 | def add_sub_tree(self, node, parent):
64 | widget_item = QTreeWidgetItem(parent)
65 | if isinstance(node, ParserRuleContext):
66 | class_name = str(type(node))
67 | # print(class_name)
68 | m = self.re_context.search(class_name)
69 | short_name = m.group(1)
70 | widget_item.setText(0, short_name)
71 | widget_item.setText(1, '')
72 | widget_item.setText(2, str(node.start))
73 | widget_item.setText(3, str(node.stop))
74 | else:
75 | widget_item.setText(0, '')
76 | widget_item.setText(1, node.getText())
77 | widget_item.setText(2, '')
78 | widget_item.setText(3, '')
79 |
80 | if hasattr(node, 'children'):
81 | for child in getattr(node, 'children'):
82 | self.add_sub_tree(child, widget_item)
83 |
84 |
85 | if __name__ == '__main__':
86 |
87 | #############################################################
88 | # parse options
89 | # not support -gui -encoding -ps
90 | #############################################################
91 | usage = "Usage: %prog [options] Grammar_Name Start_Rule"
92 | parser = optparse.OptionParser(usage=usage)
93 | # parser.add_option('-t', '--tree',
94 | # dest="out_file",
95 | # default="default.out",
96 | # help='set output file name',
97 | # )
98 | parser.add_option('-t', '--tree',
99 | dest="tree",
100 | default=False,
101 | action='store_true',
102 | help='Print AST tree'
103 | )
104 | parser.add_option('-g', '--gui',
105 | dest="gui",
106 | default=False,
107 | action='store_true',
108 | help='show AST tree in GUI'
109 | )
110 | parser.add_option('-k', '--tokens',
111 | dest="token",
112 | default=False,
113 | action='store_true',
114 | help='Show Tokens'
115 | )
116 | parser.add_option('-s', '--sll',
117 | dest="sll",
118 | default=False,
119 | action='store_true',
120 | help='Show SLL'
121 | )
122 | parser.add_option('-d', '--diagnostics',
123 | dest="diagnostics",
124 | default=False,
125 | action='store_true',
126 | help='Enable diagnostics error listener'
127 | )
128 | parser.add_option('-a', '--trace',
129 | dest="trace",
130 | default=False,
131 | action='store_true',
132 | help='Enable Trace'
133 | )
134 | parser.add_option('-p', '--path',
135 | dest="search_path",
136 | action='append',
137 | help='seach path for Lexer/Parser files'
138 | )
139 |
140 | options, remainder = parser.parse_args()
141 |
142 | if len(remainder) < 2:
143 | print('ERROR: You have to provide at least 2 arguments!')
144 | parser.print_help()
145 | exit(1)
146 | else:
147 | grammar = remainder.pop(0)
148 | start_rule = remainder.pop(0)
149 | file_list = remainder
150 |
151 | #############################################################
152 | # check and load antlr generated files
153 | #############################################################
154 | # searching for file
155 | search_path = ['.']
156 | found = False
157 | found_path = ''
158 | if options.search_path is not None:
159 | search_path += options.search_path
160 | for p in search_path:
161 | lexer_file = os.path.join(p, grammar + 'Lexer.py')
162 | parser_file = os.path.join(p, grammar + 'Parser.py')
163 | if os.path.exists(lexer_file) and os.path.exists(parser_file):
164 | found = True
165 | found_path = p
166 | break
167 | if not found:
168 | print("[ERROR] Can't find lexer file {} or parser file {}!".format(grammar + 'Lexer.py', grammar + 'Parser.py'))
169 | print("search path : ", search_path)
170 | exit(1)
171 |
172 | # dynamic load the module and class
173 | lexerName = grammar + 'Lexer'
174 | parserName = grammar + 'Parser'
175 | sys.path.append(found_path)
176 |
177 | # print("Load Lexer {}".format(lexerName))
178 | module_lexer = __import__(lexerName, globals(), locals(), lexerName)
179 | class_lexer = getattr(module_lexer, lexerName)
180 |
181 | # print("Load Parser {}".format(parserName))
182 | module_parser = __import__(parserName, globals(), locals(), parserName)
183 | class_parser = getattr(module_parser, parserName)
184 |
185 |
186 | #############################################################
187 | # main process steps.
188 | #############################################################
189 | def process(input_stream, class_lexer, class_parser):
190 | lexer = class_lexer(input_stream)
191 | token_stream = CommonTokenStream(lexer)
192 | token_stream.fill()
193 | if options.token: # need to show token
194 | for tok in token_stream.tokens:
195 | print(tok)
196 | if start_rule == 'tokens':
197 | return
198 |
199 | parser = class_parser(token_stream)
200 |
201 | if options.diagnostics:
202 | parser.addErrorListener(DiagnosticErrorListener())
203 | parser._interp.predictionMode = PredictionMode.LL_EXACT_AMBIG_DETECTION
204 | if options.tree or options.gui:
205 | parser.buildParseTrees = True
206 | if options.sll:
207 | parser._interp.predictionMode = PredictionMode.SLL
208 | # parser.setTokenStream(token_stream)
209 | parser.setTrace(options.trace)
210 | if hasattr(parser, start_rule):
211 | func_start_rule = getattr(parser, start_rule)
212 | parser_ret = func_start_rule()
213 | if options.tree:
214 | if options.gui:
215 | app = QApplication(sys.argv)
216 | viewer = AntlrParserTreeViewer()
217 | viewer.set_data(parser_ret)
218 | viewer.show()
219 | ret = app.exec_()
220 | sys.exit(ret)
221 | else:
222 | lisp_tree_str = parser_ret.toStringTree(recog=parser)
223 | print(beautify_lisp_string(lisp_tree_str))
224 | else:
225 | print("[ERROR] Can't find start rule '{}' in parser '{}'".format(start_rule, parserName))
226 |
227 |
228 | #############################################################
229 | # use stdin if not provide file as input stream
230 | #############################################################
231 | if len(file_list) == 0:
232 | input_stream = InputStream(sys.stdin.read())
233 | process(input_stream, class_lexer, class_parser)
234 | exit(0)
235 |
236 | #############################################################
237 | # iterate all input file
238 | #############################################################
239 | for file_name in file_list:
240 | if os.path.exists(file_name) and os.path.isfile(file_name):
241 | input_stream = FileStream(file_name)
242 | process(input_stream, class_lexer, class_parser)
243 | else:
244 | print("[ERROR] file {} not exist".format(os.path.normpath(file_name)))
245 |
--------------------------------------------------------------------------------
/img/IDEA.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jszheng/py3antlr4book/669f1f76d9d8ab54dfc79cf76ede2e08565abf98/img/IDEA.png
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | antlr4-python3-runtime==4.8
2 | PySide2==5.15.0
3 | shiboken2==5.15.0
4 |
--------------------------------------------------------------------------------