├── .github └── FUNDING.yml ├── GPL2.txt ├── MIT.txt ├── Makefile ├── Makefile.clang ├── Makefile.common ├── Makefile.gcc ├── README.md ├── inc ├── analyse.h ├── dhelper.h ├── json.h ├── parse.h ├── parse_atomic.h ├── parse_base.h ├── parse_text.h ├── parsedef.h ├── tokencollisionhelper.h └── vhelper.h └── test ├── test_analyse.cpp ├── test_grammar.cpp ├── test_main.cpp ├── test_pascal.cpp ├── test_rules.cpp ├── test_stream.cpp └── test_trace.cpp /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [ "MoserMichael" ] 4 | 5 | -------------------------------------------------------------------------------- /GPL2.txt: -------------------------------------------------------------------------------- 1 | cppcombinator parser generator library 2 | Copyright (C) 2020 Michael Moser 3 | 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | as published by the Free Software Foundation; either version 2 7 | of the License, or (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program; if not, write to the Free Software 16 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 | -------------------------------------------------------------------------------- /MIT.txt: -------------------------------------------------------------------------------- 1 | cppcombinator parser generator library 2 | Copyright 2020 Michael Moser 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all : 3 | make -f Makefile.gcc 4 | make -f Makefile.clang 5 | 6 | 7 | .PHONY: clean 8 | clean : 9 | make -f Makefile.gcc clean 10 | make -f Makefile.clang clean 11 | 12 | .PHONY: test 13 | test : 14 | make -f Makefile.gcc test 15 | make -f Makefile.clang test 16 | 17 | -------------------------------------------------------------------------------- /Makefile.clang: -------------------------------------------------------------------------------- 1 | 2 | OBJ_DIR:=.obj-clang 3 | COMP:=clang 4 | EXE:=test-clang 5 | 6 | #CPP_OPTS=-std=c++17 -pedantic -Wall -Wextra -Wshadow -Werror 7 | CPP_OPTS=-g -std=c++17 -pedantic -Wall -Wshadow -Werror 8 | LD_OPTS=-lstdc++ -lgtest -lpthread 9 | 10 | ifeq ($(NO_OPT),) 11 | CPP_OPTS+=-O3 12 | endif 13 | 14 | 15 | -include Makefile.common 16 | 17 | -------------------------------------------------------------------------------- /Makefile.common: -------------------------------------------------------------------------------- 1 | 2 | TEST_FILES=test/test_stream.cpp \ 3 | test/test_rules.cpp \ 4 | test/test_grammar.cpp \ 5 | test/test_trace.cpp \ 6 | test/test_analyse.cpp \ 7 | test/test_pascal.cpp \ 8 | test/test_main.cpp 9 | 10 | TEST_OBJS:=$(subst .cpp,.o,$(TEST_FILES)) 11 | 12 | LINK_OBJS:=$(patsubst %.cpp,$(OBJ_DIR)/%.o,$(TEST_FILES)) 13 | 14 | .PHOHY: all 15 | all : $(EXE) 16 | 17 | 18 | .PHONY: make-obj-dir 19 | make-obj-dir: 20 | @mkdir -p $(OBJ_DIR)/test 21 | 22 | test-clang: make-obj-dir $(LINK_OBJS) 23 | $(COMP) -g -o test-clang $(LINK_OBJS) $(LD_OPTS) 24 | 25 | test-gcc : make-obj-dir $(LINK_OBJS) 26 | $(COMP) -g -o test-gcc $(LINK_OBJS) $(LD_OPTS) 27 | 28 | .PHONY: test 29 | test : $(EXE) 30 | @echo "test with $(COMPI)" 31 | ./$(EXE) 32 | 33 | .PHONY: clean 34 | clean : 35 | rm -rf $(OBJ_DIR) $(EXE) 36 | 37 | $(OBJ_DIR)/%.o: %.cpp 38 | $(COMP) $(CPP_OPTS) -Iinc -c $< -o $@ 39 | -------------------------------------------------------------------------------- /Makefile.gcc: -------------------------------------------------------------------------------- 1 | 2 | OBJ_DIR:=.obj-gcc 3 | COMP:=gcc 4 | EXE:=test-gcc 5 | 6 | #CPP_OPTS=-std=c++17 -pedantic -Wall -Wextra -Wshadow -Werror 7 | CPP_OPTS=-g -std=c++17 -pedantic -Wall -Wshadow -Werror 8 | 9 | ifeq ($(NO_OPT),) 10 | CPP_OPTS+=-O3 11 | endif 12 | 13 | LD_OPTS=-lstdc++ -lgtest -lpthread 14 | 15 | -include Makefile.common 16 | 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # cppcombinator 3 | 4 | - [cppcombinator](#cppcombinator) 5 | - [Introduction](#introduction) 6 | - [Bounded buffer](#bounded-buffer) 7 | - [Parser reference](#parser-reference) 8 | * [Parser combinators](#parser-combinators) 9 | + [Ordered choice](#ordered-choice) 10 | + [Sequence](#sequence) 11 | + [Optional](#optional) 12 | + [Require end of file after parsing the input type.](#require-end-of-file-after-parsing-the-input-type) 13 | + [Parse repetition of term](#parse-repetition-of-term) 14 | + [Zero or more](#zero-or-more) 15 | + [One or more](#one-or-more) 16 | + [parse type T with And Predicate LookaheadType](#parse-type-t-with-and-predicate-lookaheadtype) 17 | + [parse type T with Not Predicate LookaheadType](#parse-type-t-with-not-predicate-lookaheadtype) 18 | * [Atomic parsers](#atomic-parsers) 19 | + [Fixed string token parser](#fixed-string-token-parser) 20 | + [Extension parser](#extension-parser) 21 | + [Parse unsigned integer (sequence of digits)](#parse-unsigned-integer--sequence-of-digits-) 22 | + [parse a single printable character](#parse-a-single-printable-character) 23 | + [parse a cstyle identifier](#parse-a-cstyle-identifier) 24 | - [Validation of grammar](#validation-of-grammar) 25 | - [Dumping of abstract syntax tree to json.](#dumping-of-abstract-syntax-tree-to-json) 26 | - [Tracing the generated parser](#tracing) 27 | - [What i learned from all this](#what-i-learned-from-all-this) 28 | 29 | Table of contents generated with markdown-toc 30 | 31 | 32 | 33 | # Introduction 34 | 35 | This library is a header only C++17 template library. 36 | 37 | The library provides template classes for building top-down [PEG parser](https://en.wikipedia.org/wiki/Parsing_expression_grammar), the parsers build an in-memory parse tree, 38 | Each template instantiation stands for a grammar rule, which translates directly into a node in the parse tree. 39 | 40 | 41 | # Example usage 42 | 43 | To use this library #include "parse.h" - the classes of this library are in namespace pparse. 44 | 45 | ``` 46 | #include "parse.h" 47 | 48 | using namespace pparse; 49 | 50 | ``` 51 | 52 | Lets dive into an example grammar: 53 | 54 | Each grammar rule is represented by the instantation of some template, lets look at this example grammar: 55 | The library provides template classes that can express a terminal or non terminal gramar rule. 56 | 57 | 58 | Terminal rules in this example: 59 | 60 | 61 | ``` 62 | TokInt<1> 63 | ``` 64 | 65 | terminal rule that consumes a positive integer (regex \d+). Note that the integer constant 1 will identify the node in the parse tree, this id is always the first argument to a template rule 66 | 67 | ``` 68 | PTok<3, CSTR1("*")> 69 | ``` 70 | terminal rule that consumes a fixed string * - the string is one character in length, hence CSTR1("*"). 71 | 72 | 73 | Non terminal rules in this example: 74 | 75 | ``` 76 | PAny<2, PTok<3,CSTR1("*")>, PTok<4,CSTR1("/")> > 77 | ``` 78 | 79 | PAny will match any one of the grammar rules provided as template arguments, in this examples these are either a fixed string * (expressed by PTok<3,CSTR1("*")>) or the fixed string / (expressed by PTok<4,CSTR1("/")>) ; 80 | 81 | 82 | ``` 83 | PSeq<10, PTok<11, CSTR1("-")>, TokInt<1> > 84 | ``` 85 | 86 | PSeq will match the sequence of grammar rules provided as template arguments. in this example this is the fixed string - following by a positive integer. 87 | 88 | ``` 89 | PRequireEof 90 | ``` 91 | 92 | PRequireEof will parse the argument type Expr, but will require it to be followed by whitespaces until the end of input. 93 | 94 | 95 | And example grammar for matching an arithmetic expression; note that you can give each production rule a name by deriving it from the template epression that stands for the right hand side of the production rule. 96 | 97 | ``` 98 | struct Int : PTokInt<1> {}; // terminal rule: consumes an integer (\d+) 99 | 100 | struct Expr; 101 | 102 | struct Mult : PAny<2, PTok<3,CSTR1("*")>, PTok<4,CSTR1("/")> > {};q 103 | 104 | struct Add : PAny<4, PTok<5,CSTR1("+")>, PTok<6,CSTR1("-")> > {}; 105 | 106 | struct NestedExpr : PSeq<7, PTok<8, CSTR1("(")>, Expr, PTok<9, CSTR1(")")> > {}; 107 | 108 | struct NegativeInt : PSeq<10, PTok<11, CSTR1("-")>, Int> {}; 109 | 110 | struct SimpleExpr : PAny<12, Int, NegativeInt, NestedExpr> {}; // 111 | 112 | struct MultExpr; 113 | 114 | struct MultExpr: PAny<13, PSeq<14, SimpleExpr, Mult, MultExpr >, SimpleExpr> {}; 115 | 116 | struct Expr; 117 | 118 | struct Expr: PAny<15, PSeq<16, MultExpr, Add, Expr >, MultExpr> {}; 119 | 120 | struct ExprEof : PRequireEof {}; 121 | ``` 122 | 123 | Each grammar rule (like struct Expr : PAny<12, Int, NegativeInt, NestedExpr>) has a nested AstType (Expr::AstType) ; this type defines the record in the abstract syntax tree 124 | that is produced by the parser on successfull parsing of the input text. This ast type uses c++17 helper objects for its representation, for example the PAny<ruleId,Parser1,Parser2...> parser uses std::variant to stand for any one of the argument types Parser1, Parser2 ... while PSeq<ruleId,Parser1,Parser2> uses std::tuple for the sequence of the argument parsers. 125 | 126 | Please note that the first argument of each rule is an integer constant that identifies the parser type - the rule id is returned from each ast type with getRuleId(); 127 | Note that for a real gramar you should proabably define an enumeration and use the enumeration values instead of integer constants. 128 | 129 | To call the parser you first need to create an input stream object. 130 | 131 | ``` 132 | bool isok = text_stream.open("input_file.txt"); 133 | 134 | or 135 | 136 | fd = open("input_file",O_RDONLY); 137 | bool isok = text_stream.open(fd); 138 | 139 | or you can create it with invalid file descriptor and feed it input by yourself: 140 | 141 | bool isok = text_stream.open(-1); 142 | isok = text_stream.write_tail(test_string, strlen(test_string) ); 143 | 144 | 145 | ``` 146 | 147 | The input stream is implemented as a circular buffer, by default a lookahead buffer of 4k is allocated. 148 | 149 | Then to call the parser: 150 | 151 | ``` 152 | // create a base character parser 153 | CharParser chparser(text_stream); 154 | 155 | // call the parser with the main term of the grammar 156 | Parse_result res = ExprEof::parse(chparser); 157 | ``` 158 | 159 | Note that the parse call is instantiated with the top level rule of your grammar. 160 | the result object has res.success() - this should be true if the input text has been parsed successfully. It not then res.get_start_pos() returns the line and column of the error. If parsing succeed then res.get_ast() returns the Ast type that stands for the parsed output, res.get_start_pos() and res.get_end_pos() are the location of the beginning and and of the parsed structure. 161 | 162 | Note that a CharParser is wrapping the text stream object: this way it is possible to craft a base parser for a particular grammar that consumes comments, in that case comments will not have to be dealt with by the grammar. 163 | 164 | # Bounded buffer 165 | 166 | This parser library has one optimization for the read ahead buffer: if one is parsing a top level rule then all input is discarded when the top level rule has been parsed: 167 | In the previous example the following rule is a top level rule PAny<15, PSeq<16, MultExpr, Add, Expr >, MultExpr> - it stands for a repetition of a choice of either one of: MultExpr or the nested sequence PSeq<16, MultExpr, Add, Expr >. For each instances after the first repetition has been parsed we can discard all input up to that point. 168 | This should keep the lookahead buffer bounded for most cases. (however the lookahead buffer will be reallocated if you really need a larger buffer). 169 | 170 | # Parser rule reference 171 | 172 | a reference of all parsing rules provided by this library: 173 | 174 | 175 | ## Parser combinators 176 | 177 | Given any existing parsing types, a new parsing expression can be constructed using the following operators: 178 | 179 | 180 | ### Ordered choice 181 | 182 | ``` 183 | template 184 | struct PAny 185 | ``` 186 | 187 | This type allows for recursive definitions, that means the argument types do not have to be defined when defining a PAny rule. 188 | 189 | ### Sequence 190 | 191 | ``` 192 | template 193 | struct PSeq 194 | ``` 195 | 196 | The nested AST type (PSeq::AstType) looks as follows 197 | 198 | ``` 199 | struct AstType : AstEntryBase { 200 | AstType() : AstEntryBase(ruleId) { 201 | } 202 | 203 | std::tuple...> entry_; 204 | }; 205 | ``` 206 | 207 | 208 | This type allows for recursive definitions, that means the argument types do not have to be defined when defining a PAny rule. 209 | 210 | ### Optional 211 | 212 | ``` 213 | template 214 | struct POpt 215 | ``` 216 | The nested AST type (POpt::AstType) looks as follows 217 | 218 | ``` 219 | using PTypePtr = typename std::unique_ptr; 220 | 221 | using OptionType = std::optional< PTypePtr >; 222 | 223 | struct AstType : AstEntryBase { 224 | AstType() : AstEntryBase(ruleId) { 225 | } 226 | 227 | OptionType entry_; 228 | }; 229 | 230 | ``` 231 | 232 | 233 | ### Require end of file after parsing the input type. 234 | 235 | ``` 236 | template 237 | PRequireEof 238 | ``` 239 | 240 | The AST of the type Type is forwarded by this parser. 241 | 242 | Note that this is the only parser that doesn't backtrack if parsing of Type fails. 243 | This is done on purpose - if the nested parser is a repetition parser (like zero or more or one or more), then each instance of the nested parser will advance the input and discard the buffer upon parsing an instance of the term (see Bounded Buffer section) 244 | 245 | This rule does not generate a node in the parse tree, it therefore does not need an identifying constant as the first template argument 246 | 247 | 248 | Please note that the parser for this type must be declared after the declaration of the argument type Type. Forward declarations are not possible as the Ast type of the argument type is used as is. 249 | 250 | ### Parse repetition of term 251 | 252 | ``` 253 | template 254 | struct PRepeat 255 | ``` 256 | 257 | The AST of the type Type is forwarded by this parser. 258 | 259 | ### Zero or more 260 | 261 | ``` 262 | template 263 | struct PStar 264 | ``` 265 | 266 | The AST of the type Type is forwarded by this parser. 267 | 268 | ### One or more 269 | 270 | ``` 271 | template 272 | struct PPlus 273 | ``` 274 | 275 | The AST of the type Type is forwarded by this parser. 276 | 277 | 278 | 279 | ### parse type T with And Predicate LookaheadType 280 | 281 | ``` 282 | template 283 | struct PWithAndLookahead ` 284 | ``` 285 | 286 | parsing is successfull if T parses, on condition that it is followed by input parsed by LookaheadType. 287 | The AST of the type Type is forwarded by this parser. The AST object generated upon parsing by LookaheadType is discarded. 288 | 289 | 290 | ### parse type T with Not Predicate LookaheadType 291 | 292 | 293 | ``` 294 | template 295 | struct PWithNotLookahead 296 | ``` 297 | 298 | parsing is successfull if T parses, on condition that it is not followed by input parsed by LookaheadType. 299 | The AST of the type Type is forwarded by this parser. The AST object generated upon parsing by LookaheadType is discarded. 300 | 301 | 302 | ## Atomic parsers 303 | 304 | The following parser types consume terminal symbols 305 | 306 | ### Fixed string token parser 307 | 308 | ``` 309 | template 310 | struct PTok : ParserBase { 311 | ``` 312 | 313 | Note that this parser receives a sequence of characters, as currrently there is no way to instantiate a template with a string constant (that feature was added in c++20). 314 | However thre are macros that expand an argument string into a character sequence: 315 | 316 | Example usage: 317 | ``` 318 | PTok<1,CSTR2("if")> 319 | PTok<3,CSTR4("else)> 320 | PTok<4,CSTR5("while)> 321 | ``` 322 | 323 | ### Extension parser 324 | 325 | ``` 326 | enum class Char_checker_result { 327 | error, 328 | proceed, 329 | acceptNow, 330 | acceptUnget, 331 | }; 332 | 333 | typedef Char_checker_result (PTokVar_cb_t) (Char_t current_char, bool iseof, Char_t *matched_so_far); 334 | 335 | template 336 | struct PTokVar : ParserBase { 337 | ``` 338 | 339 | The nested AST type of this parser includes the parsed expression is string entry_. 340 | 341 | ``` 342 | struct AstType : AstEntryBase { 343 | AstType() : AstEntryBase(ruleId) { 344 | } 345 | 346 | std::string entry_; 347 | }; 348 | ``` 349 | 350 | 351 | ### Parse unsigned integer (sequence of digits) 352 | 353 | ``` 354 | template 355 | struct PTokInt : PTokVar { 356 | } 357 | ``` 358 | 359 | ### parse a single printable character 360 | 361 | ``` 362 | template 363 | struct PTokChar 364 | ``` 365 | 366 | ### parse a cstyle identifier 367 | 368 | ``` 369 | template 370 | struct PTokIdentifierCStyle 371 | ``` 372 | 373 | Parses an identifier that maches the following regex [a-zA-Z][a-zA-Z0-9\_]* 374 | Special care is taken that tokens that were accepted by PTok are not accepted as identifiers. 375 | Please note that this trick works only if the top level parser is of type PRequireEof or of type PTopLevelParser. 376 | 377 | # Validation of grammar 378 | 379 | You can validate the generated grammar for left recursion errors. 380 | To enable this feature you need to place #define __PARSER_ANALYSE__ before #include "parse.h" 381 | 382 | Then call the analysis step with the following call 383 | ``` 384 | bool no_cycles = ParserChecker::check(std::cout); 385 | EXPECT_TRUE(no_cycles); 386 | ``` 387 | Note that the check call is instantiated with the top level rule of your grammar. 388 | It even tells you what is causing the left recursion, the error is written on the stream that is passed as argument to the check function 389 | 390 | # Dumping of abstract syntax tree to json. 391 | 392 | You can dump the ast into json: 393 | 394 | ``` 395 | ExprEof::dumpJson(std::cout, (ExprEof::AstType *) result.get_ast() ); 396 | ``` 397 | please note that this feature requires RTTI support enabled. 398 | 399 | 400 | 401 | # Tracing the generated parser 402 | 403 | You can trace the running of the parser, this can be quite usefull for debugging purposes. 404 | To do you need to place #define __PARSER_TRACE__ right before #include "parse.h" 405 | 406 | The execution trace is dumpedto standard output and looks like this: 407 | Please note that this feature requires RTTI support enabled. 408 | 409 | ``` 410 | (000)start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:0 offset: 0) 411 | (001) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:0 offset: 0) 412 | (002) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 413 | (003) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 414 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 415 | (005) start parsing: PSeq<1, pparse::PTok<11> Int> at: (1:0 offset: 0) 416 | (005) end parsing: PSeq<1, pparse::PTok<11> Int> SUCCESS at: (1:2 offset: 2) 417 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 418 | (003) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:0 offset: 0) 419 | (003) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 420 | (004) start parsing: PSeq<1, pparse::PTok<11> Int> at: (1:0 offset: 0) 421 | (004) end parsing: PSeq<1, pparse::PTok<11> Int> SUCCESS at: (1:2 offset: 2) 422 | (003) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 423 | (002) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 424 | (001) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:0 offset: 0) 425 | (001) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 426 | (002) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 427 | (003) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 428 | (004) start parsing: PSeq<1, pparse::PTok<11> Int> at: (1:0 offset: 0) 429 | (004) end parsing: PSeq<1, pparse::PTok<11> Int> SUCCESS at: (1:2 offset: 2) 430 | (003) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 431 | (002) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:0 offset: 0) 432 | (002) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 433 | (003) start parsing: PSeq<1, pparse::PTok<11> Int> at: (1:0 offset: 0) 434 | (003) end parsing: PSeq<1, pparse::PTok<11> Int> SUCCESS at: (1:2 offset: 2) 435 | (002) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 436 | (001) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 437 | (000)end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:2 offset: 2) 438 | (000)start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:0 offset: 0) 439 | (001) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:0 offset: 0) 440 | (002) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 441 | (003) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 442 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 443 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:1 offset: 1) 444 | (004) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 445 | (005) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 446 | (006) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 447 | (006) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 448 | (005) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 449 | (005) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 450 | (005) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 451 | (004) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:5 offset: 5) 452 | (003) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:5 offset: 5) 453 | (002) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 454 | (001) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:0 offset: 0) 455 | (001) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 456 | (002) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 457 | (003) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 458 | (003) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:1 offset: 1) 459 | (003) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 460 | (004) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 461 | (005) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 462 | (005) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 463 | (004) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 464 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 465 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 466 | (003) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:5 offset: 5) 467 | (002) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:5 offset: 5) 468 | (001) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 469 | (000)end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:5 offset: 5) 470 | (000)start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:0 offset: 0) 471 | (001) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:0 offset: 0) 472 | (002) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 473 | (003) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 474 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 475 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:1 offset: 1) 476 | (003) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:0 offset: 0) 477 | (003) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 478 | (003) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:1 offset: 1) 479 | (002) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:1 offset: 1) 480 | (002) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:3 offset: 3) 481 | (003) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:3 offset: 3) 482 | (004) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 483 | (005) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 484 | (006) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 485 | (006) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 486 | (005) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 487 | (005) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 488 | (005) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 489 | (004) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:5 offset: 5) 490 | (004) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:7 offset: 7) 491 | (005) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:7 offset: 7) 492 | (006) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:7 offset: 7) 493 | (007) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:7 offset: 7) 494 | (008) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 495 | (008) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 496 | (007) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:7 offset: 7) 497 | (007) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 498 | (007) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 499 | (006) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 500 | (005) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:7 offset: 7) 501 | (005) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:7 offset: 7) 502 | (006) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:7 offset: 7) 503 | (007) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 504 | (007) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 505 | (006) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:7 offset: 7) 506 | (006) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 507 | (006) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 508 | (005) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 509 | (004) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 510 | (003) end parsing: PSeq<6, MultExpr, Add, Expr> SUCCESS at: (1:9 offset: 9) 511 | (002) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 512 | (001) end parsing: PSeq<6, MultExpr, Add, Expr> SUCCESS at: (1:9 offset: 9) 513 | (000)end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 514 | (000)start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:0 offset: 0) 515 | (001) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:0 offset: 0) 516 | (002) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 517 | (003) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 518 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 519 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:1 offset: 1) 520 | (003) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:0 offset: 0) 521 | (003) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 522 | (003) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:1 offset: 1) 523 | (002) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:1 offset: 1) 524 | (002) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:3 offset: 3) 525 | (003) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:3 offset: 3) 526 | (004) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 527 | (005) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 528 | (006) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 529 | (006) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 530 | (005) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 531 | (005) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 532 | (005) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:5 offset: 5) 533 | (004) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:5 offset: 5) 534 | (004) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:7 offset: 7) 535 | (005) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:7 offset: 7) 536 | (006) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:7 offset: 7) 537 | (007) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:7 offset: 7) 538 | (008) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 539 | (008) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 540 | (007) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:7 offset: 7) 541 | (007) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 542 | (007) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 543 | (006) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 544 | (006) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:11 offset: 11) 545 | (007) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:11 offset: 11) 546 | (008) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:11 offset: 11) 547 | (009) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:11 offset: 11) 548 | (010) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:11 offset: 11) 549 | (010) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:13 offset: 13) 550 | (010) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:15 offset: 15) 551 | (011) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:15 offset: 15) 552 | (012) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:15 offset: 15) 553 | (012) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 554 | (011) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:15 offset: 15) 555 | (011) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:15 offset: 15) 556 | (011) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 557 | (010) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:17 offset: 17) 558 | (009) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:17 offset: 17) 559 | (008) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 560 | (007) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:11 offset: 11) 561 | (007) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:11 offset: 11) 562 | (008) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:11 offset: 11) 563 | (009) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:11 offset: 11) 564 | (009) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:13 offset: 13) 565 | (009) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:15 offset: 15) 566 | (010) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:15 offset: 15) 567 | (011) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:15 offset: 15) 568 | (011) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 569 | (010) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:15 offset: 15) 570 | (010) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:15 offset: 15) 571 | (010) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 572 | (009) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:17 offset: 17) 573 | (008) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:17 offset: 17) 574 | (007) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 575 | (006) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:17 offset: 17) 576 | (005) end parsing: PSeq<6, MultExpr, Add, Expr> SUCCESS at: (1:17 offset: 17) 577 | (004) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 578 | (003) end parsing: PSeq<6, MultExpr, Add, Expr> SUCCESS at: (1:17 offset: 17) 579 | (002) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 580 | (001) end parsing: PSeq<6, MultExpr, Add, Expr> SUCCESS at: (1:17 offset: 17) 581 | (000)end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 0 at: (1:17 offset: 17) 582 | (000)start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:0 offset: 0) 583 | (001) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:0 offset: 0) 584 | (002) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:0 offset: 0) 585 | (003) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:0 offset: 0) 586 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 587 | (005) start parsing: PSeq<1, pparse::PTok<11> Int> at: (1:0 offset: 0) 588 | (005) end parsing: PSeq<1, pparse::PTok<11> Int> FAIL at: (1:0 offset: 0) 589 | (005) start parsing: PSeq<8, pparse::PTok<0> Expr, pparse::PTok<0> > at: (1:0 offset: 0) 590 | (006) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:1 offset: 1) 591 | (007) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:1 offset: 1) 592 | (008) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:1 offset: 1) 593 | (009) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:1 offset: 1) 594 | (010) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:1 offset: 1) 595 | (010) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:2 offset: 2) 596 | (010) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 597 | (011) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 598 | (012) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 599 | (012) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 600 | (011) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 601 | (011) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 602 | (011) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 603 | (010) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:4 offset: 4) 604 | (009) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:4 offset: 4) 605 | (008) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 606 | (007) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:1 offset: 1) 607 | (007) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:1 offset: 1) 608 | (008) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:1 offset: 1) 609 | (009) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:1 offset: 1) 610 | (009) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:2 offset: 2) 611 | (009) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 612 | (010) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 613 | (011) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 614 | (011) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 615 | (010) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 616 | (010) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 617 | (010) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 618 | (009) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:4 offset: 4) 619 | (008) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:4 offset: 4) 620 | (007) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 621 | (006) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:4 offset: 4) 622 | (005) end parsing: PSeq<8, pparse::PTok<0> Expr, pparse::PTok<0> > SUCCESS at: (1:5 offset: 5) 623 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 2 at: (1:5 offset: 5) 624 | (003) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:0 offset: 0) 625 | (003) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:0 offset: 0) 626 | (004) start parsing: PSeq<1, pparse::PTok<11> Int> at: (1:0 offset: 0) 627 | (004) end parsing: PSeq<1, pparse::PTok<11> Int> FAIL at: (1:0 offset: 0) 628 | (004) start parsing: PSeq<8, pparse::PTok<0> Expr, pparse::PTok<0> > at: (1:0 offset: 0) 629 | (005) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:1 offset: 1) 630 | (006) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:1 offset: 1) 631 | (007) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:1 offset: 1) 632 | (008) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:1 offset: 1) 633 | (009) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:1 offset: 1) 634 | (009) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:2 offset: 2) 635 | (009) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 636 | (010) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 637 | (011) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 638 | (011) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 639 | (010) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 640 | (010) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 641 | (010) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 642 | (009) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:4 offset: 4) 643 | (008) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:4 offset: 4) 644 | (007) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 645 | (006) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:1 offset: 1) 646 | (006) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:1 offset: 1) 647 | (007) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:1 offset: 1) 648 | (008) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:1 offset: 1) 649 | (008) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:2 offset: 2) 650 | (008) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:3 offset: 3) 651 | (009) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:3 offset: 3) 652 | (010) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 653 | (010) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 654 | (009) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:3 offset: 3) 655 | (009) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:3 offset: 3) 656 | (009) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 657 | (008) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:4 offset: 4) 658 | (007) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> SUCCESS at: (1:4 offset: 4) 659 | (006) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 0 at: (1:4 offset: 4) 660 | (005) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:4 offset: 4) 661 | (004) end parsing: PSeq<8, pparse::PTok<0> Expr, pparse::PTok<0> > SUCCESS at: (1:5 offset: 5) 662 | (003) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 2 at: (1:5 offset: 5) 663 | (002) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:5 offset: 5) 664 | (002) start parsing: PAny<7, pparse::PSeq<6> MultExpr> at: (1:7 offset: 7) 665 | (003) start parsing: PSeq<6, MultExpr, Add, Expr> at: (1:7 offset: 7) 666 | (004) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:7 offset: 7) 667 | (005) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:7 offset: 7) 668 | (006) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 669 | (006) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 670 | (005) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:7 offset: 7) 671 | (005) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 672 | (005) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 673 | (004) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 674 | (003) end parsing: PSeq<6, MultExpr, Add, Expr> FAIL at: (1:7 offset: 7) 675 | (003) start parsing: PAny<7, pparse::PSeq<6> SimpleExpr> at: (1:7 offset: 7) 676 | (004) start parsing: PSeq<6, SimpleExpr, Mult, MultExpr> at: (1:7 offset: 7) 677 | (005) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 678 | (005) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 679 | (004) end parsing: PSeq<6, SimpleExpr, Mult, MultExpr> FAIL at: (1:7 offset: 7) 680 | (004) start parsing: PAny<6, Int, NegativeInt, NestedExpr> at: (1:7 offset: 7) 681 | (004) end parsing: PAny<6, Int, NegativeInt, NestedExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 682 | (003) end parsing: PAny<7, pparse::PSeq<6> SimpleExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 683 | (002) end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 1 at: (1:9 offset: 9) 684 | (001) end parsing: PSeq<6, MultExpr, Add, Expr> SUCCESS at: (1:9 offset: 9) 685 | (000)end parsing: PAny<7, pparse::PSeq<6> MultExpr> SUCCESS choice-index: 0 at: (1:9 offset: 9) 686 | 687 | ``` 688 | 689 | 690 | # What i learned from all this 691 | 692 | i learned that most of additions in c++17 are tooled toward template metaprogramming: things like std::tuple and std::variant are very useful in this context. 693 | i think template metaprogramming is now a doable thing ever since c++17; it takes some time to get used to the concept - this project has helped me to learn this approach well, i think. 694 | 695 | A surprising thing is that [member templates](https://en.cppreference.com/w/cpp/language/member_template) let you do recursive definition, that's because of the lazy natue of template instantiation in c++. You never stop learning with c++. 696 | 697 | Also the parser visualization feature of this projects shows that PEG grammars really do an enormous amount of backtracking; If you need a very performant parser then you should probably stick with lex and yacc - here you get something without backtracking, however the process of writing a grammar in lex and yacc is a really complicated business. Writing a PEG grammar is much easier, and more flexible in terms of the grammar that is accepted. 698 | 699 | Also i learned a few things from looking at the [PEGTL](https://github.com/taocpp/PEGTL) project. Thanks and acknowledgements. 700 | 701 | # License 702 | 703 | This code is licensed under both MIT and GPLv2 licenses. Pick any license that you like. 704 | -------------------------------------------------------------------------------- /inc/analyse.h: -------------------------------------------------------------------------------- 1 | #ifdef __PARSER_ANALYSE__ 2 | #include 3 | #include 4 | 5 | #include "dhelper.h" 6 | 7 | namespace pparse { 8 | 9 | struct CycleDetectorHelper { 10 | 11 | struct Frame { 12 | const std::type_info *info_; 13 | int subterm_; 14 | std::string typename_; 15 | }; 16 | 17 | using FrameList = std::list; 18 | 19 | bool push_and_check(std::ostream &out, const std::type_info &typid, int subterm ) { 20 | 21 | bool no_cycles = true; 22 | 23 | for(auto pos=stack_.rbegin(); pos != stack_.rend(); ++pos) { 24 | Frame &frame = *pos; 25 | if (typid == *frame.info_) { 26 | // cycle detected. 27 | out << "cycle detected: \n"; 28 | for(auto it = pos.base(); it != stack_.end(); ++it ) { 29 | out << "\t" << demangle(*it->info_); 30 | if (it->subterm_ != -1) { 31 | out << " term: " << it->subterm_; 32 | } 33 | out << "\n"; 34 | } 35 | out << demangle(typid); 36 | out << "\n"; 37 | no_cycles = false; 38 | } 39 | } 40 | stack_.push_back( Frame{ &typid, subterm } ); 41 | return no_cycles; 42 | } 43 | 44 | void pop() { 45 | stack_.pop_back(); 46 | } 47 | 48 | FrameList stack_; 49 | }; 50 | 51 | template 52 | static const std::type_info &get_tinfo(Info *ptr) { 53 | return typeid(ptr); 54 | } 55 | 56 | 57 | template 58 | struct ParserChecker { 59 | static bool check(std::ostream &stream) { 60 | CycleDetectorHelper helper; 61 | 62 | if (!helper.push_and_check(stream, get_tinfo((Grammar *) nullptr), -1)) { 63 | return false; 64 | } 65 | 66 | bool ret = Grammar::verify_no_cycles((void *) nullptr, helper, stream); 67 | 68 | helper.pop(); 69 | 70 | return !ret; 71 | } 72 | }; 73 | 74 | } //namespace 75 | 76 | #endif 77 | 78 | -------------------------------------------------------------------------------- /inc/dhelper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #if defined(__PARSER_TRACE__) || defined(__PARSER_ANALYSE__) 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | namespace pparse { 17 | 18 | // 19 | // This trick is copied from: https://stackoverflow.com/questions/37764459/getting-class-name-as-a-string-in-static-method-in-c . Thanks! 20 | // But now the tracing feature depends on the presence of RTTI. Don't know any better than this. 21 | // 22 | 23 | struct demangled_string 24 | { 25 | using ptr_type = std::unique_ptr; 26 | demangled_string(ptr_type&& ptr) noexcept; 27 | const char* c_str() const; 28 | operator std::string() const; 29 | 30 | std::ostream& write(std::ostream& os) const; 31 | private: 32 | ptr_type _ptr; 33 | }; 34 | 35 | inline std::ostream& operator<<(std::ostream& os, const demangled_string& str) 36 | { 37 | return str.write(os); 38 | } 39 | 40 | inline std::string operator+ (std::string l, const demangled_string& r) { 41 | return l + r.c_str(); 42 | } 43 | 44 | inline std::string operator+(const demangled_string& l, const std::string& r) 45 | { 46 | return std::string(l) + r; 47 | } 48 | 49 | demangled_string demangle(const char* name); 50 | demangled_string demangle(const std::type_info& type); 51 | demangled_string demangle(std::type_index type); 52 | 53 | template 54 | demangled_string demangle(T* p) { 55 | return demangle(typeid(*p)); 56 | } 57 | 58 | template 59 | demangled_string demangle() 60 | { 61 | return demangle(typeid(T)); 62 | } 63 | 64 | 65 | // implementation 66 | 67 | inline demangled_string::demangled_string(ptr_type&& ptr) noexcept 68 | : _ptr(std::move(ptr)) 69 | {} 70 | 71 | inline std::ostream& demangled_string::write(std::ostream& os) const 72 | { 73 | if (_ptr) { 74 | return os << _ptr.get(); 75 | } 76 | else { 77 | return os << "{nullptr}"; 78 | } 79 | } 80 | 81 | inline const char* demangled_string::c_str() const 82 | { 83 | if (!_ptr) 84 | { 85 | throw std::logic_error("demangled_string - zombie object"); 86 | } 87 | else { 88 | return _ptr.get(); 89 | } 90 | } 91 | 92 | inline demangled_string::operator std::string() const { 93 | return std::string(c_str()); 94 | } 95 | 96 | 97 | inline demangled_string demangle(const char* name) 98 | { 99 | using namespace std::string_literals; 100 | 101 | int status = -4; 102 | 103 | demangled_string::ptr_type ptr { 104 | abi::__cxa_demangle(name, nullptr, nullptr, &status), 105 | std::free 106 | }; 107 | 108 | if (status == 0) return { std::move(ptr) }; 109 | 110 | switch(status) 111 | { 112 | case -1: throw std::bad_alloc(); 113 | case -2: { 114 | std::string msg = "invalid mangled name~"; 115 | msg += name; 116 | auto p = (char*)std::malloc(msg.length() + 1); 117 | strcpy(p, msg.c_str()); 118 | return demangled_string::ptr_type { p, std::free }; 119 | } 120 | case -3: 121 | assert(!"invalid argument sent to __cxa_demangle"); 122 | throw std::logic_error("invalid argument sent to __cxa_demangle"); 123 | default: 124 | assert(!"PANIC! unexpected return value"); 125 | throw std::logic_error("PANIC! unexpected return value"); 126 | } 127 | } 128 | 129 | inline demangled_string demangle(const std::type_info& type) 130 | { 131 | return demangle(type.name()); 132 | } 133 | 134 | inline demangled_string demangle(std::type_index type) 135 | { 136 | return demangle(type.name()); 137 | } 138 | 139 | inline std::string method(const demangled_string& cls, const char* method) 140 | { 141 | return std::string(cls) + "::" + method; 142 | } 143 | 144 | } 145 | 146 | #endif 147 | -------------------------------------------------------------------------------- /inc/json.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "parsedef.h" 4 | 5 | namespace pparse { 6 | 7 | 8 | template 9 | struct Json { 10 | 11 | static void jsonStartTag(stream &out) { 12 | out << "{ "; 13 | } 14 | 15 | static void jsonAddField(stream &out, const char *fieldName, const std::string &fieldValue, bool isLast = false) { 16 | out << "\"" << fieldName << "\": \"" << fieldValue.c_str() << "\""; 17 | if (!isLast) { 18 | out << ","; 19 | } 20 | } 21 | template 22 | static void jsonAddField(stream &out, const char *fieldName, FType fieldValue, bool isLast = false) { 23 | out << "\"" << fieldName << "\": \"" << fieldValue << "\""; 24 | if (!isLast) { 25 | out << ","; 26 | } 27 | } 28 | 29 | static void jsonStartNested(stream &out, const char *fieldName, bool is_array = false) { 30 | out << "\"" << fieldName << "\": "; 31 | if (is_array) { 32 | out << "[ "; 33 | } 34 | } 35 | 36 | static void jsonEndNested(stream &out, bool isLast, bool is_array = false) { 37 | if (is_array) { 38 | out << "] "; 39 | } 40 | if (!isLast) { 41 | out << ","; 42 | } 43 | } 44 | 45 | static void jsonEndTag(stream &out, bool isLast = false) { 46 | out << "} "; 47 | if (!isLast) { 48 | out << ","; 49 | } 50 | } 51 | 52 | static void dumpRule(stream &out, RuleId ruleId, const char *typeName, bool isLast = false) { 53 | jsonStartTag(out); 54 | jsonAddField(out, "ruleId", ruleId); 55 | jsonAddField(out, "typeName", typeName, isLast ); 56 | } 57 | 58 | }; 59 | 60 | } 61 | -------------------------------------------------------------------------------- /inc/parse.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "parse_text.h" 12 | #include "parsedef.h" 13 | #include "tokencollisionhelper.h" 14 | #include "dhelper.h" 15 | #include "vhelper.h" 16 | #include "analyse.h" 17 | #include "json.h" 18 | 19 | namespace pparse { 20 | 21 | struct ParserBase { 22 | 23 | using AstType = void; 24 | 25 | using Char_value = Text_stream::Next_char_value; 26 | 27 | template 28 | static Char_value next_char(ParserBase &) { 29 | ERROR("Not implemented\n"); 30 | return Char_value(false,' '); 31 | } 32 | 33 | template 34 | static Char_value current_char(ParserBase &) { 35 | ERROR("Not implemented\n"); 36 | return Char_value(false,' '); 37 | } 38 | 39 | template 40 | static Parse_result parse(ParserBase &) { 41 | ERROR("Not implemented\n"); 42 | return Parse_result{false}; 43 | } 44 | 45 | template 46 | static inline Text_position current_pos_and_inc_nesting(ParserBase &parser) { 47 | ERROR("Not implemented\n"); 48 | return parser.current_pos_and_inc_nesting(); 49 | } 50 | 51 | template 52 | static inline Text_position current_pos(ParserBase &parser) { 53 | ERROR("Not implemented\n"); 54 | return parser.current_pos(); 55 | } 56 | 57 | template 58 | static inline bool backtrack(ParserBase &parser, Text_position pos) { 59 | ERROR("Not implemented\n"); 60 | return parser.backtrack(parser, pos); 61 | } 62 | 63 | // static inline Text_position dec_position_nesting(ParserBase &parser) { 64 | // ERROR("Not implemented\n"); 65 | // return parser.dec_position_nesting(parser); 66 | // } 67 | // 68 | 69 | 70 | #ifdef __PARSER_ANALYSE__ 71 | template 72 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 73 | ERROR("shouldn't get here"); 74 | return true; 75 | } 76 | 77 | template 78 | static bool can_accept_empty_input(HelperType *) { 79 | ERROR("shouldn't get here"); 80 | return false; 81 | } 82 | 83 | template 84 | static void init_collision_checker(ParserBase &base) { 85 | ERROR("shouldn't get here"); 86 | } 87 | #endif 88 | 89 | template 90 | static void dumpJson(Stream &out, const AstType *ast) { 91 | ERROR("Not implemented\n"); 92 | } 93 | 94 | void init_collission_checker() { 95 | TokenCollisionChecker *ptr = new TokenCollisionChecker(); 96 | colission_checker_.reset( ptr ); 97 | } 98 | 99 | 100 | 101 | std::unique_ptr colission_checker_; 102 | 103 | 104 | //must not be a polymorphic type, is accessed from static stuff only. don't have virtual functions here. 105 | //virtual ~ParserBase() {} 106 | }; 107 | 108 | 109 | 110 | 111 | 112 | struct CharParser : ParserBase { 113 | 114 | using AstType = void; 115 | 116 | 117 | CharParser(Text_stream &stream) : text_(stream) { 118 | } 119 | 120 | 121 | static Char_value current_char(CharParser &parser) { 122 | auto ch = parser.text_.current_char(); 123 | return ch; 124 | } 125 | 126 | static Char_value next_char(CharParser &parser) { 127 | auto ch = parser.text_.next_char(); 128 | return ch; 129 | } 130 | 131 | 132 | static inline Text_position current_pos(CharParser &parser) { 133 | return parser.text_.pos_at_cursor(); 134 | } 135 | 136 | static inline Text_position current_pos_and_inc_nesting(CharParser&parser) { 137 | return parser.text_.pos_at_cursor_and_inc_nesting(); 138 | } 139 | 140 | 141 | static inline Text_position dec_position_nesting(CharParser &parser) { 142 | return parser.text_.pos_at_cursor(); 143 | } 144 | 145 | static inline bool backtrack(CharParser &parser, Text_position pos) { 146 | return parser.text_.backtrack(pos); 147 | } 148 | 149 | 150 | 151 | #ifdef __PARSER_ANALYSE_ 152 | template 153 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 154 | ERROR("shouldn't get here"); 155 | return true; 156 | } 157 | 158 | template 159 | static bool can_accept_empty_input(HelperType *) { 160 | ERROR("shouldn't get here"); 161 | return false; 162 | } 163 | #endif 164 | 165 | template 166 | static void dumpJson(Stream &out, const AstType *) { 167 | } 168 | 169 | template 170 | static void init_collision_checker(ParserBase &base) { 171 | } 172 | 173 | private: 174 | Text_stream &text_; 175 | Text_stream::Next_char_value ch_; 176 | }; 177 | 178 | 179 | 180 | // PTopLevelParser - top level parser, initialises the collision checker; 181 | // The collision checker is used to check that variables that have the string value of a token are not counted as a variable; 182 | // the parse method initialises the collision checker and attaches it to the base parser, and after using it it is deleted. 183 | // 184 | template 185 | struct PTopLevelParser : ParserBase { 186 | 187 | using AstType = typename Type::AstType; 188 | using ThisClass = PTopLevelParser; 189 | 190 | template 191 | static Parse_result parse(ParserBase &base) { 192 | 193 | PTopLevelParser::init_collision_checker(base); 194 | //Type::init_collision_checker(base); 195 | return Type::parse(base); 196 | } 197 | 198 | template 199 | static void init_collision_checker(ParserBase &base) { 200 | 201 | base.init_collission_checker(); 202 | if (base.colission_checker_->insert_type_info(&typeid(ThisClass))) { 203 | Type::init_collision_checker(base); 204 | base.colission_checker_->remove_type_info(&typeid(ThisClass)); 205 | } 206 | } 207 | }; 208 | 209 | // 210 | // PEof - parse is ok if at end of input 211 | // 212 | 213 | template 214 | struct PRequireEof : PTopLevelParser { 215 | 216 | static inline const RuleId RULE_ID = Type::RULE_ID; 217 | 218 | using AstType = typename Type::AstType; 219 | 220 | template 221 | static Parse_result parse(ParserBase &base) { 222 | 223 | Parse_result res = PTopLevelParser::parse(base); 224 | if (!res.success_) { 225 | return res; 226 | } 227 | 228 | typename ParserBase::Char_value nchar = ParserBase::current_char(base); 229 | while (nchar.first && isspace(nchar.second)) { 230 | ParserBase::next_char(base); 231 | nchar = ParserBase::current_char(base); 232 | } 233 | 234 | Text_position end_pos = ParserBase::current_pos(base); 235 | 236 | if (!nchar.first) { 237 | return res; 238 | } 239 | return Parse_result{false, Position(end_pos), Position(end_pos) }; 240 | } 241 | 242 | #ifdef __PARSER_ANALYSE__ 243 | template 244 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 245 | bool ret; 246 | 247 | if (!helper.push_and_check(out, get_tinfo((Type *) nullptr), -1)) { 248 | return false; 249 | } 250 | 251 | ret = Type::verify_no_cycles((HelperType *) nullptr, helper, out) ; 252 | 253 | helper.pop(); 254 | 255 | return ret; 256 | } 257 | 258 | template 259 | static bool can_accept_empty_input(HelperType *) { 260 | //return Type::can_accept_empty_input(); 261 | return false; 262 | } 263 | 264 | 265 | #endif 266 | template 267 | static void dumpJson(Stream &out, const AstType *ast) { 268 | Type::dumpJson(out, ast); 269 | } 270 | 271 | 272 | 273 | }; 274 | 275 | 276 | 277 | // 278 | // PAny - ordered choice parser combinator 279 | // 280 | 281 | template 282 | struct PAny : ParserBase { 283 | 284 | static inline const RuleId RULE_ID = ruleId; 285 | 286 | using ThisClass = PAny; 287 | 288 | //typedef VariantType typename std::variant< typename std::unique_ptr...>; 289 | 290 | struct AstType : AstEntryBase { 291 | AstType() : AstEntryBase(ruleId) { 292 | } 293 | 294 | std::variant< typename std::unique_ptr...> entry_; 295 | }; 296 | 297 | 298 | template 299 | static Parse_result parse(ParserBase &base) { 300 | 301 | Position error_pos; 302 | 303 | #ifdef __PARSER_TRACE__ 304 | Text_position start_pos = ParserBase::current_pos(base); 305 | std::string short_name = VisualizeTrace::trace_start_parsing(start_pos); 306 | #endif 307 | 308 | auto ast = std::make_unique(); 309 | 310 | Parse_result res = parse_helper<0,ParserBase,Types...>(base, ast.get(), error_pos); 311 | 312 | #ifdef __PARSER_TRACE__ 313 | VisualizeTrace::end_parsing_choice(short_name, res.success_, ParserBase::current_pos(base), ast.get()->entry_.index()); 314 | #endif 315 | 316 | if (res.success_) { 317 | res.ast_.reset( ast.release() ); 318 | } 319 | 320 | 321 | return res; 322 | } 323 | 324 | #ifdef __PARSER_ANALYSE__ 325 | 326 | template 327 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 328 | return verify_no_cycles_helper(helper, out); 329 | } 330 | 331 | template 332 | static bool can_accept_empty_input(HelperType *) { 333 | return can_accept_empty_input_helper(); 334 | } 335 | 336 | #endif 337 | 338 | template 339 | static void dumpJson(Stream &out, const AstType *ast) { 340 | Json::dumpRule(out, RULE_ID,"PAny" ); 341 | 342 | dump_helper<0, Stream, Types...>(out, ast); 343 | 344 | Json::jsonEndTag(out, true); 345 | } 346 | 347 | template 348 | static void init_collision_checker(ParserBase &base) { 349 | 350 | if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) { 351 | init_collision_checker_helper(base); 352 | base.colission_checker_->remove_type_info(&typeid(ThisClass)); 353 | } 354 | } 355 | 356 | 357 | 358 | private: 359 | template 360 | static inline Parse_result parse_helper(ParserBase &base, AstType *ast, Position error_pos) { 361 | 362 | Parse_result res = PType::parse(base); 363 | typedef std::unique_ptr PTypePtr; 364 | 365 | if (res.success_) { 366 | if (res.ast_.get() != nullptr) { 367 | typename PType::AstType *retAst = (typename PType::AstType *) res.ast_.release(); 368 | typedef typename std::variant< typename std::unique_ptr...> VariantType; 369 | 370 | 371 | ast->entry_ = VariantType{ std::in_place_index, PTypePtr(retAst) }; 372 | } 373 | return res; 374 | } 375 | 376 | if (!res.success_) { 377 | if (error_pos < res.start_) { 378 | error_pos = res.start_; 379 | } 380 | } 381 | 382 | if constexpr (sizeof...(PTypes) > 0) { 383 | return parse_helper(base, ast, error_pos); 384 | } 385 | return Parse_result{false, error_pos, error_pos}; 386 | } 387 | 388 | 389 | #ifdef __PARSER_ANALYSE__ 390 | 391 | template 392 | static inline bool verify_no_cycles_helper(CycleDetectorHelper &helper, std::ostream &out) { 393 | 394 | if (!helper.push_and_check(out, get_tinfo((PType *) nullptr), FieldIndex)) { 395 | return false; 396 | } 397 | 398 | bool ret = PType::verify_no_cycles((HelperType *) nullptr, helper, out); 399 | 400 | helper.pop(); 401 | 402 | 403 | if constexpr(sizeof ...(PTypes) > 0) { 404 | 405 | ret = ret && verify_no_cycles_helper(helper, out); 406 | } 407 | 408 | return ret; 409 | } 410 | 411 | template 412 | static bool can_accept_empty_input_helper() { 413 | 414 | if (PType::can_accept_empty_input((HelperType *) nullptr)) { 415 | return true; 416 | } 417 | 418 | if constexpr(sizeof ...(PTypes) > 0) { 419 | return can_accept_empty_input_helper(); 420 | } 421 | return false; 422 | } 423 | #endif 424 | 425 | template 426 | static inline bool dump_helper( Stream &stream, const AstType *ast ) { 427 | 428 | std::stringstream sout; 429 | 430 | if (ast->entry_.index() == FieldIndex) { 431 | sout << FieldIndex; 432 | std::string sval(sout.str()); 433 | 434 | 435 | Json::jsonAddField(stream, "optionIndex", sval.c_str()); 436 | Json::jsonStartNested(stream, "data"); 437 | 438 | typename PType::AstType *retAst = (typename PType::AstType *) std::get( ast->entry_ ).get(); 439 | PType::dumpJson(stream, retAst ); 440 | 441 | Json::jsonEndNested(stream,true); 442 | 443 | } 444 | 445 | if constexpr (sizeof...(PTypes) > 0) { 446 | return dump_helper( stream, ast ); 447 | } 448 | 449 | return true; 450 | } 451 | 452 | template 453 | static void init_collision_checker_helper(ParserBase &base) { 454 | 455 | PType::init_collision_checker(base); 456 | 457 | if constexpr (sizeof...(PTypes) > 0) { 458 | return init_collision_checker_helper( base ); 459 | } 460 | } 461 | 462 | }; 463 | 464 | // 465 | // POpt - optional rule parser combinator 466 | // 467 | 468 | template 469 | struct POpt : ParserBase { 470 | 471 | static inline const RuleId RULE_ID = ruleId; 472 | 473 | using ThisClass = POpt; 474 | 475 | using PTypePtr = typename std::unique_ptr; 476 | 477 | using OptionType = std::optional< PTypePtr >; 478 | 479 | struct AstType : AstEntryBase { 480 | AstType() : AstEntryBase(ruleId) { 481 | } 482 | 483 | OptionType entry_; 484 | }; 485 | 486 | 487 | template 488 | static Parse_result parse(ParserBase &base) { 489 | 490 | #ifdef __PARSER_TRACE__ 491 | Text_position start_pos = ParserBase::current_pos(base); 492 | std::string short_name = VisualizeTrace::trace_start_parsing(start_pos); 493 | #endif 494 | 495 | 496 | auto ast = std::make_unique(); 497 | 498 | Parse_result res = PType::parse(base); 499 | 500 | if (res.success_) { 501 | typename PType::AstType *ptr = (typename PType::AstType *) res.ast_.release(); 502 | AstType *rval = ast.get(); 503 | rval->entry_ = OptionType(PTypePtr(ptr)); 504 | rval->start_ = res.start_; 505 | rval->end_ = res.end_; 506 | } 507 | res.success_ = true; 508 | res.ast_.reset( ast.release() ); 509 | 510 | #ifdef __PARSER_TRACE__ 511 | VisualizeTrace::end_parsing(short_name, res.success_, ParserBase::current_pos(base)); 512 | #endif 513 | 514 | 515 | return res; 516 | } 517 | 518 | #ifdef __PARSER_ANALYSE__ 519 | template 520 | static bool verify_no_cycles(HelperType *parg,CycleDetectorHelper &helper, std::ostream &out) { 521 | 522 | if (!helper.push_and_check(out, get_tinfo((PType *) nullptr), -1)) { 523 | return false; 524 | } 525 | 526 | bool ret = PType::verify_no_cycles(parg, helper, out) ; 527 | 528 | helper.pop(); 529 | 530 | return ret; 531 | } 532 | 533 | template 534 | static bool can_accept_empty_input(HelperType *) { 535 | return true; 536 | } 537 | 538 | #endif 539 | 540 | template 541 | static void dumpJson(Stream &out, const AstType *ast) { 542 | Json::dumpRule(out, RULE_ID,"POpt" ); 543 | 544 | Json::jsonStartNested(out, "Type"); 545 | typename PType::AstType *ptr = (typename PType::AstType *) ast; 546 | PType::dumpJson(out, ptr); 547 | Json::jsonEndNested(out,true); 548 | 549 | Json::jsonEndTag(out, true); 550 | } 551 | 552 | template 553 | static void init_collision_checker(ParserBase &base) { 554 | if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) { 555 | PType::init_collision_checker(base); 556 | base.colission_checker_->remove_type_info(&typeid(ThisClass)); 557 | } 558 | } 559 | 560 | 561 | 562 | private: 563 | 564 | }; 565 | 566 | // 567 | // PSeq - sequence parser combinator 568 | // 569 | 570 | 571 | template 572 | struct PSeq : ParserBase { 573 | 574 | using ThisClass = PSeq; 575 | 576 | static inline const RuleId RULE_ID = ruleId; 577 | 578 | struct AstType : AstEntryBase { 579 | AstType() : AstEntryBase(ruleId) { 580 | } 581 | 582 | std::tuple...> entry_; 583 | }; 584 | 585 | 586 | template 587 | static Parse_result parse(ParserBase &base) { 588 | 589 | 590 | Text_position start_pos = ParserBase::current_pos_and_inc_nesting(base); 591 | Position start_seq; 592 | 593 | #ifdef __PARSER_TRACE__ 594 | std::string short_name = VisualizeTrace::trace_start_parsing(start_pos); 595 | #endif 596 | 597 | auto ast = std::make_unique(); 598 | 599 | Parse_result res = parse_helper<0, ParserBase, Types...>(base, ast.get(), start_seq); 600 | 601 | if (!res.success_ ) { 602 | ParserBase::backtrack(base, start_pos); 603 | } 604 | 605 | if (res.success_) { 606 | res.ast_.reset( ast.release() ); 607 | ParserBase::dec_position_nesting(base); 608 | } 609 | 610 | 611 | #ifdef __PARSER_TRACE__ 612 | VisualizeTrace::end_parsing(short_name, res.success_, ParserBase::current_pos(base)); 613 | #endif 614 | 615 | 616 | return res; 617 | } 618 | 619 | #ifdef __PARSER_ANALYSE__ 620 | template 621 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 622 | return verify_no_cycles_helper(helper, out); 623 | } 624 | 625 | template 626 | static bool can_accept_empty_input(HelperType *) { 627 | return can_accept_empty_input_helper(); 628 | } 629 | 630 | #endif 631 | template 632 | static void dumpJson(Stream &out, const AstType *ast) { 633 | Json::dumpRule(out, RULE_ID,"PSeq" ); 634 | 635 | Json::jsonStartNested(out, "Type", true); 636 | dump_helper<0, Stream, Types...>(out, ast); 637 | Json::jsonEndNested(out, true, true); 638 | 639 | Json::jsonEndTag(out, true); 640 | } 641 | 642 | template 643 | static void init_collision_checker(ParserBase &base) { 644 | if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) { 645 | init_collision_checker_helper(base); 646 | base.colission_checker_->remove_type_info(&typeid(ThisClass)); 647 | } 648 | } 649 | 650 | 651 | 652 | private: 653 | 654 | template 655 | static inline bool dump_helper( Stream &stream, const AstType *ast ) { 656 | 657 | PType::dumpJson(stream, std::get( ast->entry_ ).get() ); 658 | 659 | if constexpr (sizeof...(PTypes) > 0) { 660 | 661 | Json::jsonEndNested(stream,false); 662 | 663 | return dump_helper( stream, ast ); 664 | } 665 | 666 | return true; 667 | } 668 | 669 | template 670 | static inline Parse_result parse_helper(ParserBase &base, AstType *ast, Position start_seq ) { 671 | 672 | Parse_result res = PType::parse(base); 673 | typedef std::unique_ptr PTypePtr; 674 | 675 | if (!res.success_) { 676 | return res; 677 | } 678 | 679 | if constexpr (FieldIndex == 0) { 680 | start_seq = res.start_; 681 | } 682 | 683 | if (res.ast_.get() != nullptr) { 684 | typename PType::AstType *retAst = (typename PType::AstType *) res.ast_.release(); 685 | std::get( ast->entry_ ) = PTypePtr(retAst); 686 | } 687 | 688 | if constexpr (sizeof...(PTypes) > 0) { 689 | return parse_helper( base, ast, start_seq ); 690 | } 691 | 692 | ast->start_ = start_seq; 693 | ast->end_ = res.end_; 694 | return Parse_result{true, start_seq, res.end_}; 695 | } 696 | 697 | 698 | #ifdef __PARSER_ANALYSE__ 699 | 700 | template 701 | static bool verify_no_cycles_helper(CycleDetectorHelper &helper, std::ostream &out) { 702 | 703 | if (!helper.push_and_check(out, get_tinfo((PType *) nullptr), FieldIndex)) { 704 | return false; 705 | } 706 | 707 | bool ret = PType::verify_no_cycles((HelperType *) nullptr, helper, out); 708 | 709 | helper.pop(); 710 | 711 | if (PType::can_accept_empty_input((HelperType *) nullptr)) { 712 | 713 | if constexpr(sizeof ...(PTypes) > 0) { 714 | ret = ret && verify_no_cycles_helper(helper, out); 715 | } 716 | } 717 | 718 | return ret; 719 | } 720 | 721 | template 722 | static inline bool can_accept_empty_input_helper() { 723 | 724 | if (PType::can_accept_empty_input((HelperType *) nullptr)) { 725 | if constexpr(sizeof ...(PTypes) > 0) { 726 | return can_accept_empty_input_helper(); 727 | } 728 | } 729 | return true; 730 | } 731 | #endif 732 | 733 | template 734 | static void init_collision_checker_helper(ParserBase &base) { 735 | 736 | PType::init_collision_checker(base); 737 | 738 | if constexpr (sizeof...(PTypes) > 0) { 739 | return init_collision_checker_helper( base ); 740 | } 741 | } 742 | }; 743 | 744 | 745 | // 746 | // PRepeat - repetition of element parser combinators 747 | // 748 | 749 | template 750 | struct PRepeat : ParserBase { 751 | 752 | using ThisClass = PRepeat; 753 | 754 | static inline const RuleId RULE_ID = ruleId; 755 | 756 | typedef typename std::unique_ptr< typename Type::AstType> AstTypeEntry; 757 | 758 | struct AstType : AstEntryBase { 759 | AstType() : AstEntryBase(ruleId) { 760 | } 761 | 762 | std::list entry_; 763 | }; 764 | 765 | 766 | template 767 | static Parse_result parse(ParserBase &base) { 768 | 769 | auto ast = std::make_unique(); 770 | return parse_helper(base, &ast); 771 | } 772 | 773 | template 774 | static void dumpJson(Stream &out, const AstType *ast) { 775 | Json::dumpRule(out, RULE_ID,"PRepeat" ); 776 | Json::jsonAddField(out, "minOccurance", minOccurance ); 777 | Json::jsonAddField(out, "maxOccurance", minOccurance ); 778 | 779 | Json::jsonStartNested(out, "content"); 780 | 781 | for(auto it=ast->entry_.begin(); it != ast->entry_.end(); ++it) { 782 | Type::dumpJson(out, &it->entry_); 783 | } 784 | Json::jsonEndNested(out,true); 785 | 786 | Json::jsonEndTag(out, true); 787 | } 788 | 789 | template 790 | static void init_collision_checker(ParserBase &base) { 791 | 792 | if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) { 793 | Type::init_collision_checker(base); 794 | base.colission_checker_->remove_type_info(&typeid(ThisClass)); 795 | } 796 | } 797 | 798 | #ifdef __PARSER_ANALYSE__ 799 | template 800 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 801 | 802 | if (!helper.push_and_check(out, get_tinfo((Type *) nullptr), -1)) { 803 | return false; 804 | } 805 | 806 | bool ret = Type::verify_no_cycles((HelperType *) nullptr, helper, out) ; 807 | 808 | helper.pop(); 809 | 810 | return ret; 811 | } 812 | 813 | template 814 | static bool can_accept_empty_input(HelperType *arg) { 815 | if constexpr (minOccurance == 0) { 816 | return true; 817 | } 818 | return Type::can_accept_empty_input(arg); 819 | } 820 | #endif 821 | 822 | private: 823 | template 824 | static Parse_result parse_helper(ParserBase &base, std::unique_ptr *ast) { 825 | 826 | Text_position start_pos = ParserBase::current_pos_and_inc_nesting(base); 827 | 828 | #ifdef __PARSER_TRACE__ 829 | std::string short_name = VisualizeTrace::trace_start_parsing(start_pos); 830 | #endif 831 | 832 | 833 | 834 | for(int i = 0; i < minOccurance; ++i) { 835 | Parse_result res = Type::parse(base); 836 | if (!res.success_) { 837 | ParserBase::backtrack(base, start_pos); 838 | 839 | #ifdef __PARSER_TRACE__ 840 | VisualizeTrace::end_parsing(short_name, res.success_, ParserBase::current_pos(base)); 841 | #endif 842 | 843 | 844 | return res; 845 | } 846 | if (ast != nullptr) { 847 | typename Type::AstType *retAst = (typename Type::AstType *) res.ast_.release(); 848 | ast->get()->entry_.push_back( AstTypeEntry( retAst ) ); 849 | } 850 | } 851 | 852 | ParserBase::dec_position_nesting(base); 853 | 854 | for(int i = minOccurance; ; ++i ) { 855 | 856 | Parse_result res = Type::parse(base); 857 | if (!res.success_) { 858 | break; 859 | } 860 | if (maxOccurance != 0 && i >= maxOccurance) { 861 | break; 862 | } 863 | if (ast != nullptr) { 864 | typename Type::AstType *retAst = (typename Type::AstType *) res.ast_.release(); 865 | ast->get()->entry_.push_back( AstTypeEntry( retAst ) ); 866 | } 867 | } 868 | Text_position end_pos = ParserBase::current_pos(base); 869 | 870 | #ifdef __PARSER_TRACE__ 871 | VisualizeTrace::end_parsing(short_name, true, ParserBase::current_pos(base)); 872 | #endif 873 | 874 | if (ast != nullptr) { 875 | AstType * ret = ast->release(); 876 | ret->start_ = Position(start_pos); 877 | ret->end_ = Position(end_pos); 878 | return Parse_result{true, Position(start_pos), Position(end_pos), std::unique_ptr(ret) }; 879 | } 880 | return Parse_result{true, Position(start_pos), Position(end_pos) }; 881 | } 882 | 883 | }; 884 | 885 | // 886 | // PStar - zero or more parser combinators 887 | // 888 | 889 | 890 | template 891 | struct PStar : PRepeat { 892 | }; 893 | 894 | // 895 | // PPlus - one or more parser combinator 896 | // 897 | 898 | template 899 | struct PPlus : PRepeat { 900 | }; 901 | 902 | 903 | // 904 | // Parse TPrecondition, if parsing of TPrecondition fails then try to parse Type 905 | // if TPrecondition parses, backtrack and return failure. 906 | // 907 | template 908 | struct POnPreconditionFails { 909 | 910 | using ThisClass = POnPreconditionFails; 911 | 912 | static inline const RuleId RULE_ID = Type::RULE_ID; 913 | 914 | using AstType = typename Type::AstType; 915 | 916 | template 917 | static Parse_result parse(ParserBase &base) { 918 | 919 | Text_position start_pos = ParserBase::current_pos_and_inc_nesting(base); 920 | Parse_result res = Type::parse(base); 921 | if (res.success_) { 922 | ParserBase::dec_position_nesting(base); 923 | res.success_ = false; 924 | return res; 925 | } 926 | ParserBase::backtrack(base, start_pos); 927 | 928 | Parse_result resT = Type::parse(base); 929 | if (!resT.success()) { 930 | return Parse_result{false, res.start_, res.end_ }; 931 | } 932 | return resT; 933 | } 934 | 935 | #ifdef __PARSER_ANALYSE__ 936 | template 937 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 938 | bool ret; 939 | 940 | if (!helper.push_and_check(out, get_tinfo((TPrecondition *) nullptr), -1)) { 941 | return false; 942 | } 943 | 944 | ret = Type::verify_no_cycles((Type *) nullptr, helper, out) ; 945 | 946 | helper.pop(); 947 | 948 | if (ret) { 949 | 950 | if (!helper.push_and_check(out, get_tinfo((Type *) nullptr), -1)) { 951 | return false; 952 | } 953 | 954 | ret = Type::verify_no_cycles((HelperType *) nullptr, helper, out) ; 955 | 956 | helper.pop(); 957 | } 958 | 959 | return ret; 960 | } 961 | 962 | template 963 | static bool can_accept_empty_input(HelperType *arg) { 964 | return Type::can_accept_empty_input(arg); 965 | } 966 | 967 | #endif 968 | static void init_collision_checker(ParserBase &base) { 969 | 970 | if (base.colission_checker_ != nullptr) { 971 | Type::init_collision_checker(base); 972 | } 973 | } 974 | 975 | 976 | template 977 | static void init_collision_checker(ParserBase &base) { 978 | Type::init_collision_checker(base); 979 | } 980 | 981 | }; 982 | 983 | // 984 | // PWithAndLookaheadImpl - implementation struct for lookahead parsers (don't use directly) 985 | // 986 | 987 | template 988 | struct PWithAndLookaheadImpl : ParserBase { 989 | 990 | using ThisClass = PWithAndLookaheadImpl; 991 | 992 | static inline const RuleId RULE_ID = Type::RULE_ID; 993 | 994 | using AstType = typename Type::AstType; 995 | 996 | template 997 | static Parse_result parse(ParserBase &base) { 998 | 999 | 1000 | Text_position start_pos = ParserBase::current_pos_and_inc_nesting(base); 1001 | Parse_result res = Type::parse(base); 1002 | if (!res.success_) { 1003 | ParserBase::dec_position_nesting(base); 1004 | return res; 1005 | } 1006 | 1007 | Text_position lookahead_start_pos = ParserBase::current_pos(base); 1008 | Parse_result resLookahead = LookaheadType::parse(base); 1009 | 1010 | bool isfail; 1011 | 1012 | if constexpr(AndOrNotLookahead) { 1013 | isfail = !resLookahead.success_; 1014 | } else { 1015 | isfail = resLookahead.success_; 1016 | } 1017 | 1018 | if (isfail) { 1019 | ParserBase::backtrack(base, start_pos); 1020 | return Parse_result{false, resLookahead.start_, resLookahead.end_ }; 1021 | } 1022 | ParserBase::backtrack(base, lookahead_start_pos); 1023 | 1024 | return res; 1025 | } 1026 | 1027 | #ifdef __PARSER_ANALYSE__ 1028 | template 1029 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 1030 | bool ret; 1031 | 1032 | if (!helper.push_and_check(out, get_tinfo((Type *) nullptr), -1)) { 1033 | return false; 1034 | } 1035 | 1036 | ret = Type::verify_no_cycles((HelperType *) nullptr, helper, out) ; 1037 | 1038 | helper.pop(); 1039 | 1040 | if (ret) { 1041 | 1042 | LookaheadType info; 1043 | 1044 | if (!helper.push_and_check(out, get_tinfo((LookaheadType *) nullptr), -1)) { 1045 | return false; 1046 | } 1047 | 1048 | ret = Type::verify_no_cycles((HelperType *) nullptr, helper, out) ; 1049 | 1050 | helper.pop(); 1051 | } 1052 | 1053 | return ret; 1054 | } 1055 | 1056 | template 1057 | static bool can_accept_empty_input(HelperType *arg) { 1058 | return Type::can_accept_empty_input(arg); 1059 | } 1060 | 1061 | 1062 | #endif 1063 | 1064 | template 1065 | static void dumpJson(Stream &out, const AstType *ast) { 1066 | Type::dumpJson(out, ast); 1067 | } 1068 | 1069 | template 1070 | static void init_collision_checker(ParserBase &base) { 1071 | 1072 | if (base.colission_checker_ != nullptr && base.colission_checker_->insert_type_info(&typeid(ThisClass))) { 1073 | Type::init_collision_checker(base); 1074 | LookaheadType::init_collision_checker(base); 1075 | base.colission_checker_->remove_type_info(&typeid(ThisClass)); 1076 | } 1077 | } 1078 | 1079 | 1080 | }; 1081 | 1082 | 1083 | // 1084 | // PWithAndLookahead - 1085 | // 1086 | 1087 | template 1088 | struct PWithAndLookahead : PWithAndLookaheadImpl { 1089 | }; 1090 | 1091 | // 1092 | // PWithNotLookahead - 1093 | // 1094 | 1095 | template 1096 | struct PWithNotLookahead: PWithAndLookaheadImpl { 1097 | }; 1098 | 1099 | 1100 | 1101 | 1102 | } // namespace pparse 1103 | 1104 | 1105 | #include "parse_atomic.h" 1106 | 1107 | namespace pparse { 1108 | 1109 | 1110 | // 1111 | // PAndPredicate 1112 | // 1113 | 1114 | template 1115 | struct PAndPredicate : PWithAndLookaheadImpl, Type> { 1116 | }; 1117 | 1118 | // 1119 | // PNotPredicate 1120 | // 1121 | 1122 | template 1123 | struct PNotPredicate: PWithAndLookaheadImpl, Type> { 1124 | }; 1125 | 1126 | } // namespace pparse 1127 | 1128 | 1129 | -------------------------------------------------------------------------------- /inc/parse_atomic.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace pparse { 4 | 5 | // 6 | // 7 | // Token parser 8 | // 9 | 10 | #define CSTR1(str) str[0] 11 | #define CSTR2(str) str[0], str[1] 12 | #define CSTR3(str) str[0], str[1], str[2] 13 | #define CSTR4(str) str[0], str[1], str[2], str[3] 14 | #define CSTR5(str) str[0], str[1], str[2], str[3], str[4] 15 | #define CSTR6(str) str[0], str[1], str[2], str[3], str[4], str[5] 16 | #define CSTR7(str) str[0], str[1], str[2], str[3], str[4], str[5], str[6] 17 | #define CSTR8(str) str[0], str[1], str[2], str[3], str[4], str[5], str[6], str[7] 18 | #define CSTR9(str) str[0], str[1], str[2], str[3], str[4], str[5], str[6], str[7], str[8] 19 | #define CSTR10(str) str[0], str[1], str[2], str[3], str[4], str[5], str[6], str[7], str[8], str[9] 20 | #define CSTR11(str) str[0], str[1], str[2], str[3], str[4], str[5], str[6], str[7], str[8], str[9], str[10] 21 | #define CSTR12(str) str[0], str[1], str[2], str[3], str[4], str[5], str[6], str[7], str[8], str[9], str[10], str[11] 22 | 23 | 24 | 25 | template 26 | struct PTok : ParserBase { 27 | 28 | static inline const RuleId RULE_ID = ruleId; 29 | 30 | using ThisClass = PTok; 31 | 32 | struct AstType : AstEntryBase { 33 | AstType(Position start, Position end) : AstEntryBase(ruleId) { 34 | start_ = start; 35 | end_ = end; 36 | } 37 | }; 38 | 39 | 40 | template 41 | static Parse_result parse(ParserBase &base) { 42 | 43 | Text_position start_pos = ParserBase::current_pos_and_inc_nesting(base); 44 | 45 | Char_value nchar = ParserBase::current_char(base); 46 | while (nchar.first && isspace(nchar.second)) { 47 | ParserBase::next_char(base); 48 | nchar = ParserBase::current_char(base); 49 | } 50 | 51 | Text_position token_start_pos = ParserBase::current_pos(base); 52 | 53 | #ifdef __PARSER_TRACE__ 54 | std::string short_name = VisualizeTrace::trace_start_parsing_token(token_start_pos); 55 | #endif 56 | 57 | 58 | if (! parse_helper(base)) { 59 | ParserBase::backtrack(base, start_pos); 60 | 61 | #ifdef __PARSER_TRACE__ 62 | VisualizeTrace::end_parsing(short_name, false, ParserBase::current_pos(base)); 63 | #endif 64 | 65 | 66 | return Parse_result{false, Position(token_start_pos), Position(token_start_pos) }; 67 | } else { 68 | ParserBase::dec_position_nesting(base); 69 | } 70 | 71 | 72 | #ifdef __PARSER_TRACE__ 73 | VisualizeTrace::end_parsing(short_name, true, ParserBase::current_pos(base)); 74 | #endif 75 | 76 | 77 | Text_position end_pos = ParserBase::current_pos(base); 78 | 79 | end_pos.column_ -= 1; 80 | 81 | return Parse_result{true, Position(token_start_pos), Position(end_pos), std::make_unique( Position(token_start_pos), Position(end_pos) ) }; 82 | } 83 | 84 | #ifdef __PARSER_ANALYSE__ 85 | template 86 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 87 | return true; 88 | } 89 | 90 | template 91 | static bool can_accept_empty_input(HelperType *) { 92 | return false; 93 | } 94 | 95 | 96 | #endif 97 | 98 | template 99 | static void dumpJson(Stream &out, const AstType *ast) { 100 | 101 | Json::dumpRule(out, RULE_ID, "PTok" ); 102 | 103 | Json::jsonStartNested(out, "value"); 104 | out << "\""; 105 | dump_helper(out); 106 | out << "\""; 107 | 108 | Json::jsonEndTag(out, true); 109 | } 110 | 111 | 112 | template 113 | static void init_collision_checker(ParserBase &base) { 114 | std::string sval; 115 | gather_token_value(sval); 116 | base.colission_checker_->insert(sval.c_str(), sval.size() ); 117 | } 118 | 119 | private: 120 | template 121 | static inline bool parse_helper( ParserBase &text ) { 122 | 123 | Text_stream::Text_stream::Next_char_value nchar = ParserBase::next_char(text); 124 | 125 | if (!nchar.first || nchar.second != ch) { 126 | return false; 127 | } 128 | if constexpr (sizeof...(Css) > 0) { 129 | return parse_helper( text ); 130 | } 131 | 132 | return true; 133 | } 134 | 135 | template 136 | static inline bool dump_helper( Stream &stream) { 137 | 138 | stream << ch; 139 | 140 | if constexpr (sizeof...(Css) > 0) { 141 | return dump_helper( stream ); 142 | } 143 | 144 | return true; 145 | } 146 | 147 | template 148 | static inline bool gather_token_value(std::string &strval) { 149 | 150 | strval += ch; 151 | 152 | if constexpr (sizeof...(Css) > 0) { 153 | return gather_token_value( strval ); 154 | } 155 | 156 | return true; 157 | } 158 | 159 | 160 | }; 161 | 162 | // 163 | // PTokFunc - token parser where token characters are structified by callback function 164 | // 165 | 166 | enum class Char_checker_result { 167 | error, 168 | proceed, 169 | acceptNow, 170 | acceptUnget, 171 | }; 172 | 173 | typedef Char_checker_result (PTokVar_cb_t) (Char_t current_char, bool iseof, std::string &matched_so_far); 174 | 175 | 176 | const int PTokVarCanAcceptEmptyInput = 1; 177 | const int PTokVarCheckTokenClash = 2; 178 | 179 | template 180 | struct PTokVar : ParserBase { 181 | 182 | using ThisClass = PTokVar; 183 | 184 | static inline const RuleId RULE_ID = ruleId; 185 | 186 | struct AstType : AstEntryBase { 187 | AstType() : AstEntryBase(ruleId) { 188 | } 189 | 190 | std::string entry_; 191 | }; 192 | 193 | 194 | template 195 | static Parse_result parse(ParserBase &base) { 196 | 197 | Char_value nchar = ParserBase::current_char(base); 198 | while (nchar.first && isspace(nchar.second)) { 199 | ParserBase::next_char(base); 200 | nchar = ParserBase::current_char(base); 201 | } 202 | 203 | Text_position token_start_pos = ParserBase::current_pos(base); 204 | 205 | auto ast = std::make_unique(); 206 | while( nchar.first ) { 207 | 208 | nchar = ParserBase::current_char(base); 209 | Char_checker_result res = checker(nchar.second, !nchar.first, ast.get()->entry_); 210 | switch(res) { 211 | 212 | case Char_checker_result::proceed: 213 | ast.get()->entry_ += (char) nchar.second; 214 | break; 215 | 216 | case Char_checker_result::error: 217 | token_start_pos = ParserBase::current_pos(base); 218 | return Parse_result{false, token_start_pos, token_start_pos }; 219 | 220 | case Char_checker_result::acceptNow: { 221 | ast.get()->entry_ += (char) nchar.second; 222 | Text_position end_pos = ParserBase::current_pos(base); 223 | ParserBase::next_char(base); 224 | 225 | AstEntryBase *ret = ast.release(); 226 | ret->start_ = token_start_pos; 227 | ret->end_ = end_pos; 228 | 229 | if (has_collision(base, ast.get()->entry_)) { 230 | return Parse_result{false, token_start_pos, token_start_pos}; 231 | } 232 | return Parse_result{true, token_start_pos, end_pos, std::unique_ptr(ret) }; 233 | 234 | } 235 | 236 | case Char_checker_result::acceptUnget: { 237 | Text_position end_pos = ParserBase::current_pos(base); 238 | end_pos.column_ -= 1; 239 | 240 | AstEntryBase *ret = ast.release(); 241 | ret->start_ = token_start_pos; 242 | ret->end_ = end_pos; 243 | 244 | if (has_collision(base, ast.get()->entry_)) { 245 | return Parse_result{false, token_start_pos, token_start_pos}; 246 | } 247 | return Parse_result{true, token_start_pos, end_pos, std::unique_ptr(ret) }; 248 | } 249 | 250 | } 251 | 252 | if (!nchar.first) { 253 | ERROR("eof not handled by PTokVar callback\n"); 254 | break; 255 | } 256 | 257 | nchar = ParserBase::next_char(base); 258 | } 259 | 260 | Text_position end_pos = ParserBase::current_pos(base); 261 | 262 | end_pos.column_ -= 1; 263 | 264 | return Parse_result{false, token_start_pos, token_start_pos}; 265 | } 266 | 267 | #ifdef __PARSER_ANALYSE__ 268 | template 269 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 270 | return true; 271 | } 272 | 273 | template 274 | static bool can_accept_empty_input(HelperType *) { 275 | return TokVarFlags & PTokVarCanAcceptEmptyInput; 276 | } 277 | 278 | #endif 279 | 280 | template 281 | static void dumpJson(Stream &out, const AstType *ast) { 282 | Json::dumpRule(out, RULE_ID,"PTokVar" ); 283 | 284 | //? extract the name of the callback function from type signature ? 285 | Json::jsonAddField(out, "token", ast->entry_.c_str(), true); 286 | 287 | Json::jsonEndTag(out, true); 288 | } 289 | 290 | template 291 | static void init_collision_checker(ParserBase &base) { 292 | } 293 | 294 | inline static bool has_collision(ParserBase &base, std::string &entry) { 295 | 296 | if constexpr ((TokVarFlags & PTokVarCheckTokenClash) != 0) { 297 | if (base.colission_checker_ != nullptr) { 298 | 299 | const Char_t *tok = (const Char_t *) entry.c_str(); 300 | int len = entry.size(); 301 | 302 | return base.colission_checker_->has_token(tok, len); 303 | } 304 | } 305 | return false; 306 | } 307 | 308 | }; 309 | 310 | inline Char_checker_result pparse_is_digit(Char_t current_char, bool iseof, std::string &matched_so_far) { 311 | 312 | if (!iseof && isdigit((char) current_char)) { 313 | return Char_checker_result::proceed; 314 | } 315 | if (matched_so_far.size() == 0) { 316 | return Char_checker_result::error; 317 | } 318 | return Char_checker_result::acceptUnget; 319 | } 320 | 321 | // 322 | // PTokInt - sequence of digites 323 | // 324 | template 325 | struct PTokInt : PTokVar { 326 | }; 327 | 328 | 329 | inline Char_checker_result parse_print_char(Char_t current_char, bool iseof, std::string &matched_so_far) { 330 | return !iseof && matched_so_far.size() == 0 && isprint((char) current_char) ? Char_checker_result::acceptNow : Char_checker_result::error; 331 | } 332 | 333 | // 334 | // PTokPrintChar - single printable character 335 | // 336 | template 337 | struct PTokChar : PTokVar { 338 | }; 339 | 340 | inline Char_checker_result pparse_identifier(Char_t current_char, bool iseof, std::string &matched_so_far) { 341 | 342 | ssize_t slen = matched_so_far.size(); 343 | if (iseof) { 344 | if (slen >0) { 345 | return Char_checker_result::acceptNow; 346 | } 347 | } 348 | 349 | if (slen == 0) { 350 | return isalpha(current_char) ? Char_checker_result::proceed : Char_checker_result::error; 351 | } 352 | return isalnum(current_char) || current_char == '_' ? Char_checker_result::proceed : Char_checker_result::acceptUnget; 353 | } 354 | 355 | // 356 | // PTokIdentifier 357 | // 358 | template 359 | struct PTokIdentifierCStyle : PTokVar { 360 | }; 361 | 362 | 363 | 364 | // 365 | // Always acceot or reject any input 366 | // 367 | template 368 | struct PAlways { 369 | static inline const RuleId RULE_ID = 0; 370 | 371 | using ThisClass = PAlways; 372 | 373 | struct AstType : AstEntryBase { 374 | AstType() : AstEntryBase(RULE_ID) { 375 | } 376 | }; 377 | 378 | template 379 | static Parse_result parse(ParserBase &base) { 380 | Text_position token_start_pos = ParserBase::current_pos(base); 381 | return Parse_result{acceptOrReject, Position(token_start_pos), Position(token_start_pos), std::make_unique( Position(token_start_pos), Position(token_start_pos) ) }; 382 | } 383 | 384 | #ifdef __PARSER_ANALYSE__ 385 | template 386 | static bool verify_no_cycles(HelperType *,CycleDetectorHelper &helper, std::ostream &out) { 387 | return true; 388 | } 389 | 390 | template 391 | static bool can_accept_empty_input(HelperType *) { 392 | return true; 393 | } 394 | 395 | #endif 396 | 397 | 398 | template 399 | static void dumpJson(Stream &out, const AstType *ast) { 400 | } 401 | 402 | template 403 | static void init_collision_checker(ParserBase &base) { 404 | } 405 | 406 | }; 407 | 408 | 409 | } // namespace pparse 410 | 411 | 412 | -------------------------------------------------------------------------------- /inc/parse_base.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace pparse { 7 | 8 | const uint32_t ON_RESIZE_DOUBLE_BUFFER_TILL=16*1024; 9 | 10 | using Char_t = char; 11 | using FilePos_t = long; 12 | 13 | #define ERROR(...) do { fprintf(stderr, "Error: " __VA_ARGS__); } while(false); 14 | #define INFO(...) do { fprintf(stderr, "Info: " __VA_ARGS__); } while(false); 15 | 16 | 17 | } 18 | -------------------------------------------------------------------------------- /inc/parse_text.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "parse_base.h" 15 | #include 16 | 17 | namespace pparse { 18 | 19 | struct Text_position { 20 | Text_position() : line_(1), column_(0), buffer_pos_((FilePos_t) 0) { } 21 | 22 | void next_char(Char_t ch) { 23 | if (ch == '\n') { 24 | ++line_; 25 | column_ = 0; 26 | } else { 27 | ++column_; 28 | } 29 | ++buffer_pos_; 30 | } 31 | 32 | int line_; 33 | int column_; 34 | FilePos_t buffer_pos_; 35 | }; 36 | 37 | 38 | // head_ == tail_ : buffer is empty 39 | // (tail+1) % size_ == head_ : buffer is full 40 | // cursor_ in range [ head_, (head_+1) % size_, .... tail_ ] 41 | // 42 | // tail_ - doesn't hold any data. the buffer can therefore hold size() - 1 bytes.` 43 | 44 | class Text_ringbuffer { 45 | public: 46 | 47 | bool init(uint32_t size) { 48 | buf_ = new Char_t[ size ]; 49 | size_ = size; 50 | head_ = tail_ = cursor_ = 0; 51 | return buf_ != nullptr; 52 | } 53 | 54 | void close() { 55 | delete [] buf_; 56 | buf_ = nullptr; 57 | size_ = head_ = tail_ = cursor_ = 0; 58 | 59 | } 60 | 61 | ~Text_ringbuffer() { 62 | delete [] buf_; 63 | } 64 | 65 | bool resize(uint32_t must_have = 0) { 66 | uint32_t newsize; 67 | 68 | do { 69 | if (size_ <= ON_RESIZE_DOUBLE_BUFFER_TILL) { 70 | newsize = 2 * size_; 71 | } else { 72 | newsize = size_ + ON_RESIZE_DOUBLE_BUFFER_TILL; 73 | } 74 | } while( newsize < must_have ); 75 | 76 | Char_t *newbuf = new Char_t[ newsize ]; 77 | if (newbuf == nullptr) { 78 | return false; 79 | } 80 | 81 | memcpy(newbuf, buf_, size_); 82 | delete [] buf_; 83 | buf_ = newbuf; 84 | size_ = newsize; 85 | 86 | return true; 87 | } 88 | 89 | bool empty() const { 90 | return head_ == tail_; 91 | } 92 | 93 | bool full() const { 94 | return (tail_+1) % size_ == head_; 95 | } 96 | 97 | bool eof_cursor() const { 98 | return tail_ == cursor_; 99 | } 100 | 101 | bool inc_head() { 102 | if (!empty()) { 103 | head_ = (head_ + 1) % size_; 104 | return true; 105 | } 106 | return false; 107 | } 108 | 109 | bool inc_tail() { 110 | if (!full()) { 111 | tail_ = (tail_ + 1) % size_; 112 | return true; 113 | } 114 | return false; 115 | } 116 | 117 | bool inc_tail(uint32_t inc_by) { 118 | if (inc_by > free()) { 119 | return false; 120 | } 121 | 122 | tail_ = (tail_ + inc_by) % size_; 123 | return true; 124 | } 125 | 126 | uint32_t free() const { 127 | return size_ - size() - 1; 128 | } 129 | 130 | // number of text bytes in buffer 131 | uint32_t size() const { 132 | if (head_ <= tail_) { 133 | return tail_ - head_; 134 | } 135 | return size_-head_ + tail_; 136 | } 137 | 138 | 139 | bool inc_cursor() { 140 | if (!is_valid_pos(cursor_)) { 141 | return false; 142 | } 143 | cursor_ = (cursor_ + 1) % size_; 144 | return true; 145 | } 146 | 147 | void set_cursor(FilePos_t cursor_pos) { 148 | cursor_ = cursor_pos % size_; 149 | } 150 | 151 | void set_cursor_and_head(FilePos_t cursor_pos) { 152 | head_ = cursor_ = cursor_pos % size_; 153 | } 154 | 155 | 156 | bool is_valid_pos(uint32_t cursor_pos) { 157 | auto next_tail = (tail_ + 1) % size_; 158 | if (head_ < next_tail) { 159 | if (cursor_pos < head_ || cursor_pos > next_tail) { 160 | return false; 161 | } 162 | } else { 163 | if (cursor_pos > next_tail && cursor_pos < head_) { 164 | return false; 165 | } 166 | } 167 | return true; 168 | } 169 | Char_t cursor_char() const { 170 | return buf_[cursor_]; 171 | } 172 | 173 | uint32_t size_; 174 | Char_t *buf_; 175 | uint32_t head_,tail_, cursor_; 176 | }; 177 | 178 | class Text_stream { 179 | public: 180 | static const uint32_t Default_buf_size = 4096; 181 | 182 | using Next_char_value = std::pair; 183 | 184 | Text_stream(bool resize_if_full = true) : fd_(-1), error_(0), pos_at_head_(0), resize_if_full_(resize_if_full), position_nesting_(0) { 185 | } 186 | 187 | ~Text_stream() { 188 | if (fd_ != -1) { 189 | ::close(fd_); 190 | } 191 | } 192 | 193 | bool open(const char *fname, uint32_t buf_size = Default_buf_size) { 194 | 195 | if (fd_ != -1) { 196 | ERROR("Can't open file twice\n"); 197 | return false; 198 | } 199 | 200 | int fd = ::open(fname, O_RDONLY); 201 | if (fd == -1) { 202 | ERROR("Can't open file %s error %d\n", fname, error_); 203 | error_ = errno; 204 | return false; 205 | } 206 | 207 | 208 | if (!open(fd, buf_size)) { 209 | return false; 210 | } 211 | return true; 212 | } 213 | 214 | bool open(int fd, uint32_t buf_size = Default_buf_size ) { 215 | fd_ = fd; 216 | if (!buf_.init(buf_size)) { 217 | ERROR("Can't allocate buffer error %d\n", errno); 218 | ::close(fd_); 219 | return false; 220 | } 221 | 222 | return true; 223 | } 224 | 225 | bool close() { 226 | if (fd_ != -1) { 227 | if (::close(fd_) != 0) { 228 | ERROR("close failed. errno %d\n", errno); 229 | return false; 230 | } 231 | fd_ = -1; 232 | } 233 | buf_.close(); 234 | return true; 235 | } 236 | 237 | bool write_tail(const Char_t *data, uint32_t data_len) { 238 | if (buf_.free() < data_len) { 239 | if (resize_if_full_) { 240 | if (!buf_.resize(buf_.size() + data_len + 1)) { 241 | return false; 242 | } 243 | } else { 244 | return false; 245 | } 246 | } 247 | 248 | if (buf_.tail_ >= buf_.head_) { 249 | auto next_size = buf_.size_ - buf_.tail_; 250 | auto to_copy = std::min(data_len, next_size); 251 | 252 | memcpy((uint8_t *) buf_.buf_ + buf_.tail_, data, to_copy); 253 | 254 | auto remaining = data_len - to_copy; 255 | if (remaining > 0) { 256 | memcpy((uint8_t *) buf_.buf_, data + to_copy , remaining); 257 | } 258 | } else { 259 | memcpy((uint8_t *) buf_.buf_ + buf_.tail_, data, data_len ); 260 | } 261 | 262 | return buf_.inc_tail( data_len ); 263 | } 264 | 265 | Next_char_value current_char() { 266 | 267 | if (buf_.eof_cursor()) { 268 | // if out of data then read as much as possible. 269 | if (buf_.empty()) { 270 | 271 | if (!read_empty()) { 272 | // read error 273 | return Next_char_value(false,' '); 274 | } 275 | 276 | 277 | } else if (!buf_.full()) { 278 | 279 | if (!read_next()) { 280 | // read error 281 | return Next_char_value(false,' '); 282 | } 283 | 284 | } else { 285 | if (fd_ != -1) { 286 | if (resize_if_full_) { 287 | if (!buf_.resize()) { 288 | ERROR("Failed to resize the buffer\n"); 289 | return Next_char_value(false,' '); 290 | } 291 | if (!read_next()) { 292 | // read error 293 | return Next_char_value(false,' '); 294 | } 295 | 296 | } else { 297 | ERROR("buffer is full and can't read any more data\n"); 298 | } 299 | } 300 | return Next_char_value(false,' '); 301 | } 302 | } 303 | 304 | if (buf_.empty()) { 305 | // end of text has been reached. 306 | return Next_char_value(false,' '); 307 | } 308 | 309 | if (buf_.eof_cursor()) { 310 | ERROR("HOW?\n"); 311 | exit(1); 312 | exit(1); 313 | } 314 | auto cur_char = buf_.cursor_char(); 315 | return Next_char_value(true,cur_char); 316 | } 317 | 318 | 319 | Next_char_value next_char() { 320 | 321 | auto rval = current_char(); 322 | 323 | if (rval.first) { 324 | if (!buf_.inc_cursor()) { 325 | ERROR("HOW???\n"); 326 | exit(1); 327 | 328 | } 329 | pos_at_cursor_.next_char(rval.second); 330 | } 331 | return rval; 332 | } 333 | 334 | Text_position pos_at_cursor_and_inc_nesting() { 335 | position_nesting_ += 1; 336 | return pos_at_cursor_; 337 | } 338 | 339 | Text_position pos_at_cursor() const { 340 | return pos_at_cursor_; 341 | } 342 | 343 | bool backtrack(Text_position pos) { 344 | if (pos.buffer_pos_ < pos_at_head_ || pos.buffer_pos_ > (pos_at_head_ + buf_.size()) ) { 345 | ERROR("can't set text position - out of range pos %ld head_pos %ld size %u\n", pos.buffer_pos_, pos_at_head_, buf_.size()); 346 | return true; 347 | } 348 | pos_at_cursor_ = pos; 349 | buf_.set_cursor( pos_at_cursor_.buffer_pos_ ); 350 | 351 | dec_position_nesting(); 352 | return true; 353 | } 354 | 355 | bool dec_position_nesting() { 356 | if (position_nesting_ == 0) { 357 | ERROR("position nesting droppes to negative count. parser is wrong\n"); 358 | return false; 359 | } 360 | position_nesting_ -= 1; 361 | if (position_nesting_ == 0) { 362 | return move_on( pos_at_cursor() ); 363 | } 364 | return true; 365 | } 366 | 367 | // discard all text up until the argument position 368 | bool move_on(Text_position pos) { 369 | 370 | if (pos.buffer_pos_ < pos_at_head_ || pos.buffer_pos_ > (pos_at_head_ + buf_.size()) ) { 371 | ERROR("can't set text position - out of range pos %ld head_pos %ld size %u\n", pos.buffer_pos_, pos_at_head_, buf_.size()); 372 | return true; 373 | } 374 | buf_.set_cursor_and_head(pos.buffer_pos_); 375 | pos_at_cursor_ = pos; 376 | pos_at_head_ = pos.buffer_pos_; 377 | return true; 378 | } 379 | 380 | int error() { return error_; } 381 | 382 | private: 383 | inline bool read_nextv( iovec *vec, int num_vec ) { 384 | ssize_t available = ::readv(fd_, vec, num_vec); 385 | if (available == -1) { 386 | if (errno != EAGAIN) { 387 | error_ = errno; 388 | ERROR("read error. error %d\n", error_); 389 | } 390 | return false; 391 | } 392 | if (available == 0) { 393 | // select? non blocking fd? 394 | return false; 395 | } 396 | 397 | if (!buf_.inc_tail(available)) { 398 | ERROR("HOW??\n"); 399 | return false; 400 | } 401 | return available > 0; 402 | } 403 | 404 | bool read_next() { 405 | 406 | iovec vec[2]; 407 | int num_vec = 1; 408 | 409 | if (fd_ == -1) { 410 | return false; 411 | } 412 | 413 | if (buf_.cursor_ < buf_.head_) { 414 | 415 | vec[0].iov_base = buf_.buf_ + buf_.cursor_; 416 | vec[0].iov_len = buf_.head_ - buf_.cursor_ - 1; 417 | 418 | } else { 419 | 420 | vec[0].iov_base = buf_.buf_ + buf_.cursor_; 421 | vec[0].iov_len = buf_.size_ - buf_.cursor_; 422 | 423 | if (buf_.head_ > 0) { 424 | vec[1].iov_base = buf_.buf_; 425 | vec[1].iov_len = buf_.head_ - 1; 426 | num_vec = 2; 427 | 428 | } else { 429 | 430 | vec[0].iov_len -= 1; 431 | 432 | } 433 | } 434 | 435 | return read_nextv(vec, num_vec); 436 | } 437 | 438 | 439 | bool read_empty() { 440 | iovec vec[2]; 441 | int num_vec = 1; 442 | 443 | vec[0].iov_base = buf_.buf_ + buf_.head_; 444 | vec[0].iov_len = buf_.size_ - buf_.head_; 445 | 446 | if (buf_.head_ > 1) { 447 | vec[1].iov_base = buf_.buf_; 448 | vec[1].iov_len = buf_.head_ - 1; 449 | num_vec = 2; 450 | } 451 | if (buf_.head_ == 0) { 452 | vec[0].iov_len -= 1; 453 | } 454 | 455 | return read_nextv(vec, num_vec); 456 | } 457 | 458 | 459 | int fd_; 460 | int error_; 461 | FilePos_t pos_at_head_; 462 | bool resize_if_full_; 463 | Text_position pos_at_cursor_; 464 | Text_ringbuffer buf_; 465 | int position_nesting_; 466 | }; 467 | 468 | } // namespace pparse 469 | 470 | -------------------------------------------------------------------------------- /inc/parsedef.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "parse_text.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace pparse { 10 | 11 | typedef int RuleId; 12 | 13 | struct Position { 14 | Position() : line_(0), column_(0) { 15 | } 16 | 17 | Position(int line, int column) : line_(line), column_(column) { 18 | } 19 | 20 | Position(Text_position &arg) : line_(arg.line_), column_(arg.column_) { 21 | } 22 | 23 | bool operator==(const Position &arg) const { 24 | return arg.line_ == line_ && arg.column_ == column_; 25 | } 26 | 27 | bool operator<(const Position &arg) const { 28 | return line_ < arg.line_ || (line_ == arg.line_ && column_ < arg.column_); 29 | } 30 | 31 | int line() const { 32 | return line_; 33 | } 34 | 35 | int column() const { 36 | return column_; 37 | } 38 | 39 | 40 | int line_; 41 | int column_; 42 | }; 43 | 44 | 45 | struct AstEntryBase { 46 | AstEntryBase(RuleId ruleId) : ruleId_(ruleId) { 47 | } 48 | 49 | virtual ~AstEntryBase() { 50 | } 51 | 52 | RuleId getRuleId() const { 53 | return ruleId_; 54 | } 55 | 56 | Position get_start_pos() const { 57 | return start_; 58 | } 59 | 60 | Position get_end_pos() const { 61 | return end_; 62 | } 63 | 64 | Position start_; 65 | Position end_; 66 | RuleId ruleId_; 67 | }; 68 | 69 | 70 | struct Parse_result { 71 | 72 | bool success() const { 73 | return success_; 74 | } 75 | 76 | AstEntryBase *get_ast() const { 77 | return ast_.get(); 78 | } 79 | 80 | Position get_start_pos() const { 81 | return start_; 82 | } 83 | 84 | Position get_end_pos() const { 85 | return end_; 86 | } 87 | 88 | bool success_; 89 | Position start_; 90 | Position end_; 91 | std::unique_ptr ast_; 92 | }; 93 | 94 | } // namespace pparse 95 | 96 | 97 | -------------------------------------------------------------------------------- /inc/tokencollisionhelper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "parse_base.h" 4 | #include 5 | #include 6 | #include 7 | 8 | namespace pparse { 9 | 10 | using TokenHash = unsigned long; 11 | 12 | // 13 | // The collision checker is a helper object. 14 | // it is used to check that variables that have the string value of a token are not accepted as a variable; 15 | // 16 | struct TokenCollisionChecker { 17 | 18 | static constexpr TokenHash TokenHashInitValue = 5381; 19 | using Map_type = std::unordered_multimap< TokenHash, std::string >; 20 | 21 | struct type_info_compare { 22 | bool operator() (const std::type_info *a, const std::type_info *b) const { 23 | return a->before(*b); 24 | } 25 | }; 26 | 27 | using Type_info_set_type = std::set; 28 | 29 | // pick Dan Bernstein hash from here https://stackoverflow.com/questions/7666509/hash-function-for-string 30 | static inline TokenHash calculate_hash_val(TokenHash hash, Char_t ch) { 31 | return (hash << 5) + hash + ch; 32 | } 33 | 34 | static inline TokenHash calculate_hash(const Char_t *token, uint32_t token_len) { 35 | TokenHash hash = TokenHashInitValue; 36 | 37 | for(uint32_t pos=0; pos < token_len; ++pos) { 38 | hash = calculate_hash_val( hash, token[pos]); 39 | } 40 | return hash; 41 | } 42 | 43 | 44 | bool insert(const Char_t * token, uint32_t token_len) { 45 | 46 | //printf("Insert %s(%d)\n", token,token_len); 47 | 48 | TokenHash hash = calculate_hash( token, token_len ); 49 | 50 | if (has_token_imp(token, token_len, hash)) { 51 | //printf("already exists\n"); 52 | return true; 53 | } 54 | 55 | //mapHashToToken.emplace( hash, std::string( token, token_len ) ); 56 | mapHashToToken.insert( Map_type::value_type( hash, std::string( token, token_len ) ) ); 57 | printf("inserted\n"); 58 | return true; 59 | } 60 | 61 | bool has_token(const Char_t *token, uint32_t token_len) { 62 | TokenHash hash = calculate_hash( token, token_len ); 63 | return has_token_imp( token, token_len, hash ); 64 | } 65 | 66 | 67 | bool insert_type_info(const std::type_info *info) { 68 | auto ret = tinfo_set.insert(info); 69 | return ret.second; 70 | } 71 | 72 | bool remove_type_info(const std::type_info *info) { 73 | return tinfo_set.erase(info) > 0; 74 | } 75 | 76 | private: 77 | 78 | inline bool has_token_imp(const Char_t *token, uint32_t token_len, TokenHash hash) { 79 | auto pos = mapHashToToken.find( hash ); 80 | if (pos == mapHashToToken.end()) { 81 | return false; 82 | } 83 | 84 | if (pos->second.size() != token_len) { 85 | return false; 86 | } 87 | 88 | if (strncmp(pos->second.c_str(), token, token_len) == 0) { 89 | return true; 90 | } 91 | return false; 92 | } 93 | 94 | Map_type mapHashToToken; 95 | Type_info_set_type tinfo_set; 96 | }; 97 | 98 | 99 | } // namespace pparse 100 | 101 | 102 | -------------------------------------------------------------------------------- /inc/vhelper.h: -------------------------------------------------------------------------------- 1 | #ifdef __PARSER_TRACE__ 2 | 3 | namespace pparse { 4 | 5 | const int PREFIX_NSPACE_LEN = 8; 6 | const char *CHAR_SYM="(char)"; 7 | const int CHAR_SYM_LEN=6; 8 | 9 | 10 | static inline size_t __thread nesting_ = 0; 11 | 12 | template 13 | struct VisualizeTrace { 14 | 15 | using ThisClass = VisualizeTrace; 16 | 17 | static std::string trace_start_parsing(Text_position pos) { 18 | std::string short_name = make_short_name(); 19 | printf("(%03ld)%sstart parsing: %s at: (%d:%d offset: %ld)\n", nesting_, std::string( nesting_, ' ').c_str(), short_name.c_str(), pos.line_, pos.column_, pos.buffer_pos_); 20 | nesting_ += 1; 21 | return short_name; 22 | } 23 | 24 | static void end_parsing(std::string &short_name, bool success,Text_position pos) { 25 | nesting_ -= 1; 26 | printf("(%03ld)%send parsing: %s %s at: (%d:%d offset: %ld)\n", nesting_, std::string( nesting_, ' ').c_str(), short_name.c_str(), success ? "SUCCESS" : "FAIL", pos.line_, pos.column_, pos.buffer_pos_ ); 27 | } 28 | 29 | static void end_parsing_choice(std::string &short_name, bool success,Text_position pos, size_t index) { 30 | nesting_ -= 1; 31 | 32 | if (success) { 33 | printf("(%03ld)%send parsing: %s SUCCESS choice-index: %ld at: (%d:%d offset: %ld)\n", nesting_, std::string( nesting_, ' ').c_str(), short_name.c_str(), index , pos.line_, pos.column_, pos.buffer_pos_ ); 34 | } else { 35 | 36 | printf("(%03ld)%send parsing: %s FAIL at (%d:%d offset: %ld)\n", nesting_, std::string( nesting_, ' ').c_str(), short_name.c_str(), pos.line_, pos.column_, pos.buffer_pos_ ); 37 | } 38 | } 39 | 40 | 41 | static std::string trace_start_parsing_token(Text_position pos) { 42 | std::string short_name = make_token_name(); 43 | 44 | printf("(%03ld)%sstart parsing: %s at: (%d:%d offset: %ld)\n", nesting_, std::string( nesting_, ' ').c_str(), short_name.c_str(), pos.line_, pos.column_, pos.buffer_pos_); 45 | nesting_ += 1; 46 | 47 | return short_name; 48 | } 49 | 50 | 51 | 52 | private: 53 | 54 | static std::string make_token_name() { 55 | 56 | std::string rval("token: "); 57 | 58 | size_t pos = 0; 59 | std::string cname = demangle(); 60 | while(pos < cname.size()) { 61 | size_t next_pos = cname.find( CHAR_SYM, pos); 62 | if (next_pos == std::string::npos) { 63 | break; 64 | } 65 | next_pos += CHAR_SYM_LEN; 66 | 67 | size_t eof_pos = cname.find( ' ', next_pos); 68 | if (eof_pos == std::string::npos) { 69 | eof_pos = cname.size(); 70 | } 71 | 72 | std::string tokval = cname.substr(next_pos, eof_pos - next_pos); 73 | 74 | size_t nchar = atoi(tokval.c_str()); 75 | 76 | rval += (Char_t) nchar; 77 | 78 | pos = eof_pos; 79 | } 80 | return rval; 81 | } 82 | 83 | 84 | using Component_t = std::pair; 85 | 86 | static Component_t find_component_name(std::string dname, size_t start_pos) 87 | { 88 | size_t next_pos = start_pos; 89 | 90 | if ( next_pos >= dname.size() ) { 91 | return Component_t(std::string::npos,0); 92 | } 93 | 94 | int template_nesting = 0, max_template_nesting = 0; 95 | 96 | for(next_pos += 1;next_pos < dname.size(); ++next_pos) { 97 | 98 | if (dname[next_pos] == '<') { 99 | template_nesting += 1; 100 | max_template_nesting += 1; 101 | } else if (dname[next_pos] == '>') { 102 | template_nesting -= 1; 103 | if (template_nesting == 0) { 104 | break; 105 | } 106 | } else if (dname[next_pos] == ',' && template_nesting == 0) { 107 | break; 108 | } 109 | } 110 | 111 | return Component_t(next_pos+1, max_template_nesting); 112 | 113 | } 114 | 115 | static std::string make_short_name(std::string dname) { 116 | if (dname.substr(0,PREFIX_NSPACE_LEN) != "pparse::") { 117 | return ""; 118 | } 119 | 120 | size_t npos = dname.find( '<', PREFIX_NSPACE_LEN); 121 | if (npos == std::string::npos) { 122 | return ""; 123 | } 124 | npos = dname.find( ' ', npos); 125 | if (npos == std::string::npos) { 126 | return ""; 127 | } 128 | 129 | std::string rname = dname.substr(PREFIX_NSPACE_LEN, npos - PREFIX_NSPACE_LEN ); 130 | 131 | Component_t ret; 132 | 133 | while((ret = find_component_name(dname, npos)), ret.first != std::string::npos) { 134 | 135 | std::string component = dname.substr(npos + 1, ret.first - npos - 1); 136 | 137 | if (ret.second == 0) { 138 | 139 | size_t nposc = component.rfind(':'); 140 | if (nposc != std::string::npos) { 141 | nposc += 1; 142 | rname += " " + component.substr(nposc, component.size() - nposc); 143 | } else { 144 | rname += " " + component; 145 | } 146 | } else { 147 | auto pos_start = component.find('<'); 148 | if (pos_start != std::string::npos) { 149 | pos_start = component.find(' ', pos_start+1); 150 | if (pos_start != std::string::npos) { 151 | pos_start -= 1; 152 | } else { 153 | pos_start = component.size()-1; 154 | } 155 | } 156 | 157 | rname += " " + component.substr(0,pos_start) + "> "; 158 | 159 | // auto pos_tok = component.find("::"); 160 | // if (pos_start!= std::string::npos) { 161 | // if ((pos_tok+2) < pos_start && pos_tok != std::string::npos) { 162 | // rname += " " + component.substr(pos_tok + 2, pos_start - pos_tok - 2) + "> "; 163 | // } else { 164 | // rname += " " + component.substr(0,pos_start) + "> "; 165 | // } 166 | // } 167 | } 168 | npos = dname.find(' ', ret.first); 169 | if (npos == std::string::npos) { 170 | break; 171 | } 172 | } 173 | 174 | return rname; 175 | 176 | } 177 | 178 | static std::string make_short_name() { 179 | std::string cname = demangle(); 180 | 181 | std::string shname = make_short_name(cname); 182 | if (shname == "") { 183 | return cname; 184 | } 185 | return shname; 186 | } 187 | 188 | }; 189 | 190 | 191 | } // eof namespace pparse 192 | 193 | 194 | 195 | // eof __PARSER_TRACE__ 196 | #endif 197 | 198 | -------------------------------------------------------------------------------- /test/test_analyse.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | #define __PARSER_ANALYSE__ 4 | #include "parse.h" 5 | 6 | namespace { 7 | 8 | using namespace pparse; 9 | 10 | template 11 | Parse_result test_string(Char_t *test_string) { 12 | Text_stream text_stream; 13 | 14 | bool isok = text_stream.open(-1); 15 | EXPECT_EQ(isok, true); 16 | 17 | isok = text_stream.write_tail(test_string, strlen(test_string) ); 18 | EXPECT_EQ(isok, true); 19 | 20 | 21 | CharParser chparser(text_stream); 22 | 23 | Parse_result res = Parser::parse(chparser); 24 | 25 | if (!res.success()) { 26 | printf("Error: parser error at: line: %d column: %d text: %s\n", res.get_start_pos().line(), res.get_start_pos().column(), test_string); 27 | } 28 | 29 | return res; 30 | 31 | } 32 | 33 | TEST(TestAnalyze, testCycle) { 34 | 35 | struct Int : PTokInt<1> {}; 36 | 37 | struct Expr; 38 | 39 | struct Mult : PAny<2, PTok<3,CSTR1("*")>, PTok<4,CSTR1("/")> > {}; 40 | 41 | struct Add : PAny<5, PTok<6,CSTR1("+")>, PTok<7,CSTR1("-")> > {}; 42 | 43 | struct NestedExpr : PSeq<8, /*PTok<0, CSTR1("(")>,*/ Expr, PTok<0, CSTR1(")")> > {}; 44 | 45 | struct NegativeInt : PSeq<1, PTok<11, CSTR1("-")>, Int> {}; 46 | 47 | struct SimpleExpr : PAny<6, Int, NegativeInt, NestedExpr> {}; // 48 | 49 | struct MultExpr; 50 | 51 | struct MultExpr: PAny<7, PSeq<6, SimpleExpr, Mult, MultExpr >, SimpleExpr> {}; 52 | 53 | struct Expr: PAny<7, PSeq<6, MultExpr, Add, Expr >, MultExpr> {}; 54 | 55 | bool has_cycles = ParserChecker::check(std::cout); 56 | EXPECT_TRUE(has_cycles); 57 | 58 | // 59 | // CycleDetectorHelper helper; 60 | // 61 | // helper.push_and_check(std::cout, get_tinfo((MultExpr *) nullptr), -1); 62 | // 63 | // bool is_valid = MultExpr::verify_no_cycles((void *) nullptr, helper, std::cout); 64 | // 65 | // helper.pop(); 66 | // 67 | // EXPECT_FALSE(is_valid); 68 | 69 | } 70 | 71 | TEST(TestAnalyze, testNoCycle) { 72 | 73 | struct Int : PTokInt<1> {}; 74 | 75 | struct Expr; 76 | 77 | struct Mult : PAny<2, PTok<3,CSTR1("*")>, PTok<4,CSTR1("/")> > {}; 78 | 79 | struct Add : PAny<5, PTok<6,CSTR1("+")>, PTok<7,CSTR1("-")> > {}; 80 | 81 | struct NestedExpr : PSeq<8, PTok<0, CSTR1("(")>, Expr, PTok<0, CSTR1(")")> > {}; 82 | 83 | struct NegativeInt : PSeq<1, PTok<11, CSTR1("-")>, Int> {}; 84 | 85 | struct SimpleExpr : PAny<6, Int, NegativeInt, NestedExpr> {}; // 86 | 87 | struct MultExpr; 88 | 89 | struct MultExpr: PAny<7, PSeq<6, SimpleExpr, Mult, MultExpr >, SimpleExpr> {}; 90 | 91 | struct Expr; 92 | 93 | struct Expr: PAny<7, PSeq<6, MultExpr, Add, Expr >, MultExpr> {}; 94 | 95 | 96 | bool has_cycles = ParserChecker::check(std::cout); 97 | EXPECT_FALSE(has_cycles); 98 | 99 | // CycleDetectorHelper helper; 100 | // 101 | // helper.push_and_check(std::cout, get_tinfo((MultExpr *) nullptr), -1); 102 | // 103 | // bool is_valid = MultExpr::verify_no_cycles((void *) nullptr, helper, std::cout); 104 | // 105 | // helper.pop(); 106 | // 107 | // EXPECT_TRUE(is_valid); 108 | // 109 | } 110 | 111 | }//namespace 112 | 113 | 114 | -------------------------------------------------------------------------------- /test/test_grammar.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | //enable execution trace with the next define 4 | //#define __PARSER_TRACE__ 5 | #include "parse.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | namespace { 14 | 15 | using namespace pparse; 16 | 17 | bool runJq() { 18 | int rt = system("which jq"); 19 | if (rt == 0) { 20 | int rcode = system("jq . 33 | Parse_result test_string(Char_t *test_string) { 34 | Text_stream text_stream; 35 | 36 | bool isok = text_stream.open(-1); 37 | EXPECT_EQ(isok, true); 38 | 39 | isok = text_stream.write_tail(test_string, strlen(test_string) ); 40 | EXPECT_EQ(isok, true); 41 | 42 | 43 | CharParser chparser(text_stream); 44 | 45 | Parse_result res = Parser::parse(chparser); 46 | 47 | if (!res.success()) { 48 | printf("Error: parser error at: line: %d column: %d text: %s\n", res.get_start_pos().line(), res.get_start_pos().column(), test_string); 49 | } 50 | EXPECT_TRUE(res.get_ast() != nullptr); 51 | 52 | return res; 53 | 54 | } 55 | 56 | 57 | 58 | TEST(TestGrammar, testRecursion) { 59 | 60 | struct Int : PTokInt<1> {}; 61 | 62 | struct Expr; 63 | 64 | struct Mult : PAny<2, PTok<3,CSTR1("*")>, PTok<4,CSTR1("/")> > {}; 65 | 66 | struct Add : PAny<4, PTok<5,CSTR1("+")>, PTok<6,CSTR1("-")> > {}; 67 | 68 | struct NestedExpr : PSeq<7, PTok<8, CSTR1("(")>, Expr, PTok<9, CSTR1(")")> > {}; 69 | 70 | struct NegativeInt : PSeq<10, PTok<11, CSTR1("-")>, Int> {}; 71 | 72 | struct SimpleExpr : PAny<12, Int, NegativeInt, NestedExpr> {}; // 73 | 74 | struct MultExpr; 75 | 76 | struct MultExpr: PAny<13, PSeq<14, SimpleExpr, Mult, MultExpr >, SimpleExpr> {}; 77 | 78 | struct Expr; 79 | 80 | struct Expr: PAny<15, PSeq<16, MultExpr, Add, Expr >, MultExpr> {}; 81 | 82 | struct ExprEof : PRequireEof {}; 83 | 84 | Parse_result result = test_string((Char_t *) "-1"); 85 | EXPECT_TRUE(result.success()); 86 | // 87 | // if (result.get_ast() != nullptr ) { 88 | // 89 | // 90 | // ExprEof::dumpJson(std::cout, (ExprEof::AstType *) result.get_ast() ); 91 | // } 92 | // 93 | result = test_string((Char_t *) "1 * 3"); 94 | EXPECT_TRUE(result.success()); 95 | 96 | // if (result.get_ast() != nullptr ) { 97 | // ExprEof::dumpJson(std::cout, (ExprEof::AstType *) result.get_ast() ); 98 | // } 99 | // 100 | 101 | result = test_string((Char_t *) "2 + 2 + 2"); 102 | EXPECT_TRUE(result.success()); 103 | 104 | if (result.get_ast() != nullptr ) { 105 | std::ofstream out("test.json"); 106 | // ExprEof::dumpJson(out, (ExprEof::AstType *) result.get_ast() ); 107 | out.close(); 108 | // int rcode = system("jq . ((Char_t *) "2 + 2 + 2 + 1 * 3"); 115 | EXPECT_TRUE(result.success()); 116 | 117 | if (result.get_ast() != nullptr ) { 118 | std::ofstream out("test.json"); 119 | ExprEof::dumpJson(out, (ExprEof::AstType *) result.get_ast() ); 120 | out.close(); 121 | bool rt = runJq(); 122 | EXPECT_TRUE(rt); 123 | } 124 | 125 | 126 | result = test_string((Char_t *) "(2*3) + 5"); 127 | EXPECT_TRUE(result.success()); 128 | 129 | if (result.get_ast() != nullptr ) { 130 | std::ofstream out("test.json"); 131 | ExprEof::dumpJson(out, (ExprEof::AstType *) result.get_ast() ); 132 | out.close(); 133 | bool rt = runJq(); 134 | EXPECT_TRUE(rt); 135 | } 136 | 137 | } 138 | 139 | inline Char_checker_result pparse_regex_char(Char_t current_char, bool iseof, std::string &matched_so_far) { 140 | return !iseof && 141 | matched_so_far.size() == 0 && 142 | isprint((char) current_char) && 143 | strchr("+-*()[].",current_char) == 0 ? Char_checker_result::acceptNow : Char_checker_result::error; 144 | } 145 | 146 | template 147 | struct PTokRegexChar : PTokVar { 148 | }; 149 | 150 | TEST(TestGrammar, testRegEx) { 151 | 152 | struct Regex; 153 | 154 | struct RegexAnyChar : PTok<1, CSTR1(".") > {}; 155 | 156 | struct RegexEscapeChar : PSeq<1, PTok<0,CSTR1("\\")>, PTokChar<1>> {}; 157 | 158 | struct RegexCharRangeChar : PAny<1, RegexEscapeChar, PTokRegexChar<1> > {}; 159 | 160 | struct RegexCharRange : PSeq<1, RegexCharRangeChar, PTok<1, CSTR1("-")>, RegexCharRangeChar> {}; 161 | 162 | struct RegexSetItems : PPlus<1, PAny<1, RegexCharRange, RegexAnyChar> > {}; 163 | 164 | struct RegexSet : PSeq<1, PTok<0, CSTR1("[")>, RegexSetItems, PTok<0, CSTR1("]")> > {}; 165 | 166 | struct RegexOp : PAny<1, PTok<1, CSTR1("*")>, PTok<2, CSTR1("?")>, PTok<3, CSTR1("+") > > {}; 167 | 168 | struct RegexSetWithOp : PSeq<1, RegexSet , POpt<1, RegexOp>> {}; 169 | 170 | struct RegexNestedWithOp : PSeq<1, PTok<1,CSTR1("(")>, Regex, PTok<1, CSTR1(")")>, POpt<1, RegexOp> > {}; 171 | 172 | struct RegexSetItemWithOp : PSeq<1, PAny<1, RegexCharRange, RegexAnyChar, RegexCharRangeChar>, POpt<1, RegexOp> > {}; 173 | 174 | struct Regex : PStar<1, PAny< 1, RegexSetWithOp, RegexNestedWithOp, RegexSetItemWithOp> > {}; 175 | 176 | struct RegexEof : PRequireEof {}; 177 | 178 | Parse_result result = test_string((Char_t *) "ab"); 179 | EXPECT_TRUE(result.success()); 180 | 181 | result = test_string((Char_t *) "abc123"); 182 | EXPECT_TRUE(result.success()); 183 | 184 | result = test_string((Char_t *) "[a-z]" ); 185 | EXPECT_TRUE(result.success()); 186 | 187 | result = test_string((Char_t *) "abc[a-z]+" ); 188 | EXPECT_TRUE(result.success()); 189 | 190 | result = test_string((Char_t *) "abc([0-9]+)?" ); 191 | EXPECT_TRUE(result.success()); 192 | 193 | } 194 | 195 | } 196 | -------------------------------------------------------------------------------- /test/test_main.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | 5 | testing::InitGoogleTest(&argc, argv); 6 | return RUN_ALL_TESTS(); 7 | } 8 | -------------------------------------------------------------------------------- /test/test_pascal.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | //enable execution trace with the next define 4 | //#define __PARSER_TRACE__ 5 | #include "parse.h" 6 | 7 | #include 8 | #include 9 | 10 | namespace { 11 | 12 | using namespace pparse; 13 | 14 | template 15 | Parse_result test_string(Char_t *test_string) { 16 | Text_stream text_stream; 17 | 18 | bool isok = text_stream.open(-1); 19 | EXPECT_EQ(isok, true); 20 | 21 | isok = text_stream.write_tail(test_string, strlen(test_string) ); 22 | EXPECT_EQ(isok, true); 23 | 24 | 25 | CharParser chparser(text_stream); 26 | 27 | Parse_result res = Parser::parse(chparser); 28 | 29 | if (!res.success()) { 30 | printf("Error: parser error at: line: %d column: %d text: %s\n", res.get_start_pos().line(), res.get_start_pos().column(), test_string); 31 | } 32 | 33 | return res; 34 | 35 | } 36 | 37 | [[maybe_unused]] inline Char_checker_result pparse_pascal_identifier(Char_t current_char, bool iseof, std::string &matched_so_far) { 38 | 39 | ssize_t slen = matched_so_far.size(); 40 | if (iseof) { 41 | if (slen >0) { 42 | return Char_checker_result::acceptNow; 43 | } 44 | } 45 | 46 | if (slen == 0) { 47 | return isalpha(current_char) ? Char_checker_result::proceed : Char_checker_result::error; 48 | } 49 | return isalnum(current_char) ? Char_checker_result::proceed : Char_checker_result::acceptUnget; 50 | } 51 | 52 | template 53 | struct PascalIdentifier : PTokVar { 54 | }; 55 | 56 | 57 | [[maybe_unused]] inline Char_checker_result pparse_pascal_string(Char_t current_char, bool iseof, std::string &matched_so_far) { 58 | 59 | ssize_t slen = matched_so_far.size(); 60 | 61 | if (iseof) { 62 | return Char_checker_result::error; 63 | } 64 | 65 | if (slen == 0) { 66 | if (current_char == '\'' ) { 67 | return Char_checker_result::proceed; 68 | } 69 | return Char_checker_result::error; 70 | 71 | } 72 | if (current_char == '\'' ) { 73 | return Char_checker_result::acceptNow; 74 | } 75 | return Char_checker_result::proceed; 76 | } 77 | 78 | template 79 | struct PascalStringConst : PTokVar { 80 | }; 81 | 82 | 83 | TEST(TestPascal, test) { 84 | 85 | //*** https://condor.depaul.edu/ichu/csc447/notes/wk2/pascal.html *** 86 | 87 | struct Fraction : PSeq<3, PTok<4,CSTR1(".")>, PTokInt<2> > {}; 88 | 89 | struct Exponent : PSeq<4, PTok<5,CSTR1("E")>, POpt<4, PAny<5, PTok<6, CSTR1("-")>, PTok<6, CSTR1("+")> > >, PTokInt<7> > {}; 90 | 91 | struct UnsignedNumber : PSeq<1, PTokInt<2>, POpt<2, Fraction>, POpt<3, Exponent> > {}; 92 | 93 | struct UnsignedConstant : PAny<1, PascalStringConst<1>, UnsignedNumber, PTok<7, CSTR3("NIL")>, PascalIdentifier<1> > {}; 94 | 95 | struct SimpleType : PAny<1, PascalIdentifier<2>, PSeq<3, PTok<4,CSTR1("(")>, PascalIdentifier<4>, PTok<5, CSTR1(")") > > > {}; 96 | 97 | struct ConstNum : PSeq<1, PAny<2, POpt<3, PAny<3, PTok<4,CSTR1("+")>, PTok<4,CSTR1("-")> > >, PAny<6, UnsignedNumber, UnsignedConstant> > > {}; 98 | 99 | struct Constant : PAny<1, PascalStringConst<2>, ConstNum > {}; 100 | 101 | struct PointerType : PSeq<1, PTok<2, CSTR1("^")>, PascalIdentifier<3> > {}; 102 | 103 | struct Type; 104 | 105 | struct SimpleTypeList; 106 | 107 | struct SimpleTypeList : PAny<1, PSeq<1, SimpleType, PTok<2, CSTR1(",")> , SimpleTypeList >, SimpleType > {}; 108 | 109 | struct ArrayType : PSeq<1, PTok<2, CSTR5("array")>, PTok<2, CSTR1("[")>, SimpleTypeList, PTok<4,CSTR1("]")>, PTok<5,CSTR2("of")>, Type > {}; 110 | 111 | struct FileType : PSeq<1 , PTok<2, CSTR4("file")>, PTok<3, CSTR2("of")>, Type > {}; 112 | 113 | struct SetType : PSeq<1 , PTok<2, CSTR3("set")>, PTok<3, CSTR2("of")>, SimpleType > {}; 114 | 115 | /* 116 | struct FieldList : 117 | 118 | struct RecordType : PSeq<1 , PTok<2, CSTR6("record")>, FieldList, PTok<3, CSTR3("end")> > {}; 119 | */ 120 | 121 | struct Type : PAny<1, PascalIdentifier<2>, PointerType, ArrayType, FileType, SetType > {}; 122 | 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /test/test_rules.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | //enable execution trace with the next define 4 | //#define __PARSER_TRACE__ 5 | #include "parse.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | 13 | namespace { 14 | 15 | using namespace pparse; 16 | 17 | 18 | template 19 | Parse_result test_string(Char_t *test_string) { 20 | Text_stream text_stream; 21 | 22 | bool isok = text_stream.open(-1); 23 | EXPECT_EQ(isok, true); 24 | 25 | isok = text_stream.write_tail(test_string, strlen(test_string) ); 26 | EXPECT_EQ(isok, true); 27 | 28 | 29 | CharParser chparser(text_stream); 30 | 31 | Parse_result res = Parser::parse(chparser); 32 | 33 | if (!res.success()) { 34 | printf("Error: parser error at: line: %d column: %d text: %s\n", res.get_start_pos().line(), res.get_start_pos().column(), test_string); 35 | } 36 | 37 | return res; 38 | 39 | } 40 | 41 | TEST(TestRules,testIntParser) { 42 | 43 | struct TokenIntParser : PTokInt<1> {}; 44 | 45 | auto result = test_string((Char_t *) " \t12934 " ); 46 | EXPECT_TRUE(result.success()); 47 | 48 | //printf("resul pos: start(line: %d col: %d) end(line: %d col: %d)\n", result.start_.line_, result.start_.column_, result.end_.line_, result.end_.column_ ); 49 | 50 | EXPECT_TRUE(result.start_ == Position(1,2)); 51 | EXPECT_TRUE(result.end_ == Position(1,6)); 52 | EXPECT_TRUE(result.get_ast()->getRuleId() == 1); 53 | 54 | TokenIntParser::AstType *type = (TokenIntParser::AstType *) result.get_ast(); 55 | //printf("res: [%s]\n", type->entry_.c_str()); 56 | EXPECT_TRUE(type->entry_ == "12934"); 57 | 58 | 59 | result = test_string((Char_t *) " \t12934" ); 60 | EXPECT_TRUE(result.success()); 61 | 62 | TokenIntParser::dumpJson(std::cout, (TokenIntParser::AstType *) result.get_ast() ); 63 | std::cout << "\n"; 64 | 65 | 66 | 67 | //printf("resul pos: start(line: %d col: %d) end(line: %d col: %d)\n", result.start_.line_, result.start_.column_, result.end_.line_, result.end_.column_ ); 68 | 69 | EXPECT_TRUE(result.start_ == Position(1,2)); 70 | EXPECT_TRUE(result.end_ == Position(1,6)); 71 | EXPECT_TRUE(result.get_ast()->getRuleId() == 1); 72 | 73 | type = (TokenIntParser::AstType *) result.get_ast(); 74 | //printf("res: [%s]\n", type->entry_.c_str()); 75 | EXPECT_TRUE(type->entry_ == "12934"); 76 | 77 | 78 | } 79 | 80 | 81 | TEST(TestRules,testAnyCharParser) { 82 | 83 | 84 | struct TokenCharParser : PTokChar<1> {}; 85 | 86 | auto result = test_string((Char_t *) " \tfFd" ); 87 | EXPECT_TRUE(result.success()); 88 | TokenCharParser::dumpJson(std::cout, (TokenCharParser::AstType *) result.get_ast() ); 89 | std::cout << "\n"; 90 | 91 | 92 | //printf("resul pos: start(line: %d col: %d) end(line: %d col: %d)\n", result.start_.line_, result.start_.column_, result.end_.line_, result.end_.column_ ); 93 | 94 | EXPECT_TRUE(result.start_ == Position(1,2)); 95 | EXPECT_TRUE(result.end_ == Position(1,2)); 96 | EXPECT_TRUE(result.get_ast()->getRuleId() == 1); 97 | 98 | 99 | TokenCharParser::AstType *type = (TokenCharParser::AstType *) result.get_ast(); 100 | 101 | //printf("res: %s\n", type->entry_.c_str()); 102 | 103 | EXPECT_TRUE(type->entry_ == "f"); 104 | 105 | } 106 | 107 | 108 | TEST(TestRules,testTokenParser) { 109 | 110 | Text_stream text_stream; 111 | 112 | bool isok = text_stream.open(-1,11); 113 | EXPECT_EQ(isok, true); 114 | 115 | isok = text_stream.write_tail((Char_t*)" \tif " , 5); 116 | 117 | CharParser chparser(text_stream); 118 | 119 | struct TokenIfParser : PTok<0, CSTR2("if")> {}; 120 | 121 | auto result = TokenIfParser::parse( chparser ); 122 | EXPECT_TRUE(result.success()); 123 | 124 | //printf("resul pos: start(line: %d col: %d) end(line: %d col: %d)\n", result.start_.line_, result.start_.column_, result.end_.line_, result.end_.column_ ); 125 | 126 | EXPECT_TRUE(result.start_ == Position(1,2)); 127 | EXPECT_TRUE(result.end_ == Position(1,3)); 128 | } 129 | 130 | TEST(TestRules,testAnyParser) { 131 | 132 | Text_stream text_stream; 133 | 134 | bool isok = text_stream.open(-1,11); 135 | EXPECT_EQ(isok, true); 136 | 137 | isok = text_stream.write_tail((Char_t*)" \telse" , 7); 138 | 139 | struct TokenIfParser : PTok<1, CSTR2("if")> {}; 140 | 141 | struct TokenThenParser : PTok<2, CSTR4("then")> {}; 142 | 143 | struct TokenElseParser : PTok<3, CSTR4("else")> {}; 144 | 145 | struct TokenKeywdParser : PAny<4, TokenIfParser, TokenElseParser, TokenThenParser> {}; 146 | 147 | CharParser chparser(text_stream); 148 | 149 | auto result = TokenKeywdParser::parse( chparser ); 150 | EXPECT_TRUE(result.success()); 151 | 152 | AstEntryBase *res = result.ast_.get(); 153 | EXPECT_TRUE( res != nullptr ); 154 | EXPECT_TRUE( res->ruleId_ == 4 ); 155 | 156 | TokenKeywdParser::dumpJson(std::cout, (TokenKeywdParser::AstType *) result.get_ast() ); 157 | std::cout << "\n"; 158 | 159 | 160 | TokenKeywdParser::AstType *anyAst = (TokenKeywdParser::AstType *) res; 161 | 162 | EXPECT_TRUE(anyAst->entry_.index() == 1); 163 | EXPECT_TRUE( std::get<1>(anyAst->entry_).get()->ruleId_ == 3); 164 | } 165 | 166 | TEST(TestRules,testOptParser) { 167 | 168 | struct TokenIfParser : POpt<2, PTok<1, CSTR2("if")> > {}; 169 | 170 | Parse_result result = test_string((Char_t *) "\t if\n"); 171 | EXPECT_TRUE(result.success()); 172 | 173 | TokenIfParser::AstType *ifAst = (TokenIfParser::AstType *) result.ast_.get(); 174 | EXPECT_TRUE(ifAst->entry_.has_value()); 175 | 176 | 177 | result = test_string((Char_t *) "\t else\n"); 178 | EXPECT_TRUE(result.success()); 179 | TokenIfParser::dumpJson(std::cout, (TokenIfParser::AstType *) result.get_ast() ); 180 | std::cout << "\n"; 181 | 182 | 183 | 184 | ifAst = (TokenIfParser::AstType *) result.ast_.get(); 185 | EXPECT_FALSE(ifAst->entry_.has_value()); 186 | 187 | 188 | } 189 | 190 | TEST(TestRules,testSeqParser) { 191 | 192 | Text_stream text_stream; 193 | 194 | bool isok = text_stream.open(-1,11); 195 | EXPECT_EQ(isok, true); 196 | 197 | const Char_t *str = " \tif then else"; 198 | isok = text_stream.write_tail(str, strlen(str)); 199 | 200 | struct TokenIfParser : PTok<1, CSTR2("if")> {}; 201 | 202 | struct TokenThenParser : PTok<2, CSTR4("then")> {}; 203 | 204 | struct TokenElseParser : PTok<3, CSTR4("else")> {}; 205 | 206 | struct TokenKeywdParser : PSeq<4, TokenIfParser, TokenThenParser, TokenElseParser> {}; 207 | 208 | CharParser chparser(text_stream); 209 | 210 | auto result = TokenKeywdParser::parse( chparser ); 211 | EXPECT_TRUE(result.success()); 212 | 213 | AstEntryBase *res = result.ast_.get(); 214 | EXPECT_TRUE( res != nullptr ); 215 | EXPECT_TRUE( res->ruleId_ == 4 ); 216 | 217 | TokenKeywdParser::AstType *seqAst = (TokenKeywdParser::AstType *) res; 218 | 219 | EXPECT_TRUE(std::get<0>(seqAst->entry_)->ruleId_ == 1); 220 | EXPECT_TRUE(std::get<1>(seqAst->entry_)->ruleId_ == 2); 221 | EXPECT_TRUE(std::get<2>(seqAst->entry_)->ruleId_ == 3); 222 | 223 | 224 | //printf("index %d ruleid %d\n", anyAst->entry_.index(), std::get<1>(anyAst->entry_).ruleId_ ); 225 | 226 | //EXPECT_TRUE(anyAst->entry_.index() == 1); 227 | //EXPECT_TRUE( std::get<1>(anyAst->entry_).ruleId_ == 2); 228 | } 229 | 230 | TEST(TestRules,testStarParser) { 231 | 232 | Text_stream text_stream; 233 | 234 | bool isok = text_stream.open(-1,11); 235 | EXPECT_EQ(isok, true); 236 | 237 | const Char_t *str = " \tAA A\nA A\tAA"; 238 | isok = text_stream.write_tail(str, strlen(str)); 239 | 240 | struct TokenAParser : PTok<1, CSTR1("A")> {}; 241 | 242 | struct TokenStarParser : PStar<4, TokenAParser> {}; 243 | 244 | 245 | CharParser chparser(text_stream); 246 | 247 | auto result = TokenStarParser::parse( chparser ); 248 | EXPECT_TRUE(result.success()); 249 | 250 | AstEntryBase *res = result.ast_.get(); 251 | EXPECT_TRUE( res != nullptr ); 252 | 253 | EXPECT_TRUE( res->ruleId_ == 4 ); 254 | 255 | TokenStarParser::AstType *star = (TokenStarParser::AstType *) res; 256 | 257 | for(auto &entry : star->entry_) { 258 | 259 | EXPECT_TRUE( entry.get()->ruleId_ == 1 ); 260 | } 261 | 262 | EXPECT_TRUE( star->entry_.size() == 7 ); 263 | } 264 | 265 | TEST(TestRules,testWithAndParser) { 266 | 267 | Text_stream text_stream; 268 | 269 | bool isok = text_stream.open(-1,11); 270 | EXPECT_EQ(isok, true); 271 | 272 | const Char_t *str = " \t if then"; 273 | isok = text_stream.write_tail(str, strlen(str)); 274 | 275 | struct TokenIfParser : PTok<1, CSTR2("if")> {}; 276 | struct TokenThenParser : PTok<2, CSTR4("then")> {}; 277 | struct TokenElseParser : PTok<3, CSTR4("else")> {}; 278 | 279 | 280 | struct TokenWithAndParser1 : PWithAndLookahead {}; 281 | struct TokenWithAndParser2 : PWithAndLookahead {}; 282 | 283 | CharParser chparser(text_stream); 284 | 285 | auto result2 = TokenWithAndParser2::parse( chparser ); 286 | EXPECT_FALSE(result2.success()); 287 | 288 | auto result1 = TokenWithAndParser1::parse( chparser ); 289 | EXPECT_TRUE(result1.success()); 290 | 291 | AstEntryBase *res = result1.ast_.get(); 292 | EXPECT_TRUE( res != nullptr ); 293 | 294 | //TokenWithAndParser1::AstType *ptype = (TokenWithAndParser1::AstType *) res; 295 | EXPECT_TRUE( res->ruleId_ == 1 ); 296 | } 297 | 298 | 299 | 300 | } 301 | 302 | -------------------------------------------------------------------------------- /test/test_stream.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | //enable execution trace with the next define 4 | //#define __PARSER_TRACE__ 5 | #include "parse.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace { 12 | 13 | 14 | using namespace pparse; 15 | 16 | 17 | TEST(TextStream,readCharsOneBufWithoutResize) { 18 | FILE *fp = fopen(__FILE__,"r"); 19 | EXPECT_TRUE(fp != NULL); 20 | 21 | fseek(fp, 0, SEEK_END); 22 | long file_size = ftell(fp); 23 | fseek(fp, 0, SEEK_SET); 24 | 25 | Text_stream stream(false); 26 | 27 | bool isok = stream.open(__FILE__, (uint32_t) file_size+2); 28 | EXPECT_EQ(isok, true); 29 | 30 | for(long pos=0; ; ++pos) { 31 | int ch = fgetc(fp); 32 | if (ch == EOF) { 33 | ASSERT_TRUE( pos == file_size ); 34 | } 35 | 36 | Text_stream::Next_char_value nch = stream.next_char(); 37 | if (!nch.first) { 38 | //printf("pos %ld size %ld\n",pos, file_size); 39 | ASSERT_TRUE( pos == file_size ); 40 | ASSERT_TRUE( stream.error() == 0 ); 41 | break; 42 | } 43 | 44 | if (nch.second != ch) { 45 | printf("pos %ld size %ld tch %x ch %x\n", pos, file_size, nch.second, ch); 46 | } 47 | ASSERT_TRUE(nch.second == ch); 48 | 49 | } 50 | 51 | EXPECT_TRUE(errno == 0); 52 | EXPECT_TRUE(stream.error() == 0); 53 | 54 | 55 | isok = stream.close(); 56 | EXPECT_EQ(isok, true); 57 | 58 | fclose(fp); 59 | } 60 | 61 | TEST(TextStream,readCharsOneBufWithResize) { 62 | FILE *fp = fopen(__FILE__,"r"); 63 | EXPECT_TRUE(fp != NULL); 64 | 65 | fseek(fp, 0, SEEK_END); 66 | long file_size = ftell(fp); 67 | fseek(fp, 0, SEEK_SET); 68 | 69 | Text_stream stream; 70 | 71 | bool isok = stream.open(__FILE__, (uint32_t) file_size+2); 72 | EXPECT_EQ(isok, true); 73 | 74 | for(long pos=0; ; ++pos) { 75 | int ch = fgetc(fp); 76 | if (ch == EOF) { 77 | ASSERT_TRUE( pos == file_size ); 78 | } 79 | 80 | Text_stream::Next_char_value nch = stream.next_char(); 81 | if (!nch.first) { 82 | //printf("pos %ld size %ld\n",pos, file_size); 83 | ASSERT_TRUE( pos == file_size ); 84 | ASSERT_TRUE( stream.error() == 0 ); 85 | break; 86 | } 87 | 88 | if (nch.second != ch) { 89 | printf("pos %ld size %ld tch %x ch %x\n", pos, file_size, nch.second, ch); 90 | } 91 | ASSERT_TRUE(nch.second == ch); 92 | 93 | } 94 | 95 | EXPECT_TRUE(errno == 0); 96 | EXPECT_TRUE(stream.error() == 0); 97 | 98 | 99 | isok = stream.close(); 100 | EXPECT_EQ(isok, true); 101 | 102 | fclose(fp); 103 | } 104 | 105 | 106 | TEST(TextStream,readCharsLimitedBuffer) { 107 | FILE *fp = fopen(__FILE__,"r"); 108 | EXPECT_TRUE(fp != NULL); 109 | 110 | fseek(fp, 0, SEEK_END); 111 | long file_size = ftell(fp); 112 | fseek(fp, 0, SEEK_SET); 113 | 114 | Text_stream stream; 115 | 116 | bool isok = stream.open(__FILE__, 11); 117 | EXPECT_EQ(isok, true); 118 | 119 | for(long pos=0; ; ++pos) { 120 | int ch = fgetc(fp); 121 | 122 | if (ch == EOF) { 123 | ASSERT_TRUE( pos == file_size ); 124 | } 125 | 126 | if (pos != 0 && pos % 10 == 0) { 127 | isok = stream.move_on( stream.pos_at_cursor() ); 128 | ASSERT_TRUE( isok ); 129 | } 130 | 131 | Text_stream::Next_char_value nch = stream.next_char(); 132 | if (!nch.first) { 133 | //printf("pos %ld size %ld tch %x ch %x\n", pos, file_size, nch.second, ch); 134 | ASSERT_TRUE( pos == file_size ); 135 | ASSERT_TRUE( stream.error() == 0 ); 136 | break; 137 | } 138 | 139 | //printf("%c[%c]",(char) nch.second, (char) ch); 140 | 141 | if (nch.second != ch) { 142 | printf("pos %ld size %ld tch %x ch %x\n", pos, file_size, nch.second, ch); 143 | } 144 | ASSERT_TRUE(nch.second == ch); 145 | 146 | } 147 | 148 | EXPECT_TRUE(errno == 0); 149 | EXPECT_TRUE(stream.error() == 0); 150 | 151 | 152 | isok = stream.close(); 153 | EXPECT_EQ(isok, true); 154 | 155 | fclose(fp); 156 | } 157 | 158 | TEST(TextStream,writeTest) { 159 | Text_stream stream; 160 | 161 | bool isok = stream.open(-1,11); 162 | EXPECT_EQ(isok, true); 163 | 164 | isok = stream.write_tail((Char_t*)"12345", 5); 165 | EXPECT_EQ(isok, true); 166 | 167 | long pos=0; 168 | for(; ; ++pos) { 169 | Text_stream::Next_char_value nch = stream.next_char(); 170 | if (!nch.first) { 171 | break; 172 | } 173 | ASSERT_TRUE(nch.second == '0' + (pos+1) % 10); 174 | } 175 | ASSERT_TRUE(pos==5); 176 | 177 | isok = stream.move_on( stream.pos_at_cursor() ); 178 | EXPECT_EQ(isok, true); 179 | 180 | isok = stream.write_tail((Char_t*)"6789012345", 10); 181 | EXPECT_EQ(isok, true); 182 | 183 | for(; ; ++pos) { 184 | Text_stream::Next_char_value nch = stream.next_char(); 185 | if (!nch.first) { 186 | break; 187 | } 188 | if (nch.second != '0' + (pos+1) % 10) { 189 | printf("! pos %ld\n", pos); 190 | } 191 | ASSERT_TRUE(nch.second == '0' + (pos+1) % 10); 192 | } 193 | ASSERT_TRUE(pos==15); 194 | } 195 | 196 | 197 | 198 | } // namespace 199 | 200 | 201 | 202 | -------------------------------------------------------------------------------- /test/test_trace.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | //enable execution trace with the next define 4 | #define __PARSER_TRACE__ 5 | #include "parse.h" 6 | 7 | #include 8 | #include 9 | 10 | namespace { 11 | 12 | using namespace pparse; 13 | 14 | template 15 | Parse_result test_string(Char_t *test_string) { 16 | Text_stream text_stream; 17 | 18 | bool isok = text_stream.open(-1); 19 | EXPECT_EQ(isok, true); 20 | 21 | isok = text_stream.write_tail(test_string, strlen(test_string) ); 22 | EXPECT_EQ(isok, true); 23 | 24 | 25 | CharParser chparser(text_stream); 26 | 27 | Parse_result res = Parser::parse(chparser); 28 | 29 | if (!res.success()) { 30 | printf("Error: parser error at: line: %d column: %d text: %s\n", res.get_start_pos().line(), res.get_start_pos().column(), test_string); 31 | } 32 | 33 | return res; 34 | 35 | } 36 | 37 | 38 | TEST(TestGrammarVisualization, testRecursion) { 39 | 40 | struct Int : PTokInt<1> {}; 41 | 42 | struct Expr; 43 | 44 | struct Mult : PAny<2, PTok<3,CSTR1("*")>, PTok<4,CSTR1("/")> > {}; 45 | 46 | struct Add : PAny<5, PTok<6,CSTR1("+")>, PTok<7,CSTR1("-")> > {}; 47 | 48 | struct NestedExpr : PSeq<8, PTok<0, CSTR1("(")>, Expr, PTok<0, CSTR1(")")> > {}; 49 | 50 | struct NegativeInt : PSeq<1, PTok<11, CSTR1("-")>, Int> {}; 51 | 52 | struct SimpleExpr : PAny<6, Int, NegativeInt, NestedExpr> {}; // 53 | 54 | struct MultExpr; 55 | 56 | struct MultExpr: PAny<7, PSeq<6, SimpleExpr, Mult, MultExpr >, SimpleExpr> {}; 57 | 58 | struct Expr; 59 | 60 | struct Expr: PAny<7, PSeq<6, MultExpr, Add, Expr >, MultExpr> {}; 61 | 62 | struct ExprEof : PRequireEof {}; 63 | 64 | Parse_result result = test_string((Char_t *) "-1"); 65 | EXPECT_TRUE(result.success()); 66 | 67 | result = test_string((Char_t *) "1 * 3"); 68 | EXPECT_TRUE(result.success()); 69 | 70 | result = test_string((Char_t *) "2 + 2 + 2"); 71 | EXPECT_TRUE(result.success()); 72 | 73 | result = test_string((Char_t *) "2 + 2 + 2 + 1 * 3"); 74 | EXPECT_TRUE(result.success()); 75 | 76 | result = test_string((Char_t *) "(2*3) + 5"); 77 | EXPECT_TRUE(result.success()); 78 | } 79 | 80 | #if 0 81 | inline Char_checker_result pparse_regex_char(Char_t current_char, bool iseof, Char_t *matched_so_far) { 82 | return !iseof && 83 | strlen(matched_so_far) == 0 && 84 | isprint((char) current_char) && 85 | strchr("+-*()[].",current_char) == 0 ? Char_checker_result::acceptNow : Char_checker_result::error; 86 | } 87 | 88 | template 89 | struct PTokRegexChar : PTokVar { 90 | }; 91 | 92 | TEST(TestGrammar, testRegEx) { 93 | 94 | struct Regex; 95 | 96 | struct RegexAnyChar : PTok<1, CSTR1(".") > {}; 97 | 98 | struct RegexEscapeChar : PSeq<1, PTok<0,CSTR1("\\")>, PTokChar<1>> {}; 99 | 100 | struct RegexCharRangeChar : PAny<1, RegexEscapeChar, PTokRegexChar<1> > {}; 101 | 102 | struct RegexCharRange : PSeq<1, RegexCharRangeChar, PTok<1, CSTR1("-")>, RegexCharRangeChar> {}; 103 | 104 | struct RegexSetItems : PPlus<1, PAny<1, RegexCharRange, RegexAnyChar> > {}; 105 | 106 | struct RegexSet : PSeq<1, PTok<0, CSTR1("[")>, RegexSetItems, PTok<0, CSTR1("]")> > {}; 107 | 108 | struct RegexOp : PAny<1, PTok<1, CSTR1("*")>, PTok<2, CSTR1("?")>, PTok<3, CSTR1("+") > > {}; 109 | 110 | struct RegexSetWithOp : PSeq<1, RegexSet , POpt<1, RegexOp>> {}; 111 | 112 | struct RegexNestedWithOp : PSeq<1, PTok<1,CSTR1("(")>, Regex, PTok<1, CSTR1(")")>, POpt<1, RegexOp> > {}; 113 | 114 | struct RegexSetItemWithOp : PSeq<1, PAny<1, RegexCharRange, RegexAnyChar, RegexCharRangeChar>, POpt<1, RegexOp> > {}; 115 | 116 | struct Regex : PStar<1, PAny< 1, RegexSetWithOp, RegexNestedWithOp, RegexSetItemWithOp> > {}; 117 | 118 | struct RegexEof : PSeq<8, Regex, PEof> {}; 119 | 120 | Parse_result result = test_string((Char_t *) "ab"); 121 | EXPECT_TRUE(result.success()); 122 | 123 | result = test_string((Char_t *) "abc123"); 124 | EXPECT_TRUE(result.success()); 125 | 126 | result = test_string((Char_t *) "[a-z]" ); 127 | EXPECT_TRUE(result.success()); 128 | 129 | result = test_string((Char_t *) "abc[a-z]+" ); 130 | EXPECT_TRUE(result.success()); 131 | 132 | result = test_string((Char_t *) "abc([0-9]+)?" ); 133 | EXPECT_TRUE(result.success()); 134 | 135 | } 136 | #endif 137 | 138 | } 139 | 140 | --------------------------------------------------------------------------------