├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md ├── bootstrap.sh ├── cli ├── Cargo.toml ├── README.md └── src │ ├── main.rs │ └── peg_railroad.rs ├── codegen ├── Cargo.toml ├── README.md ├── build.rs └── src │ ├── buildscript.rs │ ├── char_rule.rs │ ├── choice.rs │ ├── closure.rs │ ├── common.rs │ ├── eoi.rs │ ├── extern_rule.rs │ ├── field.rs │ ├── grammar │ ├── generated.rs │ └── mod.rs │ ├── header.rs │ ├── include_rule.rs │ ├── lib.rs │ ├── lookahead.rs │ ├── misc.rs │ ├── optional.rs │ ├── rule.rs │ ├── sequence.rs │ └── string.rs ├── doc ├── error.png ├── railroad.png ├── syntax.md └── trace.png ├── grammar.ebnf ├── macro ├── Cargo.toml ├── README.md ├── src │ └── lib.rs └── tests │ └── simple.rs ├── runtime ├── Cargo.toml ├── README.md ├── src │ ├── builtin_parsers.rs │ ├── call_hook.rs │ ├── choice_helper.rs │ ├── colored_shim.rs │ ├── error.rs │ ├── global.rs │ ├── lib.rs │ ├── parse_result.rs │ ├── peg_parser.rs │ ├── state.rs │ └── trace.rs └── syntax.md └── test ├── Cargo.toml ├── build.rs └── src ├── additional_traits ├── grammar.ebnf └── mod.rs ├── boxing ├── grammar.ebnf └── mod.rs ├── calculator_example ├── grammar.ebnf └── mod.rs ├── char_rule ├── grammar.ebnf └── mod.rs ├── check ├── grammar.ebnf └── mod.rs ├── choice ├── grammar.ebnf └── mod.rs ├── closure ├── grammar.ebnf └── mod.rs ├── custom_derives_empty ├── grammar.not_ebnf └── mod.rs ├── custom_whitespace ├── grammar.ebnf └── mod.rs ├── enums ├── grammar.ebnf └── mod.rs ├── eoi ├── grammar.ebnf └── mod.rs ├── extern_directive ├── grammar.ebnf └── mod.rs ├── field ├── grammar.ebnf └── mod.rs ├── fndef_example ├── grammar.ebnf └── mod.rs ├── include_rule ├── grammar.ebnf └── mod.rs ├── lib.rs ├── lookahead ├── grammar.ebnf └── mod.rs ├── memoization ├── grammar.ebnf └── mod.rs ├── operator_example ├── grammar.ebnf └── mod.rs ├── optional ├── grammar.ebnf └── mod.rs ├── palindrome ├── grammar.ebnf └── mod.rs ├── position ├── grammar.ebnf └── mod.rs ├── precedence ├── grammar.ebnf └── mod.rs ├── rust_keywords ├── grammar.ebnf └── mod.rs ├── sequence ├── grammar.ebnf └── mod.rs ├── simple ├── grammar.ebnf └── mod.rs ├── skip_ws ├── grammar.ebnf └── mod.rs ├── string ├── grammar.ebnf └── mod.rs ├── string_insensitive ├── grammar.ebnf └── mod.rs ├── test_utils.rs └── user_defined_state ├── grammar.not_ebnf └── mod.rs /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | 4 | target 5 | Cargo.lock 6 | 7 | test/src/**/grammar.rs 8 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["cli", "macro", "test", "runtime"] 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Alex Badics 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Peginator 2 | [![Crates.io](https://img.shields.io/crates/v/peginator.svg)](https://crates.io/crates/peginator) 3 | [![Docs.rs](https://docs.rs/peginator/badge.svg)](https://docs.rs/peginator) 4 | 5 | Peginator is a PEG (Parsing Expression Grammar) parser generator written in Rust. It 6 | is specifically made to parse into ASTs (Abstract Syntax Trees), as opposed to most, 7 | streaming-style parsers out there. 8 | 9 | It generates both the tree structure and the parsing code that can create that tree from 10 | a `&str`. The generated parsing code is deliberately very simple straightforward Rust 11 | code, which is usually optimized very well by the compiler. 12 | 13 | There is an opt-in memoization feature that makes it a proper packrat parser that can 14 | parse any input in linear time and space. 15 | 16 | Left-recursion is also supported using said memoization feature (also opt-in). 17 | 18 | ## Documentation 19 | 20 | This documentation describes how peginator implements PEGs. A basic understanding of PEGs 21 | are assumed. There are good introductions on 22 | [wikipedia](https://en.wikipedia.org/wiki/Parsing_expression_grammar) or in the 23 | [docs of other parser generators](https://pest.rs/book/grammars/syntax.html). 24 | 25 | Peginator is bootstrapped using its own 26 | [syntax and grammar file](https://github.com/badicsalex/peginator/blob/master/grammar.ebnf), 27 | which is somewhat easy-to-read. 28 | 29 | Please see [the syntax reference](doc/syntax.md) and 30 | [the API documentation](https://docs.rs/peginator) 31 | 32 | The [tests](peginator_test/src) can also be used as examples. 33 | 34 | ## Quick Start 35 | 36 | The grammars for peginator are written in a syntax similar to EBNF 37 | (extended Backus-Naur form): 38 | 39 | ```ebnf 40 | @export 41 | FunctionDef = 'fn' name:Ident '(' param_list:ParamList ')' [ '->' return_value:Type ]; 42 | 43 | ParamList = self_param:SelfParam {',' params:Param} | params:Param {',' params:Param} | ; 44 | 45 | Param = name:Ident ':' typ: Type; 46 | 47 | SelfParam = [ref_type:ReferenceMarker] 'self'; 48 | 49 | Type = [ref_type:ReferenceMarker] typename:Ident; 50 | 51 | ReferenceMarker = @:MutableReference | @:ImmutableReference; 52 | 53 | ImmutableReference = '&'; 54 | MutableReference = '&' 'mut'; 55 | 56 | @string 57 | @no_skip_ws 58 | Ident = {'a'..'z' | 'A'..'Z' | '_' | '0'..'9'}; 59 | ``` 60 | 61 | Based on the above grammar, peginator will generate the following types: 62 | 63 | ```ignore 64 | pub struct FunctionDef { 65 | pub name: Ident, 66 | pub param_list: ParamList, 67 | pub return_value: Option, 68 | } 69 | pub struct ParamList { 70 | pub self_param: Option, 71 | pub params: Vec, 72 | } 73 | pub struct Param { 74 | pub name: Ident, 75 | pub typ: Type, 76 | } 77 | pub struct SelfParam { 78 | pub ref_type: Option, 79 | } 80 | pub struct Type { 81 | pub ref_type: Option, 82 | pub typename: Ident, 83 | } 84 | pub enum ReferenceMarker { 85 | ImmutableReference(ImmutableReference), 86 | MutableReference(MutableReference), 87 | } 88 | pub struct ImmutableReference; 89 | pub struct MutableReference; 90 | pub type Ident = String; 91 | 92 | impl PegParser for FunctionDef { /* omitted */ } 93 | ``` 94 | 95 | Parsing then looks like this: 96 | ```ignore 97 | FunctionDef::parse("fn example(&self, input:&str, rectified:&mut Rect) -> ExampleResult;") 98 | ``` 99 | 100 | Which results in the following structure: 101 | ```ignore 102 | FunctionDef { 103 | name: "example", 104 | param_list: ParamList { 105 | self_param: Some(SelfParam { 106 | ref_type: Some(ImmutableReference(ImmutableReference)), 107 | }), 108 | params: [ 109 | Param { 110 | name: "input", 111 | typ: Type { 112 | ref_type: Some(ImmutableReference(ImmutableReference)), 113 | typename: "str", 114 | }, 115 | }, 116 | Param { 117 | name: "rectified", 118 | typ: Type { 119 | ref_type: Some(MutableReference(MutableReference)), 120 | typename: "Rect", 121 | }, 122 | }, 123 | ], 124 | }, 125 | return_value: Some(Type { 126 | ref_type: None, 127 | typename: "ExampleResult", 128 | }), 129 | } 130 | ``` 131 | 132 | ## Debugging 133 | 134 | We have pretty errors, based on the first failure of the longest match 135 | (a'la python's parser): 136 | 137 | ![Colors and stuff on a console](doc/error.png) 138 | 139 | And parse tracing (opt-in, no cost if not used): 140 | 141 | ![More colors and indentation](doc/trace.png) 142 | 143 | ## Integration 144 | 145 | There are multiple ways to integrate a Peginator grammar to your project: 146 | 147 | * Compile your grammars directly with the `peginator-cli` binary 148 | * Inline your grammars with the `peginate!` macro from the [peginator\_macro](https://docs.rs/peginator_macro) package 149 | * Or you can use the [buildscript helper](https://docs.rs/peginator_codegen) 150 | 151 | ## Railroad graphs 152 | 153 | The Peginator CLI can also create a nice railroad graph of your grammar: 154 | 155 | ![Railroad graph](doc/railroad.png) 156 | 157 | ## Contribution 158 | 159 | At this point, I'd be happy if simply more people used this code. Please reach out if you need any help. 160 | 161 | ## See also 162 | 163 | The project is meant to be an almost drop-in replacement for [Tatsu](https://github.com/neogeny/TatSu), 164 | and its fantastic Model Builder. This is why the grammar looks like the way it does. 165 | 166 | There are a ton of other PEG parser implementations in Rust, please check them out. Non-exhaustive list in 167 | no particular order: 168 | 169 | * [oak](https://github.com/ptal/oak) is very similar in that it generates a typed AST, but much more flexible 170 | * [pest](https://github.com/pest-parser/pest) 171 | * [rust-peg](https://github.com/kevinmehall/rust-peg) 172 | * [lrpeg](https://github.com/seanyoung/lrpeg) 173 | 174 | Special mention: [lalrpop](https://github.com/lalrpop/lalrpop) 175 | 176 | ## License 177 | 178 | Licensed under the MIT license 179 | -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | #/usr/bin/env sh 2 | 3 | OUTPUT="codegen/src/grammar/generated.rs" 4 | 5 | if cargo run -- grammar.ebnf | rustfmt > $OUTPUT.new; then 6 | if mv -f $OUTPUT.new $OUTPUT; then 7 | exit 0 8 | fi 9 | fi 10 | 11 | rm -f $OUTPUT.new 12 | exit 1 13 | -------------------------------------------------------------------------------- /cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "peginator-cli" 3 | description = "Command line interface for peginator (a PEG parser generator)" 4 | version = "0.7.0" 5 | edition = "2021" 6 | 7 | authors = ["Alex Badics "] 8 | repository = "https://github.com/badicsalex/peginator" 9 | documentation = "https://docs.rs/peginator" 10 | keywords = ["parsing", "parser", "grammar", "ast", "peg"] 11 | categories = ["parsing"] 12 | license = "MIT" 13 | 14 | [dependencies] 15 | anyhow = "1.0" 16 | clap = { version = "4.0", features = ["derive"]} 17 | colored = "2.0" 18 | peginator = { version="=0.7.0", path = "../runtime" } 19 | peginator_codegen = { version="=0.7.0", path = "../codegen" } 20 | railroad = "0.2.0" 21 | -------------------------------------------------------------------------------- /cli/README.md: -------------------------------------------------------------------------------- 1 | # Peginator CLI 2 | 3 | Command line interface for [peginator](https://github.com/badicsalex/peginator). Please find 4 | documentation and other goodies on the main project. 5 | -------------------------------------------------------------------------------- /cli/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use std::fs; 6 | 7 | use anyhow::Result; 8 | use clap::Parser; 9 | use colored::*; 10 | use peginator::{PegParser, PrettyParseError}; 11 | use peginator_codegen::Grammar; 12 | use peginator_codegen::{generate_source_header, CodegenGrammar, CodegenSettings}; 13 | 14 | use crate::peg_railroad::print_railroad_svg; 15 | 16 | mod peg_railroad; 17 | 18 | /// Compile EBNF grammar into rust parser code. 19 | #[derive(Parser, Debug)] 20 | #[clap(version, about)] 21 | struct Args { 22 | /// Print the parsed AST and exit 23 | #[clap(short, long)] 24 | ast_only: bool, 25 | 26 | /// Output a railroad graph representation of the grammar as SVG 27 | #[clap(short, long)] 28 | railroad: bool, 29 | 30 | /// Trace rule matching 31 | #[clap(short, long)] 32 | trace: bool, 33 | 34 | /// Use a custom set of derives for the generated types 35 | #[clap(short, long)] 36 | derives: Vec, 37 | 38 | grammar_file: String, 39 | } 40 | 41 | fn main_wrap() -> Result<()> { 42 | let args = Args::parse(); 43 | let grammar = fs::read_to_string(&args.grammar_file)?; 44 | let parsed_grammar = if args.trace { 45 | Grammar::parse_with_trace(&grammar) 46 | } else { 47 | Grammar::parse(&grammar) 48 | } 49 | .map_err(|err| PrettyParseError::from_parse_error(&err, &grammar, Some(&args.grammar_file)))?; 50 | if args.ast_only { 51 | println!("{:#?}", parsed_grammar); 52 | return Ok(()); 53 | } 54 | 55 | if args.railroad { 56 | print_railroad_svg(&parsed_grammar); 57 | return Ok(()); 58 | } 59 | 60 | let derives = if args.derives.is_empty() { 61 | CodegenSettings::default().derives 62 | } else { 63 | args.derives 64 | }; 65 | 66 | let settings = CodegenSettings { 67 | derives, 68 | ..Default::default() 69 | }; 70 | let generated_code = parsed_grammar.generate_code(&settings)?; 71 | println!("{}", generate_source_header(&grammar)); 72 | println!("{}", generated_code); 73 | Ok(()) 74 | } 75 | 76 | fn main() { 77 | if let Err(e) = main_wrap() { 78 | println!("{}: {}", "Error".red().bold(), e) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /cli/src/peg_railroad.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use peginator_codegen::{ 6 | grammar::{ 7 | CharRangePart, CharRule, CharRulePart, CharacterRange, Choice, Closure, 8 | DelimitedExpression, DirectiveExpression, EndOfInput, ExternRule, Field, Field_name, 9 | Grammar_rules, Group, IncludeRule, NegativeLookahead, Optional, PositiveLookahead, Rule, 10 | Sequence, StringLiteral, 11 | }, 12 | Grammar, 13 | }; 14 | use railroad::Node; 15 | 16 | pub fn print_railroad_svg(grammar: &Grammar) { 17 | let mut dia = railroad::Diagram::new(grammar.generate_railroad()); 18 | 19 | dia.add_default_css(); 20 | 21 | println!("{dia}"); 22 | } 23 | 24 | trait GenerateRailroad { 25 | fn generate_railroad(&self) -> Box; 26 | } 27 | 28 | impl GenerateRailroad for Grammar { 29 | fn generate_railroad(&self) -> Box { 30 | Box::new(railroad::Diagram::new(railroad::VerticalGrid::new( 31 | self.rules.iter().map(|r| r.generate_railroad()).collect(), 32 | ))) 33 | } 34 | } 35 | 36 | impl GenerateRailroad for Grammar_rules { 37 | fn generate_railroad(&self) -> Box { 38 | match self { 39 | Grammar_rules::CharRule(x) => x.generate_railroad(), 40 | Grammar_rules::ExternRule(x) => x.generate_railroad(), 41 | Grammar_rules::Rule(x) => x.generate_railroad(), 42 | } 43 | } 44 | } 45 | 46 | impl GenerateRailroad for Rule { 47 | fn generate_railroad(&self) -> Box { 48 | let mut seq = railroad::Sequence::>::default(); 49 | let is_exported = self 50 | .directives 51 | .iter() 52 | .any(|d| matches!(d, DirectiveExpression::ExportDirective(_))); 53 | if is_exported { 54 | seq.push(Box::new(railroad::Start)); 55 | } else { 56 | seq.push(Box::new(railroad::SimpleStart)); 57 | } 58 | seq.push(Box::new(railroad::Comment::new(self.name.clone()))); 59 | seq.push(Box::new(self.definition.generate_railroad())); 60 | if is_exported { 61 | seq.push(Box::new(railroad::End)); 62 | } else { 63 | seq.push(Box::new(railroad::SimpleEnd)); 64 | } 65 | Box::new(seq) 66 | } 67 | } 68 | 69 | impl GenerateRailroad for CharRule { 70 | fn generate_railroad(&self) -> Box { 71 | let mut seq = railroad::Sequence::>::default(); 72 | seq.push(Box::new(railroad::SimpleStart)); 73 | seq.push(Box::new(railroad::Comment::new(self.name.clone()))); 74 | seq.push(Box::new(railroad::Choice::new( 75 | self.choices.iter().map(|c| c.generate_railroad()).collect(), 76 | ))); 77 | seq.push(Box::new(railroad::SimpleEnd)); 78 | Box::new(seq) 79 | } 80 | } 81 | 82 | impl GenerateRailroad for ExternRule { 83 | fn generate_railroad(&self) -> Box { 84 | let mut seq = railroad::Sequence::>::default(); 85 | seq.push(Box::new(railroad::SimpleStart)); 86 | seq.push(Box::new(railroad::Comment::new(self.name.clone()))); 87 | seq.push(Box::new(railroad::NonTerminal::new( 88 | self.directive.function.last().unwrap().clone(), 89 | ))); 90 | seq.push(Box::new(railroad::SimpleEnd)); 91 | Box::new(seq) 92 | } 93 | } 94 | 95 | impl GenerateRailroad for CharRulePart { 96 | fn generate_railroad(&self) -> Box { 97 | match self { 98 | CharRulePart::CharRangePart(x) => x.generate_railroad(), 99 | CharRulePart::CharacterRange(x) => x.generate_railroad(), 100 | CharRulePart::Identifier(x) => Box::new(railroad::NonTerminal::new(x.clone())), 101 | } 102 | } 103 | } 104 | 105 | impl GenerateRailroad for CharRangePart { 106 | fn generate_railroad(&self) -> Box { 107 | Box::new(railroad::Terminal::new( 108 | char::try_from(self).unwrap().escape_debug().to_string(), 109 | )) 110 | } 111 | } 112 | 113 | impl GenerateRailroad for CharacterRange { 114 | fn generate_railroad(&self) -> Box { 115 | Box::new(railroad::Terminal::new(format!( 116 | "{}..{}", 117 | char::try_from(&self.from).unwrap().escape_debug(), 118 | char::try_from(&self.to).unwrap().escape_debug(), 119 | ))) 120 | } 121 | } 122 | 123 | impl GenerateRailroad for Choice { 124 | fn generate_railroad(&self) -> Box { 125 | Box::new(railroad::Choice::new( 126 | self.choices.iter().map(|c| c.generate_railroad()).collect(), 127 | )) 128 | } 129 | } 130 | 131 | impl GenerateRailroad for Sequence { 132 | fn generate_railroad(&self) -> Box { 133 | Box::new(railroad::Sequence::new( 134 | self.parts.iter().map(|c| c.generate_railroad()).collect(), 135 | )) 136 | } 137 | } 138 | 139 | impl GenerateRailroad for DelimitedExpression { 140 | fn generate_railroad(&self) -> Box { 141 | match self { 142 | DelimitedExpression::CharacterRange(x) => x.generate_railroad(), 143 | DelimitedExpression::Closure(x) => x.generate_railroad(), 144 | DelimitedExpression::EndOfInput(x) => x.generate_railroad(), 145 | DelimitedExpression::Field(x) => x.generate_railroad(), 146 | DelimitedExpression::Group(x) => x.generate_railroad(), 147 | DelimitedExpression::IncludeRule(x) => x.generate_railroad(), 148 | DelimitedExpression::NegativeLookahead(x) => x.generate_railroad(), 149 | DelimitedExpression::Optional(x) => x.generate_railroad(), 150 | DelimitedExpression::PositiveLookahead(x) => x.generate_railroad(), 151 | DelimitedExpression::StringLiteral(x) => x.generate_railroad(), 152 | } 153 | } 154 | } 155 | 156 | impl GenerateRailroad for Closure { 157 | fn generate_railroad(&self) -> Box { 158 | let closure = Box::new(railroad::Repeat::new( 159 | self.body.generate_railroad(), 160 | railroad::Empty, 161 | )); 162 | if self.at_least_one.is_some() { 163 | closure 164 | } else { 165 | Box::new(railroad::Optional::new(closure)) 166 | } 167 | } 168 | } 169 | 170 | impl GenerateRailroad for EndOfInput { 171 | fn generate_railroad(&self) -> Box { 172 | Box::new(railroad::NonTerminal::new("EOI".to_string())) 173 | } 174 | } 175 | 176 | impl GenerateRailroad for Field { 177 | fn generate_railroad(&self) -> Box { 178 | let nonterminal = Box::new(railroad::NonTerminal::new(self.typ.clone())); 179 | if let Some(name) = &self.name { 180 | match name { 181 | Field_name::Identifier(name) => Box::new(railroad::LabeledBox::new( 182 | nonterminal, 183 | railroad::Comment::new(name.clone()), 184 | )), 185 | Field_name::OverrideMarker(_) => { 186 | Box::new(railroad::NonTerminal::new(self.typ.clone())) 187 | } 188 | } 189 | } else { 190 | nonterminal 191 | } 192 | } 193 | } 194 | 195 | impl GenerateRailroad for Group { 196 | fn generate_railroad(&self) -> Box { 197 | self.body.generate_railroad() 198 | } 199 | } 200 | 201 | impl GenerateRailroad for IncludeRule { 202 | fn generate_railroad(&self) -> Box { 203 | Box::new(railroad::NonTerminal::new(self.rule.clone())) 204 | } 205 | } 206 | 207 | impl GenerateRailroad for NegativeLookahead { 208 | fn generate_railroad(&self) -> Box { 209 | Box::new(railroad::LabeledBox::new( 210 | railroad::NonTerminal::new("Inverse of the above".to_string()), 211 | self.expr.generate_railroad(), 212 | )) 213 | } 214 | } 215 | 216 | impl GenerateRailroad for PositiveLookahead { 217 | fn generate_railroad(&self) -> Box { 218 | Box::new(railroad::LabeledBox::new( 219 | railroad::NonTerminal::new("Lookahead of the above".to_string()), 220 | self.expr.generate_railroad(), 221 | )) 222 | } 223 | } 224 | 225 | impl GenerateRailroad for Optional { 226 | fn generate_railroad(&self) -> Box { 227 | Box::new(railroad::Optional::new(self.body.generate_railroad())) 228 | } 229 | } 230 | 231 | impl GenerateRailroad for StringLiteral { 232 | fn generate_railroad(&self) -> Box { 233 | Box::new(railroad::Terminal::new( 234 | String::try_from(self).unwrap().escape_debug().collect(), 235 | )) 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /codegen/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "peginator_codegen" 3 | description = "Buildscript support for peginator (a PEG parser generator)" 4 | version = "0.7.0" 5 | edition = "2021" 6 | 7 | authors = ["Alex Badics "] 8 | repository = "https://github.com/badicsalex/peginator" 9 | documentation = "https://docs.rs/peginator_codegen" 10 | keywords = ["parsing", "parser", "grammar", "ast", "peg"] 11 | categories = ["parsing"] 12 | license = "MIT" 13 | 14 | [dependencies] 15 | peginator = { version="=0.7.0", path = "../runtime" } 16 | quote = "1.0" 17 | proc-macro2 = "1.0" 18 | anyhow = "1.0" 19 | colored = "2.0" 20 | crc = "3.0" 21 | -------------------------------------------------------------------------------- /codegen/README.md: -------------------------------------------------------------------------------- 1 | # Peginator codegen 2 | 3 | Buildscript support for [peginator](https://github.com/badicsalex/peginator). Please find 4 | documentation and other goodies on the main project. 5 | -------------------------------------------------------------------------------- /codegen/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let build_time = std::time::SystemTime::now() 3 | .duration_since(std::time::UNIX_EPOCH) 4 | .unwrap() 5 | .as_secs(); 6 | 7 | println!("cargo:rerun-if-changed=src/lib.rs"); 8 | println!("cargo:rustc-env=PEGINATOR_BUILD_TIME={}", build_time); 9 | } 10 | -------------------------------------------------------------------------------- /codegen/src/buildscript.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | //! Buildscript helpers for peginator 6 | //! 7 | //! See [`Compile`] for examples. 8 | 9 | use std::{ 10 | ffi::OsStr, 11 | fs::{self, File}, 12 | io::Read, 13 | path::PathBuf, 14 | process::Command, 15 | str::FromStr, 16 | }; 17 | 18 | use anyhow::Result; 19 | use colored::*; 20 | use peginator::PrettyParseError; 21 | 22 | use crate::{generate_source_header, grammar::Grammar, CodegenGrammar, CodegenSettings}; 23 | 24 | /// Compiles peginator grammars into rust code with a builder interface. 25 | /// 26 | /// It only recompiles files if it detects (based on the generated file header in the `.rs` file) 27 | /// change in either the peginator library, or the grammar file. 28 | /// 29 | /// It is meant to be used as `peginator_codegen::Compile`, hence the generic name. 30 | /// 31 | /// Example `build.rs` for using a single grammar file and putting the result in the target directory: 32 | /// ```no_run 33 | ///# #[allow(clippy::needless_doctest_main)] 34 | /// fn main() { 35 | /// peginator_codegen::Compile::file("my_grammar.ebnf") 36 | /// .destination(format!("{}/my_grammar.rs", std::env::var("OUT_DIR").unwrap())) 37 | /// .format() 38 | /// .run_exit_on_error(); 39 | /// println!("cargo:rerun-if-changed=my_grammar.ebnf"); 40 | /// } 41 | /// ``` 42 | /// 43 | /// Importing this grammar: 44 | /// ```ignore 45 | /// mod grammar { include!(concat!(env!("OUT_DIR"), "/my_grammar.rs")); } 46 | /// ``` 47 | /// 48 | /// Example `build.rs` for multiple grammar files in the src directory, putting compiled files next to 49 | /// their grammar definitions: 50 | /// ```no_run 51 | ///# #[allow(clippy::needless_doctest_main)] 52 | /// fn main() { 53 | /// peginator_codegen::Compile::directory("src") 54 | /// .format() 55 | /// .run_exit_on_error() 56 | /// } 57 | /// ``` 58 | /// 59 | #[derive(Debug)] 60 | #[must_use] 61 | pub struct Compile { 62 | source_path: PathBuf, 63 | destination_path: Option, 64 | format: bool, 65 | recursive: bool, 66 | settings: CodegenSettings, 67 | prefix: String, 68 | } 69 | 70 | impl Compile { 71 | fn default() -> Self { 72 | Compile { 73 | source_path: PathBuf::new(), 74 | destination_path: None, 75 | format: false, 76 | recursive: false, 77 | settings: Default::default(), 78 | prefix: String::new(), 79 | } 80 | } 81 | /// Run compilation on a whole directory run. 82 | /// 83 | /// The whole directory is recursively searched for files with the `.ebnf` extension, and 84 | /// compiled to rust code with the same filename but with `.rs` extension. 85 | pub fn directory>(filename: T) -> Self { 86 | Compile { 87 | source_path: filename.into(), 88 | recursive: true, 89 | ..Self::default() 90 | } 91 | } 92 | 93 | /// Run compilation on a single file. 94 | /// 95 | /// The file will be compiled. If destination is not given, it will be the same filename in the 96 | /// same directory, but with `.rs` extension. 97 | pub fn file>(filename: T) -> Self { 98 | Compile { 99 | source_path: filename.into(), 100 | ..Self::default() 101 | } 102 | } 103 | 104 | /// Destination file name. 105 | /// 106 | /// This option only used if running on a single file. 107 | pub fn destination>(self, filename: T) -> Self { 108 | Compile { 109 | destination_path: Some(filename.into()), 110 | ..self 111 | } 112 | } 113 | 114 | /// Format .rs files with rustfmt after grammar compilation. 115 | pub fn format(self) -> Self { 116 | Compile { 117 | format: true, 118 | ..self 119 | } 120 | } 121 | 122 | /// Use a specific set of derives when declaring structs. 123 | /// 124 | /// The default set is `#[derive(Debug, Clone)]` 125 | pub fn derives(mut self, derives: Vec) -> Self { 126 | self.settings.derives = derives; 127 | self 128 | } 129 | 130 | /// Set a prefix (code) that will be pasted into the file between the header and the generated code 131 | /// 132 | /// Useful for `use` declarations or maybe custom structs. 133 | pub fn prefix(self, prefix: String) -> Self { 134 | Compile { prefix, ..self } 135 | } 136 | 137 | /// Set the type of global.user_context. Experimental feature, may change without notice. 138 | pub fn user_context_type(mut self, user_context_type: &str) -> Self { 139 | self.settings.set_user_context_type(user_context_type); 140 | self 141 | } 142 | 143 | fn run_on_single_file(&self, source: &PathBuf, destination: &PathBuf) -> Result<()> { 144 | let grammar = fs::read_to_string(source)?; 145 | let source_header = format!("{}\n{}", generate_source_header(&grammar), self.prefix); 146 | if let Ok(f) = File::open(destination) { 147 | let mut existing_header = String::new(); 148 | if f.take(source_header.len() as u64) 149 | .read_to_string(&mut existing_header) 150 | .is_ok() 151 | && source_header == existing_header 152 | { 153 | return Ok(()); 154 | } 155 | }; 156 | 157 | let parsed_grammar = Grammar::from_str(&grammar) 158 | .map_err(|err| PrettyParseError::from_parse_error(&err, &grammar, source.to_str()))?; 159 | let generated_code = format!( 160 | "{}\n{}", 161 | source_header, 162 | parsed_grammar.generate_code(&self.settings)? 163 | ); 164 | fs::write(destination, generated_code)?; 165 | if self.format { 166 | Command::new("rustfmt").arg(destination).status()?; 167 | }; 168 | Ok(()) 169 | } 170 | 171 | fn run_recursively(&self, source: &PathBuf) -> Result<()> { 172 | if source.is_dir() { 173 | source 174 | .read_dir()? 175 | .try_for_each(|c| self.run_recursively(&c?.path())) 176 | } else if source.extension().and_then(OsStr::to_str) == Some("ebnf") { 177 | self.run_on_single_file(source, &source.with_extension("rs")) 178 | } else { 179 | Ok(()) 180 | } 181 | } 182 | 183 | /// Run the compilation, returning an error. 184 | /// 185 | /// In case of a parse error, [`PrettyParseError`] is thrown, 186 | /// which will print a pretty error with `format!` or `print!`. 187 | pub fn run(self) -> Result<()> { 188 | if self.recursive { 189 | self.run_recursively(&self.source_path) 190 | } else { 191 | self.run_on_single_file( 192 | &self.source_path.clone(), 193 | &self 194 | .destination_path 195 | .clone() 196 | .unwrap_or_else(|| self.source_path.with_extension("rs")), 197 | ) 198 | } 199 | } 200 | 201 | /// Run the compilation, and exit with an exit code in case of an error. 202 | /// 203 | /// It also makes sure to pretty-print the error, should one occur. 204 | pub fn run_exit_on_error(self) { 205 | // I absolutely hate how this is a global, because there is no way to know if it was forced 206 | // already. Thankfully we are exiting right after this. 207 | colored::control::set_override(true); 208 | let result = self.run(); 209 | if let Err(error) = result { 210 | eprintln!( 211 | "{red_error}{colon} {error:?}", 212 | red_error = "error".red().bold(), 213 | colon = ":".bold().white(), 214 | error = error 215 | ); 216 | std::process::exit(1); 217 | } 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /codegen/src/char_rule.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | use quote::{format_ident, quote}; 8 | 9 | use super::{ 10 | common::{generate_rule_parse_function, safe_ident}, 11 | CodegenSettings, 12 | }; 13 | use crate::grammar::{CharRule, CharRulePart}; 14 | 15 | impl CharRulePart { 16 | pub fn generate_parse_call(&self) -> Result { 17 | match self { 18 | CharRulePart::CharRangePart(c) => { 19 | let char_literal: char = c.try_into()?; 20 | Ok(quote!(parse_character_literal(state.clone(), #char_literal))) 21 | } 22 | CharRulePart::CharacterRange(r) => { 23 | let from: char = (&r.from).try_into()?; 24 | let to: char = (&r.to).try_into()?; 25 | Ok(quote!(parse_character_range(state.clone(), #from, #to))) 26 | } 27 | CharRulePart::Identifier(ident) => { 28 | let parser_name = format_ident!("parse_{}", ident); 29 | Ok(quote!(#parser_name(state.clone(), global))) 30 | } 31 | } 32 | } 33 | } 34 | 35 | impl CharRule { 36 | pub fn generate_code(&self, settings: &CodegenSettings) -> Result { 37 | let name = &self.name; 38 | let rule_type = safe_ident(&self.name); 39 | let parser_name = format_ident!("parse_{}", self.name); 40 | let parser_calls = self 41 | .choices 42 | .iter() 43 | .map(|c| c.generate_parse_call()) 44 | .collect::>>()?; 45 | let check_calls = self.generate_check_calls()?; 46 | 47 | let parse_body = quote!( 48 | #check_calls 49 | #(if let Ok(result) = #parser_calls { return Ok(result)})* 50 | Err(state.report_error(ParseErrorSpecifics::ExpectedCharacterClass { name: #name })) 51 | ); 52 | Ok(generate_rule_parse_function( 53 | parser_name, 54 | rule_type, 55 | parse_body, 56 | settings, 57 | )) 58 | } 59 | 60 | fn generate_check_calls(&self) -> Result { 61 | if self.directives.is_empty() { 62 | return Ok(TokenStream::new()); 63 | } 64 | let name = &self.name; 65 | let check_idents = self.directives.iter().map(|d| { 66 | let part_idents = d.function.iter().map(safe_ident); 67 | quote!(#(#part_idents)::*) 68 | }); 69 | 70 | Ok(quote!( 71 | if let Some(c) = state.s().chars().next() { 72 | #( 73 | if !#check_idents(c) { 74 | return Err(state.report_error(ParseErrorSpecifics::ExpectedCharacterClass { name: #name })) 75 | } 76 | )* 77 | } else { 78 | return Err(state.report_error(ParseErrorSpecifics::ExpectedCharacterClass { name: #name })) 79 | } 80 | )) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /codegen/src/choice.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | use quote::{format_ident, quote}; 8 | 9 | use super::common::{ 10 | generate_inner_parse_function, safe_ident, Arity, CloneState, Codegen, CodegenSettings, 11 | FieldDescriptor, 12 | }; 13 | use crate::{ 14 | common::combine_field_types, 15 | grammar::{Choice, Grammar}, 16 | }; 17 | 18 | impl Codegen for Choice { 19 | fn generate_code_spec( 20 | &self, 21 | rule_fields: &[FieldDescriptor], 22 | grammar: &Grammar, 23 | settings: &CodegenSettings, 24 | ) -> Result { 25 | if self.choices.len() < 2 { 26 | return self.choices[0].generate_code_spec(rule_fields, grammar, settings); 27 | } 28 | let choice_bodies = self 29 | .choices 30 | .iter() 31 | .enumerate() 32 | .filter(|(_, choice)| { 33 | choice 34 | .generate_inline_body(rule_fields, grammar, settings, CloneState::Yes) 35 | .ok() 36 | .flatten() 37 | .is_none() 38 | }) 39 | .map(|(num, choice)| -> Result { 40 | let choice_mod = format_ident!("choice_{num}"); 41 | let sequence_body = choice.generate_code(rule_fields, grammar, settings)?; 42 | Ok(quote!( 43 | mod #choice_mod{ 44 | use super::*; 45 | #sequence_body 46 | } 47 | )) 48 | }) 49 | .collect::>()?; 50 | let parse_body = 51 | self.generate_parse_body(rule_fields, grammar, settings, CloneState::No)?; 52 | let parse_function = generate_inner_parse_function(parse_body, settings); 53 | Ok(quote!( 54 | #choice_bodies 55 | #parse_function 56 | )) 57 | } 58 | 59 | fn generate_inline_body( 60 | &self, 61 | rule_fields: &[FieldDescriptor], 62 | grammar: &Grammar, 63 | settings: &CodegenSettings, 64 | clone_state: CloneState, 65 | ) -> Result> { 66 | if self.choices.len() < 2 { 67 | self.choices[0].generate_inline_body(rule_fields, grammar, settings, clone_state) 68 | } else if self.choices.iter().all(|c| { 69 | c.generate_inline_body(rule_fields, grammar, settings, CloneState::No) 70 | .ok() 71 | .flatten() 72 | .is_some() 73 | }) && self.get_filtered_rule_fields(rule_fields, grammar)?.len() <= 1 74 | { 75 | Ok(Some(self.generate_parse_body( 76 | rule_fields, 77 | grammar, 78 | settings, 79 | clone_state, 80 | )?)) 81 | } else { 82 | Ok(None) 83 | } 84 | } 85 | 86 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 87 | let mut all_fields = Vec::::new(); 88 | let mut first_iteration = true; 89 | for choice in &self.choices { 90 | let new_fields = choice.get_fields(grammar)?; 91 | 92 | if !first_iteration { 93 | for field in &mut all_fields { 94 | if field.arity == Arity::One && !new_fields.iter().any(|f| f.name == field.name) 95 | { 96 | field.arity = Arity::Optional; 97 | } 98 | } 99 | } 100 | 101 | for new_field in new_fields { 102 | if let Some(original) = all_fields.iter_mut().find(|f| f.name == new_field.name) { 103 | original.arity = combine_arities_for_choice(&original.arity, &new_field.arity); 104 | combine_field_types(&mut original.types, &new_field.types); 105 | } else if first_iteration || new_field.arity != Arity::One { 106 | all_fields.push(new_field); 107 | } else { 108 | all_fields.push(FieldDescriptor { 109 | arity: Arity::Optional, 110 | ..new_field 111 | }); 112 | } 113 | } 114 | 115 | first_iteration = false; 116 | } 117 | Ok(all_fields) 118 | } 119 | } 120 | 121 | impl Choice { 122 | fn generate_parse_body( 123 | &self, 124 | rule_fields: &[FieldDescriptor], 125 | grammar: &Grammar, 126 | settings: &CodegenSettings, 127 | clone_state: CloneState, 128 | ) -> Result { 129 | let fields = self.get_filtered_rule_fields(rule_fields, grammar)?; 130 | let calls = self 131 | .choices 132 | .iter() 133 | .enumerate() 134 | .map(|(num, choice)| { 135 | let parse_call = if let Some(inline_body) = choice 136 | .generate_inline_body(rule_fields, grammar, settings, CloneState::No) 137 | .unwrap() 138 | { 139 | inline_body 140 | } else { 141 | let choice_mod = format_ident!("choice_{num}"); 142 | quote!(#choice_mod::parse(state, global)) 143 | }; 144 | let inner_fields = choice.get_fields(grammar).unwrap(); 145 | let postprocess = Self::generate_result_converter(&fields, &inner_fields); 146 | quote!( 147 | .choice(|state| #parse_call #postprocess) 148 | ) 149 | }) 150 | .collect::(); 151 | let state = match clone_state { 152 | CloneState::No => quote!(state), 153 | CloneState::Yes => quote!(state.clone()), 154 | }; 155 | Ok(quote!( 156 | ChoiceHelper::new(#state) 157 | #calls 158 | .end() 159 | )) 160 | } 161 | 162 | fn generate_result_converter( 163 | fields: &[FieldDescriptor], 164 | inner_fields: &[FieldDescriptor], 165 | ) -> TokenStream { 166 | if fields.is_empty() { 167 | TokenStream::new() 168 | } else if fields.len() == 1 { 169 | if inner_fields.is_empty() { 170 | let default = Self::generate_default_field(&fields[0]); 171 | quote!(.map_inner(|_| #default)) 172 | } else { 173 | TokenStream::new() 174 | } 175 | } else { 176 | let field_assignments: TokenStream = fields 177 | .iter() 178 | .map(|field| { 179 | let name = safe_ident(field.name); 180 | let inner_exists = inner_fields 181 | .iter() 182 | .any(|inner_field| inner_field.name == field.name); 183 | let value = if inner_exists { 184 | if inner_fields.len() == 1 { 185 | quote!(r) 186 | } else { 187 | quote!(r.#name) 188 | } 189 | } else { 190 | Self::generate_default_field(field) 191 | }; 192 | quote!(#name: #value,) 193 | }) 194 | .collect(); 195 | quote!(.map_inner(|r| Parsed{ #field_assignments })) 196 | } 197 | } 198 | 199 | fn generate_default_field(field: &FieldDescriptor) -> TokenStream { 200 | match field.arity { 201 | Arity::One => { 202 | panic!("Outer field ({field:?}) cannot be One if inner does not exist",) 203 | } 204 | Arity::Optional => quote!(None), 205 | Arity::Multiple => quote!(Vec::new()), 206 | } 207 | } 208 | } 209 | 210 | fn combine_arities_for_choice(left: &Arity, right: &Arity) -> Arity { 211 | match (left, right) { 212 | (Arity::One, Arity::One) => Arity::One, 213 | (Arity::One, Arity::Optional) => Arity::Optional, 214 | (Arity::One, Arity::Multiple) => Arity::Multiple, 215 | (Arity::Optional, Arity::One) => Arity::Optional, 216 | (Arity::Optional, Arity::Optional) => Arity::Optional, 217 | (Arity::Optional, Arity::Multiple) => Arity::Multiple, 218 | (Arity::Multiple, Arity::One) => Arity::Multiple, 219 | (Arity::Multiple, Arity::Optional) => Arity::Multiple, 220 | (Arity::Multiple, Arity::Multiple) => Arity::Multiple, 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /codegen/src/closure.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::{Ident, TokenStream}; 7 | use quote::quote; 8 | 9 | use super::common::{ 10 | generate_field_type, generate_inner_parse_function, safe_ident, Arity, CloneState, Codegen, 11 | CodegenSettings, FieldDescriptor, 12 | }; 13 | use crate::grammar::{Closure, Grammar}; 14 | 15 | impl Codegen for Closure { 16 | fn generate_code_spec( 17 | &self, 18 | rule_fields: &[FieldDescriptor], 19 | grammar: &Grammar, 20 | settings: &CodegenSettings, 21 | ) -> Result { 22 | let closure_body; 23 | let parse_call; 24 | if let Some(inline_body) = 25 | self.body 26 | .generate_inline_body(rule_fields, grammar, settings, CloneState::Yes)? 27 | { 28 | closure_body = TokenStream::new(); 29 | parse_call = inline_body; 30 | } else { 31 | closure_body = self.body.generate_code(rule_fields, grammar, settings)?; 32 | parse_call = quote!(closure::parse(state.clone(), global)); 33 | }; 34 | 35 | let fields = self.body.get_filtered_rule_fields(rule_fields, grammar)?; 36 | let declarations: TokenStream = fields 37 | .iter() 38 | .map(|f| { 39 | let typ = generate_field_type("Parsed", f, settings); 40 | let name_ident = safe_ident(f.name); 41 | quote!(let mut #name_ident: #typ = Vec::new();) 42 | }) 43 | .collect(); 44 | let assignments: TokenStream = fields 45 | .iter() 46 | .map(|field| { 47 | let name = safe_ident(field.name); 48 | if fields.len() == 1 { 49 | quote!(#name.extend(__result);) 50 | } else { 51 | quote!(#name.extend(__result.#name);) 52 | } 53 | }) 54 | .collect(); 55 | let field_names: Vec = fields.iter().map(|f| safe_ident(f.name)).collect(); 56 | let parse_result = if field_names.is_empty() { 57 | quote!(()) 58 | } else if field_names.len() == 1 { 59 | quote!(#( #field_names)*) 60 | } else { 61 | quote!(Parsed{ #( #field_names,)* }) 62 | }; 63 | let at_least_one_check = if self.at_least_one.is_some() { 64 | quote!(if iterations == 0 { 65 | return Err(state.report_farthest_error()); 66 | }) 67 | } else { 68 | quote!() 69 | }; 70 | let parse_body = quote!( 71 | let mut iterations:usize = 0; 72 | let mut state = state; 73 | #declarations 74 | loop { 75 | match #parse_call { 76 | Ok(ParseOk{result: __result, state:new_state, ..}) => { 77 | #assignments 78 | state = new_state; 79 | }, 80 | Err(err) => { 81 | state = state.record_error(err); 82 | break; 83 | } 84 | } 85 | iterations += 1; 86 | } 87 | #at_least_one_check 88 | Ok(ParseOk{result:#parse_result, state}) 89 | 90 | ); 91 | let parse_function = generate_inner_parse_function(parse_body, settings); 92 | 93 | Ok(quote!( 94 | mod closure{ 95 | use super::*; 96 | #closure_body 97 | } 98 | #parse_function 99 | )) 100 | } 101 | 102 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 103 | Ok(set_arity_to_multiple(self.body.get_fields(grammar)?)) 104 | } 105 | } 106 | 107 | fn set_arity_to_multiple(fields: Vec) -> Vec { 108 | let mut fields = fields; 109 | for value in &mut fields { 110 | value.arity = Arity::Multiple; 111 | } 112 | fields 113 | } 114 | -------------------------------------------------------------------------------- /codegen/src/eoi.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | 8 | use super::common::{generate_skip_ws, CloneState, Codegen, CodegenSettings, FieldDescriptor}; 9 | use crate::grammar::{EndOfInput, Grammar}; 10 | 11 | impl Codegen for EndOfInput { 12 | fn generate_inline_body( 13 | &self, 14 | _rule_fields: &[FieldDescriptor], 15 | _grammar: &Grammar, 16 | settings: &CodegenSettings, 17 | clone_state: CloneState, 18 | ) -> Result> { 19 | Ok(Some(generate_skip_ws( 20 | settings, 21 | "parse_end_of_input", 22 | TokenStream::new(), 23 | clone_state, 24 | ))) 25 | } 26 | 27 | fn get_fields(&self, _grammar: &Grammar) -> Result> { 28 | Ok(Vec::new()) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /codegen/src/extern_rule.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | use quote::{format_ident, quote}; 8 | 9 | use super::{ 10 | common::{generate_rule_parse_function, safe_ident}, 11 | CodegenSettings, 12 | }; 13 | use crate::grammar::ExternRule; 14 | 15 | impl ExternRule { 16 | pub fn generate_code(&self, settings: &CodegenSettings) -> Result<(TokenStream, TokenStream)> { 17 | let return_type = if let Some(return_type) = &self.directive.return_type { 18 | let part_idents = return_type.iter().map(safe_ident); 19 | quote!(#(#part_idents)::*) 20 | } else { 21 | quote!(String) 22 | }; 23 | let name_idents = self.directive.function.iter().map(safe_ident); 24 | let function_ident = quote!(#(#name_idents)::*); 25 | 26 | let rule_type = safe_ident(&self.name); 27 | let parser_name = format_ident!("parse_{}", self.name); 28 | let user_context_param = if settings.has_user_context { 29 | quote!(, global.user_context) 30 | } else { 31 | quote!() 32 | }; 33 | 34 | let parse_body = quote!( 35 | match #function_ident(state.s() #user_context_param) { 36 | Ok((result, advance)) => { 37 | Ok(ParseOk { 38 | result: result.into(), 39 | state: state.advance_safe(advance), 40 | }) 41 | }, 42 | Err(error_string) => { 43 | Err(state.report_error(ParseErrorSpecifics::ExternRuleFailed { 44 | error_string, 45 | })) 46 | } 47 | } 48 | ); 49 | Ok(( 50 | quote!(pub type #rule_type = #return_type;), 51 | generate_rule_parse_function(parser_name, rule_type, parse_body, settings), 52 | )) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /codegen/src/field.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | use quote::{format_ident, quote}; 8 | 9 | use super::common::{ 10 | generate_skip_ws, safe_ident, Arity, CloneState, Codegen, CodegenSettings, FieldDescriptor, 11 | }; 12 | use crate::{ 13 | common::FieldProperties, 14 | grammar::{Field, Field_name, Grammar}, 15 | }; 16 | 17 | impl Codegen for Field { 18 | fn generate_inline_body( 19 | &self, 20 | rule_fields: &[FieldDescriptor], 21 | _grammar: &Grammar, 22 | settings: &CodegenSettings, 23 | clone_state: CloneState, 24 | ) -> Result> { 25 | let postprocess = if let Some(field_name) = &self.name { 26 | generate_postprocess_calls( 27 | match field_name { 28 | Field_name::Identifier(field_name) => field_name, 29 | Field_name::OverrideMarker(_) => "_override", 30 | }, 31 | &self.typ, 32 | rule_fields, 33 | ) 34 | } else { 35 | quote!(.discard_result()) 36 | }; 37 | let parser_call = generate_skip_ws( 38 | settings, 39 | &format!("parse_{}", self.typ), 40 | quote!(&mut *global), 41 | clone_state, 42 | ); 43 | Ok(Some(quote!(#parser_call #postprocess))) 44 | } 45 | 46 | fn get_fields(&self, _grammar: &Grammar) -> Result> { 47 | if let Some(field_name) = &self.name { 48 | Ok(vec![FieldDescriptor { 49 | name: match field_name { 50 | Field_name::Identifier(field_name) => field_name, 51 | Field_name::OverrideMarker(_) => "_override", 52 | }, 53 | types: [( 54 | &self.typ as &str, 55 | FieldProperties { 56 | boxed: self.boxed.is_some(), 57 | }, 58 | )] 59 | .into(), 60 | arity: Arity::One, 61 | }]) 62 | } else { 63 | Ok(Vec::new()) 64 | } 65 | } 66 | } 67 | 68 | fn generate_postprocess_calls( 69 | field_name: &str, 70 | field_type_name: &str, 71 | rule_fields: &[FieldDescriptor], 72 | ) -> TokenStream { 73 | let field = rule_fields 74 | .iter() 75 | .find(|f| f.name == field_name) 76 | .expect("Field not found in rule_fields"); 77 | let enum_type_name = format_ident!("Parsed_{field_name}"); 78 | let field_type_ident = safe_ident(field_type_name); 79 | 80 | let field_properties = field 81 | .types 82 | .get(field_type_name) 83 | .expect("Field type not found in field"); 84 | let boxify = if field_properties.boxed { 85 | quote!(.map_inner(Box::new)) 86 | } else { 87 | TokenStream::new() 88 | }; 89 | let enumify = if field.types.len() > 1 { 90 | quote!(.map_inner(#enum_type_name::#field_type_ident)) 91 | } else { 92 | TokenStream::new() 93 | }; 94 | let aritify = match field.arity { 95 | Arity::One => TokenStream::new(), 96 | Arity::Optional => quote!(.map_inner(Some)), 97 | Arity::Multiple => quote!(.map_inner(|result| vec![result])), 98 | }; 99 | quote!( 100 | #boxify 101 | #enumify 102 | #aritify 103 | ) 104 | } 105 | -------------------------------------------------------------------------------- /codegen/src/grammar/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod generated; 6 | 7 | use std::str::FromStr; 8 | 9 | use anyhow::{Context, Result}; 10 | pub use generated::*; 11 | use peginator::{ParseError, PegParser}; 12 | use proc_macro2::TokenStream; 13 | use quote::{format_ident, quote}; 14 | 15 | use super::common::{safe_ident, CodegenGrammar, CodegenRule, CodegenSettings}; 16 | 17 | impl CodegenGrammar for Grammar { 18 | fn generate_code(&self, settings: &CodegenSettings) -> Result { 19 | let mut all_types = TokenStream::new(); 20 | let mut all_parsers = TokenStream::new(); 21 | let mut all_impls = TokenStream::new(); 22 | let mut cache_entries = TokenStream::new(); 23 | for rule_entry in &self.rules { 24 | match rule_entry { 25 | Grammar_rules::Rule(rule) => { 26 | let flags = rule.flags(); 27 | let (types, impls) = rule 28 | .generate_code(self, settings) 29 | .with_context(|| format!("Error processing rule {}", rule.name))?; 30 | all_types.extend(types); 31 | all_impls.extend(impls); 32 | let rule_ident = safe_ident(&rule.name); 33 | let internal_parser_name = format_ident!("parse_{}", rule.name); 34 | let user_context_type = &settings.user_context_type; 35 | if flags.export { 36 | all_parsers.extend(quote!( 37 | impl peginator_generated::PegParserAdvanced<#user_context_type> for #rule_ident { 38 | fn parse_advanced( 39 | s: &str, 40 | settings: &peginator_generated::ParseSettings, 41 | user_context: #user_context_type, 42 | ) -> Result { 43 | Ok(peginator_generated::#internal_parser_name( 44 | peginator_generated::ParseState::new(s, settings), 45 | &mut peginator_generated 46 | ::ParseGlobal 47 | :: 48 | ::new( 49 | Default::default(), 50 | user_context, 51 | ), 52 | )?.result) 53 | } 54 | } 55 | )) 56 | } 57 | 58 | if flags.memoize || flags.left_recursive { 59 | let cache_entry_ident = format_ident!("c_{}", rule.name); 60 | cache_entries 61 | .extend(quote!(pub #cache_entry_ident: CacheEntries<'a, #rule_ident>,)); 62 | } 63 | } 64 | Grammar_rules::CharRule(rule) => { 65 | let rule_ident = safe_ident(&rule.name); 66 | all_types.extend(quote!(pub type #rule_ident = char;)); 67 | all_impls.extend( 68 | rule.generate_code(settings).with_context(|| { 69 | format!("Error processing @char rule {}", rule.name) 70 | })?, 71 | ); 72 | } 73 | Grammar_rules::ExternRule(rule) => { 74 | let (types, impls) = rule 75 | .generate_code(settings) 76 | .with_context(|| format!("Error processing @extern rule {}", rule.name))?; 77 | all_types.extend(types); 78 | all_impls.extend(impls); 79 | } 80 | } 81 | } 82 | Ok(quote!( 83 | #all_types 84 | #all_parsers 85 | #[allow( 86 | non_snake_case, 87 | unused_variables, 88 | unused_imports, 89 | unused_mut, 90 | dead_code, 91 | )] 92 | mod peginator_generated { 93 | use super::*; 94 | pub use peginator::{ 95 | ParseError, ParseSettings, ParseState, PegParser, IndentedTracer, ParseTracer, 96 | PegPosition, ParseGlobal, PegParserAdvanced, 97 | }; 98 | use peginator::*; 99 | 100 | #[derive(Default)] 101 | pub struct ParseCache<'a> { 102 | #cache_entries 103 | _please_dont_complain: std::marker::PhantomData<&'a ()>, 104 | } 105 | #all_impls 106 | } 107 | )) 108 | } 109 | } 110 | 111 | impl FromStr for Grammar { 112 | type Err = ParseError; 113 | 114 | fn from_str(s: &str) -> Result { 115 | PegParser::parse(s) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /codegen/src/header.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::{BUILD_TIME, VERSION}; 6 | 7 | pub fn generate_source_header(grammar: &str) -> String { 8 | let crc32 = crc::Crc::::new(&crc::CRC_32_ISO_HDLC).checksum(grammar.as_bytes()); 9 | 10 | format!( 11 | "// This file was generated by Peginator v{VERSION} built at {BUILD_TIME}\n\ 12 | // CRC-32/ISO-HDLC of the grammar file: {crc32:08x}\n\ 13 | // Any changes to it will be lost on regeneration\n", 14 | ) 15 | } 16 | -------------------------------------------------------------------------------- /codegen/src/include_rule.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | 8 | use super::common::{Codegen, CodegenSettings, FieldDescriptor}; 9 | use crate::grammar::{Choice, Grammar, Grammar_rules, IncludeRule}; 10 | 11 | impl Codegen for IncludeRule { 12 | fn generate_code_spec( 13 | &self, 14 | rule_fields: &[FieldDescriptor], 15 | grammar: &Grammar, 16 | settings: &CodegenSettings, 17 | ) -> Result { 18 | self.included_rule_definition(grammar)? 19 | .generate_code_spec(rule_fields, grammar, settings) 20 | } 21 | 22 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 23 | self.included_rule_definition(grammar)?.get_fields(grammar) 24 | } 25 | } 26 | 27 | impl IncludeRule { 28 | fn included_rule_definition<'a>(&'a self, grammar: &'a Grammar) -> Result<&'a Choice> { 29 | Ok(&grammar 30 | .rules 31 | .iter() 32 | .find_map(|r| { 33 | if let Grammar_rules::Rule(r) = r { 34 | if r.name == self.rule { 35 | Some(r) 36 | } else { 37 | None 38 | } 39 | } else { 40 | None 41 | } 42 | }) 43 | .ok_or_else(|| { 44 | anyhow::anyhow!( 45 | "Could not find normal (not char or extern) rule named {}", 46 | self.rule 47 | ) 48 | })? 49 | .definition) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /codegen/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | //! This crate contains the code used for generating [`peginator`] parsing code from a 6 | //! grammar file. Unless you are using [`Compile`] in a buildscript, you 7 | //! probably want to see the [`peginator`] crate documentation instead. 8 | //! 9 | //! To integrate [`peginator`] using a buildscript, first add `peginator_codegen` as 10 | //! a build dependency in your `Cargo.toml`: 11 | //! 12 | //! ```toml 13 | //! [build-dependencies] 14 | //! peginator_codegen = "0.6" 15 | //! ``` 16 | //! 17 | //! And then in your `build.rs`: 18 | //! 19 | //! ```ignore 20 | //! use peginator_codegen::Compile; 21 | //! 22 | //! fn main() { 23 | //! let out = format!("{}/grammar.rs", std::env::var("OUT_DIR").unwrap()); 24 | //! 25 | //! peginator_codegen::Compile::file("grammar.ebnf") 26 | //! .destination(out) 27 | //! .format() 28 | //! .run_exit_on_error(); 29 | //! 30 | //! println!("cargo:rerun-if-changed=grammar.ebnf"); 31 | //! } 32 | //! ``` 33 | //! 34 | //! See the documentation of [`Compile`] for more advanced options. 35 | 36 | #[doc(hidden)] 37 | pub mod grammar; 38 | 39 | mod buildscript; 40 | mod char_rule; 41 | mod choice; 42 | mod closure; 43 | mod common; 44 | mod eoi; 45 | mod extern_rule; 46 | mod field; 47 | mod header; 48 | mod include_rule; 49 | mod lookahead; 50 | mod misc; 51 | mod optional; 52 | mod rule; 53 | mod sequence; 54 | mod string; 55 | 56 | pub use buildscript::Compile; 57 | #[doc(hidden)] 58 | pub use common::{CodegenGrammar, CodegenSettings}; 59 | #[doc(hidden)] 60 | pub use grammar::Grammar; 61 | #[doc(hidden)] 62 | pub use header::generate_source_header; 63 | 64 | #[doc(hidden)] 65 | pub const VERSION: &str = env!("CARGO_PKG_VERSION"); 66 | 67 | #[doc(hidden)] 68 | pub const BUILD_TIME: &str = env!("PEGINATOR_BUILD_TIME"); 69 | -------------------------------------------------------------------------------- /codegen/src/lookahead.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::{bail, Result}; 6 | use proc_macro2::TokenStream; 7 | use quote::quote; 8 | 9 | use super::common::{generate_inner_parse_function, Codegen, CodegenSettings, FieldDescriptor}; 10 | use crate::grammar::{Grammar, NegativeLookahead, PositiveLookahead}; 11 | 12 | impl Codegen for NegativeLookahead { 13 | fn generate_code_spec( 14 | &self, 15 | rule_fields: &[FieldDescriptor], 16 | grammar: &Grammar, 17 | settings: &CodegenSettings, 18 | ) -> Result { 19 | let body = self.expr.generate_code(rule_fields, grammar, settings)?; 20 | let parse_body = quote!(match negative_lookahead::parse(state.clone(), global) { 21 | Ok(_) => Err(state.report_error(ParseErrorSpecifics::NegativeLookaheadFailed)), 22 | Err(_) => Ok(ParseOk { result: (), state }), 23 | }); 24 | let parse_function = generate_inner_parse_function(parse_body, settings); 25 | Ok(quote!( 26 | mod negative_lookahead{ 27 | use super::*; 28 | #body 29 | } 30 | #parse_function 31 | )) 32 | } 33 | 34 | fn get_fields(&self, grammar: &Grammar) -> Result> { 35 | if !self.expr.get_fields(grammar)?.is_empty() { 36 | bail!("The body of negative lookaheads should not contain named fields") 37 | } 38 | Ok(Vec::new()) 39 | } 40 | } 41 | 42 | impl Codegen for PositiveLookahead { 43 | fn generate_code_spec( 44 | &self, 45 | rule_fields: &[FieldDescriptor], 46 | grammar: &Grammar, 47 | settings: &CodegenSettings, 48 | ) -> Result { 49 | let body = self.expr.generate_code(rule_fields, grammar, settings)?; 50 | let parse_body = quote!( 51 | positive_lookahead::parse (state.clone(), global)?; 52 | Ok(ParseOk{result:(), state}) 53 | ); 54 | let parse_function = generate_inner_parse_function(parse_body, settings); 55 | Ok(quote!( 56 | mod positive_lookahead{ 57 | use super::*; 58 | #body 59 | } 60 | #parse_function 61 | )) 62 | } 63 | 64 | fn get_fields(&self, grammar: &Grammar) -> Result> { 65 | if !self.expr.get_fields(grammar)?.is_empty() { 66 | bail!("The body of positive lookaheads should not contain named fields") 67 | } 68 | Ok(Vec::new()) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /codegen/src/misc.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | 8 | use super::common::{CloneState, Codegen, CodegenSettings, FieldDescriptor}; 9 | use crate::grammar::{DelimitedExpression, Grammar, Group}; 10 | 11 | impl Codegen for DelimitedExpression { 12 | fn generate_code_spec( 13 | &self, 14 | rule_fields: &[FieldDescriptor], 15 | grammar: &Grammar, 16 | settings: &CodegenSettings, 17 | ) -> Result { 18 | match self { 19 | DelimitedExpression::Group(a) => a.generate_code_spec(rule_fields, grammar, settings), 20 | DelimitedExpression::Optional(a) => { 21 | a.generate_code_spec(rule_fields, grammar, settings) 22 | } 23 | DelimitedExpression::Closure(a) => a.generate_code_spec(rule_fields, grammar, settings), 24 | DelimitedExpression::NegativeLookahead(a) => { 25 | a.generate_code_spec(rule_fields, grammar, settings) 26 | } 27 | DelimitedExpression::PositiveLookahead(a) => { 28 | a.generate_code_spec(rule_fields, grammar, settings) 29 | } 30 | DelimitedExpression::CharacterRange(a) => { 31 | a.generate_code_spec(rule_fields, grammar, settings) 32 | } 33 | DelimitedExpression::StringLiteral(a) => { 34 | a.generate_code_spec(rule_fields, grammar, settings) 35 | } 36 | DelimitedExpression::EndOfInput(a) => { 37 | a.generate_code_spec(rule_fields, grammar, settings) 38 | } 39 | DelimitedExpression::IncludeRule(a) => { 40 | a.generate_code_spec(rule_fields, grammar, settings) 41 | } 42 | DelimitedExpression::Field(a) => a.generate_code_spec(rule_fields, grammar, settings), 43 | } 44 | } 45 | 46 | fn generate_inline_body( 47 | &self, 48 | rule_fields: &[FieldDescriptor], 49 | grammar: &Grammar, 50 | settings: &CodegenSettings, 51 | clone_state: CloneState, 52 | ) -> Result> { 53 | match self { 54 | DelimitedExpression::CharacterRange(a) => { 55 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 56 | } 57 | DelimitedExpression::Closure(a) => { 58 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 59 | } 60 | DelimitedExpression::EndOfInput(a) => { 61 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 62 | } 63 | DelimitedExpression::Field(a) => { 64 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 65 | } 66 | DelimitedExpression::Group(a) => { 67 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 68 | } 69 | DelimitedExpression::IncludeRule(a) => { 70 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 71 | } 72 | DelimitedExpression::NegativeLookahead(a) => { 73 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 74 | } 75 | DelimitedExpression::Optional(a) => { 76 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 77 | } 78 | DelimitedExpression::PositiveLookahead(a) => { 79 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 80 | } 81 | DelimitedExpression::StringLiteral(a) => { 82 | a.generate_inline_body(rule_fields, grammar, settings, clone_state) 83 | } 84 | } 85 | } 86 | 87 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 88 | match self { 89 | DelimitedExpression::Group(a) => a.get_fields(grammar), 90 | DelimitedExpression::Optional(a) => a.get_fields(grammar), 91 | DelimitedExpression::Closure(a) => a.get_fields(grammar), 92 | DelimitedExpression::NegativeLookahead(a) => a.get_fields(grammar), 93 | DelimitedExpression::PositiveLookahead(a) => a.get_fields(grammar), 94 | DelimitedExpression::CharacterRange(a) => a.get_fields(grammar), 95 | DelimitedExpression::StringLiteral(a) => a.get_fields(grammar), 96 | DelimitedExpression::EndOfInput(a) => a.get_fields(grammar), 97 | DelimitedExpression::IncludeRule(a) => a.get_fields(grammar), 98 | DelimitedExpression::Field(a) => a.get_fields(grammar), 99 | } 100 | } 101 | } 102 | 103 | impl Codegen for Group { 104 | fn generate_code_spec( 105 | &self, 106 | rule_fields: &[FieldDescriptor], 107 | grammar: &Grammar, 108 | settings: &CodegenSettings, 109 | ) -> Result { 110 | self.body.generate_code_spec(rule_fields, grammar, settings) 111 | } 112 | 113 | fn generate_inline_body( 114 | &self, 115 | rule_fields: &[FieldDescriptor], 116 | grammar: &Grammar, 117 | settings: &CodegenSettings, 118 | clone_state: CloneState, 119 | ) -> Result> { 120 | self.body 121 | .generate_inline_body(rule_fields, grammar, settings, clone_state) 122 | } 123 | 124 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 125 | self.body.get_fields(grammar) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /codegen/src/optional.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::Result; 6 | use proc_macro2::TokenStream; 7 | use quote::quote; 8 | 9 | use super::common::{ 10 | generate_inner_parse_function, safe_ident, Arity, CloneState, Codegen, CodegenSettings, 11 | FieldDescriptor, 12 | }; 13 | use crate::grammar::{Grammar, Optional}; 14 | 15 | impl Codegen for Optional { 16 | fn generate_code_spec( 17 | &self, 18 | rule_fields: &[FieldDescriptor], 19 | grammar: &Grammar, 20 | settings: &CodegenSettings, 21 | ) -> Result { 22 | let postprocess = self.generate_postprocess_calls(rule_fields, grammar)?; 23 | let body; 24 | let parse_call; 25 | if let Some(inline_body) = 26 | self.body 27 | .generate_inline_body(rule_fields, grammar, settings, CloneState::Yes)? 28 | { 29 | body = TokenStream::new(); 30 | parse_call = inline_body; 31 | } else { 32 | let inner_body = self.body.generate_code(rule_fields, grammar, settings)?; 33 | body = quote!(mod optional{ 34 | use super::*; 35 | #inner_body 36 | }); 37 | parse_call = quote!(optional::parse(state.clone(), global)); 38 | }; 39 | let parse_body = quote!(#parse_call #postprocess); 40 | let parse_function = generate_inner_parse_function(parse_body, settings); 41 | Ok(quote!( 42 | #body 43 | #parse_function 44 | )) 45 | } 46 | 47 | fn generate_inline_body( 48 | &self, 49 | rule_fields: &[FieldDescriptor], 50 | grammar: &Grammar, 51 | settings: &CodegenSettings, 52 | clone_state: CloneState, 53 | ) -> Result> { 54 | if clone_state == CloneState::Yes { 55 | Ok(None) 56 | } else if let Some(inline_body) = 57 | self.body 58 | .generate_inline_body(rule_fields, grammar, settings, CloneState::Yes)? 59 | { 60 | let postprocess = self.generate_postprocess_calls(rule_fields, grammar)?; 61 | Ok(Some(quote!(#inline_body #postprocess))) 62 | } else { 63 | Ok(None) 64 | } 65 | } 66 | 67 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 68 | Ok(set_arity_to_optional(self.body.get_fields(grammar)?)) 69 | } 70 | } 71 | 72 | impl Optional { 73 | fn generate_postprocess_calls( 74 | &self, 75 | rule_fields: &[FieldDescriptor], 76 | grammar: &Grammar, 77 | ) -> Result { 78 | let fields = self.body.get_filtered_rule_fields(rule_fields, grammar)?; 79 | let result = if fields.is_empty() { 80 | quote!( 81 | .or_else(|err| 82 | Ok(ParseOk{ 83 | result: (), 84 | state: state.record_error(err), 85 | }) 86 | ) 87 | ) 88 | } else if fields.len() == 1 { 89 | quote!( 90 | .or_else(|err| 91 | Ok(ParseOk{ 92 | result: Default::default(), 93 | state: state.record_error(err), 94 | }) 95 | ) 96 | ) 97 | } else { 98 | let happy_case_fields: TokenStream = fields 99 | .iter() 100 | .map(|field| { 101 | let name = safe_ident(field.name); 102 | quote!(#name: result.#name,) 103 | }) 104 | .collect(); 105 | let unhappy_case_fields: TokenStream = fields 106 | .iter() 107 | .map(|field| { 108 | let name = safe_ident(field.name); 109 | quote!(#name: Default::default(),) 110 | }) 111 | .collect(); 112 | quote!( 113 | .map_inner(|result| Parsed{#happy_case_fields}) 114 | .or_else(|err| 115 | Ok(ParseOk{ 116 | result: Parsed{#unhappy_case_fields}, 117 | state: state.record_error(err), 118 | }) 119 | ) 120 | ) 121 | }; 122 | Ok(result) 123 | } 124 | } 125 | 126 | fn set_arity_to_optional(fields: Vec) -> Vec { 127 | let mut fields = fields; 128 | for value in &mut fields { 129 | value.arity = match value.arity { 130 | Arity::One => Arity::Optional, 131 | Arity::Optional => Arity::Optional, 132 | Arity::Multiple => Arity::Multiple, 133 | } 134 | } 135 | fields 136 | } 137 | -------------------------------------------------------------------------------- /codegen/src/sequence.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use std::collections::HashSet; 6 | 7 | use anyhow::Result; 8 | use proc_macro2::{Ident, TokenStream}; 9 | use quote::{format_ident, quote}; 10 | 11 | use super::common::{ 12 | generate_inner_parse_function, safe_ident, Arity, CloneState, Codegen, CodegenSettings, 13 | FieldDescriptor, 14 | }; 15 | use crate::{ 16 | common::combine_field_types, 17 | grammar::{Grammar, Sequence}, 18 | }; 19 | 20 | impl Codegen for Sequence { 21 | fn generate_code_spec( 22 | &self, 23 | rule_fields: &[FieldDescriptor], 24 | grammar: &Grammar, 25 | settings: &CodegenSettings, 26 | ) -> Result { 27 | if self.parts.is_empty() { 28 | return Ok(generate_inner_parse_function( 29 | quote!(Ok(ParseOk { result: (), state })), 30 | settings, 31 | )); 32 | } 33 | if self.parts.len() < 2 { 34 | return self.parts[0].generate_code_spec(rule_fields, grammar, settings); 35 | } 36 | let part_bodies = self 37 | .parts 38 | .iter() 39 | .enumerate() 40 | .filter(|(_, part)| { 41 | part.generate_inline_body(rule_fields, grammar, settings, CloneState::No) 42 | .ok() 43 | .flatten() 44 | .is_none() 45 | }) 46 | .map(|(num, part)| -> Result { 47 | let part_mod = format_ident!("part_{num}"); 48 | let part_body = part.generate_code(rule_fields, grammar, settings)?; 49 | Ok(quote!( 50 | mod #part_mod{ 51 | use super::*; 52 | #part_body 53 | } 54 | )) 55 | }) 56 | .collect::>()?; 57 | let parse_function = self.generate_parse_function(rule_fields, grammar, settings)?; 58 | Ok(quote!( 59 | #part_bodies 60 | #parse_function 61 | )) 62 | } 63 | 64 | fn generate_inline_body( 65 | &self, 66 | rule_fields: &[FieldDescriptor], 67 | grammar: &Grammar, 68 | settings: &CodegenSettings, 69 | clone_state: CloneState, 70 | ) -> Result> { 71 | if self.parts.is_empty() { 72 | let state = match clone_state { 73 | CloneState::No => quote!(state), 74 | CloneState::Yes => quote!(state: state.clone()), 75 | }; 76 | Ok(Some(quote!(Ok(ParseOk { result: (), #state })))) 77 | } else if self.parts.len() < 2 { 78 | self.parts[0].generate_inline_body(rule_fields, grammar, settings, clone_state) 79 | } else { 80 | Ok(None) 81 | } 82 | } 83 | 84 | fn get_fields<'a>(&'a self, grammar: &'a Grammar) -> Result>> { 85 | let mut all_fields = Vec::::new(); 86 | for part in &self.parts { 87 | let new_fields = part.get_fields(grammar)?; 88 | for new_field in new_fields { 89 | if let Some(original) = all_fields.iter_mut().find(|f| f.name == new_field.name) { 90 | original.arity = Arity::Multiple; 91 | combine_field_types(&mut original.types, &new_field.types); 92 | } else { 93 | all_fields.push(new_field); 94 | } 95 | } 96 | } 97 | Ok(all_fields) 98 | } 99 | } 100 | 101 | impl Sequence { 102 | fn generate_parse_function( 103 | &self, 104 | rule_fields: &[FieldDescriptor], 105 | grammar: &Grammar, 106 | settings: &CodegenSettings, 107 | ) -> Result { 108 | let fields = self.get_filtered_rule_fields(rule_fields, grammar)?; 109 | let mut calls = TokenStream::new(); 110 | let mut fields_seen = HashSet::<&str>::new(); 111 | for (num, part) in self.parts.iter().enumerate() { 112 | let inner_fields = part.get_filtered_rule_fields(rule_fields, grammar)?; 113 | let part_mod = format_ident!("part_{num}"); 114 | let parse_call = if let Some(inline_body) = 115 | part.generate_inline_body(rule_fields, grammar, settings, CloneState::No)? 116 | { 117 | inline_body 118 | } else { 119 | quote!(#part_mod::parse(state, global)) 120 | }; 121 | let call = if inner_fields.is_empty() { 122 | quote!( 123 | let ParseOk{state, ..} = #parse_call?; 124 | ) 125 | } else { 126 | let mut field_assignments = TokenStream::new(); 127 | let mut extend_calls = TokenStream::new(); 128 | if inner_fields.len() == 1 { 129 | let field = &inner_fields[0]; 130 | let field_name = &field.name; 131 | let name = safe_ident(field_name); 132 | if !fields_seen.contains(field_name) { 133 | fields_seen.insert(field_name); 134 | if field.arity == Arity::Multiple { 135 | field_assignments.extend(quote!(mut #name)) 136 | } else { 137 | field_assignments.extend(quote!(#name)) 138 | } 139 | } else { 140 | assert_eq!(field.arity, Arity::Multiple); 141 | let extend_name = format_ident!("extend_{field_name}_with"); 142 | field_assignments.extend(quote!(#extend_name)); 143 | extend_calls.extend(quote!(#name.extend(#extend_name);)); 144 | } 145 | } else { 146 | for field in &inner_fields { 147 | let field_name = &field.name; 148 | let name = safe_ident(field_name); 149 | if !fields_seen.contains(field_name) { 150 | fields_seen.insert(field_name); 151 | if field.arity == Arity::Multiple { 152 | field_assignments.extend(quote!(mut #name,)) 153 | } else { 154 | field_assignments.extend(quote!(#name,)) 155 | } 156 | } else { 157 | assert_eq!(field.arity, Arity::Multiple); 158 | let extend_name = format_ident!("extend_{field_name}_with"); 159 | field_assignments.extend(quote!(#name: #extend_name,)); 160 | extend_calls.extend(quote!(#name.extend(#extend_name);)); 161 | } 162 | } 163 | field_assignments = quote!( #part_mod::Parsed { #field_assignments } ) 164 | } 165 | quote!( 166 | let ParseOk{result: #field_assignments, state} = #parse_call?; 167 | #extend_calls 168 | ) 169 | }; 170 | calls.extend(call); 171 | } 172 | let field_names: Vec = fields.iter().map(|f| safe_ident(f.name)).collect(); 173 | let parse_result = if field_names.is_empty() { 174 | quote!(()) 175 | } else if field_names.len() == 1 { 176 | quote!(#( #field_names )* ) 177 | } else { 178 | quote!(Parsed{ #( #field_names,)* }) 179 | }; 180 | let parse_body = quote!( 181 | #calls 182 | Ok(ParseOk{result:#parse_result, state}) 183 | ); 184 | Ok(generate_inner_parse_function(parse_body, settings)) 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /codegen/src/string.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use anyhow::{anyhow, bail, Result}; 6 | use proc_macro2::TokenStream; 7 | use quote::quote; 8 | 9 | use super::common::{generate_skip_ws, CloneState, Codegen, CodegenSettings, FieldDescriptor}; 10 | use crate::grammar::{ 11 | CharacterRange, Grammar, HexaEscape, SimpleEscape, StringItem, StringLiteral, Utf8Escape, 12 | }; 13 | 14 | impl TryFrom<&StringItem> for char { 15 | type Error = anyhow::Error; 16 | 17 | fn try_from(value: &StringItem) -> Result { 18 | match value { 19 | StringItem::HexaEscape(x) => Ok(x.into()), 20 | StringItem::SimpleEscape(x) => Ok(x.into()), 21 | StringItem::Utf8Escape(x) => x.try_into(), 22 | StringItem::char(x) => Ok(*x), 23 | } 24 | } 25 | } 26 | 27 | impl From<&HexaEscape> for char { 28 | fn from(value: &HexaEscape) -> Self { 29 | let c1i = value.c1.to_digit(16).unwrap() as u8; 30 | let c2i = value.c2.to_digit(16).unwrap() as u8; 31 | (c1i * 16 + c2i).into() 32 | } 33 | } 34 | 35 | impl From<&SimpleEscape> for char { 36 | fn from(value: &SimpleEscape) -> Self { 37 | match value { 38 | SimpleEscape::SimpleEscapeBackslash(_) => '\\', 39 | SimpleEscape::SimpleEscapeCarriageReturn(_) => '\r', 40 | SimpleEscape::SimpleEscapeDQuote(_) => '"', 41 | SimpleEscape::SimpleEscapeNewline(_) => '\n', 42 | SimpleEscape::SimpleEscapeQuote(_) => '\'', 43 | SimpleEscape::SimpleEscapeTab(_) => '\t', 44 | } 45 | } 46 | } 47 | 48 | impl TryFrom<&Utf8Escape> for char { 49 | type Error = anyhow::Error; 50 | 51 | fn try_from(value: &Utf8Escape) -> Result { 52 | let mut result: u32 = value.c1.to_digit(16).unwrap(); 53 | if let Some(v) = &value.c2 { 54 | result = result * 16 + v.to_digit(16).unwrap(); 55 | }; 56 | if let Some(v) = &value.c3 { 57 | result = result * 16 + v.to_digit(16).unwrap(); 58 | }; 59 | if let Some(v) = &value.c4 { 60 | result = result * 16 + v.to_digit(16).unwrap(); 61 | }; 62 | if let Some(v) = &value.c5 { 63 | result = result * 16 + v.to_digit(16).unwrap(); 64 | }; 65 | if let Some(v) = &value.c6 { 66 | result = result * 16 + v.to_digit(16).unwrap(); 67 | }; 68 | char::from_u32(result).ok_or_else(|| anyhow!("Invalid utf-8 codepoint {result:#x}")) 69 | } 70 | } 71 | 72 | impl TryFrom<&StringLiteral> for String { 73 | type Error = anyhow::Error; 74 | 75 | fn try_from(value: &StringLiteral) -> Result { 76 | value 77 | .body 78 | .iter() 79 | .map(|item| -> Result { item.try_into() }) 80 | .collect::>() 81 | } 82 | } 83 | 84 | impl Codegen for CharacterRange { 85 | fn generate_inline_body( 86 | &self, 87 | _rule_fields: &[FieldDescriptor], 88 | _grammar: &Grammar, 89 | settings: &CodegenSettings, 90 | clone_state: CloneState, 91 | ) -> Result> { 92 | let from: char = (&self.from).try_into()?; 93 | let to: char = (&self.to).try_into()?; 94 | let parse_call = generate_skip_ws( 95 | settings, 96 | "parse_character_range", 97 | quote!(#from, #to), 98 | clone_state, 99 | ); 100 | Ok(Some(quote!(#parse_call .discard_result()))) 101 | } 102 | 103 | fn get_fields(&self, _grammar: &Grammar) -> Result> { 104 | Ok(Vec::new()) 105 | } 106 | } 107 | 108 | impl Codegen for StringLiteral { 109 | fn generate_inline_body( 110 | &self, 111 | _rule_fields: &[FieldDescriptor], 112 | _grammar: &Grammar, 113 | settings: &CodegenSettings, 114 | clone_state: CloneState, 115 | ) -> Result> { 116 | let parser_name; 117 | let additional_params; 118 | let literal = String::try_from(self)?; 119 | if self.insensitive.is_some() { 120 | if !literal.is_ascii() { 121 | bail!("Case insensitive matching only works for ascii strings. ({literal:?} was not ascii)"); 122 | } 123 | let literal = literal.to_ascii_lowercase(); 124 | if literal.chars().count() == 1 { 125 | let char_literal = literal.chars().next().unwrap(); 126 | parser_name = "parse_character_literal_insensitive"; 127 | additional_params = quote!(#char_literal); 128 | } else { 129 | parser_name = "parse_string_literal_insensitive"; 130 | additional_params = quote!(#literal); 131 | } 132 | } else if literal.chars().count() == 1 { 133 | let char_literal = literal.chars().next().unwrap(); 134 | parser_name = "parse_character_literal"; 135 | additional_params = quote!(#char_literal); 136 | } else { 137 | parser_name = "parse_string_literal"; 138 | additional_params = quote!(#literal); 139 | }; 140 | let parse_call = generate_skip_ws(settings, parser_name, additional_params, clone_state); 141 | Ok(Some(quote!(#parse_call .discard_result()))) 142 | } 143 | 144 | fn get_fields(&self, _grammar: &Grammar) -> Result> { 145 | Ok(Vec::new()) 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /doc/error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badicsalex/peginator/ec66eb255dfac1b4a49dce2604fdc0a575919b07/doc/error.png -------------------------------------------------------------------------------- /doc/railroad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badicsalex/peginator/ec66eb255dfac1b4a49dce2604fdc0a575919b07/doc/railroad.png -------------------------------------------------------------------------------- /doc/trace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badicsalex/peginator/ec66eb255dfac1b4a49dce2604fdc0a575919b07/doc/trace.png -------------------------------------------------------------------------------- /grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Grammar = {(rules:Rule | rules:CharRule | rules:ExternRule) ";"} $ ; 7 | 8 | Rule = {directives:DirectiveExpression} name:Identifier "=" definition:Choice; 9 | 10 | CharRule = {directives:CheckDirective} CharDirective {directives:CheckDirective} name:Identifier "=" choices:CharRulePart {"|" choices:CharRulePart }; 11 | 12 | CharRulePart = @:CharacterRange | @:CharRangePart | @:Identifier; 13 | 14 | ExternRule = directive:ExternDirective name:Identifier; 15 | 16 | Choice = choices:Sequence {"|" choices:Sequence}; 17 | 18 | Sequence = { parts:DelimitedExpression }; 19 | 20 | Group = "(" body:Choice ")"; 21 | 22 | Optional = "[" body:Choice "]"; 23 | 24 | Closure = "{" body:Choice "}" [at_least_one:AtLeastOneMarker]; 25 | AtLeastOneMarker = '+'; 26 | 27 | NegativeLookahead = "!" expr:*DelimitedExpression; 28 | 29 | PositiveLookahead = "&" expr:*DelimitedExpression; 30 | 31 | CharacterRange = from:CharRangePart ".." to:CharRangePart; 32 | 33 | @no_skip_ws 34 | CharRangePart = "'" @:StringItem "'"; 35 | 36 | @no_skip_ws 37 | StringLiteral = 38 | [insensitive:CaseInsensitiveMarker] 39 | ( 40 | '"' {!'"' body:StringItem } '"' | 41 | "'" {!"'" body:StringItem } "'" 42 | ) 43 | ; 44 | 45 | CaseInsensitiveMarker = 'i'; 46 | 47 | Field = [(name:Identifier | name:OverrideMarker) ":" [boxed:BoxMarker]] typ:Identifier; 48 | 49 | BoxMarker = '*'; 50 | 51 | OverrideMarker = "@"; 52 | 53 | IncludeRule = ">" rule:Identifier; 54 | 55 | DelimitedExpression = 56 | @:Group | 57 | @:Optional | 58 | @:Closure | 59 | @:NegativeLookahead | 60 | @:PositiveLookahead | 61 | @:CharacterRange | 62 | @:StringLiteral | 63 | @:EndOfInput | 64 | @:IncludeRule | 65 | @:Field 66 | ; 67 | 68 | 69 | @string 70 | @no_skip_ws 71 | Identifier = {IdentifierChar}+; 72 | 73 | @char 74 | IdentifierChar = 'a'..'z' | 'A'..'Z' | '0'..'9' | '_'; 75 | 76 | @no_skip_ws 77 | StringItem = '\\' (@:SimpleEscape | @:HexaEscape | @:Utf8Escape) | !'\\' @:char ; 78 | 79 | @no_skip_ws 80 | SimpleEscape = 81 | @:SimpleEscapeNewline | 82 | @:SimpleEscapeCarriageReturn | 83 | @:SimpleEscapeTab | 84 | @:SimpleEscapeBackslash | 85 | @:SimpleEscapeQuote | 86 | @:SimpleEscapeDQuote 87 | ; 88 | 89 | @no_skip_ws 90 | SimpleEscapeNewline = 'n'; 91 | @no_skip_ws 92 | SimpleEscapeCarriageReturn = 'r'; 93 | @no_skip_ws 94 | SimpleEscapeTab = 't'; 95 | @no_skip_ws 96 | SimpleEscapeBackslash = '\\'; 97 | @no_skip_ws 98 | SimpleEscapeQuote = "'"; 99 | @no_skip_ws 100 | SimpleEscapeDQuote = '"'; 101 | 102 | @no_skip_ws 103 | HexaEscape = 'x' c1:HexChar c2:HexChar; 104 | 105 | @char 106 | HexChar = '0'..'9' | 'a'..'f' | 'A'..'F'; 107 | 108 | @no_skip_ws 109 | Utf8Escape = 110 | 'u' '{' c1:HexChar [c2:HexChar [c3:HexChar [c4:HexChar [c5:HexChar [c6:HexChar ]]]]] '}' | 111 | 'u' c1:HexChar c2:HexChar c3:HexChar c4:HexChar | 112 | 'U' '0' '0' c1:HexChar c2:HexChar c3:HexChar c4:HexChar c5:HexChar c6:HexChar 113 | ; 114 | 115 | DirectiveExpression = 116 | @:StringDirective | 117 | @:NoSkipWsDirective | 118 | @:ExportDirective | 119 | @:PositionDirective | 120 | @:MemoizeDirective | 121 | @:LeftrecDirective | 122 | @:CheckDirective 123 | ; 124 | 125 | StringDirective = "@string"; 126 | CharDirective = "@char"; 127 | NoSkipWsDirective = "@no_skip_ws"; 128 | ExportDirective = "@export"; 129 | PositionDirective = "@position"; 130 | MemoizeDirective = "@memoize"; 131 | LeftrecDirective = "@leftrec"; 132 | CheckDirective = "@check" "(" function:NamespacedRustName ")"; 133 | ExternDirective = 134 | "@extern" "(" 135 | function:NamespacedRustName 136 | [ '->' return_type:NamespacedRustName ] 137 | ")" 138 | ; 139 | 140 | NamespacedRustName = @:RustNamePart { '::' @:RustNamePart }; 141 | 142 | @string 143 | RustNamePart = {!( '-' | ')' |':' ) char}+; 144 | 145 | EndOfInput = '$'; 146 | 147 | @no_skip_ws 148 | Whitespace = {Comment | '\t' | '\n' | '\x0C' | '\r' | ' '}; 149 | 150 | @no_skip_ws 151 | Comment = '#' {!'\n' char} '\n'; 152 | -------------------------------------------------------------------------------- /macro/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "peginator_macro" 3 | description = "Convenience macros for using peginator (a PEG parser generator)" 4 | version = "0.7.0" 5 | edition = "2021" 6 | 7 | authors = ["Alex Badics "] 8 | repository = "https://github.com/badicsalex/peginator" 9 | documentation = "https://docs.rs/peginator_macro" 10 | keywords = ["parsing", "parser", "grammar", "ast", "peg"] 11 | categories = ["parsing"] 12 | license = "MIT" 13 | 14 | [lib] 15 | proc-macro = true 16 | 17 | [dependencies] 18 | peginator_codegen = { version="=0.7.0", path = "../codegen" } 19 | syn = { version = "2.0", default-features = false, features = ["parsing", "proc-macro"] } 20 | 21 | [dev-dependencies] 22 | peginator = { version="=0.7.0", path = "../runtime" } 23 | -------------------------------------------------------------------------------- /macro/README.md: -------------------------------------------------------------------------------- 1 | # Peginator macro 2 | 3 | Proc-macro support for [peginator](https://github.com/badicsalex/peginator). Please find 4 | documentation and other goodies on the main project. 5 | 6 | Usage: 7 | ```rust 8 | use peginator_macro::peginate; 9 | use peginator::PegParser; 10 | 11 | peginate!(" 12 | @export 13 | PizzaRule = 14 | 'Pizza' 'with' 15 | toppings:Topping 16 | {',' toppings:Topping} 17 | ['and' toppings:Topping] 18 | ; 19 | @string 20 | Topping = 'sausage' | 'pineapple' | 'bacon' | 'cheese'; 21 | "); 22 | 23 | fn main() { 24 | let result = PizzaRule::parse("Pizza with sausage, bacon and cheese").unwrap(); 25 | println!("{:?}", result.toppings); 26 | } 27 | ``` 28 | -------------------------------------------------------------------------------- /macro/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #![doc = include_str!("../README.md")] 6 | #![warn(missing_docs)] 7 | 8 | use std::str::FromStr; 9 | 10 | use peginator_codegen::{CodegenGrammar, CodegenSettings, Grammar}; 11 | use proc_macro::TokenStream; 12 | 13 | /// Compile peginator grammar in-place 14 | /// 15 | /// The parameter is the grammar as a string, and will generate the parser in-place. 16 | /// 17 | /// The `PegParser` trait needs to be `use`-d separately. 18 | #[proc_macro] 19 | pub fn peginate(input: TokenStream) -> TokenStream { 20 | let param: syn::LitStr = 21 | syn::parse(input).expect("peginate!() expects a single string as a parameter"); 22 | let parsed_grammar = Grammar::from_str(¶m.value()).unwrap(); 23 | parsed_grammar 24 | .generate_code(&CodegenSettings::default()) 25 | .unwrap() 26 | .into() 27 | } 28 | -------------------------------------------------------------------------------- /macro/tests/simple.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use peginator::PegParser; 6 | use peginator_macro::peginate; 7 | 8 | peginate!("@export Simple = c:char;"); 9 | 10 | #[test] 11 | fn test_macro() { 12 | let s: Simple = Simple::parse("xyz").unwrap(); 13 | assert_eq!(s.c, 'x'); 14 | } 15 | -------------------------------------------------------------------------------- /runtime/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "peginator" 3 | description = "PEG parser generator for creating ASTs in Rust (runtime)" 4 | version = "0.7.0" 5 | edition = "2021" 6 | 7 | authors = ["Alex Badics "] 8 | repository = "https://github.com/badicsalex/peginator" 9 | documentation = "https://docs.rs/peginator" 10 | keywords = ["parsing", "parser", "grammar", "ast", "peg"] 11 | categories = ["parsing"] 12 | license = "MIT" 13 | 14 | # All dependencies of this crate will appear in every dependency tree of every 15 | # crate which uses `peginator` to generate code. Therefore, dependencies should 16 | # be strenuously avoided in this crate. 17 | [dependencies] 18 | nohash-hasher = "0.2.0" 19 | colored = { version = "2.0", optional = true } 20 | 21 | [features] 22 | default = ["colored"] 23 | -------------------------------------------------------------------------------- /runtime/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /runtime/src/builtin_parsers.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::{ParseErrorSpecifics, ParseOk, ParseResult, ParseState}; 6 | 7 | /// Hand-written 'rule parser' for parsing a single cahracter. 8 | /// 9 | /// Should always look just like all the other generated parse functions. 10 | #[inline(always)] 11 | pub fn parse_char<_GT>(state: ParseState, _global: _GT) -> ParseResult { 12 | let result = state.s().chars().next().ok_or_else(|| { 13 | state 14 | .clone() 15 | .report_error(ParseErrorSpecifics::ExpectedAnyCharacter) 16 | })?; 17 | // SAFETY: 18 | // Callers of this function are responsible that these preconditions are satisfied: 19 | // Indexes must lie on UTF-8 sequence boundaries. 20 | // 21 | // We are skipping a full character, so we should be OK. 22 | let state = unsafe { state.advance(result.len_utf8()) }; 23 | Ok(ParseOk { result, state }) 24 | } 25 | 26 | /// Hand-written 'rule parser' for skipping whitespace. Shadowed by grammar-generated implementations, if there is one. 27 | /// 28 | /// Should always look just like all the other generated parse functions. 29 | #[inline] 30 | #[allow(non_snake_case)] 31 | pub fn parse_Whitespace<_GT>(state: ParseState, _global: _GT) -> ParseResult<()> { 32 | let mut state = state; 33 | while !state.is_empty() { 34 | if state.s().as_bytes()[0].is_ascii_whitespace() { 35 | // SAFETY: 36 | // Callers of this function are responsible that these preconditions are satisfied: 37 | // Indexes must lie on UTF-8 sequence boundaries. 38 | // 39 | // The byte we are skipping is ASCII, so we are OK. 40 | state = unsafe { state.advance(1) }; 41 | } else { 42 | break; 43 | } 44 | } 45 | Ok(ParseOk { result: (), state }) 46 | } 47 | 48 | #[inline(always)] 49 | pub fn parse_string_literal<'a>( 50 | state: ParseState<'a>, 51 | s: &'static str, 52 | ) -> ParseResult<'a, &'static str> { 53 | if !state.s().starts_with(s) { 54 | Err(state.report_error(ParseErrorSpecifics::ExpectedString { s })) 55 | } else { 56 | // SAFETY: 57 | // Callers of this function are responsible that these preconditions are satisfied: 58 | // Indexes must lie on UTF-8 sequence boundaries. 59 | // 60 | // We are skipping a correct string's length, so we should be OK. 61 | let state = unsafe { state.advance(s.len()) }; 62 | Ok(ParseOk { result: s, state }) 63 | } 64 | } 65 | 66 | #[inline(always)] 67 | pub fn parse_character_literal(state: ParseState, c: char) -> ParseResult { 68 | if c.is_ascii() { 69 | // fast path 70 | if state.is_empty() || state.s().as_bytes()[0] != c as u8 { 71 | Err(state.report_error(ParseErrorSpecifics::ExpectedCharacter { c })) 72 | } else { 73 | // SAFETY: 74 | // Callers of this function are responsible that these preconditions are satisfied: 75 | // Indexes must lie on UTF-8 sequence boundaries. 76 | // 77 | // The byte we are skipping is ASCII, so we are OK. 78 | let state = unsafe { state.advance(1) }; 79 | Ok(ParseOk { result: c, state }) 80 | } 81 | } else if !state.s().starts_with(c) { 82 | // utf-8 path 83 | Err(state.report_error(ParseErrorSpecifics::ExpectedCharacter { c })) 84 | } else { 85 | // SAFETY: 86 | // Callers of this function are responsible that these preconditions are satisfied: 87 | // Indexes must lie on UTF-8 sequence boundaries. 88 | // 89 | // We are skipping a full character, so we should be OK. 90 | let state = unsafe { state.advance(c.len_utf8()) }; 91 | Ok(ParseOk { result: c, state }) 92 | } 93 | } 94 | 95 | #[inline(always)] 96 | pub fn parse_character_range(state: ParseState, from: char, to: char) -> ParseResult { 97 | if from.is_ascii() && to.is_ascii() { 98 | // fast path 99 | if state.is_empty() { 100 | return Err( 101 | state.report_error(ParseErrorSpecifics::ExpectedCharacterRange { from, to }) 102 | ); 103 | } 104 | 105 | let first_byte = state.s().as_bytes()[0]; 106 | if first_byte < from as u8 || first_byte > to as u8 { 107 | return Err( 108 | state.report_error(ParseErrorSpecifics::ExpectedCharacterRange { from, to }) 109 | ); 110 | } 111 | // SAFETY: 112 | // Callers of this function are responsible that these preconditions are satisfied: 113 | // Indexes must lie on UTF-8 sequence boundaries. 114 | // 115 | // The byte we are skipping is ASCII, so we are OK. 116 | let state = unsafe { state.advance(1) }; 117 | Ok(ParseOk { 118 | result: first_byte as char, 119 | state, 120 | }) 121 | } else { 122 | // utf-8 path 123 | let c = state.s().chars().next().ok_or_else(|| { 124 | state 125 | .clone() 126 | .report_error(ParseErrorSpecifics::ExpectedCharacterRange { from, to }) 127 | })?; 128 | if c < from || c > to { 129 | Err(state.report_error(ParseErrorSpecifics::ExpectedCharacterRange { from, to })) 130 | } else { 131 | // SAFETY: 132 | // Callers of this function are responsible that these preconditions are satisfied: 133 | // Indexes must lie on UTF-8 sequence boundaries. 134 | // 135 | // We are skipping a full character, so we should be OK. 136 | let state = unsafe { state.advance(c.len_utf8()) }; 137 | Ok(ParseOk { result: c, state }) 138 | } 139 | } 140 | } 141 | 142 | #[inline(always)] 143 | pub fn parse_string_literal_insensitive<'a>( 144 | state: ParseState<'a>, 145 | s: &'static str, 146 | ) -> ParseResult<'a, &'static str> { 147 | let prefix = state 148 | .s() 149 | .bytes() 150 | .map(|c| c.to_ascii_lowercase()) 151 | .take(s.len()); 152 | if !s.bytes().eq(prefix) { 153 | Err(state.report_error(ParseErrorSpecifics::ExpectedString { s })) 154 | } else { 155 | // SAFETY: 156 | // Callers of this function are responsible that these preconditions are satisfied: 157 | // Indexes must lie on UTF-8 sequence boundaries. 158 | // 159 | // We are skipping a correct string's length, so we should be OK. 160 | let state = unsafe { state.advance(s.len()) }; 161 | Ok(ParseOk { result: s, state }) 162 | } 163 | } 164 | 165 | #[inline(always)] 166 | pub fn parse_character_literal_insensitive(state: ParseState, c: char) -> ParseResult { 167 | // ASCII Only ! 168 | if state.is_empty() || state.s().as_bytes()[0].to_ascii_lowercase() != c as u8 { 169 | Err(state.report_error(ParseErrorSpecifics::ExpectedCharacter { c })) 170 | } else { 171 | // SAFETY: 172 | // Callers of this function are responsible that these preconditions are satisfied: 173 | // Indexes must lie on UTF-8 sequence boundaries. 174 | // 175 | // The byte we are skipping is ASCII, so we are OK. 176 | let state = unsafe { state.advance(1) }; 177 | Ok(ParseOk { result: c, state }) 178 | } 179 | } 180 | 181 | #[inline(always)] 182 | pub fn parse_end_of_input(state: ParseState) -> ParseResult<()> { 183 | if state.is_empty() { 184 | Ok(ParseOk { result: (), state }) 185 | } else { 186 | Err(state.report_error(ParseErrorSpecifics::ExpectedEoi)) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /runtime/src/call_hook.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | pub trait CallHook { 6 | fn call_as_extern(&self, s: &str, user_context: TU) -> Result<(TR, usize), &'static str>; 7 | fn call_as_check(&self, s: &str, user_context: TU) -> bool; 8 | } 9 | 10 | impl CallHook for TF 11 | where 12 | TF: Fn(&str) -> Result<(TR, usize), &'static str>, 13 | { 14 | fn call_as_extern(&self, s: &str, _user_context: TU) -> Result<(TR, usize), &'static str> { 15 | self(s) 16 | } 17 | 18 | fn call_as_check(&self, s: &str, _user_context: TU) -> bool { 19 | self(s).is_ok() 20 | } 21 | } 22 | 23 | impl CallHook for TF 24 | where 25 | TF: Fn(&str, TU) -> Result<(TR, usize), &'static str>, 26 | { 27 | fn call_as_extern(&self, s: &str, user_context: TU) -> Result<(TR, usize), &'static str> { 28 | self(s, user_context) 29 | } 30 | 31 | fn call_as_check(&self, s: &str, user_context: TU) -> bool { 32 | self(s, user_context).is_ok() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /runtime/src/choice_helper.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::{ParseOk, ParseResult, ParseState}; 6 | 7 | pub struct ChoiceHelper<'a, T> { 8 | state: ParseState<'a>, 9 | result: Option>, 10 | } 11 | 12 | impl<'a, T> ChoiceHelper<'a, T> { 13 | #[inline] 14 | pub fn new(state: ParseState<'a>) -> Self { 15 | Self { 16 | state, 17 | result: None, 18 | } 19 | } 20 | 21 | #[inline] 22 | pub fn choice(mut self, parse_fn: impl FnOnce(ParseState<'a>) -> ParseResult<'a, T>) -> Self { 23 | if self.result.is_none() { 24 | match parse_fn(self.state.clone()) { 25 | Ok(ok_result) => self.result = Some(ok_result), 26 | Err(err) => self.state = self.state.record_error(err), 27 | } 28 | } 29 | self 30 | } 31 | 32 | #[inline] 33 | pub fn end(self) -> ParseResult<'a, T> { 34 | match self.result { 35 | Some(ok) => Ok(ok), 36 | None => Err(self.state.report_farthest_error()), 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /runtime/src/colored_shim.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | pub trait FakeColored: Sized { 6 | fn bold(self) -> Self { 7 | self 8 | } 9 | fn red(self) -> Self { 10 | self 11 | } 12 | fn white(self) -> Self { 13 | self 14 | } 15 | fn blue(self) -> Self { 16 | self 17 | } 18 | fn cyan(self) -> Self { 19 | self 20 | } 21 | fn green(self) -> Self { 22 | self 23 | } 24 | fn yellow(self) -> Self { 25 | self 26 | } 27 | } 28 | 29 | impl FakeColored for &str {} 30 | 31 | impl FakeColored for String {} 32 | -------------------------------------------------------------------------------- /runtime/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use std::error::Error; 6 | 7 | #[cfg(feature = "colored")] 8 | use colored::*; 9 | 10 | #[cfg(not(feature = "colored"))] 11 | use super::colored_shim::*; 12 | 13 | /// The type and specifics of the atomic match, used by [`ParseError`]. 14 | #[derive(Debug, Clone)] 15 | pub enum ParseErrorSpecifics { 16 | /// Expected any character, but found end of input. 17 | ExpectedAnyCharacter, 18 | /// Expected a specific character. 19 | ExpectedCharacter { 20 | /// The character that was expected 21 | c: char, 22 | }, 23 | /// Expected a character from a specific range. 24 | ExpectedCharacterRange { 25 | /// Start of the character range that was expected 26 | from: char, 27 | /// End of the character range that was expected (inclusive) 28 | to: char, 29 | }, 30 | /// Expected a specific string. 31 | ExpectedString { 32 | /// The string that was expected 33 | s: &'static str, 34 | }, 35 | /// Expected to match a @char rule. 36 | ExpectedCharacterClass { 37 | /// The character class that was expected 38 | name: &'static str, 39 | }, 40 | /// Expected the end of file, but found additional characters. 41 | ExpectedEoi, 42 | /// A negative lookahead (`!`) rule part failed. 43 | NegativeLookaheadFailed, 44 | /// A custom check function failed 45 | CheckFunctionFailed { 46 | /// The name of the function that failed 47 | function_name: &'static str, 48 | }, 49 | /// A custom extern rule failed 50 | ExternRuleFailed { 51 | /// The string that was returned as an error 52 | error_string: &'static str, 53 | }, 54 | /// Left recursion error reached. Should not be returned from a normal parse call. 55 | LeftRecursionSentinel, 56 | 57 | /// An unknown error happened. Usually means there is a problem with peginator itself. 58 | Other, 59 | } 60 | 61 | impl ToString for ParseErrorSpecifics { 62 | fn to_string(&self) -> String { 63 | match self { 64 | ParseErrorSpecifics::ExpectedAnyCharacter => { 65 | "expected any character (found end of input)".to_string() 66 | } 67 | ParseErrorSpecifics::ExpectedCharacter { c } => format!("expected character '{c}'"), 68 | ParseErrorSpecifics::ExpectedCharacterRange { from, to } => { 69 | format!("expected character from range '{from}'-'{to}'") 70 | } 71 | ParseErrorSpecifics::ExpectedCharacterClass { name } => { 72 | format!("expected character from character class {name}") 73 | } 74 | ParseErrorSpecifics::ExpectedString { s } => format!("expected string \"{s}\""), 75 | ParseErrorSpecifics::ExpectedEoi => "expected end of input".to_string(), 76 | ParseErrorSpecifics::NegativeLookaheadFailed => { 77 | "negative lookahead condition failed".to_string() 78 | } 79 | ParseErrorSpecifics::CheckFunctionFailed { function_name } => { 80 | format!("check function '{function_name}' failed") 81 | } 82 | ParseErrorSpecifics::ExternRuleFailed { error_string } => { 83 | format!("extern function failed with '{error_string}'") 84 | } 85 | ParseErrorSpecifics::LeftRecursionSentinel => { 86 | "Left recursion sentinel reached, will probably retry.".to_string() 87 | } 88 | ParseErrorSpecifics::Other => "Unknown error. Sorry :(".to_string(), 89 | } 90 | } 91 | } 92 | 93 | /// An error happened during parsing (compact version). 94 | /// 95 | /// During parsing, the parser records the furthest it got without encountering a match failure. The 96 | /// error will contain both this furthest position and the first unmatched atomic matcher at this 97 | /// position. 98 | /// 99 | /// Convert to [`PrettyParseError`] before showing it to a user. 100 | #[derive(Debug, Clone)] 101 | pub struct ParseError { 102 | /// The byte-position of the furthest match failure. 103 | pub position: usize, 104 | /// The atomic match that was unsuccessful at the furthest parsing position. 105 | pub specifics: ParseErrorSpecifics, 106 | } 107 | 108 | impl std::fmt::Display for ParseError { 109 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 110 | let specifics = self.specifics.to_string(); 111 | write!( 112 | f, 113 | "Parse error on byte position {} while parsing: {}", 114 | self.position, specifics 115 | ) 116 | } 117 | } 118 | 119 | impl Error for ParseError {} 120 | 121 | #[derive(Debug, Clone)] 122 | struct IndexedStringLine<'a> { 123 | pub s: &'a str, 124 | pub lineno: usize, 125 | pub start_offset: usize, 126 | pub end_offset: usize, 127 | } 128 | 129 | struct IndexedStringLineIterator<'a> { 130 | source: &'a str, 131 | lineno: usize, 132 | byte_offset: usize, 133 | } 134 | 135 | impl<'a> IndexedStringLineIterator<'a> { 136 | fn new(s: &'a str) -> IndexedStringLineIterator<'a> { 137 | IndexedStringLineIterator { 138 | source: s, 139 | lineno: 0, 140 | byte_offset: 0, 141 | } 142 | } 143 | } 144 | 145 | impl<'a> Iterator for IndexedStringLineIterator<'a> { 146 | type Item = IndexedStringLine<'a>; 147 | 148 | fn next(&mut self) -> Option { 149 | if self.byte_offset >= self.source.bytes().len() { 150 | return None; 151 | } 152 | let next_offset = self.source[self.byte_offset..] 153 | .bytes() 154 | .position(|b| b == b'\n') 155 | .map(|p| p + self.byte_offset) 156 | .unwrap_or_else(|| self.source.bytes().len()) 157 | + 1; 158 | let result = IndexedStringLine { 159 | s: &self.source[self.byte_offset..next_offset - 1], 160 | lineno: self.lineno, 161 | start_offset: self.byte_offset, 162 | end_offset: next_offset, 163 | }; 164 | self.lineno += 1; 165 | self.byte_offset = next_offset; 166 | Some(result) 167 | } 168 | } 169 | 170 | /// An error happened during parsing (pretty version). 171 | /// 172 | /// Converted from [`ParseError`], produces a very pretty, colored error message when printed with 173 | /// regular [`std::fmt::Display`]. 174 | #[derive(Debug, Clone)] 175 | pub struct PrettyParseError { 176 | err_string: String, 177 | } 178 | 179 | impl PrettyParseError { 180 | /// Convert from [`ParseError`] 181 | /// 182 | /// The parsed `text` needs to be supplied to show the context of the error. 183 | /// 184 | /// The `source_file` parameter is used to print the error with the same format `rustc` does. 185 | pub fn from_parse_error(err: &ParseError, text: &str, source_file: Option<&str>) -> Self { 186 | let target_line = IndexedStringLineIterator::new(text) 187 | .find(|l| l.start_offset <= err.position && l.end_offset >= err.position) 188 | .unwrap(); 189 | let character_position = target_line 190 | .s 191 | .char_indices() 192 | .map(|(cp, _c)| cp) 193 | .position(|cp| cp == err.position - target_line.start_offset) 194 | .unwrap_or(0); 195 | let position = if let Some(f) = source_file { 196 | format!( 197 | "{}:{:?}:{:?}", 198 | f, 199 | target_line.lineno + 1, 200 | character_position + 1 201 | ) 202 | } else { 203 | format!( 204 | "Line {} character {}", 205 | target_line.lineno + 1, 206 | character_position + 1 207 | ) 208 | }; 209 | let err_string = format!( 210 | "{err}\n{arrow}{position}\n{pipe}\n{pipe}{the_line}\n{pipe}{caret:>caret_offset$}\n", 211 | err = err.specifics.to_string().bold().white(), 212 | position = position, 213 | the_line = target_line.s.trim_end(), 214 | caret = "^".bold().red(), 215 | caret_offset = character_position + 1, 216 | arrow = "--> ".bold().blue(), 217 | pipe = " | ".bold().blue(), 218 | ); 219 | Self { err_string } 220 | } 221 | } 222 | 223 | impl std::fmt::Display for PrettyParseError { 224 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 225 | write!(f, "{}", self.err_string) 226 | } 227 | } 228 | 229 | impl Error for PrettyParseError {} 230 | -------------------------------------------------------------------------------- /runtime/src/global.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::ParseTracer; 6 | 7 | #[derive(Debug, Clone)] 8 | pub struct ParseGlobal { 9 | pub tracer: TT, 10 | pub cache: TC, 11 | pub user_context: TUD, 12 | } 13 | 14 | impl ParseGlobal { 15 | pub fn new(cache: TC, user_context: TUD) -> Self { 16 | Self { 17 | tracer: TT::new(), 18 | cache, 19 | user_context, 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /runtime/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | //! # Table of contents 6 | //! * [Description](#description) 7 | //! * [Quickstart](#quickstart) 8 | //! * [Integration](#integration) 9 | //! * [Features and optional dependencies](#features-and-optional-dependencies) 10 | //! * [Syntax](#peginator-syntax) 11 | //! * [Rules](#rules) 12 | //! * [Expressions](#expressions) 13 | //! * [Fields](#fields) 14 | //! * [Directives](#directives) 15 | //! * [Whitespace skipping](#whitespace-skipping) 16 | //! 17 | //! 18 | //! # Description 19 | //! 20 | //! PEG parser generator for creating ASTs in Rust 21 | //! 22 | //! Peginator is a PEG (Parsing Expression Grammar) parser generator written in Rust. 23 | //! It is specifically made to parse into ASTs (Abstract Syntax Trees), as opposed to 24 | //! most, streaming-style parsers out there. 25 | //! 26 | //! It generates both the tree structure and the parsing code that can create that tree from any 27 | //! `&str`. The generated parsing code is deliberately very simple straightforward Rust 28 | //! code, which is usually optimized very well by the compiler. 29 | //! 30 | //! There is an opt-in memoization feature that makes it a proper packrat parser that can parse 31 | //! any input in linear time and space. 32 | //! 33 | //! Left-recursion is also supported using said memoization feature (also opt-in). 34 | //! 35 | //! #### Crate structure 36 | //! 37 | //! The crates in peginator are: 38 | //! - [`peginator`](crate): The runtime crate used by generated code and the crates calling it. 39 | //! It should be a "normal" dependency of all `peginator`-using code. 40 | //! - [`peginator_codegen`]: Helper crate with reduced dependencies for direct buildscript 41 | //! integration. It should be a build-depenednecy if used. 42 | //! - [`peginator_macro`]: Macro support for small grammars directly included in code. 43 | //! - [`peginator_cli`]: A binary crate providing the `peginator-cli` cli command. 44 | //! 45 | //! 46 | //! #### About PEGs 47 | //! 48 | //! This documentation describes how peginator implements PEGs. A basic understanding of PEGs 49 | //! are assumed. There are good introductions on 50 | //! [wikipedia](https://en.wikipedia.org/wiki/Parsing_expression_grammar) or in the 51 | //! [docs of other parser generators](https://pest.rs/book/grammars/syntax.html). 52 | //! 53 | //! Peginator is bootstrapped using its own 54 | //! [syntax and grammar file](https://github.com/badicsalex/peginator/blob/master/grammar.ebnf), 55 | //! which is somewhat easy-to-read. 56 | //! 57 | //! # Quickstart 58 | //! 59 | //! The grammars for peginator are written in a syntax similar to EBNF 60 | //! (extended Backus-Naur form): 61 | //! 62 | //! ```ebnf 63 | //! @export 64 | //! FunctionDef = 'fn' name:Ident '(' param_list:ParamList ')' [ '->' return_value:Type ]; 65 | //! 66 | //! ParamList = self_param:SelfParam {',' params:Param} | params:Param {',' params:Param} | ; 67 | //! 68 | //! Param = name:Ident ':' typ: Type; 69 | //! 70 | //! SelfParam = [ref_type:ReferenceMarker] 'self'; 71 | //! 72 | //! Type = [ref_type:ReferenceMarker] typename:Ident; 73 | //! 74 | //! ReferenceMarker = @:MutableReference | @:ImmutableReference; 75 | //! 76 | //! ImmutableReference = '&'; 77 | //! MutableReference = '&' 'mut'; 78 | //! 79 | //! @string 80 | //! @no_skip_ws 81 | //! Ident = {'a'..'z' | 'A'..'Z' | '_' | '0'..'9'}; 82 | //! ``` 83 | //! 84 | //! Based on the above grammar, peginator will generate the following types: 85 | //! 86 | //! ```ignore 87 | //! pub struct FunctionDef { 88 | //! pub name: Ident, 89 | //! pub param_list: ParamList, 90 | //! pub return_value: Option, 91 | //! } 92 | //! pub struct ParamList { 93 | //! pub self_param: Option, 94 | //! pub params: Vec, 95 | //! } 96 | //! pub struct Param { 97 | //! pub name: Ident, 98 | //! pub typ: Type, 99 | //! } 100 | //! pub struct SelfParam { 101 | //! pub ref_type: Option, 102 | //! } 103 | //! pub struct Type { 104 | //! pub ref_type: Option, 105 | //! pub typename: Ident, 106 | //! } 107 | //! pub enum ReferenceMarker { 108 | //! ImmutableReference(ImmutableReference), 109 | //! MutableReference(MutableReference), 110 | //! } 111 | //! pub struct ImmutableReference; 112 | //! pub struct MutableReference; 113 | //! pub type Ident = String; 114 | //! 115 | //! impl PegParser for FunctionDef { /* omitted */ } 116 | //! ``` 117 | //! 118 | //! Parsing then looks like this: 119 | //! ```ignore 120 | //! FunctionDef::parse("fn example(&self, input:&str, rectified:&mut Rect) -> ExampleResult;") 121 | //! ``` 122 | //! 123 | //! Which results in the folowing structure: 124 | //! ```ignore 125 | //! FunctionDef { 126 | //! name: "example", 127 | //! param_list: ParamList { 128 | //! self_param: Some(SelfParam { 129 | //! ref_type: Some(ImmutableReference(ImmutableReference)), 130 | //! }), 131 | //! params: [ 132 | //! Param { 133 | //! name: "input", 134 | //! typ: Type { 135 | //! ref_type: Some(ImmutableReference(ImmutableReference)), 136 | //! typename: "str", 137 | //! }, 138 | //! }, 139 | //! Param { 140 | //! name: "rectified", 141 | //! typ: Type { 142 | //! ref_type: Some(MutableReference(MutableReference)), 143 | //! typename: "Rect", 144 | //! }, 145 | //! }, 146 | //! ], 147 | //! }, 148 | //! return_value: Some(Type { 149 | //! ref_type: None, 150 | //! typename: "ExampleResult", 151 | //! }), 152 | //! } 153 | //! ``` 154 | //! 155 | //! # Integration 156 | //! 157 | //! To use peginator grammars in your project, you should write the grammar and save it in the 158 | //! project directory with the `.ebnf` extension. 159 | //! 160 | //! Then run the `peginator-cli` command (from the `peginator_cli` crate) on it, 161 | //! and (optionally) rustfmt: 162 | //! 163 | //! ```text 164 | //! peginator-cli your_grammar.ebnf | rustfmt >src/grammar.rs 165 | //! ``` 166 | //! 167 | //! Make sure you also add the `peginator` crate as a dependency in your `Cargo.toml`: 168 | //! 169 | //! ```toml 170 | //! [dependencies] 171 | //! peginator = "0.6" 172 | //! ``` 173 | //! 174 | //! Once you have compiled your grammar, you can import the types, and the 175 | //! [`PegParser`] trait, and you can start parsing strings: 176 | //! ```ignore 177 | //! use crate::grammar::YourRootType; 178 | //! use peginator::PegParser; 179 | //! 180 | //! let parse_result = YourRootType::parse("some string (maybe?)"); 181 | //! println!("{:?}", parse_result); 182 | //! ``` 183 | //! 184 | //! Alternatively, you can use a buildscript using the [`peginator_codegen`] crate, 185 | //! or a macro using the [`peginator_macro`] crate. 186 | //! 187 | //! [`peginator_macro`]: https://docs.rs/peginator_macro 188 | //! [`peginator_codegen`]: https://docs.rs/peginator_codegen 189 | //! [`peginator_cli`]: https://crates.io/crates/peginator_cli 190 | //! 191 | //! ## Features and optional dependencies 192 | //! 193 | //! The following features can be enabled: 194 | //! - `colored` (on by default): Adds a dependency to the [`colored`] crate, and makes error reports 195 | //! and traces colorful. 196 | //! 197 | #![doc = include_str!("../syntax.md")] 198 | use std::collections::HashMap; 199 | 200 | use nohash_hasher::BuildNoHashHasher; 201 | 202 | mod builtin_parsers; 203 | mod choice_helper; 204 | mod colored_shim; 205 | mod error; 206 | mod global; 207 | mod parse_result; 208 | mod peg_parser; 209 | mod state; 210 | mod trace; 211 | 212 | #[doc(hidden)] 213 | pub use builtin_parsers::{ 214 | parse_Whitespace, parse_char, parse_character_literal, parse_character_literal_insensitive, 215 | parse_character_range, parse_end_of_input, parse_string_literal, 216 | parse_string_literal_insensitive, 217 | }; 218 | #[doc(hidden)] 219 | pub use choice_helper::ChoiceHelper; 220 | pub use error::{ParseError, ParseErrorSpecifics, PrettyParseError}; 221 | #[doc(hidden)] 222 | pub use global::ParseGlobal; 223 | #[doc(hidden)] 224 | pub use parse_result::{ParseOk, ParseResult, ParseResultExtras}; 225 | pub use peg_parser::{ParseSettings, PegParser, PegParserAdvanced}; 226 | #[doc(hidden)] 227 | pub use state::ParseState; 228 | #[doc(hidden)] 229 | pub use trace::{IndentedTracer, NoopTracer, ParseTracer}; 230 | 231 | #[doc(hidden)] 232 | pub type CacheEntries<'a, T> = HashMap, BuildNoHashHasher>; 233 | 234 | /// Helper trait to get the parse position of the parsed rule 235 | /// 236 | /// Useful for creating general error reporting functions or 237 | /// similar functionality where the position of multiple generated 238 | /// types are used. 239 | pub trait PegPosition { 240 | /// The parsed position of the rule in bytes (not characters) 241 | fn position(&self) -> &std::ops::Range; 242 | } 243 | -------------------------------------------------------------------------------- /runtime/src/parse_result.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::{ParseError, ParseState}; 6 | 7 | #[derive(Debug, Clone)] 8 | pub struct ParseOk<'a, T> { 9 | pub result: T, 10 | pub state: ParseState<'a>, 11 | } 12 | 13 | impl<'a, T> ParseOk<'a, T> { 14 | #[inline] 15 | pub fn map(self, f: F) -> ParseOk<'a, T2> 16 | where 17 | F: Fn(T) -> T2, 18 | { 19 | ParseOk:: { 20 | result: f(self.result), 21 | state: self.state, 22 | } 23 | } 24 | 25 | #[inline] 26 | pub fn map_with_state(self, f: F) -> ParseOk<'a, T2> 27 | where 28 | F: Fn(T, &ParseState) -> T2, 29 | { 30 | ParseOk:: { 31 | result: f(self.result, &self.state), 32 | state: self.state, 33 | } 34 | } 35 | } 36 | 37 | pub type ParseResult<'a, T> = Result, ParseError>; 38 | 39 | pub trait ParseResultExtras<'a, T> { 40 | fn discard_result(self) -> ParseResult<'a, ()>; 41 | fn map_inner(self, f: F) -> ParseResult<'a, T2> 42 | where 43 | F: Fn(T) -> T2; 44 | } 45 | 46 | impl<'a, T> ParseResultExtras<'a, T> for ParseResult<'a, T> { 47 | #[inline] 48 | fn discard_result(self) -> ParseResult<'a, ()> { 49 | self.map_inner(|_| ()) 50 | } 51 | 52 | #[inline] 53 | fn map_inner(self, f: F) -> ParseResult<'a, T2> 54 | where 55 | F: Fn(T) -> T2, 56 | { 57 | self.map(|ok| ok.map(f)) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /runtime/src/peg_parser.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::{IndentedTracer, NoopTracer, ParseError, ParseTracer}; 6 | 7 | /// The main trait for interfacing with peginator. Implemented by `@export`-ed rules. 8 | pub trait PegParser: Sized { 9 | /// Parse a string into the AST. 10 | fn parse(s: &str) -> Result; 11 | 12 | /// Parse a string into the AST, print a colored trace of the parse process. 13 | /// 14 | /// The printing happens with regular `eprintln!()`. 15 | fn parse_with_trace(s: &str) -> Result; 16 | } 17 | 18 | impl> PegParser for T { 19 | fn parse(s: &str) -> Result { 20 | Self::parse_advanced::(s, &ParseSettings::default(), ()) 21 | } 22 | fn parse_with_trace(s: &str) -> Result { 23 | Self::parse_advanced::(s, &ParseSettings::default(), ()) 24 | } 25 | } 26 | 27 | /// The trait that is actually implemented by the generated code. 28 | /// Should only be used in very specific cases, and [`PegParser`] should be 29 | /// used in all normal cases. 30 | pub trait PegParserAdvanced: Sized { 31 | /// Internal function that is actually generated by the grammar compiler, used by the more 32 | /// friendly functions. 33 | fn parse_advanced( 34 | s: &str, 35 | settings: &ParseSettings, 36 | user_context: TUD, 37 | ) -> Result; 38 | } 39 | 40 | /// Parse settings (for future compatibility) 41 | #[derive(Debug, Default)] 42 | #[non_exhaustive] 43 | pub struct ParseSettings {} 44 | -------------------------------------------------------------------------------- /runtime/src/state.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | use super::{ParseError, ParseErrorSpecifics, ParseSettings}; 6 | 7 | #[derive(Debug, Clone)] 8 | pub struct ParseState<'a> { 9 | partial_string: &'a str, 10 | start_index: usize, 11 | farthest_error: Option, 12 | } 13 | 14 | impl<'a> ParseState<'a> { 15 | #[inline] 16 | pub fn new(s: &'a str, _settings: &ParseSettings) -> ParseState<'a> { 17 | Self { 18 | partial_string: s, 19 | start_index: 0, 20 | farthest_error: None, 21 | } 22 | } 23 | 24 | #[inline] 25 | pub fn s(&self) -> &str { 26 | self.partial_string 27 | } 28 | 29 | #[inline] 30 | pub fn is_empty(&self) -> bool { 31 | self.s().is_empty() 32 | } 33 | 34 | /// Advance the parsing pointer n bytes 35 | /// 36 | /// # Safety 37 | /// Callers of this function are responsible that these preconditions are satisfied: 38 | /// Indexes must lie on UTF-8 sequence boundaries. 39 | #[inline] 40 | #[warn(unsafe_op_in_unsafe_fn)] 41 | pub unsafe fn advance(self, length: usize) -> Self { 42 | // We are eschewing mainly the utf-8 codepoint check here, 43 | // because the caller can be sure that everything is fine. 44 | 45 | if length > self.partial_string.len() { 46 | // This should be optimized out in most cases 47 | panic!("String length overrun in advance()") 48 | }; 49 | Self { 50 | start_index: self.start_index + length, 51 | // SAFETY: 52 | // Callers of this function are responsible that these preconditions are satisfied: 53 | // Indexes must lie on UTF-8 sequence boundaries. 54 | // The starting index must not exceed the ending index; 55 | // Indexes must be within bounds of the original slice; 56 | partial_string: unsafe { self.partial_string.get_unchecked(length..) }, 57 | ..self 58 | } 59 | } 60 | 61 | /// Advance the parsing pointer n chars. Panics if length indexes into a character 62 | #[inline] 63 | pub fn advance_safe(self, length: usize) -> Self { 64 | if length > self.partial_string.len() { 65 | // This should be optimized out in most cases 66 | panic!("String length overrun in advance()") 67 | }; 68 | Self { 69 | start_index: self.start_index + length, 70 | partial_string: &self.partial_string[length..], 71 | ..self 72 | } 73 | } 74 | 75 | #[inline] 76 | pub fn slice_until(&self, other: &ParseState) -> &str { 77 | &self.partial_string[..(other.start_index - self.start_index)] 78 | } 79 | 80 | #[inline] 81 | pub fn range_until(&self, other: &ParseState) -> std::ops::Range { 82 | self.start_index..other.start_index 83 | } 84 | 85 | #[inline] 86 | pub fn cache_key(&self) -> usize { 87 | self.start_index 88 | } 89 | 90 | #[inline] 91 | pub fn report_error(self, specifics: ParseErrorSpecifics) -> ParseError { 92 | let position = self.start_index; 93 | self.record_error(ParseError { 94 | position, 95 | specifics, 96 | }) 97 | .report_farthest_error() 98 | } 99 | 100 | #[inline] 101 | pub fn record_error(mut self, error: ParseError) -> Self { 102 | match &mut self.farthest_error { 103 | Some(farthest_error) => { 104 | if farthest_error.position <= error.position { 105 | *farthest_error = error 106 | } 107 | } 108 | None => self.farthest_error = Some(error), 109 | } 110 | self 111 | } 112 | 113 | #[inline] 114 | pub fn report_farthest_error(self) -> ParseError { 115 | self.farthest_error.unwrap_or(ParseError { 116 | position: self.start_index, 117 | specifics: ParseErrorSpecifics::Other, 118 | }) 119 | } 120 | 121 | pub fn first_n_chars(&self, n: usize) -> String { 122 | self.s().chars().take(n).collect() 123 | } 124 | 125 | #[inline] 126 | pub fn is_further_than(&self, other: &Self) -> bool { 127 | self.start_index > other.start_index 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /runtime/src/trace.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | #[cfg(feature = "colored")] 5 | use colored::*; 6 | 7 | #[cfg(not(feature = "colored"))] 8 | use super::colored_shim::*; 9 | use super::{ParseResult, ParseState}; 10 | 11 | pub trait ParseTracer: Clone + Copy { 12 | fn print_informative(&mut self, s: &str) { 13 | let _ = s; 14 | } 15 | fn print_trace_start(&mut self, state: &ParseState, name: &str) { 16 | let _ = (state, name); 17 | } 18 | fn print_trace_result(&mut self, result: &ParseResult) { 19 | let _ = result; 20 | } 21 | 22 | fn new() -> Self; 23 | } 24 | 25 | #[derive(Debug, Clone, Copy)] 26 | pub struct IndentedTracer { 27 | indentation_level: usize, 28 | } 29 | 30 | impl ParseTracer for IndentedTracer { 31 | #[inline] 32 | fn print_informative(&mut self, s: &str) { 33 | let indentation = " ".repeat(self.indentation_level); 34 | eprintln!("{indentation}{}", s.cyan()); 35 | } 36 | 37 | #[inline] 38 | fn print_trace_start(&mut self, state: &ParseState, name: &str) { 39 | let indentation = " ".repeat(self.indentation_level); 40 | eprintln!("{indentation}{:?}", state.first_n_chars(50)); 41 | eprintln!("{indentation}{}?", name.yellow()); 42 | self.indentation_level += 1; 43 | } 44 | 45 | #[inline] 46 | fn print_trace_result(&mut self, result: &ParseResult) { 47 | let indentation = " ".repeat(self.indentation_level); 48 | match &result { 49 | Ok(ok_result) => { 50 | eprintln!("{indentation}{:?}", ok_result.state.first_n_chars(50)); 51 | eprintln!("{indentation}{}", "Ok".green()); 52 | } 53 | Err(err) => eprintln!( 54 | "{indentation}{} {}", 55 | "Error:".red(), 56 | err.specifics.to_string() 57 | ), 58 | }; 59 | self.indentation_level -= 1; 60 | } 61 | 62 | #[inline] 63 | fn new() -> Self { 64 | Self { 65 | indentation_level: 0, 66 | } 67 | } 68 | } 69 | 70 | #[derive(Debug, Clone, Copy)] 71 | pub struct NoopTracer {} 72 | 73 | impl ParseTracer for NoopTracer { 74 | #[inline] 75 | fn new() -> Self { 76 | Self {} 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /runtime/syntax.md: -------------------------------------------------------------------------------- 1 | ../doc/syntax.md -------------------------------------------------------------------------------- /test/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "peginator_test" 3 | description = "Test package for peginator" 4 | version = "0.7.0" 5 | edition = "2021" 6 | 7 | authors = ["Alex Badics "] 8 | repository = "https://github.com/badicsalex/peginator" 9 | license = "MIT" 10 | publish = false 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [dependencies] 15 | peginator = { version="=0.7.0", path = "../runtime" } 16 | 17 | [build-dependencies] 18 | peginator_codegen = { version="=0.7.0", path = "../codegen" } 19 | 20 | [dev-dependencies] 21 | ntest = "0.9" 22 | -------------------------------------------------------------------------------- /test/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | fn main() { 6 | peginator_codegen::Compile::directory("src") 7 | .format() 8 | .derives(vec![ 9 | "Debug".into(), 10 | "Clone".into(), 11 | "PartialEq".into(), 12 | "Eq".into(), 13 | ]) 14 | .run_exit_on_error(); 15 | peginator_codegen::Compile::file("src/custom_derives_empty/grammar.not_ebnf") 16 | .format() 17 | .derives(vec![]) 18 | .prefix("pub struct ImJustHereToConfuse;".into()) 19 | .run_exit_on_error(); 20 | peginator_codegen::Compile::file("src/user_defined_state/grammar.not_ebnf") 21 | .format() 22 | .derives(vec![ 23 | "Debug".into(), 24 | "Clone".into(), 25 | "PartialEq".into(), 26 | "Eq".into(), 27 | ]) 28 | .user_context_type("crate::user_defined_state::TheState") 29 | .run_exit_on_error(); 30 | } 31 | -------------------------------------------------------------------------------- /test/src/additional_traits/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export Simple = c:Tag1 | c:Tag2; 6 | Tag1 = '1'; 7 | Tag2 = '2'; 8 | -------------------------------------------------------------------------------- /test/src/additional_traits/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | trait CustomTrait { 10 | fn numeric(self) -> i32; 11 | } 12 | 13 | impl CustomTrait for Tag1 { 14 | fn numeric(self) -> i32 { 15 | 1 16 | } 17 | } 18 | 19 | impl CustomTrait for Tag2 { 20 | fn numeric(self) -> i32 { 21 | 2 22 | } 23 | } 24 | 25 | #[test] 26 | fn test_macro() { 27 | let s = Simple::parse("1").unwrap(); 28 | if let Simple_c::Tag1(parsed) = s.c { 29 | assert_eq!(parsed.numeric(), 1); 30 | } else { 31 | panic!("Wat."); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /test/src/boxing/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | BoxedSimple = p:*Point; 7 | 8 | @export 9 | BoxedOptional = [p:*Point]; 10 | 11 | @export 12 | BoxedVec = {p:*Point}; 13 | 14 | @export 15 | HalfBoxedEnum = p:Point | p:*Coordinate; 16 | 17 | @export 18 | FullBoxedEnum = p:*Point | p:*Coordinate; 19 | 20 | @export 21 | Point = '(' x:Coordinate ';' y:Coordinate ')'; 22 | 23 | @string 24 | @no_skip_ws 25 | Coordinate = {'0'..'9'}+; 26 | 27 | @export 28 | OverrideTest = s:SimpleOverride e:EnumOverride o:OptionOverride v:VecOverride; 29 | 30 | SimpleOverride = @:*Point; 31 | 32 | EnumOverride = @:*Point | @:Coordinate; 33 | 34 | OptionOverride = [@:*Point]; 35 | 36 | VecOverride = {@:*Point}; 37 | -------------------------------------------------------------------------------- /test/src/boxing/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #[allow(clippy::module_inception, clippy::vec_box)] 6 | mod grammar; 7 | use grammar::*; 8 | use peginator::PegParser; 9 | 10 | use crate::test_utils::assert_type_eq; 11 | 12 | #[test] 13 | fn test_enums() { 14 | match HalfBoxedEnum::parse("(123;456)").unwrap().p { 15 | HalfBoxedEnum_p::Point(parsed) => assert_eq!( 16 | parsed, 17 | Point { 18 | x: "123".to_string(), 19 | y: "456".to_string() 20 | } 21 | ), 22 | _ => panic!("Invalid parse"), 23 | } 24 | match HalfBoxedEnum::parse("5").unwrap().p { 25 | HalfBoxedEnum_p::Coordinate(parsed) => assert_eq!(parsed, Box::new("5".into())), 26 | _ => panic!("Invalid parse"), 27 | } 28 | 29 | match FullBoxedEnum::parse("(123;456)").unwrap().p { 30 | FullBoxedEnum_p::Point(parsed) => assert_eq!( 31 | parsed, 32 | Box::new(Point { 33 | x: "123".to_string(), 34 | y: "456".to_string() 35 | }) 36 | ), 37 | _ => panic!("Invalid parse"), 38 | } 39 | } 40 | 41 | #[test] 42 | fn test_overrides() { 43 | let result = OverrideTest::parse("(1;2) (3;4)").unwrap(); 44 | assert_eq!( 45 | result.s, 46 | Box::new(Point { 47 | x: "1".to_string(), 48 | y: "2".to_string() 49 | }) 50 | ); 51 | match result.e { 52 | EnumOverride::Point(p) => assert_eq!( 53 | p, 54 | Box::new(Point { 55 | x: "3".to_string(), 56 | y: "4".to_string() 57 | }) 58 | ), 59 | _ => panic!("Invalid parse"), 60 | } 61 | assert_eq!(result.o, None); 62 | assert_eq!(result.v, vec![]); 63 | let result = OverrideTest::parse("(1;2) (3;4) (5;6) (7;8) (9;10)").unwrap(); 64 | assert_eq!( 65 | result.o, 66 | Some(Box::new(Point { 67 | x: "5".to_string(), 68 | y: "6".to_string() 69 | })) 70 | ); 71 | assert_eq!( 72 | result.v, 73 | vec![ 74 | Box::new(Point { 75 | x: "7".to_string(), 76 | y: "8".to_string() 77 | }), 78 | Box::new(Point { 79 | x: "9".to_string(), 80 | y: "10".to_string() 81 | }) 82 | ] 83 | ); 84 | } 85 | 86 | #[test] 87 | #[allow(clippy::vec_box)] 88 | fn test_boxes() { 89 | assert_type_eq!(BoxedSimple, p, Box); 90 | assert_type_eq!(BoxedOptional, p, Option>); 91 | assert_type_eq!(BoxedVec, p, Vec>); 92 | 93 | assert_eq!( 94 | *BoxedSimple::parse("(3;45)").unwrap().p, 95 | Point { 96 | x: "3".to_string(), 97 | y: "45".to_string() 98 | } 99 | ); 100 | 101 | assert_eq!( 102 | *BoxedOptional::parse("(3;45)").unwrap().p.unwrap(), 103 | Point { 104 | x: "3".to_string(), 105 | y: "45".to_string() 106 | } 107 | ); 108 | 109 | assert_eq!(BoxedOptional::parse("()").unwrap().p, None); 110 | 111 | assert_eq!( 112 | BoxedVec::parse("(3;45)").unwrap().p, 113 | vec![Box::new(Point { 114 | x: "3".to_string(), 115 | y: "45".to_string() 116 | })] 117 | ); 118 | } 119 | -------------------------------------------------------------------------------- /test/src/calculator_example/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | @leftrec 7 | Expression = @:Add | @:Sub | @:Term; 8 | Add = left:*Expression '+' right:Term; 9 | Sub = left:*Expression '-' right:Term; 10 | 11 | @leftrec 12 | Term = @:Mul | @:Div | @:Factor; 13 | Mul = left:*Term '*' right:Factor; 14 | Div = left:*Term '/' right:Factor; 15 | 16 | @memoize 17 | Factor = @:Group | @:Number; 18 | 19 | Group = '(' body:*Expression ')'; 20 | 21 | @string 22 | @no_skip_ws 23 | Number = {'0'..'9'}+; 24 | -------------------------------------------------------------------------------- /test/src/calculator_example/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | trait Calculate { 10 | fn calculate(&self) -> i64; 11 | } 12 | 13 | impl Calculate for Number { 14 | fn calculate(&self) -> i64 { 15 | self.parse().unwrap() 16 | } 17 | } 18 | 19 | impl Calculate for Group { 20 | fn calculate(&self) -> i64 { 21 | self.body.calculate() 22 | } 23 | } 24 | 25 | impl Calculate for Add { 26 | fn calculate(&self) -> i64 { 27 | self.left.calculate() + self.right.calculate() 28 | } 29 | } 30 | 31 | impl Calculate for Sub { 32 | fn calculate(&self) -> i64 { 33 | self.left.calculate() - self.right.calculate() 34 | } 35 | } 36 | impl Calculate for Mul { 37 | fn calculate(&self) -> i64 { 38 | self.left.calculate() * self.right.calculate() 39 | } 40 | } 41 | impl Calculate for Div { 42 | fn calculate(&self) -> i64 { 43 | self.left.calculate() / self.right.calculate() 44 | } 45 | } 46 | 47 | // ==== Boilerplate. This could be replaced with enum_dispatch ==== 48 | impl Calculate for Expression { 49 | fn calculate(&self) -> i64 { 50 | match self { 51 | Expression::Add(x) => x.calculate(), 52 | Expression::Sub(x) => x.calculate(), 53 | Expression::Term(x) => x.calculate(), 54 | } 55 | } 56 | } 57 | impl Calculate for Term { 58 | fn calculate(&self) -> i64 { 59 | match self { 60 | Term::Div(x) => x.calculate(), 61 | Term::Factor(x) => x.calculate(), 62 | Term::Mul(x) => x.calculate(), 63 | } 64 | } 65 | } 66 | impl Calculate for Factor { 67 | fn calculate(&self) -> i64 { 68 | match self { 69 | Factor::Group(x) => x.calculate(), 70 | Factor::Number(x) => x.calculate(), 71 | } 72 | } 73 | } 74 | // ============ 75 | 76 | #[test] 77 | fn test() { 78 | assert_eq!( 79 | Expression::parse_with_trace("1 - 2 + 3").unwrap(), 80 | Expression::Add(Add { 81 | left: Box::new(Expression::Sub(Sub { 82 | left: Box::new(Expression::Term(Term::Factor(Factor::Number("1".into())))), 83 | right: Term::Factor(Factor::Number("2".into())), 84 | })), 85 | right: Term::Factor(Factor::Number("3".into())), 86 | }) 87 | ); 88 | assert_eq!( 89 | Expression::parse_with_trace("(1 - 2) * 3 / 4 + 5").unwrap(), 90 | Expression::Add(Add { 91 | left: Box::new(Expression::Term(Term::Div(Div { 92 | left: Box::new(Term::Mul(Mul { 93 | left: Box::new(Term::Factor(Factor::Group(Group { 94 | body: Box::new(Expression::Sub(Sub { 95 | left: Box::new(Expression::Term(Term::Factor(Factor::Number( 96 | "1".into() 97 | )))), 98 | right: Term::Factor(Factor::Number("2".into())) 99 | })) 100 | }))), 101 | right: Factor::Number("3".into()) 102 | })), 103 | right: Factor::Number("4".into()) 104 | }))), 105 | right: Term::Factor(Factor::Number("5".into())) 106 | }) 107 | ); 108 | assert_eq!( 109 | Expression::parse("(15-27) * 3 / 4 + 123 - (5*7 + 2)") 110 | .unwrap() 111 | .calculate(), 112 | 77 113 | ); 114 | } 115 | -------------------------------------------------------------------------------- /test/src/char_rule/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | HexNumTester = num:HexNum; 7 | 8 | @string 9 | @no_skip_ws 10 | HexNum = {Hexadecimal}+; 11 | 12 | @char 13 | Hexadecimal = '0'..'9' | 'a'..'f' | 'A'..'F'; 14 | 15 | @export 16 | @no_skip_ws 17 | WeirdTest = '.' a:CharA w:Weird; 18 | 19 | @char 20 | CharA = 'a'; 21 | 22 | @char 23 | Weird = 'x' | '1'..'8' | CharA | '\u{2700}'; 24 | -------------------------------------------------------------------------------- /test/src/char_rule/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_hex() { 11 | assert_eq!( 12 | HexNumTester::parse("123ABFZ1").unwrap().num, 13 | "123ABF".to_string() 14 | ); 15 | assert_eq!( 16 | HexNumTester::parse("123 ABFZ1").unwrap().num, 17 | "123".to_string() 18 | ); 19 | assert!(HexNumTester::parse("X").is_err()); 20 | } 21 | 22 | #[test] 23 | fn test_weird() { 24 | assert_eq!( 25 | WeirdTest::parse(".axz").unwrap(), 26 | WeirdTest { a: 'a', w: 'x' } 27 | ); 28 | assert_eq!( 29 | WeirdTest::parse(".a2z").unwrap(), 30 | WeirdTest { a: 'a', w: '2' } 31 | ); 32 | assert_eq!( 33 | WeirdTest::parse(".aaz").unwrap(), 34 | WeirdTest { a: 'a', w: 'a' } 35 | ); 36 | assert_eq!( 37 | WeirdTest::parse(".a✀z").unwrap(), 38 | WeirdTest { a: 'a', w: '✀' } 39 | ); 40 | assert!(WeirdTest::parse(".b✀z").is_err()); 41 | assert!(WeirdTest::parse(".a0z").is_err()); 42 | assert!(WeirdTest::parse(".a").is_err()); 43 | assert!(WeirdTest::parse(".a ✀z").is_err()); 44 | assert!(WeirdTest::parse(". a✀z").is_err()); 45 | } 46 | -------------------------------------------------------------------------------- /test/src/check/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Test = 7 | 'u' u:UnitRule | 8 | 'f' f:FailingUnitRule | 9 | 'c' c:ComplexRule | 10 | 's' s:SmallNumber | 11 | 'd' d:DualNumber | 12 | 'l' l:LowerCase 13 | ; 14 | 15 | @check(crate::check::unit_checker) 16 | UnitRule = 'a'; 17 | 18 | @check(crate::check::unit_checker_failing) 19 | FailingUnitRule = 'a'; 20 | 21 | @check(crate::check::complex_checker) 22 | ComplexRule = '(' x:Number ',' y:Number ')'; 23 | 24 | @check(crate::check::override_check) 25 | SmallNumber = 's' @:Number; 26 | 27 | @check(crate::check::enum_override_check) 28 | DualNumber = @:HexNumber | @:Number ; 29 | 30 | @string 31 | Number = { '0'..'9' }+; 32 | 33 | @string 34 | HexNumber = '0x' {'0'..'9' | 'a'..'f'}; 35 | 36 | @check(char::is_lowercase) 37 | @char 38 | LowerCase = char; 39 | -------------------------------------------------------------------------------- /test/src/check/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | assert_eq!(Test::parse("ua").unwrap().u.unwrap(), UnitRule); 12 | assert!(Test::parse("fa").is_err()); 13 | 14 | assert_eq!( 15 | Test::parse("c(123,456)").unwrap().c.unwrap(), 16 | ComplexRule { 17 | x: "123".to_string(), 18 | y: "456".to_string() 19 | } 20 | ); 21 | assert!(Test::parse("c(23,456)").is_err()); 22 | 23 | assert_eq!(Test::parse("ss123").unwrap().s.unwrap(), "123".to_string()); 24 | assert!(Test::parse("ss23").is_err()); 25 | 26 | assert_eq!( 27 | Test::parse("d0xabc").unwrap().d.unwrap(), 28 | DualNumber::HexNumber("0xabc".to_string()) 29 | ); 30 | assert!(Test::parse("d0xbc").is_err()); 31 | 32 | assert_eq!(Test::parse("lű").unwrap().l.unwrap(), 'ű'); 33 | assert!(Test::parse("lŰ").is_err()); 34 | } 35 | 36 | pub fn unit_checker(_: &UnitRule) -> bool { 37 | true 38 | } 39 | 40 | pub fn unit_checker_failing(_: &FailingUnitRule) -> bool { 41 | false 42 | } 43 | 44 | pub fn complex_checker(cr: &ComplexRule) -> bool { 45 | cr.x.contains('1') 46 | } 47 | 48 | pub fn override_check(sn: &SmallNumber) -> bool { 49 | sn.contains('1') 50 | } 51 | 52 | pub fn enum_override_check(dn: &DualNumber) -> bool { 53 | match dn { 54 | DualNumber::HexNumber(hn) => hn.contains('a'), 55 | DualNumber::Number(n) => n.contains('1'), 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /test/src/choice/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Simple = f:FieldA | f:FieldB; 7 | 8 | @export 9 | ContainsOverridden = f:Overridden; 10 | 11 | Overridden = @:FieldA | @:FieldB; 12 | 13 | @export 14 | Arity11 = '1' f:FieldA | '2' f:FieldA; 15 | @export 16 | Arity1O = '1' f:FieldA | '2' [f:FieldA]; 17 | @export 18 | Arity1M = '1' f:FieldA | '2' {f:FieldA}; 19 | @export 20 | Arity1X = '1' f:FieldA | '2'; 21 | 22 | @export 23 | ArityO1 = '1' [f:FieldA] | '2' f:FieldA; 24 | @export 25 | ArityOO = '1' [f:FieldA] | '2' [f:FieldA]; 26 | @export 27 | ArityOM = '1' [f:FieldA] | '2' {f:FieldA}; 28 | @export 29 | ArityOX = '1' [f:FieldA] | '2' ; 30 | 31 | @export 32 | ArityM1 = '1' {f:FieldA} | '2' f:FieldA; 33 | @export 34 | ArityMO = '1' {f:FieldA} | '2' [f:FieldA]; 35 | @export 36 | ArityMM = '1' {f:FieldA} | '2' {f:FieldA}; 37 | @export 38 | ArityMX = '1' {f:FieldA} | '2'; 39 | 40 | @export 41 | ArityX1 = '1' | '2' f:FieldA; 42 | @export 43 | ArityXO = '1' | '2' [f:FieldA]; 44 | @export 45 | ArityXM = '1' | '2' {f:FieldA}; 46 | @export 47 | ArityXX = '1' | '2'; 48 | 49 | @export 50 | DefaultO = '1' [f:FieldA] | '2'; 51 | @export 52 | DefaultM = '1' {f:FieldA} | '2'; 53 | 54 | @export 55 | Empty = '1' | '2'; 56 | 57 | @export 58 | MultipleFields = f1:FieldA '1' f2:FieldB | f2:FieldA '2' f3:FieldB; 59 | 60 | FieldA = 'a'; 61 | FieldB = 'b'; 62 | -------------------------------------------------------------------------------- /test/src/choice/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | use crate::test_utils::assert_type_eq; 10 | 11 | #[test] 12 | fn test_simple() { 13 | let s = Simple::parse("a").unwrap(); 14 | assert_eq!(s.f, Simple_f::FieldA(FieldA)); 15 | 16 | assert!(Simple::parse("x").is_err()) 17 | } 18 | 19 | #[test] 20 | fn test_types() { 21 | assert_type_eq!(Arity11, f, FieldA); 22 | assert_type_eq!(Arity1O, f, Option); 23 | assert_type_eq!(Arity1M, f, Vec); 24 | assert_type_eq!(ArityO1, f, Option); 25 | assert_type_eq!(ArityOO, f, Option); 26 | assert_type_eq!(ArityOM, f, Vec); 27 | assert_type_eq!(ArityM1, f, Vec); 28 | assert_type_eq!(ArityMO, f, Vec); 29 | assert_type_eq!(ArityMM, f, Vec); 30 | 31 | assert_type_eq!(DefaultO, f, Option); 32 | assert_type_eq!(DefaultM, f, Vec); 33 | } 34 | 35 | #[test] 36 | fn test_arities_1() { 37 | assert_eq!(Arity11::parse("1a").unwrap(), Arity11 { f: FieldA }); 38 | assert_eq!(Arity11::parse("2a").unwrap(), Arity11 { f: FieldA }); 39 | assert!(Arity11::parse("1").is_err()); 40 | assert!(Arity11::parse("2").is_err()); 41 | assert!(Arity11::parse("3a").is_err()); 42 | 43 | assert_eq!(Arity1O::parse("1a").unwrap(), Arity1O { f: Some(FieldA) }); 44 | assert_eq!(Arity1O::parse("2a").unwrap(), Arity1O { f: Some(FieldA) }); 45 | assert_eq!(Arity1O::parse("2").unwrap(), Arity1O { f: None }); 46 | assert!(Arity1O::parse("1").is_err()); 47 | assert!(Arity1O::parse("3a").is_err()); 48 | 49 | assert_eq!(Arity1M::parse("1a").unwrap(), Arity1M { f: vec![FieldA] }); 50 | assert_eq!(Arity1M::parse("2a").unwrap(), Arity1M { f: vec![FieldA] }); 51 | assert_eq!( 52 | Arity1M::parse("2aaa").unwrap(), 53 | Arity1M { 54 | f: vec![FieldA, FieldA, FieldA] 55 | } 56 | ); 57 | assert_eq!(Arity1M::parse("2").unwrap(), Arity1M { f: vec![] }); 58 | assert!(Arity1M::parse("1").is_err()); 59 | assert!(Arity1M::parse("3a").is_err()); 60 | 61 | assert_eq!(Arity1X::parse("1a").unwrap(), Arity1X { f: Some(FieldA) }); 62 | assert_eq!(Arity1X::parse("2a").unwrap(), Arity1X { f: None }); 63 | assert_eq!(Arity1X::parse("2").unwrap(), Arity1X { f: None }); 64 | assert!(Arity1X::parse("1").is_err()); 65 | assert!(Arity1X::parse("3a").is_err()); 66 | } 67 | 68 | #[test] 69 | fn test_arities_o() { 70 | assert_eq!(ArityO1::parse("1a").unwrap(), ArityO1 { f: Some(FieldA) }); 71 | assert_eq!(ArityO1::parse("1").unwrap(), ArityO1 { f: None }); 72 | assert_eq!(ArityO1::parse("2a").unwrap(), ArityO1 { f: Some(FieldA) }); 73 | assert!(ArityO1::parse("2").is_err()); 74 | assert!(ArityO1::parse("3a").is_err()); 75 | 76 | assert_eq!(ArityOO::parse("1a").unwrap(), ArityOO { f: Some(FieldA) }); 77 | assert_eq!(ArityOO::parse("1").unwrap(), ArityOO { f: None }); 78 | assert_eq!(ArityOO::parse("2a").unwrap(), ArityOO { f: Some(FieldA) }); 79 | assert_eq!(ArityOO::parse("2").unwrap(), ArityOO { f: None }); 80 | assert!(ArityOO::parse("3a").is_err()); 81 | 82 | assert_eq!(ArityOM::parse("1a").unwrap(), ArityOM { f: vec![FieldA] }); 83 | // Not an error, because no $ at end of grammar 84 | assert_eq!(ArityOM::parse("1aaa").unwrap(), ArityOM { f: vec![FieldA] }); 85 | assert_eq!(ArityOM::parse("1").unwrap(), ArityOM { f: vec![] }); 86 | assert_eq!(ArityOM::parse("2a").unwrap(), ArityOM { f: vec![FieldA] }); 87 | assert_eq!( 88 | ArityOM::parse("2aaa").unwrap(), 89 | ArityOM { 90 | f: vec![FieldA, FieldA, FieldA] 91 | } 92 | ); 93 | assert_eq!(ArityOM::parse("2").unwrap(), ArityOM { f: vec![] }); 94 | assert!(ArityOM::parse("3a").is_err()); 95 | 96 | assert_eq!(ArityOX::parse("1a").unwrap(), ArityOX { f: Some(FieldA) }); 97 | assert_eq!(ArityOX::parse("1").unwrap(), ArityOX { f: None }); 98 | assert_eq!(ArityOX::parse("2a").unwrap(), ArityOX { f: None }); 99 | assert_eq!(ArityOX::parse("2").unwrap(), ArityOX { f: None }); 100 | assert!(ArityOX::parse("3a").is_err()); 101 | } 102 | 103 | #[test] 104 | fn test_arities_m() { 105 | assert_eq!(ArityM1::parse("1a").unwrap(), ArityM1 { f: vec![FieldA] }); 106 | assert_eq!(ArityM1::parse("1").unwrap(), ArityM1 { f: vec![] }); 107 | assert_eq!( 108 | ArityM1::parse("1aaa").unwrap(), 109 | ArityM1 { 110 | f: vec![FieldA, FieldA, FieldA] 111 | } 112 | ); 113 | assert_eq!(ArityM1::parse("2a").unwrap(), ArityM1 { f: vec![FieldA] }); 114 | assert!(ArityM1::parse("2").is_err()); 115 | assert!(ArityM1::parse("3a").is_err()); 116 | 117 | assert_eq!(ArityMO::parse("1a").unwrap(), ArityMO { f: vec![FieldA] }); 118 | assert_eq!(ArityMO::parse("1").unwrap(), ArityMO { f: vec![] }); 119 | assert_eq!( 120 | ArityMO::parse("1aaa").unwrap(), 121 | ArityMO { 122 | f: vec![FieldA, FieldA, FieldA] 123 | } 124 | ); 125 | assert_eq!(ArityMO::parse("2a").unwrap(), ArityMO { f: vec![FieldA] }); 126 | // Not an error, because no $ at end of grammar 127 | assert_eq!(ArityMO::parse("2aaa").unwrap(), ArityMO { f: vec![FieldA] }); 128 | assert_eq!(ArityMO::parse("2").unwrap(), ArityMO { f: vec![] }); 129 | assert!(ArityMO::parse("3a").is_err()); 130 | 131 | assert_eq!(ArityMM::parse("1a").unwrap(), ArityMM { f: vec![FieldA] }); 132 | assert_eq!( 133 | ArityMM::parse("1aaa").unwrap(), 134 | ArityMM { 135 | f: vec![FieldA, FieldA, FieldA] 136 | } 137 | ); 138 | assert_eq!(ArityMM::parse("1").unwrap(), ArityMM { f: vec![] }); 139 | assert_eq!(ArityMM::parse("2a").unwrap(), ArityMM { f: vec![FieldA] }); 140 | assert_eq!( 141 | ArityMM::parse("2aaa").unwrap(), 142 | ArityMM { 143 | f: vec![FieldA, FieldA, FieldA] 144 | } 145 | ); 146 | assert_eq!(ArityMM::parse("2").unwrap(), ArityMM { f: vec![] }); 147 | assert!(ArityMM::parse("3a").is_err()); 148 | 149 | assert_eq!(ArityMX::parse("1a").unwrap(), ArityMX { f: vec![FieldA] }); 150 | assert_eq!(ArityMX::parse("1").unwrap(), ArityMX { f: vec![] }); 151 | assert_eq!( 152 | ArityMX::parse("1aaa").unwrap(), 153 | ArityMX { 154 | f: vec![FieldA, FieldA, FieldA] 155 | } 156 | ); 157 | // Not an error, because no $ at end of grammar 158 | assert_eq!(ArityMX::parse("2a").unwrap(), ArityMX { f: vec![] }); 159 | assert_eq!(ArityMX::parse("2aaa").unwrap(), ArityMX { f: vec![] }); 160 | assert_eq!(ArityMX::parse("2").unwrap(), ArityMX { f: vec![] }); 161 | assert!(ArityMX::parse("3a").is_err()); 162 | } 163 | 164 | #[test] 165 | fn test_arities_x() { 166 | assert_eq!(ArityX1::parse("1a").unwrap(), ArityX1 { f: None }); 167 | assert_eq!(ArityX1::parse("1").unwrap(), ArityX1 { f: None }); 168 | assert_eq!(ArityX1::parse("2a").unwrap(), ArityX1 { f: Some(FieldA) }); 169 | assert!(ArityX1::parse("2").is_err()); 170 | assert!(ArityX1::parse("3a").is_err()); 171 | 172 | assert_eq!(ArityXO::parse("1a").unwrap(), ArityXO { f: None }); 173 | assert_eq!(ArityXO::parse("1").unwrap(), ArityXO { f: None }); 174 | assert_eq!(ArityXO::parse("2a").unwrap(), ArityXO { f: Some(FieldA) }); 175 | assert_eq!(ArityXO::parse("2").unwrap(), ArityXO { f: None }); 176 | assert!(ArityXO::parse("3a").is_err()); 177 | 178 | assert_eq!(ArityXM::parse("1a").unwrap(), ArityXM { f: vec![] }); 179 | // Not an error, because no $ at end of grammar 180 | assert_eq!(ArityXM::parse("1aaa").unwrap(), ArityXM { f: vec![] }); 181 | assert_eq!(ArityXM::parse("1").unwrap(), ArityXM { f: vec![] }); 182 | assert_eq!(ArityXM::parse("2a").unwrap(), ArityXM { f: vec![FieldA] }); 183 | assert_eq!( 184 | ArityXM::parse("2aaa").unwrap(), 185 | ArityXM { 186 | f: vec![FieldA, FieldA, FieldA] 187 | } 188 | ); 189 | assert_eq!(ArityXM::parse("2").unwrap(), ArityXM { f: vec![] }); 190 | assert!(ArityXM::parse("3a").is_err()); 191 | 192 | assert_eq!(ArityXX::parse("1a").unwrap(), ArityXX {}); 193 | assert_eq!(ArityXX::parse("1").unwrap(), ArityXX {}); 194 | assert_eq!(ArityXX::parse("2a").unwrap(), ArityXX {}); 195 | assert_eq!(ArityXX::parse("2").unwrap(), ArityXX {}); 196 | assert!(ArityXX::parse("3a").is_err()); 197 | } 198 | 199 | #[test] 200 | fn test_defaults() { 201 | assert_eq!(DefaultO::parse("1").unwrap(), DefaultO { f: None }); 202 | assert_eq!(DefaultO::parse("1a").unwrap(), DefaultO { f: Some(FieldA) }); 203 | assert_eq!(DefaultO::parse("2").unwrap(), DefaultO { f: None }); 204 | 205 | assert_eq!(DefaultM::parse("1").unwrap(), DefaultM { f: vec![] }); 206 | assert_eq!( 207 | DefaultM::parse("1aaa").unwrap(), 208 | DefaultM { 209 | f: vec![FieldA, FieldA, FieldA] 210 | } 211 | ); 212 | assert_eq!(DefaultM::parse("2").unwrap(), DefaultM { f: vec![] }); 213 | } 214 | 215 | #[test] 216 | fn test_multiple_fields() { 217 | // MultipleFields = f1:FieldA '1' f2:FieldB | f2:FieldA '2' f3:FieldB; 218 | assert_eq!( 219 | MultipleFields::parse("a1b").unwrap(), 220 | MultipleFields { 221 | f1: Some(FieldA), 222 | f2: MultipleFields_f2::FieldB(FieldB), 223 | f3: None 224 | } 225 | ); 226 | assert_eq!( 227 | MultipleFields::parse("a2b").unwrap(), 228 | MultipleFields { 229 | f1: None, 230 | f2: MultipleFields_f2::FieldA(FieldA), 231 | f3: Some(FieldB) 232 | } 233 | ); 234 | } 235 | -------------------------------------------------------------------------------- /test/src/closure/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Simple = {f:FieldA}; 7 | @export 8 | SimplePlus = {f:FieldA}+; 9 | 10 | @export 11 | Arity1 = {'.' f:FieldA}; 12 | @export 13 | ArityO = {'.' [f:FieldA]}; 14 | @export 15 | ArityM = {'.' {f:FieldA}}; 16 | @export 17 | ArityX = {'.'}; 18 | 19 | @export 20 | Arity1P = {'.' f:FieldA}+; 21 | @export 22 | ArityOP = {'.' [f:FieldA]}+; 23 | @export 24 | ArityMP = {'.' {f:FieldA}}+; 25 | @export 26 | ArityXP = {'.'}+; 27 | 28 | @export 29 | MultiField = {f1:FieldA f2:FieldB [f3:FieldC] {f4:FieldD}}; 30 | 31 | FieldA = 'a'; 32 | FieldB = 'b'; 33 | FieldC = 'c'; 34 | FieldD = 'd'; 35 | 36 | @export 37 | MultiPart = (f1:FieldA | f2:FieldB ) f3:FieldC; 38 | -------------------------------------------------------------------------------- /test/src/closure/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_simple() { 11 | assert_eq!(Simple::parse("").unwrap(), Simple { f: vec![] }); 12 | assert_eq!(Simple::parse("a").unwrap(), Simple { f: vec![FieldA] }); 13 | assert_eq!( 14 | Simple::parse("aaa").unwrap(), 15 | Simple { 16 | f: vec![FieldA, FieldA, FieldA] 17 | } 18 | ); 19 | 20 | assert!(SimplePlus::parse("").is_err()); 21 | assert_eq!( 22 | SimplePlus::parse("a").unwrap(), 23 | SimplePlus { f: vec![FieldA] } 24 | ); 25 | assert_eq!( 26 | SimplePlus::parse("aaa").unwrap(), 27 | SimplePlus { 28 | f: vec![FieldA, FieldA, FieldA] 29 | } 30 | ); 31 | } 32 | 33 | #[test] 34 | fn test_arities() { 35 | assert_eq!(Arity1::parse("").unwrap(), Arity1 { f: vec![] }); 36 | assert_eq!(Arity1::parse(".").unwrap(), Arity1 { f: vec![] }); 37 | assert_eq!(Arity1::parse(".a").unwrap(), Arity1 { f: vec![FieldA] }); 38 | assert_eq!(Arity1::parse(".aa").unwrap(), Arity1 { f: vec![FieldA] }); 39 | assert_eq!(Arity1::parse(".a..a").unwrap(), Arity1 { f: vec![FieldA] }); 40 | assert_eq!( 41 | Arity1::parse(".a.a.a").unwrap(), 42 | Arity1 { 43 | f: vec![FieldA, FieldA, FieldA] 44 | } 45 | ); 46 | assert_eq!( 47 | Arity1::parse(".a.aaa.a").unwrap(), 48 | Arity1 { 49 | f: vec![FieldA, FieldA] 50 | } 51 | ); 52 | 53 | assert_eq!(ArityO::parse("").unwrap(), ArityO { f: vec![] }); 54 | assert_eq!(ArityO::parse(".").unwrap(), ArityO { f: vec![] }); 55 | assert_eq!(ArityO::parse(".a").unwrap(), ArityO { f: vec![FieldA] }); 56 | assert_eq!(ArityO::parse(".aa").unwrap(), ArityO { f: vec![FieldA] }); 57 | assert_eq!( 58 | ArityO::parse(".a..a").unwrap(), 59 | ArityO { 60 | f: vec![FieldA, FieldA] 61 | } 62 | ); 63 | assert_eq!( 64 | ArityO::parse(".a.a.a").unwrap(), 65 | ArityO { 66 | f: vec![FieldA, FieldA, FieldA] 67 | } 68 | ); 69 | assert_eq!( 70 | ArityO::parse(".a.aaa.a").unwrap(), 71 | ArityO { 72 | f: vec![FieldA, FieldA] 73 | } 74 | ); 75 | assert_eq!(ArityM::parse("").unwrap(), ArityM { f: vec![] }); 76 | assert_eq!(ArityM::parse(".").unwrap(), ArityM { f: vec![] }); 77 | assert_eq!(ArityM::parse(".a").unwrap(), ArityM { f: vec![FieldA] }); 78 | assert_eq!( 79 | ArityM::parse(".aa").unwrap(), 80 | ArityM { 81 | f: vec![FieldA, FieldA] 82 | } 83 | ); 84 | assert_eq!( 85 | ArityM::parse(".a..a").unwrap(), 86 | ArityM { 87 | f: vec![FieldA, FieldA] 88 | } 89 | ); 90 | assert_eq!( 91 | ArityM::parse(".a.a.a").unwrap(), 92 | ArityM { 93 | f: vec![FieldA, FieldA, FieldA] 94 | } 95 | ); 96 | assert_eq!( 97 | ArityM::parse(".a.aaa.a").unwrap(), 98 | ArityM { 99 | f: vec![FieldA, FieldA, FieldA, FieldA, FieldA] 100 | } 101 | ); 102 | 103 | assert_eq!(ArityX::parse("").unwrap(), ArityX {}); 104 | assert_eq!(ArityX::parse(".").unwrap(), ArityX {}); 105 | assert_eq!(ArityX::parse("...").unwrap(), ArityX {}); 106 | } 107 | 108 | #[test] 109 | fn test_arities_plus() { 110 | assert!(Arity1P::parse("").is_err()); 111 | assert!(Arity1P::parse(".").is_err()); 112 | assert_eq!(Arity1P::parse(".a").unwrap(), Arity1P { f: vec![FieldA] }); 113 | assert_eq!(Arity1P::parse(".aa").unwrap(), Arity1P { f: vec![FieldA] }); 114 | assert_eq!( 115 | Arity1P::parse(".a..a").unwrap(), 116 | Arity1P { f: vec![FieldA] } 117 | ); 118 | assert_eq!( 119 | Arity1P::parse(".a.a.a").unwrap(), 120 | Arity1P { 121 | f: vec![FieldA, FieldA, FieldA] 122 | } 123 | ); 124 | assert_eq!( 125 | Arity1P::parse(".a.aaa.a").unwrap(), 126 | Arity1P { 127 | f: vec![FieldA, FieldA] 128 | } 129 | ); 130 | 131 | assert!(ArityOP::parse("").is_err()); 132 | assert_eq!(ArityOP::parse(".").unwrap(), ArityOP { f: vec![] }); 133 | assert_eq!(ArityOP::parse(".a").unwrap(), ArityOP { f: vec![FieldA] }); 134 | assert_eq!(ArityOP::parse(".aa").unwrap(), ArityOP { f: vec![FieldA] }); 135 | assert_eq!( 136 | ArityOP::parse(".a..a").unwrap(), 137 | ArityOP { 138 | f: vec![FieldA, FieldA] 139 | } 140 | ); 141 | assert_eq!( 142 | ArityOP::parse(".a.a.a").unwrap(), 143 | ArityOP { 144 | f: vec![FieldA, FieldA, FieldA] 145 | } 146 | ); 147 | assert_eq!( 148 | ArityOP::parse(".a.aaa.a").unwrap(), 149 | ArityOP { 150 | f: vec![FieldA, FieldA] 151 | } 152 | ); 153 | 154 | assert!(ArityMP::parse("").is_err()); 155 | assert_eq!(ArityMP::parse(".").unwrap(), ArityMP { f: vec![] }); 156 | assert_eq!(ArityMP::parse(".a").unwrap(), ArityMP { f: vec![FieldA] }); 157 | assert_eq!( 158 | ArityMP::parse(".aa").unwrap(), 159 | ArityMP { 160 | f: vec![FieldA, FieldA] 161 | } 162 | ); 163 | assert_eq!( 164 | ArityMP::parse(".a..a").unwrap(), 165 | ArityMP { 166 | f: vec![FieldA, FieldA] 167 | } 168 | ); 169 | assert_eq!( 170 | ArityMP::parse(".a.a.a").unwrap(), 171 | ArityMP { 172 | f: vec![FieldA, FieldA, FieldA] 173 | } 174 | ); 175 | assert_eq!( 176 | ArityMP::parse(".a.aaa.a").unwrap(), 177 | ArityMP { 178 | f: vec![FieldA, FieldA, FieldA, FieldA, FieldA] 179 | } 180 | ); 181 | 182 | assert!(ArityXP::parse("").is_err()); 183 | assert_eq!(ArityXP::parse(".").unwrap(), ArityXP {}); 184 | assert_eq!(ArityXP::parse("...").unwrap(), ArityXP {}); 185 | } 186 | 187 | #[test] 188 | fn test_multi_field() { 189 | //MultiField = {f1:FieldA f2:FieldB [f3:FieldC] {f4:FieldD}}; 190 | assert_eq!( 191 | MultiField::parse("").unwrap(), 192 | MultiField { 193 | f1: vec![], 194 | f2: vec![], 195 | f3: vec![], 196 | f4: vec![] 197 | } 198 | ); 199 | assert_eq!( 200 | MultiField::parse("ab").unwrap(), 201 | MultiField { 202 | f1: vec![FieldA], 203 | f2: vec![FieldB], 204 | f3: vec![], 205 | f4: vec![] 206 | } 207 | ); 208 | assert_eq!( 209 | MultiField::parse("abab").unwrap(), 210 | MultiField { 211 | f1: vec![FieldA, FieldA], 212 | f2: vec![FieldB, FieldB], 213 | f3: vec![], 214 | f4: vec![] 215 | } 216 | ); 217 | assert_eq!( 218 | MultiField::parse("abcddd").unwrap(), 219 | MultiField { 220 | f1: vec![FieldA], 221 | f2: vec![FieldB], 222 | f3: vec![FieldC], 223 | f4: vec![FieldD, FieldD, FieldD] 224 | } 225 | ); 226 | assert_eq!( 227 | MultiField::parse("abcddabddabccabc").unwrap(), 228 | MultiField { 229 | f1: vec![FieldA, FieldA, FieldA], 230 | f2: vec![FieldB, FieldB, FieldB], 231 | f3: vec![FieldC, FieldC], 232 | f4: vec![FieldD, FieldD, FieldD, FieldD] 233 | } 234 | ); 235 | } 236 | 237 | #[test] 238 | fn test_multi_part() { 239 | assert_eq!( 240 | MultiPart::parse("ac").unwrap(), 241 | MultiPart { 242 | f1: Some(FieldA), 243 | f2: None, 244 | f3: FieldC 245 | } 246 | ); 247 | } 248 | -------------------------------------------------------------------------------- /test/src/custom_derives_empty/grammar.not_ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export Simple = a:Field; 6 | Field = 'a'; 7 | -------------------------------------------------------------------------------- /test/src/custom_derives_empty/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use std::fmt::Debug; 7 | 8 | use grammar::*; 9 | use peginator::PegParser; 10 | 11 | impl Clone for Simple { 12 | fn clone(&self) -> Self { 13 | Self { a: Field } 14 | } 15 | } 16 | 17 | impl Debug for Field { 18 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 19 | f.write_str("Haha, no") 20 | } 21 | } 22 | 23 | impl PartialEq for Field { 24 | fn eq(&self, _other: &Self) -> bool { 25 | true 26 | } 27 | } 28 | 29 | #[test] 30 | fn test() { 31 | let s = Simple::parse("a").unwrap(); 32 | assert_eq!(s.a, Field); 33 | 34 | let _check_custom_struct = ImJustHereToConfuse; 35 | } 36 | -------------------------------------------------------------------------------- /test/src/custom_whitespace/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Simple = 'a' 'b' $; 7 | 8 | @no_skip_ws 9 | Whitespace = {Comment | 'x' | ' '}; 10 | 11 | @no_skip_ws 12 | Comment = '#' {!'\n' char} '\n'; 13 | -------------------------------------------------------------------------------- /test/src/custom_whitespace/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | assert!(Simple::parse("ab").is_ok()); 12 | assert!(Simple::parse("a\nb").is_err()); 13 | assert!(Simple::parse(" a b ").is_ok()); 14 | assert!(Simple::parse("x a xx b x").is_ok()); 15 | assert!(Simple::parse("# ayy, his is some good stuff\na b # haha\n").is_ok()); 16 | assert!(Simple::parse("# ayy, this is some good stuff\na b # haha").is_err()); 17 | } 18 | -------------------------------------------------------------------------------- /test/src/enums/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Simple = (a:EnumVal1 | a:EnumVal2) | (a:EnumVal3 | a:EnumVal4); 7 | 8 | @export 9 | WithOpt = (a:EnumVal1 | a:EnumVal2) | (a:EnumVal3 | a:EnumVal4|); 10 | 11 | @export 12 | WithMult = {a:EnumVal1 | a:EnumVal2}+ | (a:EnumVal3 | a:EnumVal4); 13 | 14 | @export 15 | WithMultOpt = {a:EnumVal1 | a:EnumVal2}+ | (a:EnumVal3 | a:EnumVal4|); 16 | 17 | @export 18 | OneOpt = a:EnumVal1 [a:EnumVal2]; 19 | 20 | EnumVal1 = '1'; 21 | EnumVal2 = '2'; 22 | EnumVal3 = '3'; 23 | EnumVal4 = '4'; 24 | -------------------------------------------------------------------------------- /test/src/enums/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_simple() { 11 | let s = Simple::parse("1").unwrap(); 12 | assert_eq!(s.a, Simple_a::EnumVal1(EnumVal1)); 13 | let s = Simple::parse("2").unwrap(); 14 | assert_eq!(s.a, Simple_a::EnumVal2(EnumVal2)); 15 | let s = Simple::parse("3").unwrap(); 16 | assert_eq!(s.a, Simple_a::EnumVal3(EnumVal3)); 17 | let s = Simple::parse("4").unwrap(); 18 | assert_eq!(s.a, Simple_a::EnumVal4(EnumVal4)); 19 | 20 | assert!(Simple::parse("5").is_err()) 21 | } 22 | 23 | #[test] 24 | fn test_with_opt() { 25 | let s = WithOpt::parse("1").unwrap(); 26 | assert_eq!(s.a, Some(WithOpt_a::EnumVal1(EnumVal1))); 27 | let s = WithOpt::parse("2").unwrap(); 28 | assert_eq!(s.a, Some(WithOpt_a::EnumVal2(EnumVal2))); 29 | let s = WithOpt::parse("3").unwrap(); 30 | assert_eq!(s.a, Some(WithOpt_a::EnumVal3(EnumVal3))); 31 | let s = WithOpt::parse("4").unwrap(); 32 | assert_eq!(s.a, Some(WithOpt_a::EnumVal4(EnumVal4))); 33 | 34 | let s = WithOpt::parse("5").unwrap(); 35 | assert_eq!(s.a, None); 36 | } 37 | 38 | #[test] 39 | fn test_with_mult() { 40 | let s = WithMult::parse("1122").unwrap(); 41 | assert_eq!( 42 | s.a, 43 | vec![ 44 | WithMult_a::EnumVal1(EnumVal1), 45 | WithMult_a::EnumVal1(EnumVal1), 46 | WithMult_a::EnumVal2(EnumVal2), 47 | WithMult_a::EnumVal2(EnumVal2), 48 | ] 49 | ); 50 | let s = WithMult::parse("21").unwrap(); 51 | assert_eq!( 52 | s.a, 53 | vec![ 54 | WithMult_a::EnumVal2(EnumVal2), 55 | WithMult_a::EnumVal1(EnumVal1), 56 | ] 57 | ); 58 | let s = WithMult::parse("3").unwrap(); 59 | assert_eq!(s.a, vec![WithMult_a::EnumVal3(EnumVal3)]); 60 | 61 | let s = WithMult::parse("4").unwrap(); 62 | assert_eq!(s.a, vec![WithMult_a::EnumVal4(EnumVal4)]); 63 | 64 | assert!(WithMult::parse("5").is_err()) 65 | } 66 | 67 | #[test] 68 | fn test_with_mult_opt() { 69 | let s = WithMultOpt::parse("1122").unwrap(); 70 | assert_eq!( 71 | s.a, 72 | vec![ 73 | WithMultOpt_a::EnumVal1(EnumVal1), 74 | WithMultOpt_a::EnumVal1(EnumVal1), 75 | WithMultOpt_a::EnumVal2(EnumVal2), 76 | WithMultOpt_a::EnumVal2(EnumVal2), 77 | ] 78 | ); 79 | let s = WithMultOpt::parse("21").unwrap(); 80 | assert_eq!( 81 | s.a, 82 | vec![ 83 | WithMultOpt_a::EnumVal2(EnumVal2), 84 | WithMultOpt_a::EnumVal1(EnumVal1), 85 | ] 86 | ); 87 | let s = WithMultOpt::parse("3").unwrap(); 88 | assert_eq!(s.a, vec![WithMultOpt_a::EnumVal3(EnumVal3)]); 89 | 90 | let s = WithMultOpt::parse("4").unwrap(); 91 | assert_eq!(s.a, vec![WithMultOpt_a::EnumVal4(EnumVal4)]); 92 | 93 | let s = WithMultOpt::parse("5").unwrap(); 94 | assert_eq!(s.a, vec![]); 95 | } 96 | 97 | #[test] 98 | fn test_one_opt() { 99 | let s = OneOpt::parse("1").unwrap(); 100 | assert_eq!(s.a, vec![OneOpt_a::EnumVal1(EnumVal1)]); 101 | let s = OneOpt::parse("12").unwrap(); 102 | assert_eq!( 103 | s.a, 104 | vec![OneOpt_a::EnumVal1(EnumVal1), OneOpt_a::EnumVal2(EnumVal2),] 105 | ); 106 | 107 | assert!(OneOpt::parse("2").is_err()) 108 | } 109 | -------------------------------------------------------------------------------- /test/src/eoi/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Simple = Hmm $; 7 | 8 | @export 9 | Choice = Hmm $ | "thonk"; 10 | 11 | @export 12 | Optional = "well" [f:Hmm $]; 13 | 14 | @export 15 | @no_skip_ws 16 | NoSkipWs = Hmm $; 17 | 18 | Hmm = "hmm"; 19 | -------------------------------------------------------------------------------- /test/src/eoi/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | assert!(Simple::parse("").is_err()); 12 | assert!(Simple::parse("hmm").is_ok()); 13 | assert!(Simple::parse(" hmm").is_ok()); 14 | assert!(Simple::parse(" hmm ").is_ok()); 15 | assert!(Simple::parse("hmm ").is_ok()); 16 | assert!(Simple::parse("hmmhmm").is_err()); 17 | 18 | assert!(Choice::parse("").is_err()); 19 | assert!(Choice::parse("hmm").is_ok()); 20 | assert!(Choice::parse(" hmm ").is_ok()); 21 | assert!(Choice::parse("hmmhmm").is_err()); 22 | assert!(Choice::parse("thonk").is_ok()); 23 | assert!(Choice::parse("thonkhmm").is_ok()); 24 | 25 | assert_eq!(Optional::parse("well").unwrap(), Optional { f: None }); 26 | assert_eq!( 27 | Optional::parse("well hmm").unwrap(), 28 | Optional { f: Some(Hmm) } 29 | ); 30 | assert_eq!( 31 | Optional::parse("well hmm hmm").unwrap(), 32 | Optional { f: None } 33 | ); 34 | 35 | assert!(NoSkipWs::parse("").is_err()); 36 | assert!(NoSkipWs::parse("hmm").is_ok()); 37 | assert!(NoSkipWs::parse(" hmm ").is_err()); 38 | assert!(NoSkipWs::parse("hmm ").is_err()); 39 | assert!(NoSkipWs::parse("hmmhmm").is_err()); 40 | } 41 | -------------------------------------------------------------------------------- /test/src/extern_directive/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Test = a:RustRawString b:OtherType; 7 | 8 | @extern(crate::extern_directive::raw_string_parser) 9 | RustRawString; 10 | 11 | @extern(crate::extern_directive::struct_maker -> crate::extern_directive::TheStruct) 12 | OtherType; 13 | -------------------------------------------------------------------------------- /test/src/extern_directive/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #[allow(clippy::useless_conversion)] 6 | mod grammar; 7 | use grammar::*; 8 | use peginator::PegParser; 9 | 10 | #[test] 11 | fn test() { 12 | assert_eq!( 13 | Test::parse("r###\"éáűú\"## \"###other").unwrap(), 14 | Test { 15 | a: "éáűú\"## ".into(), 16 | b: TheStruct { s: "other".into() } 17 | } 18 | ); 19 | } 20 | 21 | pub fn raw_string_parser(s: &str) -> Result<(&str, usize), &'static str> { 22 | let sb = s.as_bytes(); 23 | if *sb.first().ok_or("expected 'r', found end of string")? != b'r' { 24 | return Err("expected 'r'"); 25 | } 26 | let num_hashes = sb[1..] 27 | .iter() 28 | .position(|c| *c != b'#') 29 | .ok_or("expected '#' or '\"', found end of string ")?; 30 | let hashes = &sb[1..1 + num_hashes]; 31 | if *sb 32 | .get(1 + num_hashes) 33 | .ok_or("expected '\"', found end of string")? 34 | != b'"' 35 | { 36 | return Err("expected '\"'"); 37 | } 38 | for i in num_hashes + 2..sb.len() { 39 | if sb[i] == b'"' && sb[i + 1..].starts_with(hashes) { 40 | return Ok((&s[num_hashes + 2..i], i + 1 + num_hashes)); 41 | } 42 | } 43 | Err("could not find end of raw string marker") 44 | } 45 | 46 | #[derive(Debug, Clone, PartialEq, Eq)] 47 | pub struct TheStruct { 48 | s: String, 49 | } 50 | 51 | pub fn struct_maker(s: &str) -> Result<(TheStruct, usize), &'static str> { 52 | Ok((TheStruct { s: s.into() }, s.len())) 53 | } 54 | -------------------------------------------------------------------------------- /test/src/field/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | EnumTest = 'a' a:Point | 'b' a:NoFieldName; 7 | 8 | @export 9 | EnumTestComplex = 'a' a:Point | ('b' a:NoFieldName | 'c' a:OverrideTest); 10 | 11 | @export 12 | OverrideTest = s:SimpleOverride e:EnumOverride o:OptionOverride v:VecOverride; 13 | 14 | SimpleOverride = @:Point; 15 | 16 | EnumOverride = @:Point | @:NoFieldName; 17 | 18 | OptionOverride = [@:Point]; 19 | 20 | VecOverride = {@:Point}; 21 | 22 | @export 23 | NoFieldName = "dontcare:" Coordinate f:Coordinate; 24 | 25 | @export 26 | Point = '(' x:Coordinate ';' y:Coordinate ')'; 27 | 28 | @string 29 | @no_skip_ws 30 | Coordinate = {'0'..'9'}+; 31 | -------------------------------------------------------------------------------- /test/src/field/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #[allow(clippy::module_inception)] 6 | mod grammar; 7 | use grammar::*; 8 | use peginator::PegParser; 9 | 10 | #[test] 11 | fn test_enums() { 12 | match EnumTest::parse("a(123;456)").unwrap().a { 13 | EnumTest_a::Point(parsed) => assert_eq!( 14 | parsed, 15 | Point { 16 | x: "123".to_string(), 17 | y: "456".to_string() 18 | } 19 | ), 20 | _ => panic!("Invalid parse"), 21 | } 22 | 23 | match EnumTestComplex::parse("b dontcare: 123 456").unwrap().a { 24 | EnumTestComplex_a::NoFieldName(parsed) => assert_eq!( 25 | parsed, 26 | NoFieldName { 27 | f: "456".to_string() 28 | } 29 | ), 30 | _ => panic!("Invalid parse"), 31 | } 32 | } 33 | 34 | #[test] 35 | fn test_overrides() { 36 | let result = OverrideTest::parse("(1;2) (3;4)").unwrap(); 37 | assert_eq!( 38 | result.s, 39 | Point { 40 | x: "1".to_string(), 41 | y: "2".to_string() 42 | } 43 | ); 44 | match result.e { 45 | EnumOverride::Point(p) => assert_eq!( 46 | p, 47 | Point { 48 | x: "3".to_string(), 49 | y: "4".to_string() 50 | } 51 | ), 52 | _ => panic!("Invalid parse"), 53 | } 54 | assert_eq!(result.o, None); 55 | assert_eq!(result.v, vec![]); 56 | let result = OverrideTest::parse("(1;2) (3;4) (5;6) (7;8) (9;10)").unwrap(); 57 | assert_eq!( 58 | result.o, 59 | Some(Point { 60 | x: "5".to_string(), 61 | y: "6".to_string() 62 | }) 63 | ); 64 | assert_eq!( 65 | result.v, 66 | vec![ 67 | Point { 68 | x: "7".to_string(), 69 | y: "8".to_string() 70 | }, 71 | Point { 72 | x: "9".to_string(), 73 | y: "10".to_string() 74 | } 75 | ] 76 | ); 77 | } 78 | -------------------------------------------------------------------------------- /test/src/fndef_example/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | FunctionDef = 'fn' name:Ident '(' param_list:ParamList ')' [ '->' return_value:Type ]; 7 | 8 | ParamList = self_param:SelfParam {',' params:Param} | params:Param {',' params:Param} | ; 9 | 10 | Param = name:Ident ':' typ: Type; 11 | 12 | SelfParam = [ref_type:ReferenceMarker] 'self'; 13 | 14 | Type = [ref_type:ReferenceMarker] typename:Ident; 15 | 16 | ReferenceMarker = @:MutableReference | @:ImmutableReference; 17 | 18 | ImmutableReference = '&'; 19 | MutableReference = '&' 'mut'; 20 | 21 | @string 22 | @no_skip_ws 23 | Ident = {'a'..'z' | 'A'..'Z' | '_' | '0'..'9'}; 24 | -------------------------------------------------------------------------------- /test/src/fndef_example/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::{PegParser, PrettyParseError}; 8 | 9 | const PARSE_ME: &str = "fn example(&self, input:&str, rectified:&mut Rect) -> ExampleResult;"; 10 | 11 | #[test] 12 | fn test() { 13 | match FunctionDef::parse_with_trace(PARSE_ME) { 14 | Ok(x) => println!("{:#?}", x), 15 | Err(e) => { 16 | println!("{}", PrettyParseError::from_parse_error(&e, PARSE_ME, None)); 17 | panic!(); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /test/src/include_rule/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Test = a:Field >IncludeMe; 7 | 8 | Field = 'a'; 9 | 10 | IncludeMe = '(' x:Number ',' y:Number ')'; 11 | 12 | Number = {'0'..'9'}; 13 | 14 | @export 15 | MultiTest = a:Field >IncludeMe >IncludeMe {>IncludeMe2}; 16 | 17 | IncludeMe2 = '[' y:Number ',' z:Number ']'; 18 | 19 | 20 | @export 21 | OverrideTest = o:Overridden; 22 | 23 | Overridden = @:Field | >OverrideInclude; 24 | 25 | OverrideInclude = @:Number; 26 | -------------------------------------------------------------------------------- /test/src/include_rule/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | assert_eq!( 12 | Test::parse("a(123,456)").unwrap(), 13 | Test { 14 | a: Field, 15 | x: Number, 16 | y: Number 17 | } 18 | ); 19 | assert_eq!( 20 | MultiTest::parse("a(123,456)(123,456)[123,456][123,456]").unwrap(), 21 | MultiTest { 22 | a: Field, 23 | x: vec![Number, Number], 24 | y: vec![Number, Number, Number, Number], 25 | z: vec![Number, Number] 26 | } 27 | ); 28 | assert_eq!( 29 | OverrideTest::parse("123").unwrap(), 30 | OverrideTest { 31 | o: Overridden::Number(Number) 32 | } 33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /test/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #![cfg(test)] 6 | pub(crate) mod test_utils; 7 | 8 | mod additional_traits; 9 | mod boxing; 10 | mod calculator_example; 11 | mod char_rule; 12 | pub mod check; 13 | mod choice; 14 | mod closure; 15 | mod custom_derives_empty; 16 | mod custom_whitespace; 17 | mod enums; 18 | mod eoi; 19 | pub mod extern_directive; 20 | mod field; 21 | mod fndef_example; 22 | mod include_rule; 23 | mod lookahead; 24 | mod memoization; 25 | mod operator_example; 26 | mod optional; 27 | mod palindrome; 28 | mod position; 29 | mod precedence; 30 | mod rust_keywords; 31 | mod sequence; 32 | mod simple; 33 | mod skip_ws; 34 | mod string; 35 | mod string_insensitive; 36 | pub mod user_defined_state; 37 | -------------------------------------------------------------------------------- /test/src/lookahead/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | AnyButB = !'b' c:char; 7 | 8 | @export 9 | AnyButASDF = !('a' | 's' | 'd' | 'f') c:char; 10 | 11 | @export 12 | @no_skip_ws 13 | SplitLast = { !(',' {!',' char} $) list_rest:char } ',' {list_last:char} $; 14 | 15 | @export 16 | List = {parts:ListPart (',' | &(('.' | '!') $))}+ ('.' | '!')$; 17 | ListPart = a:char '*' b:char; 18 | 19 | @export 20 | Intersection = &'a'..'f' 'e'..'h'; 21 | -------------------------------------------------------------------------------- /test/src/lookahead/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_skips() { 11 | assert_eq!(AnyButB::parse("a").unwrap().c, 'a'); 12 | assert!(AnyButB::parse("b").is_err()); 13 | 14 | assert_eq!(AnyButASDF::parse("x").unwrap().c, 'x'); 15 | assert!(AnyButASDF::parse("s").is_err()); 16 | } 17 | 18 | fn split_last(s: &str) -> (String, String) { 19 | let result = SplitLast::parse_with_trace(s).unwrap(); 20 | ( 21 | result.list_rest.iter().collect(), 22 | result.list_last.iter().collect(), 23 | ) 24 | } 25 | 26 | #[test] 27 | fn test_split_last() { 28 | assert_eq!( 29 | split_last("abc, def, ghi"), 30 | ("abc, def".to_string(), " ghi".to_string()) 31 | ); 32 | assert_eq!(split_last(",,,,"), (",,,".to_string(), "".to_string())); 33 | assert!(SplitLast::parse_with_trace("asd").is_err()); 34 | } 35 | 36 | #[test] 37 | fn test_list() { 38 | let result = List::parse("1*2, 2*3!").unwrap(); 39 | assert_eq!( 40 | result, 41 | List { 42 | parts: vec![ListPart { a: '1', b: '2' }, ListPart { a: '2', b: '3' }] 43 | } 44 | ); 45 | assert!(List::parse("1*2, 2*3, 5*6.").is_ok()); 46 | assert!(List::parse("1*2! 2*3!").is_err()); 47 | assert!(List::parse("1*2, 2*3,").is_err()); 48 | assert!(List::parse("1*2,").is_err()); 49 | assert!(List::parse("1*2").is_err()); 50 | } 51 | 52 | #[test] 53 | fn test_intersection() { 54 | assert!(Intersection::parse("d").is_err()); 55 | assert!(Intersection::parse("e").is_ok()); 56 | assert!(Intersection::parse("f").is_ok()); 57 | assert!(Intersection::parse("g").is_err()); 58 | } 59 | -------------------------------------------------------------------------------- /test/src/memoization/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Root = parsed:Recursive '.' $; 7 | 8 | @memoize 9 | Recursive = 'a' inner:*Recursive 'b' | 'a' inner:*Recursive 'c'|; 10 | -------------------------------------------------------------------------------- /test/src/memoization/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use ntest::timeout; 8 | use peginator::PegParser; 9 | 10 | #[test] 11 | #[timeout(1000)] 12 | fn test_memoization() { 13 | let s = Root::parse(&format!("{:a>100}{:c>100}.", "", "")) 14 | .unwrap() 15 | .parsed; 16 | let expected: Option> = 17 | (0..101).fold(None, |r, _| Some(Box::new(Recursive { inner: r }))); 18 | assert_eq!(s, *expected.unwrap()); 19 | } 20 | -------------------------------------------------------------------------------- /test/src/operator_example/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Assignment = lvalue:Identifier '=' expr:Expression ';'$; 7 | 8 | Expression = @:Additive | @:Term; 9 | Additive = parts:Term {(parts:Plus | parts:Minus) parts:Term}+; 10 | Plus = '+'; 11 | Minus = '-'; 12 | 13 | Term = @:Multiplicative | @:Factor; 14 | Multiplicative = parts:Factor {(parts:Mul | parts:Div) parts:Factor}+; 15 | Mul = '*'; 16 | Div = '/'; 17 | 18 | Factor = @:Group | @:Number; 19 | Group = '(' body:*Expression ')'; 20 | 21 | @string 22 | @no_skip_ws 23 | Number = {'0'..'9'}+; 24 | 25 | @string 26 | @no_skip_ws 27 | Identifier = {'a'..'z' | '0'..'9' | '_'}+; 28 | -------------------------------------------------------------------------------- /test/src/operator_example/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::{PegParser, PrettyParseError}; 8 | 9 | const PARSE_ME: &str = "result = (1 - 2 + 3) * (13 - 37 * 4 + 20);"; 10 | 11 | #[test] 12 | fn test() { 13 | match Assignment::parse_with_trace(PARSE_ME) { 14 | Ok(x) => println!("{:#?}", x), 15 | Err(e) => { 16 | println!("{}", PrettyParseError::from_parse_error(&e, PARSE_ME, None)); 17 | panic!(); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /test/src/optional/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Tester = [simple:FieldA] [',' [optional:FieldA]] ['.' {multi:FieldA}]; 7 | 8 | FieldA = 'a'; 9 | -------------------------------------------------------------------------------- /test/src/optional/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | assert_eq!( 12 | Tester::parse("").unwrap(), 13 | Tester { 14 | simple: None, 15 | optional: None, 16 | multi: vec!() 17 | } 18 | ); 19 | assert_eq!( 20 | Tester::parse(",.").unwrap(), 21 | Tester { 22 | simple: None, 23 | optional: None, 24 | multi: vec!() 25 | } 26 | ); 27 | assert_eq!( 28 | Tester::parse("a,a.aaa").unwrap(), 29 | Tester { 30 | simple: Some(FieldA), 31 | optional: Some(FieldA), 32 | multi: vec![FieldA, FieldA, FieldA] 33 | } 34 | ); 35 | assert_eq!( 36 | Tester::parse("a,aa.aaa").unwrap(), 37 | Tester { 38 | simple: Some(FieldA), 39 | optional: Some(FieldA), 40 | multi: vec![], 41 | } 42 | ); 43 | } 44 | -------------------------------------------------------------------------------- /test/src/palindrome/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Palindrome = 7 | letter:A inner:*Palindrome 'a' | 8 | letter:B inner:*Palindrome 'b' | 9 | letter:A 'a' | 10 | letter:B 'b'; 11 | A = 'a'; 12 | B = 'b'; 13 | -------------------------------------------------------------------------------- /test/src/palindrome/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | fn to_palindrome_struct(s: &str) -> Palindrome { 10 | let result: Option> = s.chars().rev().fold(None, |p, c| { 11 | Some(Box::new(Palindrome { 12 | letter: match c { 13 | 'a' => Palindrome_letter::A(A), 14 | _ => Palindrome_letter::B(B), 15 | }, 16 | inner: p, 17 | })) 18 | }); 19 | *result.unwrap() 20 | } 21 | 22 | #[test] 23 | fn test_parsing() { 24 | let s = Palindrome::parse("aabbaaaabbaabbaaabab").unwrap(); 25 | assert_eq!(s, to_palindrome_struct("aabbaa")); 26 | let s = Palindrome::parse("aabbaaaabbaaaabbaaabab").unwrap(); 27 | assert_eq!(s, to_palindrome_struct("aabbaaaab")); 28 | } 29 | -------------------------------------------------------------------------------- /test/src/position/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | @position 7 | Root = "nice" field:Field field2:OverriddenField "rulerino"; 8 | 9 | @position 10 | Field = "f" ":" s:Ident; 11 | 12 | OverriddenField = "f" ":" @:Ident; 13 | 14 | @position 15 | @no_skip_ws 16 | Ident = {'a'..'z'}+; 17 | 18 | @export 19 | Specials = n:Number e:EnumField; 20 | 21 | @string 22 | @no_skip_ws 23 | @position 24 | Number = {'0'..'9'}+; 25 | 26 | @position 27 | EnumField = @:FieldA | @:FieldB; 28 | 29 | @position 30 | FieldA = {'a'}+; 31 | 32 | @position 33 | FieldB = {'b'}+; 34 | -------------------------------------------------------------------------------- /test/src/position/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::{PegParser, PegPosition}; 8 | 9 | #[test] 10 | fn test_positions() { 11 | let parsed_str = "nice f:abcd f:z rulerino and some junk"; 12 | let s = Root::parse(parsed_str).unwrap(); 13 | assert_eq!(parsed_str[s.field.s.position.clone()], *"abcd"); 14 | assert_eq!( 15 | s, 16 | Root { 17 | field: Field { 18 | s: Ident { position: 7..11 }, 19 | position: 5..11 20 | }, 21 | field2: Ident { position: 14..15 }, 22 | position: 0..24 23 | } 24 | ); 25 | assert_eq!(*s.position(), 0..24); 26 | assert_eq!(*s.field2.position(), 14..15); 27 | 28 | let s2 = Specials::parse("123aaaa").unwrap(); 29 | assert_eq!( 30 | s2, 31 | Specials { 32 | n: Number { 33 | string: "123".into(), 34 | position: 0..3 35 | }, 36 | e: EnumField::FieldA(FieldA { position: 3..7 }) 37 | } 38 | ); 39 | assert_eq!(*s2.e.position(), 3..7); 40 | } 41 | -------------------------------------------------------------------------------- /test/src/precedence/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Simple = Inner $; 7 | 8 | Inner = 'a' | 'b' 'c' | 'd' ('e' | 'f'); 9 | -------------------------------------------------------------------------------- /test/src/precedence/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | assert!(Simple::parse("").is_err()); 12 | assert!(Simple::parse("a").is_ok()); 13 | assert!(Simple::parse("b").is_err()); 14 | assert!(Simple::parse("bc").is_ok()); 15 | assert!(Simple::parse("de").is_ok()); 16 | assert!(Simple::parse("df").is_ok()); 17 | assert!(Simple::parse("ace").is_err()); 18 | } 19 | -------------------------------------------------------------------------------- /test/src/rust_keywords/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export Simple = type:type | fn:fn; 6 | type = 'type'; 7 | fn = 'fn'; 8 | -------------------------------------------------------------------------------- /test/src/rust_keywords/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #[allow(non_camel_case_types)] 6 | mod grammar; 7 | use grammar::*; 8 | use peginator::PegParser; 9 | 10 | #[test] 11 | fn test_macro() { 12 | let s = Simple::parse("type").unwrap(); 13 | assert_eq!(s.r#type, Some(r#type)); 14 | } 15 | -------------------------------------------------------------------------------- /test/src/sequence/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | MergeSimple = x:FieldA (a:FieldA b:FieldA ) ["empty"] y:FieldA; 7 | 8 | @export 9 | MultiSimple = 10 | x:FieldX x:FieldX 11 | y:FieldY ('.' y:FieldY ) y:FieldY 12 | z:FieldZ (',' z:FieldZ z:FieldZ) z:FieldZ 13 | (w:FieldW w:FieldW) w:FieldW 14 | ; 15 | 16 | @export 17 | MultiOpt = 18 | x:FieldX ['.' x:FieldX] x:FieldX 19 | [y:FieldY] ['.' y:FieldY ] y:FieldY 20 | z:FieldZ [',' z:FieldZ z:FieldZ] z:FieldZ 21 | [w:FieldW w:FieldW] w:FieldW 22 | ; 23 | 24 | FieldA = 'a'; 25 | 26 | FieldX = 'x'; 27 | FieldY = 'y'; 28 | FieldZ = 'z'; 29 | FieldW = 'w'; 30 | -------------------------------------------------------------------------------- /test/src/sequence/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_merge() { 11 | assert_eq!( 12 | MergeSimple::parse("a a a empty a").unwrap(), 13 | MergeSimple { 14 | x: FieldA, 15 | y: FieldA, 16 | a: FieldA, 17 | b: FieldA 18 | } 19 | ); 20 | } 21 | 22 | #[test] 23 | fn test_multi_simple() { 24 | assert_eq!( 25 | MultiSimple::parse("xxy.yyz,zzzwww").unwrap(), 26 | MultiSimple { 27 | x: vec![FieldX, FieldX], 28 | y: vec![FieldY, FieldY, FieldY], 29 | z: vec![FieldZ, FieldZ, FieldZ, FieldZ], 30 | w: vec![FieldW, FieldW, FieldW], 31 | } 32 | ) 33 | } 34 | 35 | #[test] 36 | fn test_multi_opt() { 37 | assert_eq!( 38 | MultiOpt::parse("xxy.yyz,zzzwww").unwrap(), 39 | MultiOpt { 40 | x: vec![FieldX, FieldX], 41 | y: vec![FieldY, FieldY, FieldY], 42 | z: vec![FieldZ, FieldZ, FieldZ, FieldZ], 43 | w: vec![FieldW, FieldW, FieldW], 44 | } 45 | ); 46 | assert_eq!( 47 | MultiOpt::parse("xxyyzzwww").unwrap(), 48 | MultiOpt { 49 | x: vec![FieldX, FieldX], 50 | y: vec![FieldY, FieldY], 51 | z: vec![FieldZ, FieldZ], 52 | w: vec![FieldW, FieldW, FieldW], 53 | } 54 | ) 55 | } 56 | -------------------------------------------------------------------------------- /test/src/simple/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export Simple = a:Field; 6 | Field = 'a'; 7 | -------------------------------------------------------------------------------- /test/src/simple/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test() { 11 | let s = Simple::parse("a").unwrap(); 12 | assert_eq!(s.a, Field); 13 | } 14 | -------------------------------------------------------------------------------- /test/src/skip_ws/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | SkipWsString = 'aaa' 'bbb'; 7 | @export 8 | @no_skip_ws 9 | NoSkipWsString = 'aaa' 'bbb'; 10 | 11 | @export 12 | SkipWsCharRange = 'a'..'b' 'c'..'d'; 13 | @export 14 | @no_skip_ws 15 | NoSkipWsCharRange = 'a'..'b' 'c'..'d'; 16 | 17 | @export 18 | SkipWsEoi = 'a' $; 19 | @export 20 | @no_skip_ws 21 | NoSkipWsEoi = 'a' $; 22 | 23 | @export 24 | SkipWsField = a:NoSkipWsString b:NoSkipWsCharRange; 25 | @export 26 | @no_skip_ws 27 | NoSkipWsField = a:NoSkipWsString b:NoSkipWsCharRange; 28 | 29 | @export 30 | @no_skip_ws 31 | OverrideTester = s:SkipWsOField ns:NoSkipWsOField s2:SkipWsOField; 32 | 33 | SkipWsOField = @:NoSkipWsString | @:NoSkipWsCharRange; 34 | @no_skip_ws 35 | NoSkipWsOField = @:NoSkipWsString | @:NoSkipWsCharRange; 36 | 37 | @export 38 | @no_skip_ws 39 | MixedSkipWsField = a:SkipWsString b:SkipWsCharRange; 40 | 41 | 42 | @export 43 | SkipWsPropagation = 'a' ['.' {'b'}+] 'c'; 44 | 45 | @export 46 | @no_skip_ws 47 | NoSkipWsPropagation = 'a' ['.' {'b'}+] 'c'; 48 | 49 | @export 50 | @no_skip_ws 51 | OptBack = 'a' Whitespace 'b' 'c'; 52 | -------------------------------------------------------------------------------- /test/src/skip_ws/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_string() { 11 | assert!(SkipWsString::parse("aaabbb").is_ok()); 12 | assert!(SkipWsString::parse(" \n\taaa bbb ").is_ok()); 13 | 14 | assert!(NoSkipWsString::parse("aaabbb").is_ok()); 15 | assert!(NoSkipWsString::parse(" \n\taaa bbb ").is_err()); 16 | assert!(NoSkipWsString::parse(" aaabbb").is_err()); 17 | assert!(NoSkipWsString::parse("aaa bbb").is_err()); 18 | assert!(NoSkipWsString::parse(" aaa bbb").is_err()); 19 | } 20 | 21 | #[test] 22 | fn test_char_range() { 23 | assert!(SkipWsCharRange::parse("ac").is_ok()); 24 | assert!(SkipWsCharRange::parse(" \n\ta c ").is_ok()); 25 | 26 | assert!(NoSkipWsCharRange::parse("ac").is_ok()); 27 | assert!(NoSkipWsCharRange::parse(" \n\ta c ").is_err()); 28 | assert!(NoSkipWsCharRange::parse(" ac").is_err()); 29 | assert!(NoSkipWsCharRange::parse(" a c").is_err()); 30 | assert!(NoSkipWsCharRange::parse("a c").is_err()); 31 | } 32 | 33 | #[test] 34 | fn test_eoi() { 35 | assert!(SkipWsEoi::parse("a").is_ok()); 36 | assert!(SkipWsEoi::parse(" a\n\t ").is_ok()); 37 | 38 | assert!(NoSkipWsEoi::parse("a").is_ok()); 39 | assert!(NoSkipWsEoi::parse(" a\n\t ").is_err()); 40 | assert!(NoSkipWsEoi::parse(" a").is_err()); 41 | assert!(NoSkipWsEoi::parse("a ").is_err()); 42 | } 43 | 44 | #[test] 45 | fn test_field() { 46 | assert!(SkipWsField::parse("aaabbbac").is_ok()); 47 | assert!(SkipWsField::parse(" aaabbb ac ").is_ok()); 48 | assert!(SkipWsField::parse(" aaa bbb ac ").is_err()); 49 | assert!(SkipWsField::parse(" aaabbb a c ").is_err()); 50 | 51 | assert!(NoSkipWsField::parse("aaabbbac").is_ok()); 52 | assert!(NoSkipWsField::parse(" aaabbb ac ").is_err()); 53 | assert!(NoSkipWsField::parse("aaabbb ac").is_err()); 54 | assert!(NoSkipWsField::parse(" aaabbbac").is_err()); 55 | assert!(NoSkipWsField::parse("aaa bbbac").is_err()); 56 | assert!(NoSkipWsField::parse("aaabbba c").is_err()); 57 | } 58 | 59 | #[test] 60 | fn test_override_field() { 61 | assert!(OverrideTester::parse("aaabbbaaabbbaaabbb").is_ok()); 62 | assert!(OverrideTester::parse(" aaabbbaaabbbaaabbb").is_ok()); 63 | assert!(OverrideTester::parse("aaa bbbaaabbbaaabbb").is_err()); 64 | assert!(OverrideTester::parse("aaabbb aaabbbaaabbb").is_err()); 65 | assert!(OverrideTester::parse("aaabbbaaa bbbaaabbb").is_err()); 66 | assert!(OverrideTester::parse("aaabbbaaabbb aaabbb").is_ok()); 67 | assert!(OverrideTester::parse("aaabbbaaabbbaaa bbb").is_err()); 68 | 69 | assert!(OverrideTester::parse("acacac").is_ok()); 70 | assert!(OverrideTester::parse(" acacac").is_ok()); 71 | assert!(OverrideTester::parse("a cacac").is_err()); 72 | assert!(OverrideTester::parse("ac acac").is_err()); 73 | assert!(OverrideTester::parse("aca cac").is_err()); 74 | assert!(OverrideTester::parse("acac ac").is_ok()); 75 | assert!(OverrideTester::parse("acaca c").is_err()); 76 | } 77 | 78 | #[test] 79 | fn test_propagation() { 80 | assert!(SkipWsPropagation::parse("a.bbbc").is_ok()); 81 | assert!(SkipWsPropagation::parse(" a . b b b c ").is_ok()); 82 | 83 | assert!(NoSkipWsPropagation::parse("a.bbbc").is_ok()); 84 | assert!(NoSkipWsPropagation::parse(" a.bbbc").is_err()); 85 | assert!(NoSkipWsPropagation::parse("a .bbbc").is_err()); 86 | assert!(NoSkipWsPropagation::parse("a. bbbc").is_err()); 87 | assert!(NoSkipWsPropagation::parse("a.b bbc").is_err()); 88 | assert!(NoSkipWsPropagation::parse("a.bb bc").is_err()); 89 | assert!(NoSkipWsPropagation::parse("a.bbb c").is_err()); 90 | } 91 | 92 | #[test] 93 | fn test_opt_back() { 94 | assert!(OptBack::parse("abc").is_ok()); 95 | assert!(OptBack::parse("a \nbc").is_ok()); 96 | assert!(OptBack::parse("ab c").is_err()); 97 | assert!(OptBack::parse(" abc").is_err()); 98 | } 99 | -------------------------------------------------------------------------------- /test/src/string/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | String1 = "almafa"; 7 | @export 8 | String2 = 'almafa'; 9 | 10 | @export 11 | Char1 = "a"; 12 | @export 13 | Char2 = 'a'; 14 | @export 15 | UtfChar = "☃"; 16 | 17 | @export 18 | CharRange = 'b'..'g'; 19 | @export 20 | UtfRange = '\u{2191}'..'\u{2193}'; 21 | @export 22 | HalfRange = 'x'..'\u074a'; 23 | 24 | @export 25 | Spaces = "ab cd ef"; 26 | 27 | @export 28 | SimpleEscapes1 = ".\n\t\r\\'\""; 29 | @export 30 | SimpleEscapes2 = '.\n\t\r\\\'"'; 31 | 32 | @export 33 | HexEscape = "a\x55b"; 34 | 35 | @export 36 | UnicodeEscapeSimple = "\xC10" "\uFE560" "\U00010abcd"; 37 | 38 | @export 39 | UnicodeEscapeRust = "(" 40 | "\u{7}" "\u{000007}" 41 | "\u{C1}" "\u{0000C1}" 42 | "\u{171}" "\u{000171}" 43 | "\u{2031}" "\u{002031}" 44 | "\u{1d410}" "\u{01d410}" 45 | "\u{101337}" 46 | ")"; 47 | 48 | @export 49 | Unescaped = "a 50 | 🫃🏿_"; 51 | -------------------------------------------------------------------------------- /test/src/string/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_simple() { 11 | assert!(String1::parse("almafa").is_ok()); 12 | assert!(String1::parse("almafa2").is_ok()); 13 | assert!(String1::parse("blmafa").is_err()); 14 | assert!(String1::parse("almbfa").is_err()); 15 | assert!(String1::parse("almafb").is_err()); 16 | 17 | assert!(String2::parse("almafa").is_ok()); 18 | assert!(String2::parse("blmafa").is_err()); 19 | 20 | assert!(Char1::parse("a").is_ok()); 21 | assert!(Char1::parse("b").is_err()); 22 | 23 | assert!(Char2::parse("a").is_ok()); 24 | assert!(Char2::parse("b").is_err()); 25 | 26 | assert!(UtfChar::parse("☃").is_ok()); 27 | assert!(UtfChar::parse("a").is_err()); 28 | assert!(UtfChar::parse("☄").is_err()); 29 | } 30 | 31 | #[test] 32 | fn test_char_range() { 33 | assert!(CharRange::parse("b").is_ok()); 34 | assert!(CharRange::parse("d").is_ok()); 35 | assert!(CharRange::parse("g").is_ok()); 36 | assert!(CharRange::parse("a").is_err()); 37 | assert!(CharRange::parse("h").is_err()); 38 | assert!(CharRange::parse("á").is_err()); 39 | 40 | assert!(UtfRange::parse("←").is_err()); 41 | assert!(UtfRange::parse("↑").is_ok()); 42 | assert!(UtfRange::parse("→").is_ok()); 43 | assert!(UtfRange::parse("↓").is_ok()); 44 | assert!(UtfRange::parse("↔").is_err()); 45 | assert!(UtfRange::parse("a").is_err()); 46 | 47 | assert!(HalfRange::parse("a").is_err()); 48 | assert!(HalfRange::parse("w").is_err()); 49 | assert!(HalfRange::parse("x").is_ok()); 50 | assert!(HalfRange::parse("y").is_ok()); 51 | assert!(HalfRange::parse("}").is_ok()); 52 | assert!(HalfRange::parse("á").is_ok()); 53 | assert!(HalfRange::parse("\u{0749}").is_ok()); 54 | assert!(HalfRange::parse("\u{074A}").is_ok()); 55 | assert!(HalfRange::parse("\u{074B}").is_err()); 56 | assert!(HalfRange::parse("☃").is_err()); 57 | } 58 | 59 | #[test] 60 | fn test_spaces() { 61 | assert!(Spaces::parse("ab cd ef").is_ok()); 62 | assert!(Spaces::parse(" ab cd ef").is_ok()); 63 | assert!(Spaces::parse("ab cd ef ").is_ok()); 64 | 65 | assert!(Spaces::parse("ab cdef").is_err()); 66 | assert!(Spaces::parse("abcd ef").is_err()); 67 | assert!(Spaces::parse("abcdef").is_err()); 68 | assert!(Spaces::parse("ab c d ef").is_err()); 69 | assert!(Spaces::parse("abc d ef").is_err()); 70 | assert!(Spaces::parse("ab cde f").is_err()); 71 | } 72 | 73 | #[test] 74 | fn test_escapes() { 75 | assert!(SimpleEscapes1::parse(".\n\t\r\\'\"").is_ok()); 76 | assert!(SimpleEscapes1::parse(".\n\t\r\\'").is_err()); 77 | 78 | assert!(SimpleEscapes2::parse(".\n\t\r\\'\"").is_ok()); 79 | assert!(SimpleEscapes2::parse(".\n\t\r\\'").is_err()); 80 | 81 | assert!(HexEscape::parse("aUb").is_ok()); 82 | assert!(HexEscape::parse("aU").is_err()); 83 | assert!(HexEscape::parse("aVb").is_err()); 84 | 85 | assert!(UnicodeEscapeSimple::parse("Á0 ﹖0 \u{10abc}d").is_ok()); 86 | assert!(UnicodeEscapeSimple::parse("Á0 ﹖0 \u{10abc}").is_err()); 87 | assert!(UnicodeEscapeSimple::parse("0 ﹖0 \u{10abc}d").is_err()); 88 | assert!(UnicodeEscapeSimple::parse("Á0 ﹖ 0 \u{10abc}d").is_err()); 89 | 90 | assert!(UnicodeEscapeRust::parse("(\u{7}\u{7}ÁÁűű‱‱𝐐𝐐\u{101337})").is_ok()); 91 | assert!(UnicodeEscapeRust::parse("(\u{7} \u{7} Á Á ű ű ‱ ‱ 𝐐 𝐐 \u{101337} )").is_ok()); 92 | assert!(UnicodeEscapeRust::parse("(\u{7}\u{7}ÁÁűű‱‱𝐐𝐐\u{101338})").is_err()); 93 | assert!(UnicodeEscapeRust::parse("(\u{7}\u{7}ÁÁŰű‱‱𝐐𝐐\u{101337})").is_err()); 94 | assert!(UnicodeEscapeRust::parse("(\u{6}\u{7}ÁÁűű‱‱𝐐𝐐\u{101337})").is_err()); 95 | } 96 | 97 | #[test] 98 | fn test_unescaped() { 99 | assert!(Unescaped::parse("a\n 🫃🏿_").is_ok()); 100 | assert!(Unescaped::parse("a\n 🫃🏿 _").is_err()); 101 | assert!(Unescaped::parse("a\n\t🫃🏿_").is_err()); 102 | assert!(Unescaped::parse("b\n 🫃🏿_").is_err()); 103 | assert!(Unescaped::parse("a\n 🫃🏿>").is_err()); 104 | } 105 | -------------------------------------------------------------------------------- /test/src/string_insensitive/grammar.ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | String1 = i"almafa"; 7 | @export 8 | String2 = i'ALMAFA'; 9 | 10 | @export 11 | Char1 = i"a"; 12 | @export 13 | Char2 = i'A'; 14 | -------------------------------------------------------------------------------- /test/src/string_insensitive/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | mod grammar; 6 | use grammar::*; 7 | use peginator::PegParser; 8 | 9 | #[test] 10 | fn test_simple() { 11 | assert!(String1::parse("almafa").is_ok()); 12 | assert!(String1::parse("almafa2").is_ok()); 13 | assert!(String1::parse("blmafa").is_err()); 14 | assert!(String1::parse("almbfa").is_err()); 15 | assert!(String1::parse("almafb").is_err()); 16 | assert!(String1::parse("almafű").is_err()); 17 | 18 | assert!(String2::parse("almafa").is_ok()); 19 | assert!(String2::parse("blmafa").is_err()); 20 | 21 | assert!(Char1::parse("a").is_ok()); 22 | assert!(Char1::parse("b").is_err()); 23 | assert!(Char1::parse("ű").is_err()); 24 | 25 | assert!(Char2::parse("a").is_ok()); 26 | assert!(Char2::parse("b").is_err()); 27 | assert!(Char2::parse("ű").is_err()); 28 | } 29 | 30 | #[test] 31 | fn test_sensitivity() { 32 | assert!(String1::parse("AlMaFa").is_ok()); 33 | assert!(String1::parse("alMafa2").is_ok()); 34 | assert!(String1::parse("bLMAfa").is_err()); 35 | assert!(String1::parse("aLMBfa").is_err()); 36 | assert!(String1::parse("aLMAfb").is_err()); 37 | assert!(String1::parse("aLMAfŰ").is_err()); 38 | 39 | assert!(String2::parse("alMAFA").is_ok()); 40 | assert!(String2::parse("blMAFA").is_err()); 41 | 42 | assert!(Char1::parse("A").is_ok()); 43 | assert!(Char1::parse("B").is_err()); 44 | assert!(Char1::parse("Ű").is_err()); 45 | 46 | assert!(Char2::parse("A").is_ok()); 47 | assert!(Char2::parse("B").is_err()); 48 | assert!(Char2::parse("Ű").is_err()); 49 | } 50 | -------------------------------------------------------------------------------- /test/src/test_utils.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | macro_rules! assert_type_eq ( 6 | ($parent:ident, $field:ident, $type:ty) => { 7 | let _ = |a:$parent| -> $type {a.$field}; 8 | } 9 | ); 10 | 11 | pub(crate) use assert_type_eq; 12 | -------------------------------------------------------------------------------- /test/src/user_defined_state/grammar.not_ebnf: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2022, Alex Badics 2 | # This file is part of peginator 3 | # Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | @export 6 | Test = f1: UserDefined f2:UserDefined {f3:ManyAs}+; 7 | 8 | @extern(crate::user_defined_state::stateful_parser -> u32) 9 | UserDefined; 10 | 11 | @check(crate::user_defined_state::stateful_checker) 12 | ManyAs = 'a'; 13 | -------------------------------------------------------------------------------- /test/src/user_defined_state/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022, Alex Badics 2 | // This file is part of peginator 3 | // Licensed under the MIT license. See LICENSE file in the project root for details. 4 | 5 | #[allow(clippy::useless_conversion)] 6 | mod grammar; 7 | use grammar::*; 8 | use peginator::{NoopTracer, ParseSettings, PegParserAdvanced}; 9 | 10 | pub struct TheState { 11 | retval: u32, 12 | a_count: u32, 13 | } 14 | 15 | impl TheState { 16 | pub fn do_parse(&mut self, _s: &str) -> Result<(u32, usize), &'static str> { 17 | self.retval += 1; 18 | Ok((self.retval, 1)) 19 | } 20 | } 21 | 22 | pub fn stateful_parser(s: &str, state: &mut TheState) -> Result<(u32, usize), &'static str> { 23 | state.do_parse(s) 24 | } 25 | pub fn stateful_checker(_a: &ManyAs, state: &mut TheState) -> bool { 26 | if state.a_count > 0 { 27 | state.a_count -= 1; 28 | true 29 | } else { 30 | false 31 | } 32 | } 33 | 34 | #[test] 35 | fn test() { 36 | let mut state = TheState { 37 | retval: 42, 38 | a_count: 4, 39 | }; 40 | assert_eq!( 41 | Test::parse_advanced::("ab a a a a a a", &ParseSettings::default(), &mut state) 42 | .unwrap(), 43 | Test { 44 | f1: 43, 45 | f2: 44, 46 | f3: vec![ManyAs, ManyAs, ManyAs, ManyAs] 47 | } 48 | ); 49 | assert_eq!(state.retval, 44); 50 | } 51 | --------------------------------------------------------------------------------