├── tests └── csv.rs ├── examples ├── csv.csv ├── csv.pest └── csv.rs ├── derive ├── examples │ ├── simple_enum_derives.pest │ ├── function_defaults.pest │ ├── defaults_showcase.pest │ ├── simple_struct_derives.pest │ ├── simple_struct_derives.rs │ ├── simple_enum_derives.rs │ ├── function_defaults.rs │ └── defaults_showcase.rs ├── Cargo.toml ├── src │ ├── lib.rs │ ├── from_pest │ │ ├── field.rs │ │ └── mod.rs │ └── attributes.rs └── README.md ├── .idea ├── vcs.xml ├── modules.xml └── misc.xml ├── Cargo.toml ├── SECURITY.md ├── .github └── workflows │ └── test.yml ├── LICENSE-MIT ├── pest-deconstruct.iml ├── .gitignore ├── README.md ├── src └── lib.rs └── LICENSE-APACHE /tests/csv.rs: -------------------------------------------------------------------------------- 1 | ../examples/csv.rs -------------------------------------------------------------------------------- /examples/csv.csv: -------------------------------------------------------------------------------- 1 | 65279,1179403647,1463895090 2 | 3.1415927,2.7182817,1.618034 3 | -40,-273.15 4 | 13,42 5 | 65537 6 | -------------------------------------------------------------------------------- /derive/examples/simple_enum_derives.pest: -------------------------------------------------------------------------------- 1 | a = { "a" } 2 | b = { "b" } 3 | c = { "c" } 4 | 5 | abc = { a | b | c } 6 | ABC = { abc* } 7 | -------------------------------------------------------------------------------- /examples/csv.pest: -------------------------------------------------------------------------------- 1 | field = { (ASCII_DIGIT | "." | "-")+ } 2 | record = { field ~ ("," ~ field)* } 3 | file = { SOI ~ (record ~ ("\r\n" | "\n"))* ~ EOI } 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 12 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["derive"] 3 | 4 | [package] 5 | name = "from-pest" 6 | version = "0.3.4" 7 | edition = "2021" 8 | authors = ["cad97 "] 9 | readme = "./README.md" 10 | description = "Convert from a pest grammar to a typed AST" 11 | license = "MIT/Apache-2.0" 12 | repository = "https://github.com/pest-parser/pest_deconstruct" 13 | 14 | [dependencies] 15 | void = "1.0" 16 | pest = "2.5" 17 | log = "0.4.6" 18 | 19 | [dev-dependencies] 20 | pest_derive = "2.5" 21 | pest-ast = { version = "0.3", path = "derive" } 22 | -------------------------------------------------------------------------------- /derive/examples/function_defaults.pest: -------------------------------------------------------------------------------- 1 | // Grammar for a simple language with function declarations 2 | // Functions can have optional return types that default to "void" 3 | 4 | WHITESPACE = _{ " " | "\t" | "\n" | "\r" } 5 | 6 | // Basic types 7 | id = { ASCII_ALPHA ~ (ASCII_ALPHANUMERIC | "_")* } 8 | type_name = { "void" | "int" | "string" | id } 9 | 10 | // Function parameter 11 | param = { id ~ ":" ~ type_name } 12 | params = { param ~ ("," ~ param)* } 13 | 14 | // Function declaration with optional return type 15 | function = { "fn" ~ id ~ "(" ~ params? ~ ")" ~ ("->" ~ type_name)? ~ "{" ~ "}" } 16 | 17 | // Program is a sequence of functions 18 | program = { function* } 19 | -------------------------------------------------------------------------------- /derive/examples/defaults_showcase.pest: -------------------------------------------------------------------------------- 1 | // Simple grammar showcasing default values 2 | 3 | WHITESPACE = _{ " " | "\t" | "\n" | "\r" } 4 | 5 | // Variable declarations with optional types and initialization 6 | var_decl = { var_kind ~ id ~ type_annotation? ~ initializer? ~ ";" } 7 | var_kind = { "let" | "const" } 8 | type_annotation = { ":" ~ type_name } 9 | initializer = { "=" ~ expr } 10 | 11 | // Expressions 12 | expr = { number | string | id } 13 | number = { ASCII_DIGIT+ } 14 | string = { "\"" ~ (!("\"") ~ ANY)* ~ "\"" } 15 | 16 | // Basic constructs 17 | id = { ASCII_ALPHA ~ (ASCII_ALPHANUMERIC | "_")* } 18 | type_name = { "int" | "string" | "bool" | "void" } 19 | 20 | // Program 21 | program = { var_decl* } 22 | -------------------------------------------------------------------------------- /derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pest-ast" 3 | version = "0.3.6" 4 | edition = "2021" 5 | authors = ["cad97 "] 6 | description = "Derive to convert from pest parse tree to typed syntax tree" 7 | license = "MIT/Apache-2.0" 8 | readme = "./README.md" 9 | repository = "https://github.com/pest-parser/pest_deconstruct" 10 | 11 | [lib] 12 | name = "pest_ast" 13 | proc-macro = true 14 | 15 | [dependencies] 16 | syn = { version = "2", features = ["extra-traits"] } 17 | quote = "1" 18 | proc-macro2 = "1" 19 | itertools = "0.10" 20 | 21 | [dev-dependencies] 22 | from-pest = { version = "0.3", path = ".." } 23 | pest = "2.5" 24 | pest_derive = "2.5" 25 | 26 | [features] 27 | default = ["trace"] 28 | trace = [] 29 | -------------------------------------------------------------------------------- /derive/examples/simple_struct_derives.pest: -------------------------------------------------------------------------------- 1 | // 2 | // 3 | // The parsing expression ('a'/'b')* matches and consumes an arbitrary-length 4 | // sequence of a's and b's. The production rule S ← 'a' S? 'b' describes the 5 | // simple context-free "matching language" $\{ `a^n b^n : n \geq 1 \}`$. 6 | // The following parsing expression grammar describes the classic 7 | // non-context-free language $`\{ a^n b^n c^n : n \geq 1 \}`$: 8 | // 9 | // ```peg 10 | // S ← &(A 'c') 'a'+ B !. 11 | // A ← 'a' A? 'b' 12 | // B ← 'b' B? 'c' 13 | // ``` 14 | 15 | a = { "a" } 16 | b = { "b" } 17 | c = { "c" } 18 | 19 | S = { &(A ~ c) ~ a+ ~ B ~ !ANY } 20 | A = _{ a ~ A? ~ b } 21 | B = _{ b ~ B? ~ c } 22 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Only the most recent minor version is supported. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | 0.3.x | :white_check_mark: | 10 | | < 0.3.x | :x: | 11 | 12 | 13 | ## Reporting a Vulnerability 14 | 15 | Please use the [GitHub private reporting functionality](https://github.com/pest-parser/ast/security/advisories/new) 16 | to submit potential security bug reports. If the bug report is reproduced and valid, we'll then: 17 | 18 | - Prepare a fix and regression tests. 19 | - Make a patch release for the most recent release. 20 | - Submit an advisory to [rustsec/advisory-db](https://github.com/RustSec/advisory-db). 21 | - Refer to the advisory in the release notes. 22 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | name: cargo test and examples 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: dtolnay/rust-toolchain@stable 11 | - run: cargo test --all --all-features 12 | - run: cargo run --example csv 13 | - run: cd derive && cargo run --example simple_enum_derives && cargo run --example simple_struct_derives 14 | fmt: 15 | name: cargo fmt 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - uses: dtolnay/rust-toolchain@stable 20 | with: 21 | components: rustfmt 22 | - run: cargo fmt --all -- --check 23 | lint: 24 | name: cargo clippy 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: dtolnay/rust-toolchain@stable 29 | with: 30 | components: clippy 31 | - run: cargo clippy --all --all-features --all-targets -- -Dwarnings -------------------------------------------------------------------------------- /derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_snake_case)] 2 | #![recursion_limit = "150"] 3 | 4 | extern crate itertools; 5 | extern crate proc_macro; 6 | extern crate proc_macro2; 7 | #[macro_use] 8 | extern crate syn; 9 | #[macro_use] 10 | extern crate quote; 11 | 12 | #[allow(non_snake_case)] 13 | #[proc_macro_derive(FromPest, attributes(pest_ast))] 14 | pub fn derive_FromPest(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 15 | // let x = 16 | syn::parse(input) 17 | .and_then(from_pest::derive) 18 | .unwrap_or_else(|err| err.to_compile_error()) 19 | // .to_string(); 20 | // quote!(compile_error!(#x);) 21 | .into() 22 | } 23 | 24 | mod attributes; 25 | mod from_pest; 26 | 27 | #[cfg(feature = "trace")] 28 | fn trace(t: proc_macro2::TokenStream) -> proc_macro2::TokenStream { 29 | quote! { ::from_pest::log::trace!( #t ); } 30 | } 31 | 32 | #[cfg(not(feature = "trace"))] 33 | fn trace(_t: proc_macro2::TokenStream) -> proc_macro2::TokenStream { 34 | quote! {} 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pest-deconstruct.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /derive/examples/simple_struct_derives.rs: -------------------------------------------------------------------------------- 1 | #![allow( 2 | bad_style, 3 | dead_code, 4 | clippy::clone_on_copy, 5 | clippy::upper_case_acronyms 6 | )] 7 | 8 | #[macro_use] 9 | extern crate pest_derive; 10 | extern crate from_pest; 11 | #[macro_use] 12 | extern crate pest_ast; 13 | extern crate pest; 14 | 15 | use from_pest::FromPest; 16 | use pest::Parser; 17 | 18 | #[derive(Parser)] 19 | #[grammar = "../examples/simple_struct_derives.pest"] 20 | pub struct SimpleParser; 21 | 22 | #[derive(FromPest, Debug)] 23 | #[pest_ast(rule(Rule::S))] 24 | struct S<'pest> { 25 | #[pest_ast(outer())] 26 | span: pest::Span<'pest>, 27 | a: Vec>, 28 | b: Vec>, 29 | c: Vec>, 30 | } 31 | 32 | #[derive(FromPest, Debug)] 33 | #[pest_ast(rule(Rule::a))] 34 | struct a<'pest> { 35 | #[pest_ast(outer())] 36 | span: pest::Span<'pest>, 37 | } 38 | 39 | #[derive(FromPest, Debug)] 40 | #[pest_ast(rule(Rule::b))] 41 | struct b<'pest> { 42 | #[pest_ast(outer())] 43 | span: pest::Span<'pest>, 44 | } 45 | 46 | #[derive(FromPest, Debug)] 47 | #[pest_ast(rule(Rule::c))] 48 | struct c<'pest> { 49 | #[pest_ast(outer())] 50 | span: pest::Span<'pest>, 51 | } 52 | 53 | fn main() { 54 | let source = "aaabbbccc"; 55 | 56 | let mut parse_tree = SimpleParser::parse(Rule::S, source).expect("parse success"); 57 | println!("parse tree = {parse_tree:#?}"); 58 | 59 | let syntax_tree = S::from_pest(&mut parse_tree).expect("infallible"); 60 | println!("syntax tree = {syntax_tree:#?}"); 61 | } 62 | -------------------------------------------------------------------------------- /derive/examples/simple_enum_derives.rs: -------------------------------------------------------------------------------- 1 | #![allow( 2 | bad_style, 3 | dead_code, 4 | clippy::clone_on_copy, 5 | clippy::upper_case_acronyms 6 | )] 7 | 8 | #[macro_use] 9 | extern crate pest_derive; 10 | extern crate from_pest; 11 | #[macro_use] 12 | extern crate pest_ast; 13 | extern crate pest; 14 | 15 | use from_pest::FromPest; 16 | use pest::Parser; 17 | 18 | #[derive(Parser)] 19 | #[grammar = "../examples/simple_enum_derives.pest"] 20 | pub struct SimpleParser; 21 | 22 | #[derive(FromPest, Debug)] 23 | #[pest_ast(rule(Rule::a))] 24 | struct a<'pest> { 25 | #[pest_ast(outer())] 26 | span: pest::Span<'pest>, 27 | } 28 | 29 | #[derive(FromPest, Debug)] 30 | #[pest_ast(rule(Rule::b))] 31 | struct b<'pest> { 32 | #[pest_ast(outer())] 33 | span: pest::Span<'pest>, 34 | } 35 | 36 | #[derive(FromPest, Debug)] 37 | #[pest_ast(rule(Rule::c))] 38 | struct c<'pest> { 39 | #[pest_ast(outer())] 40 | span: pest::Span<'pest>, 41 | } 42 | 43 | #[derive(FromPest, Debug)] 44 | #[pest_ast(rule(Rule::abc))] 45 | enum abc<'pest> { 46 | a(a<'pest>), 47 | b(b<'pest>), 48 | c(c<'pest>), 49 | } 50 | 51 | #[derive(FromPest, Debug)] 52 | #[pest_ast(rule(Rule::ABC))] 53 | struct ABC<'pest> { 54 | abc: Vec>, 55 | } 56 | 57 | fn main() { 58 | let source = "aaabbbccc"; 59 | 60 | let mut parse_tree = SimpleParser::parse(Rule::ABC, source).expect("parse success"); 61 | println!("parse tree = {parse_tree:#?}"); 62 | 63 | let syntax_tree = ABC::from_pest(&mut parse_tree).expect("infallible"); 64 | println!("syntax tree = {syntax_tree:#?}"); 65 | } 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### JetBrains template 2 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 3 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 4 | 5 | # User-specific stuff 6 | .idea/**/workspace.xml 7 | .idea/**/tasks.xml 8 | .idea/**/usage.statistics.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # Sensitive or high-churn files 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | .idea/**/dbnavigator.xml 20 | 21 | # Gradle 22 | .idea/**/gradle.xml 23 | .idea/**/libraries 24 | 25 | # Gradle and Maven with auto-import 26 | # When using Gradle or Maven with auto-import, you should exclude module files, 27 | # since they will be recreated, and may cause churn. Uncomment if using 28 | # auto-import. 29 | # .idea/modules.xml 30 | # .idea/*.iml 31 | # .idea/modules 32 | 33 | # CMake 34 | cmake-build-*/ 35 | 36 | # Mongo Explorer plugin 37 | .idea/**/mongoSettings.xml 38 | 39 | # File-based project format 40 | *.iws 41 | 42 | # IntelliJ 43 | out/ 44 | 45 | # mpeltonen/sbt-idea plugin 46 | .idea_modules/ 47 | 48 | # JIRA plugin 49 | atlassian-ide-plugin.xml 50 | 51 | # Cursive Clojure plugin 52 | .idea/replstate.xml 53 | 54 | # Crashlytics plugin (for Android Studio and IntelliJ) 55 | com_crashlytics_export_strings.xml 56 | crashlytics.properties 57 | crashlytics-build.properties 58 | fabric.properties 59 | 60 | # Editor-based Rest Client 61 | .idea/httpRequests 62 | ### Rust template 63 | # Generated by Cargo 64 | # will have compiled files and executables 65 | /target/ 66 | 67 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 68 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 69 | Cargo.lock 70 | 71 | # These are backup files generated by rustfmt 72 | **/*.rs.bk 73 | 74 | -------------------------------------------------------------------------------- /examples/csv.rs: -------------------------------------------------------------------------------- 1 | // Unfortunately, you currently have to import all four of these. 2 | // We're considering what it would look like to make this redundant, 3 | // and then you'd only need pest and pest-ast. 4 | 5 | #[macro_use] 6 | extern crate pest_derive; 7 | extern crate from_pest; 8 | #[macro_use] 9 | extern crate pest_ast; 10 | extern crate pest; 11 | 12 | mod csv { 13 | #[derive(Parser)] 14 | #[grammar = "../examples/csv.pest"] 15 | pub struct Parser; 16 | } 17 | 18 | mod ast { 19 | use super::csv::Rule; 20 | use pest::Span; 21 | 22 | fn span_into_str(span: Span) -> &str { 23 | span.as_str() 24 | } 25 | 26 | #[derive(Debug, FromPest)] 27 | #[pest_ast(rule(Rule::field))] 28 | pub struct Field { 29 | #[pest_ast(outer(with(span_into_str), with(str::parse), with(Result::unwrap)))] 30 | pub value: f64, 31 | } 32 | 33 | #[derive(Debug, FromPest)] 34 | #[pest_ast(rule(Rule::record))] 35 | pub struct Record { 36 | pub fields: Vec, 37 | } 38 | 39 | #[derive(Debug, FromPest)] 40 | #[pest_ast(rule(Rule::file))] 41 | pub struct File { 42 | pub records: Vec, 43 | _eoi: Eoi, 44 | } 45 | 46 | #[derive(Debug, FromPest)] 47 | #[pest_ast(rule(Rule::EOI))] 48 | struct Eoi; 49 | } 50 | 51 | fn main() -> Result<(), Box> { 52 | use crate::ast::File; 53 | use from_pest::FromPest; 54 | use pest::Parser; 55 | use std::fs; 56 | 57 | let source = String::from_utf8(fs::read("./examples/csv.csv")?)?; 58 | let mut parse_tree = csv::Parser::parse(csv::Rule::file, &source)?; 59 | println!("parse tree = {parse_tree:#?}"); 60 | let syntax_tree: File = File::from_pest(&mut parse_tree).expect("infallible"); 61 | println!("syntax tree = {syntax_tree:#?}"); 62 | println!(); 63 | 64 | let mut field_sum = 0.0; 65 | let mut record_count = 0; 66 | 67 | for record in syntax_tree.records { 68 | record_count += 1; 69 | for field in record.fields { 70 | field_sum += field.value; 71 | } 72 | } 73 | 74 | println!("Sum of fields: {field_sum}"); 75 | println!("Number of records: {record_count}"); 76 | 77 | Ok(()) 78 | } 79 | 80 | #[test] 81 | fn csv_example_runs() { 82 | main().unwrap() 83 | } 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pest-ast 2 | 3 | This is an in-development add-on to the pest parsing library. 4 | 5 | Pest-ast provides a structured manner to go from the "dynamically typed" Pest Parse Tree 6 | to a strongly typed (Abstract) Syntax Tree, as well as a derive to do so automatically. 7 | In the future, it's planned to optionally additionally check the source grammar to statically 8 | prevent issues that are currently detected at runtime. 9 | 10 | In the _future_ 🦄, pest-ast may provide a way of defining grammar directly on the AST nodes. 11 | 12 | ## Note: 13 | 14 | This crate is actually `from-pest`, which provides the trait framework for the said conversion. 15 | [`pest-ast`](./derive/README.md) provides the actual derive for the conversion. 16 | 17 | This README is the root of the repository for legacy reasons. This will be corrected in a future reorganization. 18 | 19 | ## Contributing 20 | 21 | Check out the [issue tracker](https://github.com/pest-parser/ast); 22 | we try to keep it stocked with [good-first-issue](https://github.com/pest-parser/ast/labels/good%20first%20issue) 23 | and [help-wanted](https://github.com/pest-parser/ast/issues?q=is%3Aopen+label%3A%22help+wanted%22) opportunities. 24 | If you have questions, don't be afraid to @ the author (CAD97) 25 | [on Gitter](https://gitter.im/pest-parser/pest) or [on Discord](https://discord.gg/FuPE9JE). 26 | The best thing you can probably do for this library currently is to use it! 27 | More than anything else, I just want eyes on the interface trying it out and seeing where it shines and wher it falters. 28 | 29 | ## License 30 | 31 | pest-deconstruct is licensed under both the MIT license and the Apache License 2.0. 32 | Either terms may be used at your option. All PRs are understood to be agreeing to 33 | contribution under these terms as defined in the Apache license. 34 | 35 | See [LICENSE-APACHE] and [LICENSE-MIT] for details. 36 | 37 | Copyright 2018 Christopher Durham (aka CAD97) 38 | 39 | Dual licensed under the Apache License, Version 2.0 and the MIT License 40 | (collectively, the "License"); you may not use this file except in 41 | compliance with the License. You may obtain a copy of the License at 42 | 43 | 44 | 45 | 46 | Unless required by applicable law or agreed to in writing, software 47 | distributed under the License is distributed on an "AS IS" BASIS, 48 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 49 | See the License for the specific language governing permissions and 50 | limitations under the License. 51 | -------------------------------------------------------------------------------- /derive/examples/function_defaults.rs: -------------------------------------------------------------------------------- 1 | #![allow( 2 | bad_style, 3 | dead_code, 4 | clippy::clone_on_copy, 5 | clippy::upper_case_acronyms 6 | )] 7 | 8 | #[macro_use] 9 | extern crate pest_derive; 10 | extern crate from_pest; 11 | #[macro_use] 12 | extern crate pest_ast; 13 | extern crate pest; 14 | 15 | use from_pest::FromPest; 16 | use pest::Parser; 17 | 18 | #[derive(Parser)] 19 | #[grammar = "../examples/function_defaults.pest"] 20 | pub struct FunctionParser; 21 | 22 | // Define a simple enum for types that can have a default 23 | #[derive(Debug, Clone, PartialEq, Default)] 24 | pub enum Type { 25 | #[default] 26 | Void, 27 | Int, 28 | String, 29 | } 30 | 31 | // Implement FromPest for Type 32 | impl<'pest> FromPest<'pest> for Type { 33 | type Rule = Rule; 34 | type FatalError = from_pest::Void; 35 | 36 | fn from_pest( 37 | pest: &mut pest::iterators::Pairs<'pest, Rule>, 38 | ) -> Result> { 39 | let pair = pest.next().ok_or(from_pest::ConversionError::NoMatch)?; 40 | if pair.as_rule() == Rule::type_name { 41 | let span = pair.as_span(); 42 | match span.as_str() { 43 | "void" => Ok(Type::Void), 44 | "int" => Ok(Type::Int), 45 | "string" => Ok(Type::String), 46 | _ => Err(from_pest::ConversionError::NoMatch), 47 | } 48 | } else { 49 | Err(from_pest::ConversionError::NoMatch) 50 | } 51 | } 52 | } 53 | 54 | // Define the AST types 55 | 56 | #[derive(FromPest, Debug, Clone, PartialEq)] 57 | #[pest_ast(rule(Rule::id))] 58 | pub struct Id<'pest> { 59 | #[pest_ast(outer())] 60 | pub span: pest::Span<'pest>, 61 | } 62 | 63 | #[derive(FromPest, Debug, Clone, PartialEq)] 64 | #[pest_ast(rule(Rule::param))] 65 | pub struct Param<'pest> { 66 | pub id: Id<'pest>, 67 | pub type_name: Type, 68 | } 69 | 70 | // This is the key example: function return type with default 71 | #[derive(FromPest, Debug, Clone, PartialEq)] 72 | #[pest_ast(rule(Rule::function))] 73 | pub struct Function<'pest> { 74 | pub id: Id<'pest>, 75 | pub params: Vec>, 76 | 77 | // This demonstrates the new default feature! 78 | // Instead of Option, we use Type with a default 79 | #[pest_ast(default(Type::Void))] 80 | pub return_type: Type, 81 | } 82 | 83 | #[derive(FromPest, Debug, Clone, PartialEq)] 84 | #[pest_ast(rule(Rule::program))] 85 | pub struct Program<'pest> { 86 | pub functions: Vec>, 87 | } 88 | 89 | fn main() -> Result<(), Box> { 90 | // Test with a function that has no return type (should default) 91 | let input1 = "fn main() {}"; 92 | let pairs1 = FunctionParser::parse(Rule::program, input1)?; 93 | let program1: Program = Program::from_pest(&mut pairs1.clone())?; 94 | println!("Program 1 (no return type): {program1:#?}"); 95 | 96 | // Test with a function that has an explicit return type 97 | let input2 = "fn add() -> int {}"; 98 | let pairs2 = FunctionParser::parse(Rule::program, input2)?; 99 | let program2: Program = Program::from_pest(&mut pairs2.clone())?; 100 | println!("Program 2 (explicit return type): {program2:#?}"); 101 | 102 | Ok(()) 103 | } 104 | 105 | #[cfg(test)] 106 | mod tests { 107 | use super::*; 108 | 109 | #[test] 110 | fn test_function_with_default_return_type() { 111 | let input = "fn main() {}"; 112 | let pairs = FunctionParser::parse(Rule::program, input).unwrap(); 113 | let program: Program = Program::from_pest(&mut pairs.clone()).unwrap(); 114 | 115 | assert_eq!(program.functions.len(), 1); 116 | let function = &program.functions[0]; 117 | 118 | // The return type should be Void (the default) even though it wasn't specified 119 | assert_eq!(function.return_type, Type::Void); 120 | println!("Function return type: {:?}", function.return_type); 121 | } 122 | 123 | #[test] 124 | fn test_function_with_explicit_return_type() { 125 | let input = "fn add() -> int {}"; 126 | let pairs = FunctionParser::parse(Rule::program, input).unwrap(); 127 | let program: Program = Program::from_pest(&mut pairs.clone()).unwrap(); 128 | 129 | assert_eq!(program.functions.len(), 1); 130 | let function = &program.functions[0]; 131 | 132 | // The return type should be Int 133 | assert_eq!(function.return_type, Type::Int); 134 | println!("Function return type: {:?}", function.return_type); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The [`FromPest`] conversion framework to convert from pest trees into typed structure. 2 | 3 | #[doc(hidden)] 4 | pub extern crate log; 5 | #[doc(hidden)] 6 | pub extern crate pest; 7 | extern crate void; 8 | 9 | #[doc(inline)] 10 | pub use void::Void; 11 | 12 | use { 13 | pest::{ 14 | iterators::{Pair, Pairs}, 15 | RuleType, 16 | }, 17 | std::marker::PhantomData, 18 | }; 19 | 20 | /// An error that occurs during conversion. 21 | #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] 22 | pub enum ConversionError { 23 | /// No match occurred: this node is not present here 24 | NoMatch, 25 | /// Fatal error: this node is present but malformed 26 | Malformed(FatalError), 27 | /// Found unexpected tokens at the end 28 | Extraneous { current_node: &'static str }, 29 | } 30 | 31 | use std::fmt; 32 | 33 | impl fmt::Display for ConversionError 34 | where 35 | FatalError: fmt::Display, 36 | { 37 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 38 | match self { 39 | ConversionError::NoMatch => write!(f, "Rule did not match, failed to convert node"), 40 | ConversionError::Malformed(fatalerror) => write!(f, "Malformed node: {fatalerror}"), 41 | ConversionError::Extraneous { current_node, .. } => { 42 | write!(f, "when converting {current_node}, found extraneous tokens") 43 | } 44 | } 45 | } 46 | } 47 | 48 | use std::error; 49 | 50 | impl error::Error for ConversionError 51 | where 52 | FatalError: error::Error + 'static, 53 | { 54 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 55 | match self { 56 | ConversionError::NoMatch => None, 57 | ConversionError::Extraneous { .. } => None, 58 | ConversionError::Malformed(ref fatalerror) => Some(fatalerror), 59 | } 60 | } 61 | } 62 | 63 | /// Potentially borrowing conversion from a pest parse tree. 64 | pub trait FromPest<'pest>: Sized { 65 | /// The rule type for the parse tree this type corresponds to. 66 | type Rule: RuleType; 67 | /// A fatal error during conversion. 68 | type FatalError; 69 | /// Convert from a Pest parse tree. 70 | /// 71 | /// # Return type semantics 72 | /// 73 | /// - `Err(ConversionError::NoMatch)` => node not at head of the cursor, cursor unchanged 74 | /// - `Err(ConversionError::Malformed)` => fatal error; node at head of the cursor but malformed 75 | /// - `Ok` => success; the cursor has been updated past this node 76 | fn from_pest( 77 | pest: &mut Pairs<'pest, Self::Rule>, 78 | ) -> Result>; 79 | } 80 | 81 | /// Convert a production without storing it. 82 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for PhantomData { 83 | type Rule = Rule; 84 | type FatalError = T::FatalError; 85 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 86 | T::from_pest(pest).map(|_| PhantomData) 87 | } 88 | } 89 | 90 | /// For recursive grammars. 91 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for Box { 92 | type Rule = Rule; 93 | type FatalError = T::FatalError; 94 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 95 | T::from_pest(pest).map(Box::new) 96 | } 97 | } 98 | 99 | /// Convert an optional production. 100 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for Option { 101 | type Rule = Rule; 102 | type FatalError = T::FatalError; 103 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 104 | match T::from_pest(pest) { 105 | Err(ConversionError::NoMatch) => Ok(None), 106 | result => result.map(Some), 107 | } 108 | } 109 | } 110 | 111 | /// Convert many productions. (If `` is non-advancing, this will be non-terminating.) 112 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for Vec { 113 | type Rule = Rule; 114 | type FatalError = T::FatalError; 115 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 116 | let mut acc = vec![]; 117 | loop { 118 | match T::from_pest(pest) { 119 | Ok(t) => acc.push(t), 120 | Err(ConversionError::NoMatch) => break, 121 | Err(error) => return Err(error), 122 | } 123 | } 124 | Ok(acc) 125 | } 126 | } 127 | 128 | /// Consume a production without doing any processing. 129 | impl<'pest, Rule: RuleType> FromPest<'pest> for Pair<'pest, Rule> { 130 | type Rule = Rule; 131 | type FatalError = Void; 132 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 133 | pest.next().ok_or(ConversionError::NoMatch) 134 | } 135 | } 136 | 137 | macro_rules! impl_for_tuple { 138 | () => {}; 139 | ($ty1:ident $($ty:ident)*) => { 140 | impl<'pest, $ty1, $($ty,)* Rule: RuleType, FatalError> FromPest<'pest> for ($ty1, $($ty),*) 141 | where 142 | $ty1: FromPest<'pest, Rule=Rule, FatalError=FatalError>, 143 | $($ty: FromPest<'pest, Rule=Rule, FatalError=FatalError>,)* 144 | { 145 | type Rule = Rule; 146 | type FatalError = FatalError; 147 | fn from_pest(pest: &mut Pairs<'pest, Rule>) 148 | -> Result> 149 | { 150 | let mut clone = pest.clone(); 151 | let this = ( 152 | $ty1::from_pest(&mut clone)?, 153 | $($ty::from_pest(&mut clone)?),* 154 | ); 155 | *pest = clone; 156 | Ok(this) 157 | } 158 | } 159 | impl_for_tuple!($($ty)*); 160 | }; 161 | } 162 | 163 | impl_for_tuple!(A B C D); 164 | -------------------------------------------------------------------------------- /derive/src/from_pest/field.rs: -------------------------------------------------------------------------------- 1 | use syn::Variant; 2 | 3 | use { 4 | proc_macro2::{Span, TokenStream}, 5 | syn::{ 6 | parse::Error, parse::Result, parse_quote, spanned::Spanned, Fields, Index, Member, Path, 7 | }, 8 | }; 9 | 10 | use crate::attributes::FieldAttribute; 11 | use crate::trace; 12 | 13 | #[derive(Clone, Debug)] 14 | enum ConversionStrategy { 15 | FromPest, 16 | Outer(Span, Vec), 17 | Inner(Span, Vec, Option), 18 | Default(Span, syn::Expr), 19 | } 20 | 21 | impl ConversionStrategy { 22 | fn from_attrs(attrs: Vec) -> Result { 23 | let mut attrs = attrs.into_iter(); 24 | Ok(match (attrs.next(), attrs.next()) { 25 | (Some(_), Some(attr)) => Err(Error::new( 26 | attr.span(), 27 | "only a single field attribute allowed", 28 | ))?, 29 | (None, None) => ConversionStrategy::FromPest, 30 | (Some(FieldAttribute::Outer(attr)), None) => ConversionStrategy::Outer( 31 | attr.span(), 32 | attr.with.into_iter().map(|attr| attr.path).collect(), 33 | ), 34 | (Some(FieldAttribute::Inner(attr)), None) => ConversionStrategy::Inner( 35 | attr.span(), 36 | attr.with.into_iter().map(|attr| attr.path).collect(), 37 | attr.rule.map(|attr| { 38 | let path = attr.path; 39 | let variant = attr.variant; 40 | parse_quote!(#path::#variant) 41 | }), 42 | ), 43 | (Some(FieldAttribute::Default(attr)), None) => { 44 | ConversionStrategy::Default(attr.span(), attr.expr) 45 | } 46 | _ => unreachable!(), 47 | }) 48 | } 49 | 50 | fn apply(self, member: Member) -> TokenStream { 51 | let conversion = match self { 52 | ConversionStrategy::FromPest => quote!(::from_pest::FromPest::from_pest(inner)?), 53 | ConversionStrategy::Outer(span, mods) => with_mods(quote_spanned!(span=>span), mods), 54 | ConversionStrategy::Inner(span, mods, rule) => { 55 | let pair = quote!(inner.next().ok_or(::from_pest::ConversionError::NoMatch)?); 56 | let get_span = if let Some(rule) = rule { 57 | let error_msg = trace(quote! { 58 | concat!( 59 | "in ", 60 | stringify!(#member), 61 | ", expected `", 62 | stringify!(#rule), 63 | "` but found `{:?}`" 64 | ), 65 | pair.as_rule(), 66 | }); 67 | quote_spanned! {span=>{ 68 | let pair = #pair; 69 | if pair.as_rule() == #rule { 70 | pair.as_span() 71 | } else { 72 | #error_msg 73 | return Err(::from_pest::ConversionError::NoMatch) 74 | // TODO: Should this be panicking instead? 75 | // panic!( 76 | // concat!( 77 | // "in ", 78 | // stringify!(#name), 79 | // ".", 80 | // stringify!(#member), 81 | // ", expected `", 82 | // stringify!(#rule), 83 | // "` but found `{:?}`" 84 | // ), 85 | // pair.as_rule(), 86 | // ) 87 | } 88 | }} 89 | } else { 90 | quote_spanned!(span=>#pair.as_span()) 91 | }; 92 | with_mods(get_span, mods) 93 | } 94 | ConversionStrategy::Default(span, default_expr) => { 95 | // For default strategy, we try to parse as normal FromPest, 96 | // but if it fails (NoMatch), we use the default value 97 | quote_spanned! {span=> { 98 | // Try to parse using FromPest first 99 | match ::from_pest::FromPest::from_pest(inner) { 100 | Ok(value) => value, 101 | Err(::from_pest::ConversionError::NoMatch) => #default_expr, 102 | Err(e) => return Err(e), 103 | } 104 | }} 105 | } 106 | }; 107 | if let Member::Named(name) = member { 108 | quote!(#name : #conversion) 109 | } else { 110 | conversion 111 | } 112 | } 113 | } 114 | 115 | fn with_mods(stream: TokenStream, mods: Vec) -> TokenStream { 116 | mods.into_iter() 117 | .fold(stream, |stream, path| quote!(#path(#stream))) 118 | } 119 | 120 | pub fn enum_convert(name: &Path, variant: &Variant) -> Result { 121 | let fields = variant.fields.clone(); 122 | Ok(match fields { 123 | Fields::Named(fields) => { 124 | let fields: Vec<_> = fields 125 | .named 126 | .into_iter() 127 | .map(|field| { 128 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 129 | Ok(ConversionStrategy::from_attrs(attrs)? 130 | .apply(Member::Named(field.ident.unwrap()))) 131 | }) 132 | .collect::>()?; 133 | quote!(#name{#(#fields,)*}) 134 | } 135 | Fields::Unnamed(fields) => { 136 | let fields: Vec<_> = fields 137 | .unnamed 138 | .into_iter() 139 | .enumerate() 140 | .map(|(i, field)| { 141 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 142 | Ok(ConversionStrategy::from_attrs(attrs)? 143 | .apply(Member::Unnamed(Index::from(i)))) 144 | }) 145 | .collect::>()?; 146 | quote!(#name(#(#fields),*)) 147 | } 148 | Fields::Unit => { 149 | let attrs = FieldAttribute::from_attributes(variant.attrs.clone())?; 150 | let real_name = 151 | ConversionStrategy::from_attrs(attrs)?.apply(Member::Unnamed(Index::from(0))); 152 | quote!(#real_name) 153 | } 154 | }) 155 | } 156 | 157 | pub fn struct_convert(name: &Path, fields: Fields) -> Result { 158 | Ok(match fields { 159 | Fields::Named(fields) => { 160 | let fields: Vec<_> = fields 161 | .named 162 | .into_iter() 163 | .map(|field| { 164 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 165 | Ok(ConversionStrategy::from_attrs(attrs)? 166 | .apply(Member::Named(field.ident.unwrap()))) 167 | }) 168 | .collect::>()?; 169 | quote!(#name{#(#fields,)*}) 170 | } 171 | Fields::Unnamed(fields) => { 172 | let fields: Vec<_> = fields 173 | .unnamed 174 | .into_iter() 175 | .enumerate() 176 | .map(|(i, field)| { 177 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 178 | Ok(ConversionStrategy::from_attrs(attrs)? 179 | .apply(Member::Unnamed(Index::from(i)))) 180 | }) 181 | .collect::>()?; 182 | quote!(#name(#(#fields),*)) 183 | } 184 | Fields::Unit => { 185 | quote!(#name) 186 | } 187 | }) 188 | } 189 | -------------------------------------------------------------------------------- /derive/src/from_pest/mod.rs: -------------------------------------------------------------------------------- 1 | //! Machinery in charge of deriving `FromPest` for a type. 2 | //! 3 | //! Documentation in this module and submodules describes the requirement placed on _child_ 4 | //! functions. This is important as manipulation is done over untyped `TokenStream`. 5 | 6 | use { 7 | proc_macro2::TokenStream, 8 | std::path::PathBuf as FilePath, 9 | syn::{ 10 | parse::Error, parse::Result, spanned::Spanned, Data, DataEnum, DataStruct, DeriveInput, 11 | Ident, Path, 12 | }, 13 | }; 14 | 15 | use crate::attributes::DeriveAttribute; 16 | 17 | mod field; 18 | 19 | /// Creates implementation of `FromPest` for given derive input. 20 | /// 21 | /// For child functions, sets up an environment with: 22 | /// 23 | /// ```text 24 | /// type Self::Rule; 25 | /// type Self::FatalError = Void; 26 | /// let pest: &mut Pairs; 27 | /// ``` 28 | /// 29 | /// Child function is required to produce a number of statements that implement the semantics of 30 | /// `FromPest::from_pest`; that is: `Ok(Self)` => success, `pest` updated past the node; 31 | /// `Err(NoMatch)` => failure, `pest` is unchanged; `Err(Malformed)` impossible, panic instead. 32 | /// `?` and `return` may be used for early exit of failed matches. 33 | pub(crate) fn derive( 34 | DeriveInput { 35 | attrs, 36 | ident: name, 37 | generics, 38 | data, 39 | .. 40 | }: DeriveInput, 41 | ) -> Result { 42 | let attrs = DeriveAttribute::from_attributes(attrs)?; 43 | 44 | let grammar = { 45 | let mut grammar_attrs = attrs.iter().filter_map(|attr| match attr { 46 | DeriveAttribute::Grammar(attr) => Some(attr), 47 | _ => None, 48 | }); 49 | match (grammar_attrs.next(), grammar_attrs.next()) { 50 | (Some(_), Some(attr)) => Err(Error::new( 51 | attr.span(), 52 | "duplicate #[pest_ast(grammar)] not allowed", 53 | ))?, 54 | (None, None) => None, 55 | (Some(attr), None) => Some(FilePath::from(attr.lit.value())), 56 | _ => unreachable!(), 57 | } 58 | }; 59 | 60 | let (rule_enum, rule_variant) = { 61 | let mut rule_attrs = attrs.into_iter().filter_map(|attr| match attr { 62 | DeriveAttribute::Rule(attr) => Some(attr), 63 | _ => None, 64 | }); 65 | match (rule_attrs.next(), rule_attrs.next()) { 66 | (Some(_), Some(attr)) => Err(Error::new( 67 | attr.span(), 68 | "duplicate #[pest_ast(rule)] not allowed", 69 | ))?, 70 | (None, None) => Err(Error::new(name.span(), "#[pest_ast(rule)] required here"))?, 71 | (Some(attr), None) => (attr.path, attr.variant), 72 | _ => unreachable!(), 73 | } 74 | }; 75 | 76 | let (from_pest_lifetime, did_synthesize_lifetime) = generics 77 | .lifetimes() 78 | .next() 79 | .map(|def| (def.lifetime.clone(), false)) 80 | .unwrap_or_else(|| (parse_quote!('unique_lifetime_name), true)); 81 | 82 | let mut generics_ = generics.clone(); 83 | let (_, ty_generics, where_clause) = generics.split_for_impl(); 84 | if did_synthesize_lifetime { 85 | let lt = from_pest_lifetime.clone(); 86 | generics_.params.push(parse_quote!(#lt)); 87 | } 88 | let (impl_generics, _, _) = generics_.split_for_impl(); 89 | 90 | let implementation = match data { 91 | Data::Union(data) => Err(Error::new( 92 | data.union_token.span(), 93 | "Cannot derive FromPest for union", 94 | )), 95 | Data::Struct(data) => derive_for_struct(grammar, &name, &rule_enum, &rule_variant, data), 96 | Data::Enum(data) => derive_for_enum(grammar, &name, &rule_enum, &rule_variant, data), 97 | }?; 98 | 99 | Ok(quote! { 100 | impl #impl_generics ::from_pest::FromPest<#from_pest_lifetime> for #name #ty_generics #where_clause { 101 | type Rule = #rule_enum; 102 | type FatalError = ::from_pest::Void; 103 | 104 | fn from_pest( 105 | pest: &mut ::from_pest::pest::iterators::Pairs<#from_pest_lifetime, #rule_enum> 106 | ) -> ::std::result::Result> { 107 | #implementation 108 | } 109 | } 110 | }) 111 | } 112 | 113 | /// Implements `FromPest::from_pest` for some struct. 114 | /// 115 | /// For child functions, sets up an environment with: 116 | /// 117 | /// ```text 118 | /// let span: Span; // the span of this production 119 | /// let inner: &mut Pairs; // the contents of this production 120 | /// ``` 121 | /// 122 | /// Child function is required to produce an _expression_ which constructs an instance of `Self` 123 | /// from the `Pair`s in `inner` or early returns a `NoMatch`. `inner` and `span` are free working 124 | /// space, but `inner` should represent the point past all consumed productions afterwards. 125 | fn derive_for_struct( 126 | grammar: Option, 127 | name: &Ident, 128 | rule_enum: &Path, 129 | rule_variant: &Ident, 130 | DataStruct { fields, .. }: DataStruct, 131 | ) -> Result { 132 | if let Some(_path) = grammar { 133 | unimplemented!("Grammar introspection not implemented yet") 134 | } 135 | 136 | let construct = field::struct_convert(&parse_quote!(#name), fields)?; 137 | 138 | let extraneous = crate::trace( 139 | quote! { "when converting {}, found extraneous {:?}", stringify!(#name), inner}, 140 | ); 141 | 142 | Ok(quote! { 143 | let mut clone = pest.clone(); 144 | let pair = clone.next().ok_or(::from_pest::ConversionError::NoMatch)?; 145 | if pair.as_rule() == #rule_enum::#rule_variant { 146 | let span = pair.as_span(); 147 | let mut inner = pair.into_inner(); 148 | let inner = &mut inner; 149 | let this = #construct; 150 | if inner.clone().next().is_some() { 151 | #extraneous 152 | Err(::from_pest::ConversionError::Extraneous { 153 | current_node: stringify!(#name), 154 | })?; 155 | } 156 | *pest = clone; 157 | Ok(this) 158 | } else { 159 | Err(::from_pest::ConversionError::NoMatch) 160 | } 161 | }) 162 | } 163 | 164 | #[allow(unused)] 165 | #[allow(clippy::needless_pass_by_value)] 166 | fn derive_for_enum( 167 | grammar: Option, 168 | name: &Ident, 169 | rule_enum: &Path, 170 | rule_variant: &Ident, 171 | DataEnum { variants, .. }: DataEnum, 172 | ) -> Result { 173 | if let Some(_path) = grammar { 174 | unimplemented!("Grammar introspection not implemented yet") 175 | } 176 | 177 | let convert_variants: Vec = variants 178 | .into_iter() 179 | .map(|variant| { 180 | let variant_name = &variant.ident; 181 | let construct_variant = field::enum_convert(&parse_quote!(#name::#variant_name), &variant)?; 182 | let extraneous = crate::trace(quote! { 183 | "when converting {}, found extraneous {:?}", stringify!(#name), stringify!(#variant_name) 184 | }); 185 | 186 | Ok(quote! { 187 | let span = pair.as_span(); 188 | let mut inner = pair.clone().into_inner(); 189 | let inner = &mut inner; 190 | let this = #construct_variant; 191 | if inner.clone().next().is_some() { 192 | #extraneous 193 | Err(::from_pest::ConversionError::Extraneous { 194 | current_node: stringify!(#variant_name), 195 | })?; 196 | } 197 | Ok(this) 198 | }) 199 | }) 200 | .collect::>()?; 201 | 202 | Ok(quote! { 203 | let mut clone = pest.clone(); 204 | let pair = clone.next().ok_or(::from_pest::ConversionError::NoMatch)?; 205 | if pair.as_rule() == #rule_enum::#rule_variant { 206 | let this = Err(::from_pest::ConversionError::NoMatch) 207 | #(.or_else(|_: ::from_pest::ConversionError<::from_pest::Void>| { 208 | #convert_variants 209 | }))*?; 210 | *pest = clone; 211 | Ok(this) 212 | } else { 213 | Err(::from_pest::ConversionError::NoMatch) 214 | } 215 | }) 216 | } 217 | -------------------------------------------------------------------------------- /derive/examples/defaults_showcase.rs: -------------------------------------------------------------------------------- 1 | #![allow( 2 | bad_style, 3 | dead_code, 4 | clippy::clone_on_copy, 5 | clippy::upper_case_acronyms 6 | )] 7 | 8 | #[macro_use] 9 | extern crate pest_derive; 10 | extern crate from_pest; 11 | #[macro_use] 12 | extern crate pest_ast; 13 | extern crate pest; 14 | 15 | use from_pest::FromPest; 16 | use pest::Parser; 17 | 18 | #[derive(Parser)] 19 | #[grammar = "../examples/defaults_showcase.pest"] 20 | pub struct ShowcaseParser; 21 | 22 | // Define enum types that can have defaults 23 | #[derive(Debug, Clone, PartialEq)] 24 | pub enum Type { 25 | Int, 26 | String, 27 | Bool, 28 | Void, 29 | } 30 | 31 | impl<'pest> FromPest<'pest> for Type { 32 | type Rule = Rule; 33 | type FatalError = from_pest::Void; 34 | 35 | fn from_pest( 36 | pest: &mut pest::iterators::Pairs<'pest, Rule>, 37 | ) -> Result> { 38 | let pair = pest.next().ok_or(from_pest::ConversionError::NoMatch)?; 39 | if pair.as_rule() == Rule::type_name { 40 | match pair.as_str() { 41 | "int" => Ok(Type::Int), 42 | "string" => Ok(Type::String), 43 | "bool" => Ok(Type::Bool), 44 | "void" => Ok(Type::Void), 45 | _ => Err(from_pest::ConversionError::NoMatch), 46 | } 47 | } else { 48 | Err(from_pest::ConversionError::NoMatch) 49 | } 50 | } 51 | } 52 | 53 | #[derive(Debug, Clone, PartialEq)] 54 | pub enum Expr { 55 | Number(i32), 56 | String(String), 57 | Id(String), 58 | } 59 | 60 | impl<'pest> FromPest<'pest> for Expr { 61 | type Rule = Rule; 62 | type FatalError = from_pest::Void; 63 | 64 | fn from_pest( 65 | pest: &mut pest::iterators::Pairs<'pest, Rule>, 66 | ) -> Result> { 67 | let pair = pest.next().ok_or(from_pest::ConversionError::NoMatch)?; 68 | match pair.as_rule() { 69 | Rule::expr => { 70 | // The expr rule contains nested rules, so we need to look at its inner content 71 | let mut inner = pair.into_inner(); 72 | let inner_pair = inner.next().ok_or(from_pest::ConversionError::NoMatch)?; 73 | match inner_pair.as_rule() { 74 | Rule::number => Ok(Expr::Number(inner_pair.as_str().parse().unwrap())), 75 | Rule::string => { 76 | let s = inner_pair.as_str(); 77 | Ok(Expr::String(s[1..s.len() - 1].to_string())) // Remove quotes 78 | } 79 | Rule::id => Ok(Expr::Id(inner_pair.as_str().to_string())), 80 | _ => Err(from_pest::ConversionError::NoMatch), 81 | } 82 | } 83 | Rule::number => Ok(Expr::Number(pair.as_str().parse().unwrap())), 84 | Rule::string => { 85 | let s = pair.as_str(); 86 | Ok(Expr::String(s[1..s.len() - 1].to_string())) // Remove quotes 87 | } 88 | Rule::id => Ok(Expr::Id(pair.as_str().to_string())), 89 | _ => Err(from_pest::ConversionError::NoMatch), 90 | } 91 | } 92 | } 93 | 94 | #[derive(Debug, Clone, PartialEq)] 95 | pub enum VarKind { 96 | Let, 97 | Const, 98 | } 99 | 100 | impl<'pest> FromPest<'pest> for VarKind { 101 | type Rule = Rule; 102 | type FatalError = from_pest::Void; 103 | 104 | fn from_pest( 105 | pest: &mut pest::iterators::Pairs<'pest, Rule>, 106 | ) -> Result> { 107 | let pair = pest.next().ok_or(from_pest::ConversionError::NoMatch)?; 108 | if pair.as_rule() == Rule::var_kind { 109 | match pair.as_str() { 110 | "let" => Ok(VarKind::Let), 111 | "const" => Ok(VarKind::Const), 112 | _ => Err(from_pest::ConversionError::NoMatch), 113 | } 114 | } else { 115 | Err(from_pest::ConversionError::NoMatch) 116 | } 117 | } 118 | } 119 | 120 | #[derive(FromPest, Debug, Clone, PartialEq)] 121 | #[pest_ast(rule(Rule::id))] 122 | pub struct Id<'pest> { 123 | #[pest_ast(outer())] 124 | pub span: pest::Span<'pest>, 125 | } 126 | 127 | impl<'pest> Id<'pest> { 128 | pub fn name(&self) -> &str { 129 | self.span.as_str() 130 | } 131 | } 132 | 133 | #[derive(FromPest, Debug, Clone, PartialEq)] 134 | #[pest_ast(rule(Rule::type_annotation))] 135 | pub struct TypeAnnotation { 136 | pub type_name: Type, 137 | } 138 | 139 | #[derive(FromPest, Debug, Clone, PartialEq)] 140 | #[pest_ast(rule(Rule::initializer))] 141 | pub struct Initializer { 142 | pub expr: Expr, 143 | } 144 | 145 | // Variable declaration showcasing multiple default values 146 | #[derive(FromPest, Debug, Clone, PartialEq)] 147 | #[pest_ast(rule(Rule::var_decl))] 148 | pub struct VarDecl<'pest> { 149 | // Now this should parse correctly from the var_kind rule 150 | #[pest_ast(default(VarKind::Let))] 151 | pub kind: VarKind, 152 | 153 | pub id: Id<'pest>, 154 | 155 | // Type annotation defaults to 'void' if not specified 156 | #[pest_ast(default(TypeAnnotation { type_name: Type::Void }))] 157 | pub type_annotation: TypeAnnotation, 158 | 159 | // Initialization defaults to a placeholder value 160 | #[pest_ast(default(Initializer { expr: Expr::Number(0) }))] 161 | pub initializer: Initializer, 162 | } 163 | 164 | #[derive(FromPest, Debug, Clone, PartialEq)] 165 | #[pest_ast(rule(Rule::program))] 166 | pub struct Program<'pest> { 167 | pub declarations: Vec>, 168 | } 169 | 170 | fn main() -> Result<(), Box> { 171 | println!("=== Default Values Showcase ===\n"); 172 | 173 | // Test 1: Minimal declaration (all defaults) 174 | let input1 = "let x;"; 175 | println!("Input 1: {input1}"); 176 | let pairs1 = ShowcaseParser::parse(Rule::program, input1)?; 177 | let program1: Program = Program::from_pest(&mut pairs1.clone())?; 178 | println!("Parsed: {program1:#?}\n"); 179 | 180 | // Test 2: With type annotation 181 | let input2 = "let y: int;"; 182 | println!("Input 2: {input2}"); 183 | let pairs2 = ShowcaseParser::parse(Rule::program, input2)?; 184 | let program2: Program = Program::from_pest(&mut pairs2.clone())?; 185 | println!("Parsed: {program2:#?}\n"); 186 | 187 | // Test 3: With initialization 188 | let input3 = "let z = 42;"; 189 | println!("Input 3: {input3}"); 190 | let pairs3 = ShowcaseParser::parse(Rule::program, input3)?; 191 | let program3: Program = Program::from_pest(&mut pairs3.clone())?; 192 | println!("Parsed: {program3:#?}\n"); 193 | 194 | // Test 4: Fully specified 195 | let input4 = "const w: string = \"hello\";"; 196 | println!("Input 4: {input4}"); 197 | let pairs4 = ShowcaseParser::parse(Rule::program, input4)?; 198 | let program4: Program = Program::from_pest(&mut pairs4.clone())?; 199 | println!("Parsed: {program4:#?}\n"); 200 | 201 | Ok(()) 202 | } 203 | 204 | #[cfg(test)] 205 | mod tests { 206 | use super::*; 207 | 208 | #[test] 209 | fn test_defaults_applied() { 210 | let input = "let x;"; 211 | let pairs = ShowcaseParser::parse(Rule::program, input).unwrap(); 212 | let program: Program = Program::from_pest(&mut pairs.clone()).unwrap(); 213 | 214 | assert_eq!(program.declarations.len(), 1); 215 | let decl = &program.declarations[0]; 216 | 217 | // All defaults should be applied 218 | assert_eq!(decl.kind, VarKind::Let); 219 | assert_eq!(decl.type_annotation.type_name, Type::Void); 220 | assert_eq!(decl.initializer.expr, Expr::Number(0)); 221 | assert_eq!(decl.id.name(), "x"); 222 | } 223 | 224 | #[test] 225 | fn test_explicit_values_override_defaults() { 226 | let input = "const y: int = 42;"; 227 | let pairs = ShowcaseParser::parse(Rule::program, input).unwrap(); 228 | let program: Program = Program::from_pest(&mut pairs.clone()).unwrap(); 229 | 230 | assert_eq!(program.declarations.len(), 1); 231 | let decl = &program.declarations[0]; 232 | 233 | // Explicit values should override defaults 234 | assert_eq!(decl.kind, VarKind::Const); // Now should be correctly parsed 235 | assert_eq!(decl.type_annotation.type_name, Type::Int); // Explicit 236 | assert_eq!(decl.initializer.expr, Expr::Number(42)); // Explicit 237 | assert_eq!(decl.id.name(), "y"); 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /derive/README.md: -------------------------------------------------------------------------------- 1 | # pest-ast 2 | 3 | Convert from [pest](https://pest.rs) parse trees into typed syntax trees with ease! 4 | 5 | Which would you rather have? 6 | 7 |
Pest Parse Tree 8 | 9 | ``` 10 | [ 11 | Pair { 12 | rule: file, 13 | span: Span { 14 | str: "65279,1179403647,1463895090\n3.1415927,2.7182817,1.618034\n-40,-273.15\n13,42\n65537\n", 15 | start: 0, 16 | end: 81 17 | }, 18 | inner: [ 19 | Pair { 20 | rule: record, 21 | span: Span { 22 | str: "65279,1179403647,1463895090", 23 | start: 0, 24 | end: 27 25 | }, 26 | inner: [ 27 | Pair { 28 | rule: field, 29 | span: Span { 30 | str: "65279", 31 | start: 0, 32 | end: 5 33 | }, 34 | inner: [] 35 | }, 36 | Pair { 37 | rule: field, 38 | span: Span { 39 | str: "1179403647", 40 | start: 6, 41 | end: 16 42 | }, 43 | inner: [] 44 | }, 45 | Pair { 46 | rule: field, 47 | span: Span { 48 | str: "1463895090", 49 | start: 17, 50 | end: 27 51 | }, 52 | inner: [] 53 | } 54 | ] 55 | }, 56 | Pair { 57 | rule: record, 58 | span: Span { 59 | str: "3.1415927,2.7182817,1.618034", 60 | start: 28, 61 | end: 56 62 | }, 63 | inner: [ 64 | Pair { 65 | rule: field, 66 | span: Span { 67 | str: "3.1415927", 68 | start: 28, 69 | end: 37 70 | }, 71 | inner: [] 72 | }, 73 | Pair { 74 | rule: field, 75 | span: Span { 76 | str: "2.7182817", 77 | start: 38, 78 | end: 47 79 | }, 80 | inner: [] 81 | }, 82 | Pair { 83 | rule: field, 84 | span: Span { 85 | str: "1.618034", 86 | start: 48, 87 | end: 56 88 | }, 89 | inner: [] 90 | } 91 | ] 92 | }, 93 | Pair { 94 | rule: record, 95 | span: Span { 96 | str: "-40,-273.15", 97 | start: 57, 98 | end: 68 99 | }, 100 | inner: [ 101 | Pair { 102 | rule: field, 103 | span: Span { 104 | str: "-40", 105 | start: 57, 106 | end: 60 107 | }, 108 | inner: [] 109 | }, 110 | Pair { 111 | rule: field, 112 | span: Span { 113 | str: "-273.15", 114 | start: 61, 115 | end: 68 116 | }, 117 | inner: [] 118 | } 119 | ] 120 | }, 121 | Pair { 122 | rule: record, 123 | span: Span { 124 | str: "13,42", 125 | start: 69, 126 | end: 74 127 | }, 128 | inner: [ 129 | Pair { 130 | rule: field, 131 | span: Span { 132 | str: "13", 133 | start: 69, 134 | end: 71 135 | }, 136 | inner: [] 137 | }, 138 | Pair { 139 | rule: field, 140 | span: Span { 141 | str: "42", 142 | start: 72, 143 | end: 74 144 | }, 145 | inner: [] 146 | } 147 | ] 148 | }, 149 | Pair { 150 | rule: record, 151 | span: Span { 152 | str: "65537", 153 | start: 75, 154 | end: 80 155 | }, 156 | inner: [ 157 | Pair { 158 | rule: field, 159 | span: Span { 160 | str: "65537", 161 | start: 75, 162 | end: 80 163 | }, 164 | inner: [] 165 | } 166 | ] 167 | }, 168 | Pair { 169 | rule: EOI, 170 | span: Span { 171 | str: "", 172 | start: 81, 173 | end: 81 174 | }, 175 | inner: [] 176 | } 177 | ] 178 | } 179 | ] 180 | ``` 181 |
182 |
Typed Syntax Tree 183 | 184 | ``` 185 | File { 186 | records: [ 187 | Record { 188 | fields: [ 189 | Field { 190 | value: 65279.0 191 | }, 192 | Field { 193 | value: 1179403647.0 194 | }, 195 | Field { 196 | value: 1463895090.0 197 | } 198 | ] 199 | }, 200 | Record { 201 | fields: [ 202 | Field { 203 | value: 3.1415927 204 | }, 205 | Field { 206 | value: 2.7182817 207 | }, 208 | Field { 209 | value: 1.618034 210 | } 211 | ] 212 | }, 213 | Record { 214 | fields: [ 215 | Field { 216 | value: -40.0 217 | }, 218 | Field { 219 | value: -273.15 220 | } 221 | ] 222 | }, 223 | Record { 224 | fields: [ 225 | Field { 226 | value: 13.0 227 | }, 228 | Field { 229 | value: 42.0 230 | } 231 | ] 232 | }, 233 | Record { 234 | fields: [ 235 | Field { 236 | value: 65537.0 237 | } 238 | ] 239 | } 240 | ], 241 | eoi: EOI 242 | } 243 | ``` 244 |
245 | 246 | ----- 247 | 248 | The above parse tree is produced by the following pest grammar: 249 | 250 | ```pest 251 | field = { (ASCII_DIGIT | "." | "-")+ } 252 | record = { field ~ ("," ~ field)* } 253 | file = { SOI ~ (record ~ ("\r\n" | "\n"))* ~ EOI } 254 | ``` 255 | 256 | parsing this csv: 257 | 258 | ```csv 259 | 65279,1179403647,1463895090 260 | 3.1415927,2.7182817,1.618034 261 | -40,-273.15 262 | 13,42 263 | 65537 264 | ``` 265 | 266 | And converting it to a typed syntax tree is as simple as the following code: 267 | 268 | ```rust 269 | mod ast { 270 | use super::csv::Rule; 271 | use pest::Span; 272 | 273 | fn span_into_str(span: Span) -> &str { 274 | span.as_str() 275 | } 276 | 277 | #[derive(Debug, FromPest)] 278 | #[pest_ast(rule(Rule::field))] 279 | pub struct Field { 280 | #[pest_ast(outer(with(span_into_str), with(str::parse), with(Result::unwrap)))] 281 | pub value: f64, 282 | } 283 | 284 | #[derive(Debug, FromPest)] 285 | #[pest_ast(rule(Rule::record))] 286 | pub struct Record { 287 | pub fields: Vec, 288 | } 289 | 290 | #[derive(Debug, FromPest)] 291 | #[pest_ast(rule(Rule::file))] 292 | pub struct File { 293 | pub records: Vec, 294 | eoi: EOI, 295 | } 296 | 297 | #[derive(Debug, FromPest)] 298 | #[pest_ast(rule(Rule::EOI))] 299 | struct EOI; 300 | } 301 | ``` 302 | 303 | And doing the actual parse is as simple as 304 | 305 | ```rust 306 | let mut parse_tree = csv::Parser::parse(csv::Rule::file, &source)?; 307 | let syntax_tree = File::from_pest(&mut parse_tree).expect("infallible"); 308 | ``` 309 | 310 | ## Default Values for Optional Rules 311 | 312 | A powerful feature for handling optional grammar rules without requiring `Option` in your AST is the `#[pest_ast(default(...))]` attribute. This allows you to specify default values that will be used when optional rules are not present in the input. 313 | 314 | ### The Problem 315 | 316 | When using optional rules in Pest grammar, you typically need `Option` in your AST: 317 | 318 | ```rust 319 | // Grammar: function = { "fn" ~ id ~ ("->" ~ type)? ~ "{" ~ "}" } 320 | 321 | #[derive(FromPest, Debug)] 322 | #[pest_ast(rule(Rule::function))] 323 | pub struct Function { 324 | pub name: String, 325 | pub return_type: Option, // Optional field 326 | } 327 | ``` 328 | 329 | ### The Solution 330 | 331 | With the `default` attribute, you can eliminate `Option` and specify a default value: 332 | 333 | ```rust 334 | #[derive(FromPest, Debug)] 335 | #[pest_ast(rule(Rule::function))] 336 | pub struct Function { 337 | pub name: String, 338 | 339 | #[pest_ast(default(Type::Void))] // Specify default value 340 | pub return_type: Type, // No Option needed! 341 | } 342 | ``` 343 | 344 | ### Usage Examples 345 | 346 | ```rust 347 | // Simple defaults 348 | #[pest_ast(default(Type::Void))] 349 | pub return_type: Type, 350 | 351 | // Complex defaults with expressions 352 | #[pest_ast(default(Vec::new()))] 353 | pub parameters: Vec, 354 | 355 | #[pest_ast(default({ 356 | Config { 357 | debug: false, 358 | optimization_level: 2, 359 | } 360 | }))] 361 | pub config: Config, 362 | ``` 363 | 364 | ### How It Works 365 | 366 | The `default` attribute generates code that: 367 | 1. First tries to parse the field normally using `FromPest` 368 | 2. If conversion fails with `NoMatch` (optional rule not present), uses the default value 369 | 3. If parsing fails with other errors, propagates the error 370 | 371 | This provides a clean, type-safe way to handle optional grammar elements while keeping your AST representation simple and avoiding the complexity of `Option` handling. 372 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /derive/src/attributes.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::mixed_read_write_in_expression)] // syn patterns 2 | 3 | use proc_macro2::TokenTree; 4 | 5 | use { 6 | itertools::Itertools, 7 | proc_macro2::TokenStream, 8 | quote::ToTokens, 9 | syn::{ 10 | parse::{Error, Parse, ParseStream, Parser, Result}, 11 | punctuated::Punctuated, 12 | spanned::Spanned, 13 | token::Paren, 14 | Attribute, Ident, LitStr, Path, 15 | }, 16 | }; 17 | 18 | mod kw { 19 | custom_keyword!(grammar); 20 | custom_keyword!(outer); 21 | custom_keyword!(inner); 22 | custom_keyword!(with); 23 | custom_keyword!(rule); 24 | custom_keyword!(default); 25 | } 26 | 27 | /// `#[pest_ast(..)]` for the outer `#[derive(FromPest)]` 28 | #[derive(Debug)] 29 | pub(crate) enum DeriveAttribute { 30 | /// `grammar = "grammar.rs"` 31 | Grammar(GrammarAttribute), 32 | /// `rule(path::to)` 33 | Rule(RuleAttribute), 34 | } 35 | 36 | #[derive(Debug)] 37 | /// `#[pest_ast(..)]` for fields in `#[derive(FromPest)]` 38 | pub(crate) enum FieldAttribute { 39 | /// `outer(with(path::to),*)` 40 | Outer(OuterAttribute), 41 | /// `inner(rule(path::to), with(path::to),*)` 42 | Inner(InnerAttribute), 43 | /// `default(expr)` 44 | Default(DefaultAttribute), 45 | } 46 | 47 | #[derive(Debug)] 48 | pub(crate) struct GrammarAttribute { 49 | pub(crate) grammar: kw::grammar, 50 | pub(crate) eq: Token![=], 51 | pub(crate) lit: LitStr, 52 | } 53 | 54 | #[derive(Debug)] 55 | pub(crate) struct OuterAttribute { 56 | pub(crate) outer: kw::outer, 57 | pub(crate) paren: Paren, 58 | pub(crate) with: Punctuated, 59 | } 60 | 61 | #[derive(Debug)] 62 | pub(crate) struct InnerAttribute { 63 | pub(crate) inner: kw::inner, 64 | pub(crate) paren: Paren, 65 | pub(crate) rule: Option, 66 | pub(crate) comma: Option, 67 | pub(crate) with: Punctuated, 68 | } 69 | 70 | #[derive(Debug)] 71 | pub(crate) struct WithAttribute { 72 | pub(crate) with: kw::with, 73 | pub(crate) paren: Paren, 74 | pub(crate) path: Path, 75 | } 76 | 77 | #[derive(Debug)] 78 | pub(crate) struct RuleAttribute { 79 | pub(crate) rule: kw::rule, 80 | pub(crate) paren: Paren, 81 | pub(crate) path: Path, 82 | pub(crate) sep: Token![::], 83 | pub(crate) variant: Ident, 84 | } 85 | 86 | #[derive(Debug)] 87 | pub(crate) struct DefaultAttribute { 88 | pub(crate) default: kw::default, 89 | pub(crate) paren: Paren, 90 | pub(crate) expr: syn::Expr, 91 | } 92 | 93 | impl DeriveAttribute { 94 | pub(crate) fn from_attributes(attrs: impl IntoIterator) -> Result> { 95 | attrs 96 | .into_iter() 97 | .map(DeriveAttribute::from_attribute) 98 | .fold_ok(vec![], |mut acc, t| { 99 | acc.extend(t); 100 | acc 101 | }) 102 | } 103 | 104 | pub(crate) fn from_attribute(attr: Attribute) -> Result> { 105 | if attr.path() != &parse_quote!(pest_ast) { 106 | return Ok(vec![]); 107 | } 108 | 109 | Parser::parse2( 110 | |input: ParseStream| { 111 | let content; 112 | input.parse::()?; 113 | bracketed!(content in input); 114 | content.step(|cursor| { 115 | if let Some((tt, next)) = cursor.token_tree() { 116 | match tt { 117 | TokenTree::Ident(id) if id == "pest_ast" => Ok(((), next)), 118 | _ => Err(cursor.error("expected `pest_ast`")), 119 | } 120 | } else { 121 | Err(cursor.error("expected `pest_ast`")) 122 | } 123 | })?; 124 | let content2; 125 | parenthesized!(content2 in content); 126 | let punctuated: Punctuated<_, Token![,]> = 127 | content2.parse_terminated(Parse::parse, Token![,])?; 128 | Ok(punctuated.into_iter().collect_vec()) 129 | }, 130 | attr.to_token_stream(), 131 | ) 132 | } 133 | } 134 | 135 | impl FieldAttribute { 136 | pub(crate) fn from_attributes(attrs: impl IntoIterator) -> Result> { 137 | attrs 138 | .into_iter() 139 | .map(FieldAttribute::from_attribute) 140 | .fold_ok(vec![], |mut acc, t| { 141 | acc.extend(t); 142 | acc 143 | }) 144 | } 145 | 146 | pub(crate) fn from_attribute(attr: Attribute) -> Result> { 147 | if attr.path() != &parse_quote!(pest_ast) { 148 | return Ok(vec![]); 149 | } 150 | 151 | Parser::parse2( 152 | |input: ParseStream| { 153 | input.parse::()?; 154 | let content; 155 | bracketed!(content in input); 156 | content.step(|cursor| { 157 | if let Some((tt, next)) = cursor.token_tree() { 158 | match tt { 159 | TokenTree::Ident(id) if id == "pest_ast" => Ok(((), next)), 160 | _ => Err(cursor.error("expected `pest_ast`")), 161 | } 162 | } else { 163 | Err(cursor.error("expected `pest_ast`")) 164 | } 165 | })?; 166 | let content2; 167 | parenthesized!(content2 in content); 168 | let punctuated: Punctuated<_, Token![,]> = 169 | content2.parse_terminated(Parse::parse, Token![,])?; 170 | Ok(punctuated.into_iter().collect_vec()) 171 | }, 172 | attr.to_token_stream(), 173 | ) 174 | } 175 | } 176 | 177 | impl Parse for DeriveAttribute { 178 | fn parse(input: ParseStream) -> Result { 179 | let lookahead = input.lookahead1(); 180 | if lookahead.peek(kw::grammar) { 181 | GrammarAttribute::parse(input).map(DeriveAttribute::Grammar) 182 | } else if lookahead.peek(kw::rule) { 183 | RuleAttribute::parse(input).map(DeriveAttribute::Rule) 184 | } else { 185 | Err(lookahead.error()) 186 | } 187 | } 188 | } 189 | 190 | impl Parse for FieldAttribute { 191 | fn parse(input: ParseStream) -> Result { 192 | let lookahead = input.lookahead1(); 193 | if lookahead.peek(kw::outer) { 194 | OuterAttribute::parse(input).map(FieldAttribute::Outer) 195 | } else if lookahead.peek(kw::inner) { 196 | InnerAttribute::parse(input).map(FieldAttribute::Inner) 197 | } else if lookahead.peek(kw::default) { 198 | DefaultAttribute::parse(input).map(FieldAttribute::Default) 199 | } else { 200 | Err(lookahead.error()) 201 | } 202 | } 203 | } 204 | 205 | impl ToTokens for DeriveAttribute { 206 | fn to_tokens(&self, tokens: &mut TokenStream) { 207 | match self { 208 | DeriveAttribute::Grammar(attr) => attr.to_tokens(tokens), 209 | DeriveAttribute::Rule(attr) => attr.to_tokens(tokens), 210 | } 211 | } 212 | } 213 | 214 | impl ToTokens for FieldAttribute { 215 | fn to_tokens(&self, tokens: &mut TokenStream) { 216 | match self { 217 | FieldAttribute::Outer(attr) => attr.to_tokens(tokens), 218 | FieldAttribute::Inner(attr) => attr.to_tokens(tokens), 219 | FieldAttribute::Default(attr) => attr.to_tokens(tokens), 220 | } 221 | } 222 | } 223 | 224 | impl Parse for GrammarAttribute { 225 | fn parse(input: ParseStream) -> Result { 226 | Ok(GrammarAttribute { 227 | grammar: input.parse()?, 228 | eq: input.parse()?, 229 | lit: input.parse()?, 230 | }) 231 | } 232 | } 233 | 234 | impl ToTokens for GrammarAttribute { 235 | fn to_tokens(&self, tokens: &mut TokenStream) { 236 | self.grammar.to_tokens(tokens); 237 | self.eq.to_tokens(tokens); 238 | self.lit.to_tokens(tokens); 239 | } 240 | } 241 | 242 | impl Parse for OuterAttribute { 243 | fn parse(input: ParseStream) -> Result { 244 | let content; 245 | Ok(OuterAttribute { 246 | outer: input.parse()?, 247 | paren: parenthesized!(content in input), 248 | with: content.parse_terminated(WithAttribute::parse, Token![,])?, 249 | }) 250 | } 251 | } 252 | 253 | impl ToTokens for OuterAttribute { 254 | fn to_tokens(&self, tokens: &mut TokenStream) { 255 | self.outer.to_tokens(tokens); 256 | self.paren.surround(tokens, |tokens| { 257 | self.with.to_tokens(tokens); 258 | }) 259 | } 260 | } 261 | 262 | impl Parse for InnerAttribute { 263 | fn parse(input: ParseStream) -> Result { 264 | let content; 265 | let inner = input.parse()?; 266 | let paren = parenthesized!(content in input); 267 | let (rule, comma) = if content.peek(kw::rule) { 268 | (Some(content.parse()?), content.parse().ok()) 269 | } else { 270 | (None, None) 271 | }; 272 | let with = content.parse_terminated(WithAttribute::parse, Token![,])?; 273 | Ok(InnerAttribute { 274 | inner, 275 | paren, 276 | rule, 277 | comma, 278 | with, 279 | }) 280 | } 281 | } 282 | 283 | impl ToTokens for InnerAttribute { 284 | fn to_tokens(&self, tokens: &mut TokenStream) { 285 | self.inner.to_tokens(tokens); 286 | self.paren.surround(tokens, |tokens| { 287 | if let Some(rule) = &self.rule { 288 | rule.to_tokens(tokens); 289 | } 290 | if let Some(comma) = &self.comma { 291 | comma.to_tokens(tokens); 292 | } 293 | self.with.to_tokens(tokens); 294 | }) 295 | } 296 | } 297 | 298 | impl Parse for WithAttribute { 299 | fn parse(input: ParseStream) -> Result { 300 | let content; 301 | Ok(WithAttribute { 302 | with: input.parse()?, 303 | paren: parenthesized!(content in input), 304 | path: content.parse()?, 305 | }) 306 | } 307 | } 308 | 309 | impl ToTokens for WithAttribute { 310 | fn to_tokens(&self, tokens: &mut TokenStream) { 311 | self.with.to_tokens(tokens); 312 | self.paren 313 | .surround(tokens, |tokens| self.path.to_tokens(tokens)); 314 | } 315 | } 316 | 317 | impl Parse for RuleAttribute { 318 | fn parse(input: ParseStream) -> Result { 319 | let content; 320 | let rule = input.parse()?; 321 | let paren = parenthesized!(content in input); 322 | let mut path: Path = content.parse()?; 323 | let (variant, _) = path.segments.pop().unwrap().into_tuple(); 324 | let sep = if path.segments.trailing_punct() { 325 | // fix trailing punct 326 | let (head, sep) = path.segments.pop().unwrap().into_tuple(); 327 | path.segments.push(head); 328 | sep.unwrap() 329 | } else { 330 | Err(Error::new( 331 | path.span(), 332 | "must be a path to enum variant (both enum and variant)", 333 | ))? 334 | }; 335 | if variant.arguments.is_empty() { 336 | Ok(RuleAttribute { 337 | rule, 338 | paren, 339 | path, 340 | sep, 341 | variant: variant.ident, 342 | }) 343 | } else { 344 | Err(Error::new(path.span(), "must be a path to enum variant")) 345 | } 346 | } 347 | } 348 | 349 | impl ToTokens for RuleAttribute { 350 | fn to_tokens(&self, tokens: &mut TokenStream) { 351 | self.rule.to_tokens(tokens); 352 | self.paren.surround(tokens, |tokens| { 353 | self.path.to_tokens(tokens); 354 | self.sep.to_tokens(tokens); 355 | self.variant.to_tokens(tokens); 356 | }); 357 | } 358 | } 359 | 360 | impl Parse for DefaultAttribute { 361 | fn parse(input: ParseStream) -> Result { 362 | let content; 363 | Ok(DefaultAttribute { 364 | default: input.parse()?, 365 | paren: parenthesized!(content in input), 366 | expr: content.parse()?, 367 | }) 368 | } 369 | } 370 | 371 | impl ToTokens for DefaultAttribute { 372 | fn to_tokens(&self, tokens: &mut TokenStream) { 373 | self.default.to_tokens(tokens); 374 | self.paren.surround(tokens, |tokens| { 375 | self.expr.to_tokens(tokens); 376 | }); 377 | } 378 | } 379 | --------------------------------------------------------------------------------