├── .github └── workflows │ └── test.yml ├── .gitignore ├── .idea ├── misc.xml ├── modules.xml └── vcs.xml ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── SECURITY.md ├── derive ├── Cargo.toml ├── README.md ├── examples │ ├── simple_enum_derives.pest │ ├── simple_enum_derives.rs │ ├── simple_struct_derives.pest │ └── simple_struct_derives.rs └── src │ ├── attributes.rs │ ├── from_pest │ ├── field.rs │ └── mod.rs │ └── lib.rs ├── examples ├── csv.csv ├── csv.pest └── csv.rs ├── pest-deconstruct.iml ├── src └── lib.rs └── tests └── csv.rs /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | name: cargo test and examples 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: dtolnay/rust-toolchain@stable 11 | - run: cargo test --all --all-features 12 | - run: cargo run --example csv 13 | - run: cd derive && cargo run --example simple_enum_derives && cargo run --example simple_struct_derives 14 | fmt: 15 | name: cargo fmt 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - uses: dtolnay/rust-toolchain@stable 20 | with: 21 | components: rustfmt 22 | - run: cargo fmt --all -- --check 23 | lint: 24 | name: cargo clippy 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: dtolnay/rust-toolchain@stable 29 | with: 30 | components: clippy 31 | - run: cargo clippy --all --all-features --all-targets -- -Dwarnings -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### JetBrains template 2 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 3 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 4 | 5 | # User-specific stuff 6 | .idea/**/workspace.xml 7 | .idea/**/tasks.xml 8 | .idea/**/usage.statistics.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # Sensitive or high-churn files 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | .idea/**/dbnavigator.xml 20 | 21 | # Gradle 22 | .idea/**/gradle.xml 23 | .idea/**/libraries 24 | 25 | # Gradle and Maven with auto-import 26 | # When using Gradle or Maven with auto-import, you should exclude module files, 27 | # since they will be recreated, and may cause churn. Uncomment if using 28 | # auto-import. 29 | # .idea/modules.xml 30 | # .idea/*.iml 31 | # .idea/modules 32 | 33 | # CMake 34 | cmake-build-*/ 35 | 36 | # Mongo Explorer plugin 37 | .idea/**/mongoSettings.xml 38 | 39 | # File-based project format 40 | *.iws 41 | 42 | # IntelliJ 43 | out/ 44 | 45 | # mpeltonen/sbt-idea plugin 46 | .idea_modules/ 47 | 48 | # JIRA plugin 49 | atlassian-ide-plugin.xml 50 | 51 | # Cursive Clojure plugin 52 | .idea/replstate.xml 53 | 54 | # Crashlytics plugin (for Android Studio and IntelliJ) 55 | com_crashlytics_export_strings.xml 56 | crashlytics.properties 57 | crashlytics-build.properties 58 | fabric.properties 59 | 60 | # Editor-based Rest Client 61 | .idea/httpRequests 62 | ### Rust template 63 | # Generated by Cargo 64 | # will have compiled files and executables 65 | /target/ 66 | 67 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 68 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 69 | Cargo.lock 70 | 71 | # These are backup files generated by rustfmt 72 | **/*.rs.bk 73 | 74 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 12 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["derive"] 3 | 4 | [package] 5 | name = "from-pest" 6 | version = "0.3.3" 7 | edition = "2021" 8 | authors = ["cad97 "] 9 | readme = "./README.md" 10 | description = "Convert from a pest grammar to a typed AST" 11 | license = "MIT/Apache-2.0" 12 | repository = "https://github.com/pest-parser/pest_deconstruct" 13 | 14 | [dependencies] 15 | void = "1.0" 16 | pest = "2.5" 17 | log = "0.4.6" 18 | 19 | [dev-dependencies] 20 | pest_derive = "2.5" 21 | pest-ast = { version = "0.3", path = "derive" } 22 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pest-ast 2 | 3 | This is an in-development add-on to the pest parsing library. 4 | 5 | Pest-ast provides a structured manner to go from the "dynamically typed" Pest Parse Tree 6 | to a strongly typed (Abstract) Syntax Tree, as well as a derive to do so automatically. 7 | In the future, it's planned to optionally additionally check the source grammar to statically 8 | prevent issues that are currently detected at runtime. 9 | 10 | In the _future_ 🦄, pest-ast may provide a way of defining grammar directly on the AST nodes. 11 | 12 | ## Note: 13 | 14 | This crate is actually `from-pest`, which provides the trait framework for the said conversion. 15 | [`pest-ast`](./derive/README.md) provides the actual derive for the conversion. 16 | 17 | This README is the root of the repository for legacy reasons. This will be corrected in a future reorganization. 18 | 19 | ## Contributing 20 | 21 | Check out the [issue tracker](https://github.com/pest-parser/ast); 22 | we try to keep it stocked with [good-first-issue](https://github.com/pest-parser/ast/labels/good%20first%20issue) 23 | and [help-wanted](https://github.com/pest-parser/ast/issues?q=is%3Aopen+label%3A%22help+wanted%22) opportunities. 24 | If you have questions, don't be afraid to @ the author (CAD97) 25 | [on Gitter](https://gitter.im/pest-parser/pest) or [on Discord](https://discord.gg/FuPE9JE). 26 | The best thing you can probably do for this library currently is to use it! 27 | More than anything else, I just want eyes on the interface trying it out and seeing where it shines and wher it falters. 28 | 29 | ## License 30 | 31 | pest-deconstruct is licensed under both the MIT license and the Apache License 2.0. 32 | Either terms may be used at your option. All PRs are understood to be agreeing to 33 | contribution under these terms as defined in the Apache license. 34 | 35 | See [LICENSE-APACHE] and [LICENSE-MIT] for details. 36 | 37 | Copyright 2018 Christopher Durham (aka CAD97) 38 | 39 | Dual licensed under the Apache License, Version 2.0 and the MIT License 40 | (collectively, the "License"); you may not use this file except in 41 | compliance with the License. You may obtain a copy of the License at 42 | 43 | 44 | 45 | 46 | Unless required by applicable law or agreed to in writing, software 47 | distributed under the License is distributed on an "AS IS" BASIS, 48 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 49 | See the License for the specific language governing permissions and 50 | limitations under the License. 51 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Only the most recent minor version is supported. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | 0.3.x | :white_check_mark: | 10 | | < 0.3.x | :x: | 11 | 12 | 13 | ## Reporting a Vulnerability 14 | 15 | Please use the [GitHub private reporting functionality](https://github.com/pest-parser/ast/security/advisories/new) 16 | to submit potential security bug reports. If the bug report is reproduced and valid, we'll then: 17 | 18 | - Prepare a fix and regression tests. 19 | - Make a patch release for the most recent release. 20 | - Submit an advisory to [rustsec/advisory-db](https://github.com/RustSec/advisory-db). 21 | - Refer to the advisory in the release notes. 22 | -------------------------------------------------------------------------------- /derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pest-ast" 3 | version = "0.3.5" 4 | edition = "2021" 5 | authors = ["cad97 "] 6 | description = "Derive to convert from pest parse tree to typed syntax tree" 7 | license = "MIT/Apache-2.0" 8 | readme = "./README.md" 9 | repository = "https://github.com/pest-parser/pest_deconstruct" 10 | 11 | [lib] 12 | name = "pest_ast" 13 | proc-macro = true 14 | 15 | [dependencies] 16 | syn = { version = "2", features = ["extra-traits"] } 17 | quote = "1" 18 | proc-macro2 = "1" 19 | itertools = "0.10" 20 | 21 | [dev-dependencies] 22 | from-pest = { version = "0.3", path = ".." } 23 | pest = "2.5" 24 | pest_derive = "2.5" 25 | 26 | [features] 27 | default = ["trace"] 28 | trace = [] 29 | -------------------------------------------------------------------------------- /derive/README.md: -------------------------------------------------------------------------------- 1 | # pest-ast 2 | 3 | Convert from [pest](https://pest.rs) parse trees into typed syntax trees with ease! 4 | 5 | Which would you rather have? 6 | 7 |
Pest Parse Tree 8 | 9 | ``` 10 | [ 11 | Pair { 12 | rule: file, 13 | span: Span { 14 | str: "65279,1179403647,1463895090\n3.1415927,2.7182817,1.618034\n-40,-273.15\n13,42\n65537\n", 15 | start: 0, 16 | end: 81 17 | }, 18 | inner: [ 19 | Pair { 20 | rule: record, 21 | span: Span { 22 | str: "65279,1179403647,1463895090", 23 | start: 0, 24 | end: 27 25 | }, 26 | inner: [ 27 | Pair { 28 | rule: field, 29 | span: Span { 30 | str: "65279", 31 | start: 0, 32 | end: 5 33 | }, 34 | inner: [] 35 | }, 36 | Pair { 37 | rule: field, 38 | span: Span { 39 | str: "1179403647", 40 | start: 6, 41 | end: 16 42 | }, 43 | inner: [] 44 | }, 45 | Pair { 46 | rule: field, 47 | span: Span { 48 | str: "1463895090", 49 | start: 17, 50 | end: 27 51 | }, 52 | inner: [] 53 | } 54 | ] 55 | }, 56 | Pair { 57 | rule: record, 58 | span: Span { 59 | str: "3.1415927,2.7182817,1.618034", 60 | start: 28, 61 | end: 56 62 | }, 63 | inner: [ 64 | Pair { 65 | rule: field, 66 | span: Span { 67 | str: "3.1415927", 68 | start: 28, 69 | end: 37 70 | }, 71 | inner: [] 72 | }, 73 | Pair { 74 | rule: field, 75 | span: Span { 76 | str: "2.7182817", 77 | start: 38, 78 | end: 47 79 | }, 80 | inner: [] 81 | }, 82 | Pair { 83 | rule: field, 84 | span: Span { 85 | str: "1.618034", 86 | start: 48, 87 | end: 56 88 | }, 89 | inner: [] 90 | } 91 | ] 92 | }, 93 | Pair { 94 | rule: record, 95 | span: Span { 96 | str: "-40,-273.15", 97 | start: 57, 98 | end: 68 99 | }, 100 | inner: [ 101 | Pair { 102 | rule: field, 103 | span: Span { 104 | str: "-40", 105 | start: 57, 106 | end: 60 107 | }, 108 | inner: [] 109 | }, 110 | Pair { 111 | rule: field, 112 | span: Span { 113 | str: "-273.15", 114 | start: 61, 115 | end: 68 116 | }, 117 | inner: [] 118 | } 119 | ] 120 | }, 121 | Pair { 122 | rule: record, 123 | span: Span { 124 | str: "13,42", 125 | start: 69, 126 | end: 74 127 | }, 128 | inner: [ 129 | Pair { 130 | rule: field, 131 | span: Span { 132 | str: "13", 133 | start: 69, 134 | end: 71 135 | }, 136 | inner: [] 137 | }, 138 | Pair { 139 | rule: field, 140 | span: Span { 141 | str: "42", 142 | start: 72, 143 | end: 74 144 | }, 145 | inner: [] 146 | } 147 | ] 148 | }, 149 | Pair { 150 | rule: record, 151 | span: Span { 152 | str: "65537", 153 | start: 75, 154 | end: 80 155 | }, 156 | inner: [ 157 | Pair { 158 | rule: field, 159 | span: Span { 160 | str: "65537", 161 | start: 75, 162 | end: 80 163 | }, 164 | inner: [] 165 | } 166 | ] 167 | }, 168 | Pair { 169 | rule: EOI, 170 | span: Span { 171 | str: "", 172 | start: 81, 173 | end: 81 174 | }, 175 | inner: [] 176 | } 177 | ] 178 | } 179 | ] 180 | ``` 181 |
182 |
Typed Syntax Tree 183 | 184 | ``` 185 | File { 186 | records: [ 187 | Record { 188 | fields: [ 189 | Field { 190 | value: 65279.0 191 | }, 192 | Field { 193 | value: 1179403647.0 194 | }, 195 | Field { 196 | value: 1463895090.0 197 | } 198 | ] 199 | }, 200 | Record { 201 | fields: [ 202 | Field { 203 | value: 3.1415927 204 | }, 205 | Field { 206 | value: 2.7182817 207 | }, 208 | Field { 209 | value: 1.618034 210 | } 211 | ] 212 | }, 213 | Record { 214 | fields: [ 215 | Field { 216 | value: -40.0 217 | }, 218 | Field { 219 | value: -273.15 220 | } 221 | ] 222 | }, 223 | Record { 224 | fields: [ 225 | Field { 226 | value: 13.0 227 | }, 228 | Field { 229 | value: 42.0 230 | } 231 | ] 232 | }, 233 | Record { 234 | fields: [ 235 | Field { 236 | value: 65537.0 237 | } 238 | ] 239 | } 240 | ], 241 | eoi: EOI 242 | } 243 | ``` 244 |
245 | 246 | ----- 247 | 248 | The above parse tree is produced by the following pest grammar: 249 | 250 | ```pest 251 | field = { (ASCII_DIGIT | "." | "-")+ } 252 | record = { field ~ ("," ~ field)* } 253 | file = { SOI ~ (record ~ ("\r\n" | "\n"))* ~ EOI } 254 | ``` 255 | 256 | parsing this csv: 257 | 258 | ```csv 259 | 65279,1179403647,1463895090 260 | 3.1415927,2.7182817,1.618034 261 | -40,-273.15 262 | 13,42 263 | 65537 264 | ``` 265 | 266 | And converting it to a typed syntax tree is as simple as the following code: 267 | 268 | ```rust 269 | mod ast { 270 | use super::csv::Rule; 271 | use pest::Span; 272 | 273 | fn span_into_str(span: Span) -> &str { 274 | span.as_str() 275 | } 276 | 277 | #[derive(Debug, FromPest)] 278 | #[pest_ast(rule(Rule::field))] 279 | pub struct Field { 280 | #[pest_ast(outer(with(span_into_str), with(str::parse), with(Result::unwrap)))] 281 | pub value: f64, 282 | } 283 | 284 | #[derive(Debug, FromPest)] 285 | #[pest_ast(rule(Rule::record))] 286 | pub struct Record { 287 | pub fields: Vec, 288 | } 289 | 290 | #[derive(Debug, FromPest)] 291 | #[pest_ast(rule(Rule::file))] 292 | pub struct File { 293 | pub records: Vec, 294 | eoi: EOI, 295 | } 296 | 297 | #[derive(Debug, FromPest)] 298 | #[pest_ast(rule(Rule::EOI))] 299 | struct EOI; 300 | } 301 | ``` 302 | 303 | And doing the actual parse is as simple as 304 | 305 | ```rust 306 | let mut parse_tree = csv::Parser::parse(csv::Rule::file, &source)?; 307 | let syntax_tree = File::from_pest(&mut parse_tree).expect("infallible"); 308 | ``` 309 | -------------------------------------------------------------------------------- /derive/examples/simple_enum_derives.pest: -------------------------------------------------------------------------------- 1 | a = { "a" } 2 | b = { "b" } 3 | c = { "c" } 4 | 5 | abc = { a | b | c } 6 | ABC = { abc* } 7 | -------------------------------------------------------------------------------- /derive/examples/simple_enum_derives.rs: -------------------------------------------------------------------------------- 1 | #![allow( 2 | bad_style, 3 | dead_code, 4 | clippy::clone_on_copy, 5 | clippy::upper_case_acronyms 6 | )] 7 | 8 | #[macro_use] 9 | extern crate pest_derive; 10 | extern crate from_pest; 11 | #[macro_use] 12 | extern crate pest_ast; 13 | extern crate pest; 14 | 15 | use from_pest::FromPest; 16 | use pest::Parser; 17 | 18 | #[derive(Parser)] 19 | #[grammar = "../examples/simple_enum_derives.pest"] 20 | pub struct SimpleParser; 21 | 22 | #[derive(FromPest, Debug)] 23 | #[pest_ast(rule(Rule::a))] 24 | struct a<'pest> { 25 | #[pest_ast(outer())] 26 | span: pest::Span<'pest>, 27 | } 28 | 29 | #[derive(FromPest, Debug)] 30 | #[pest_ast(rule(Rule::b))] 31 | struct b<'pest> { 32 | #[pest_ast(outer())] 33 | span: pest::Span<'pest>, 34 | } 35 | 36 | #[derive(FromPest, Debug)] 37 | #[pest_ast(rule(Rule::c))] 38 | struct c<'pest> { 39 | #[pest_ast(outer())] 40 | span: pest::Span<'pest>, 41 | } 42 | 43 | #[derive(FromPest, Debug)] 44 | #[pest_ast(rule(Rule::abc))] 45 | enum abc<'pest> { 46 | a(a<'pest>), 47 | b(b<'pest>), 48 | c(c<'pest>), 49 | } 50 | 51 | #[derive(FromPest, Debug)] 52 | #[pest_ast(rule(Rule::ABC))] 53 | struct ABC<'pest> { 54 | abc: Vec>, 55 | } 56 | 57 | fn main() { 58 | let source = "aaabbbccc"; 59 | 60 | let mut parse_tree = SimpleParser::parse(Rule::ABC, source).expect("parse success"); 61 | println!("parse tree = {:#?}", parse_tree); 62 | 63 | let syntax_tree = ABC::from_pest(&mut parse_tree).expect("infallible"); 64 | println!("syntax tree = {:#?}", syntax_tree); 65 | } 66 | -------------------------------------------------------------------------------- /derive/examples/simple_struct_derives.pest: -------------------------------------------------------------------------------- 1 | // 2 | // 3 | // The parsing expression ('a'/'b')* matches and consumes an arbitrary-length 4 | // sequence of a's and b's. The production rule S ← 'a' S? 'b' describes the 5 | // simple context-free "matching language" $\{ `a^n b^n : n \geq 1 \}`$. 6 | // The following parsing expression grammar describes the classic 7 | // non-context-free language $`\{ a^n b^n c^n : n \geq 1 \}`$: 8 | // 9 | // ```peg 10 | // S ← &(A 'c') 'a'+ B !. 11 | // A ← 'a' A? 'b' 12 | // B ← 'b' B? 'c' 13 | // ``` 14 | 15 | a = { "a" } 16 | b = { "b" } 17 | c = { "c" } 18 | 19 | S = { &(A ~ c) ~ a+ ~ B ~ !ANY } 20 | A = _{ a ~ A? ~ b } 21 | B = _{ b ~ B? ~ c } 22 | -------------------------------------------------------------------------------- /derive/examples/simple_struct_derives.rs: -------------------------------------------------------------------------------- 1 | #![allow( 2 | bad_style, 3 | dead_code, 4 | clippy::clone_on_copy, 5 | clippy::upper_case_acronyms 6 | )] 7 | 8 | #[macro_use] 9 | extern crate pest_derive; 10 | extern crate from_pest; 11 | #[macro_use] 12 | extern crate pest_ast; 13 | extern crate pest; 14 | 15 | use from_pest::FromPest; 16 | use pest::Parser; 17 | 18 | #[derive(Parser)] 19 | #[grammar = "../examples/simple_struct_derives.pest"] 20 | pub struct SimpleParser; 21 | 22 | #[derive(FromPest, Debug)] 23 | #[pest_ast(rule(Rule::S))] 24 | struct S<'pest> { 25 | #[pest_ast(outer())] 26 | span: pest::Span<'pest>, 27 | a: Vec>, 28 | b: Vec>, 29 | c: Vec>, 30 | } 31 | 32 | #[derive(FromPest, Debug)] 33 | #[pest_ast(rule(Rule::a))] 34 | struct a<'pest> { 35 | #[pest_ast(outer())] 36 | span: pest::Span<'pest>, 37 | } 38 | 39 | #[derive(FromPest, Debug)] 40 | #[pest_ast(rule(Rule::b))] 41 | struct b<'pest> { 42 | #[pest_ast(outer())] 43 | span: pest::Span<'pest>, 44 | } 45 | 46 | #[derive(FromPest, Debug)] 47 | #[pest_ast(rule(Rule::c))] 48 | struct c<'pest> { 49 | #[pest_ast(outer())] 50 | span: pest::Span<'pest>, 51 | } 52 | 53 | fn main() { 54 | let source = "aaabbbccc"; 55 | 56 | let mut parse_tree = SimpleParser::parse(Rule::S, source).expect("parse success"); 57 | println!("parse tree = {:#?}", parse_tree); 58 | 59 | let syntax_tree = S::from_pest(&mut parse_tree).expect("infallible"); 60 | println!("syntax tree = {:#?}", syntax_tree); 61 | } 62 | -------------------------------------------------------------------------------- /derive/src/attributes.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::mixed_read_write_in_expression)] // syn patterns 2 | 3 | use proc_macro2::TokenTree; 4 | 5 | use { 6 | itertools::Itertools, 7 | proc_macro2::TokenStream, 8 | quote::ToTokens, 9 | syn::{ 10 | parse::{Error, Parse, ParseStream, Parser, Result}, 11 | punctuated::Punctuated, 12 | spanned::Spanned, 13 | token::Paren, 14 | Attribute, Ident, LitStr, Path, 15 | }, 16 | }; 17 | 18 | mod kw { 19 | custom_keyword!(grammar); 20 | custom_keyword!(outer); 21 | custom_keyword!(inner); 22 | custom_keyword!(with); 23 | custom_keyword!(rule); 24 | } 25 | 26 | /// `#[pest_ast(..)]` for the outer `#[derive(FromPest)]` 27 | #[derive(Debug)] 28 | pub(crate) enum DeriveAttribute { 29 | /// `grammar = "grammar.rs"` 30 | Grammar(GrammarAttribute), 31 | /// `rule(path::to)` 32 | Rule(RuleAttribute), 33 | } 34 | 35 | #[derive(Debug)] 36 | /// `#[pest_ast(..)]` for fields in `#[derive(FromPest)]` 37 | pub(crate) enum FieldAttribute { 38 | /// `outer(with(path::to),*)` 39 | Outer(OuterAttribute), 40 | /// `inner(rule(path::to), with(path::to),*)` 41 | Inner(InnerAttribute), 42 | } 43 | 44 | #[derive(Debug)] 45 | pub(crate) struct GrammarAttribute { 46 | pub(crate) grammar: kw::grammar, 47 | pub(crate) eq: Token![=], 48 | pub(crate) lit: LitStr, 49 | } 50 | 51 | #[derive(Debug)] 52 | pub(crate) struct OuterAttribute { 53 | pub(crate) outer: kw::outer, 54 | pub(crate) paren: Paren, 55 | pub(crate) with: Punctuated, 56 | } 57 | 58 | #[derive(Debug)] 59 | pub(crate) struct InnerAttribute { 60 | pub(crate) inner: kw::inner, 61 | pub(crate) paren: Paren, 62 | pub(crate) rule: Option, 63 | pub(crate) comma: Option, 64 | pub(crate) with: Punctuated, 65 | } 66 | 67 | #[derive(Debug)] 68 | pub(crate) struct WithAttribute { 69 | pub(crate) with: kw::with, 70 | pub(crate) paren: Paren, 71 | pub(crate) path: Path, 72 | } 73 | 74 | #[derive(Debug)] 75 | pub(crate) struct RuleAttribute { 76 | pub(crate) rule: kw::rule, 77 | pub(crate) paren: Paren, 78 | pub(crate) path: Path, 79 | pub(crate) sep: Token![::], 80 | pub(crate) variant: Ident, 81 | } 82 | 83 | impl DeriveAttribute { 84 | pub(crate) fn from_attributes(attrs: impl IntoIterator) -> Result> { 85 | attrs 86 | .into_iter() 87 | .map(DeriveAttribute::from_attribute) 88 | .fold_ok(vec![], |mut acc, t| { 89 | acc.extend(t); 90 | acc 91 | }) 92 | } 93 | 94 | pub(crate) fn from_attribute(attr: Attribute) -> Result> { 95 | if attr.path() != &parse_quote!(pest_ast) { 96 | return Ok(vec![]); 97 | } 98 | 99 | Parser::parse2( 100 | |input: ParseStream| { 101 | let content; 102 | input.parse::()?; 103 | bracketed!(content in input); 104 | content.step(|cursor| { 105 | if let Some((tt, next)) = cursor.token_tree() { 106 | match tt { 107 | TokenTree::Ident(id) if id == "pest_ast" => Ok(((), next)), 108 | _ => Err(cursor.error("expected `pest_ast`")), 109 | } 110 | } else { 111 | Err(cursor.error("expected `pest_ast`")) 112 | } 113 | })?; 114 | let content2; 115 | parenthesized!(content2 in content); 116 | let punctuated: Punctuated<_, Token![,]> = 117 | content2.parse_terminated(Parse::parse, Token![,])?; 118 | Ok(punctuated.into_iter().collect_vec()) 119 | }, 120 | attr.to_token_stream(), 121 | ) 122 | } 123 | } 124 | 125 | impl FieldAttribute { 126 | pub(crate) fn from_attributes(attrs: impl IntoIterator) -> Result> { 127 | attrs 128 | .into_iter() 129 | .map(FieldAttribute::from_attribute) 130 | .fold_ok(vec![], |mut acc, t| { 131 | acc.extend(t); 132 | acc 133 | }) 134 | } 135 | 136 | pub(crate) fn from_attribute(attr: Attribute) -> Result> { 137 | if attr.path() != &parse_quote!(pest_ast) { 138 | return Ok(vec![]); 139 | } 140 | 141 | Parser::parse2( 142 | |input: ParseStream| { 143 | input.parse::()?; 144 | let content; 145 | bracketed!(content in input); 146 | content.step(|cursor| { 147 | if let Some((tt, next)) = cursor.token_tree() { 148 | match tt { 149 | TokenTree::Ident(id) if id == "pest_ast" => Ok(((), next)), 150 | _ => Err(cursor.error("expected `pest_ast`")), 151 | } 152 | } else { 153 | Err(cursor.error("expected `pest_ast`")) 154 | } 155 | })?; 156 | let content2; 157 | parenthesized!(content2 in content); 158 | let punctuated: Punctuated<_, Token![,]> = 159 | content2.parse_terminated(Parse::parse, Token![,])?; 160 | Ok(punctuated.into_iter().collect_vec()) 161 | }, 162 | attr.to_token_stream(), 163 | ) 164 | } 165 | } 166 | 167 | impl Parse for DeriveAttribute { 168 | fn parse(input: ParseStream) -> Result { 169 | let lookahead = input.lookahead1(); 170 | if lookahead.peek(kw::grammar) { 171 | GrammarAttribute::parse(input).map(DeriveAttribute::Grammar) 172 | } else if lookahead.peek(kw::rule) { 173 | RuleAttribute::parse(input).map(DeriveAttribute::Rule) 174 | } else { 175 | Err(lookahead.error()) 176 | } 177 | } 178 | } 179 | 180 | impl Parse for FieldAttribute { 181 | fn parse(input: ParseStream) -> Result { 182 | let lookahead = input.lookahead1(); 183 | if lookahead.peek(kw::outer) { 184 | OuterAttribute::parse(input).map(FieldAttribute::Outer) 185 | } else if lookahead.peek(kw::inner) { 186 | InnerAttribute::parse(input).map(FieldAttribute::Inner) 187 | } else { 188 | Err(lookahead.error()) 189 | } 190 | } 191 | } 192 | 193 | impl ToTokens for DeriveAttribute { 194 | fn to_tokens(&self, tokens: &mut TokenStream) { 195 | match self { 196 | DeriveAttribute::Grammar(attr) => attr.to_tokens(tokens), 197 | DeriveAttribute::Rule(attr) => attr.to_tokens(tokens), 198 | } 199 | } 200 | } 201 | 202 | impl ToTokens for FieldAttribute { 203 | fn to_tokens(&self, tokens: &mut TokenStream) { 204 | match self { 205 | FieldAttribute::Outer(attr) => attr.to_tokens(tokens), 206 | FieldAttribute::Inner(attr) => attr.to_tokens(tokens), 207 | } 208 | } 209 | } 210 | 211 | impl Parse for GrammarAttribute { 212 | fn parse(input: ParseStream) -> Result { 213 | Ok(GrammarAttribute { 214 | grammar: input.parse()?, 215 | eq: input.parse()?, 216 | lit: input.parse()?, 217 | }) 218 | } 219 | } 220 | 221 | impl ToTokens for GrammarAttribute { 222 | fn to_tokens(&self, tokens: &mut TokenStream) { 223 | self.grammar.to_tokens(tokens); 224 | self.eq.to_tokens(tokens); 225 | self.lit.to_tokens(tokens); 226 | } 227 | } 228 | 229 | impl Parse for OuterAttribute { 230 | fn parse(input: ParseStream) -> Result { 231 | let content; 232 | Ok(OuterAttribute { 233 | outer: input.parse()?, 234 | paren: parenthesized!(content in input), 235 | with: content.parse_terminated(WithAttribute::parse, Token![,])?, 236 | }) 237 | } 238 | } 239 | 240 | impl ToTokens for OuterAttribute { 241 | fn to_tokens(&self, tokens: &mut TokenStream) { 242 | self.outer.to_tokens(tokens); 243 | self.paren.surround(tokens, |tokens| { 244 | self.with.to_tokens(tokens); 245 | }) 246 | } 247 | } 248 | 249 | impl Parse for InnerAttribute { 250 | fn parse(input: ParseStream) -> Result { 251 | let content; 252 | let inner = input.parse()?; 253 | let paren = parenthesized!(content in input); 254 | let (rule, comma) = if content.peek(kw::rule) { 255 | (Some(content.parse()?), content.parse().ok()) 256 | } else { 257 | (None, None) 258 | }; 259 | let with = content.parse_terminated(WithAttribute::parse, Token![,])?; 260 | Ok(InnerAttribute { 261 | inner, 262 | paren, 263 | rule, 264 | comma, 265 | with, 266 | }) 267 | } 268 | } 269 | 270 | impl ToTokens for InnerAttribute { 271 | fn to_tokens(&self, tokens: &mut TokenStream) { 272 | self.inner.to_tokens(tokens); 273 | self.paren.surround(tokens, |tokens| { 274 | if let Some(rule) = &self.rule { 275 | rule.to_tokens(tokens); 276 | } 277 | if let Some(comma) = &self.comma { 278 | comma.to_tokens(tokens); 279 | } 280 | self.with.to_tokens(tokens); 281 | }) 282 | } 283 | } 284 | 285 | impl Parse for WithAttribute { 286 | fn parse(input: ParseStream) -> Result { 287 | let content; 288 | Ok(WithAttribute { 289 | with: input.parse()?, 290 | paren: parenthesized!(content in input), 291 | path: content.parse()?, 292 | }) 293 | } 294 | } 295 | 296 | impl ToTokens for WithAttribute { 297 | fn to_tokens(&self, tokens: &mut TokenStream) { 298 | self.with.to_tokens(tokens); 299 | self.paren 300 | .surround(tokens, |tokens| self.path.to_tokens(tokens)); 301 | } 302 | } 303 | 304 | impl Parse for RuleAttribute { 305 | fn parse(input: ParseStream) -> Result { 306 | let content; 307 | let rule = input.parse()?; 308 | let paren = parenthesized!(content in input); 309 | let mut path: Path = content.parse()?; 310 | let (variant, _) = path.segments.pop().unwrap().into_tuple(); 311 | let sep = if path.segments.trailing_punct() { 312 | // fix trailing punct 313 | let (head, sep) = path.segments.pop().unwrap().into_tuple(); 314 | path.segments.push(head); 315 | sep.unwrap() 316 | } else { 317 | Err(Error::new( 318 | path.span(), 319 | "must be a path to enum variant (both enum and variant)", 320 | ))? 321 | }; 322 | if variant.arguments.is_empty() { 323 | Ok(RuleAttribute { 324 | rule, 325 | paren, 326 | path, 327 | sep, 328 | variant: variant.ident, 329 | }) 330 | } else { 331 | Err(Error::new(path.span(), "must be a path to enum variant")) 332 | } 333 | } 334 | } 335 | 336 | impl ToTokens for RuleAttribute { 337 | fn to_tokens(&self, tokens: &mut TokenStream) { 338 | self.rule.to_tokens(tokens); 339 | self.paren.surround(tokens, |tokens| { 340 | self.path.to_tokens(tokens); 341 | self.sep.to_tokens(tokens); 342 | self.variant.to_tokens(tokens); 343 | }); 344 | } 345 | } 346 | -------------------------------------------------------------------------------- /derive/src/from_pest/field.rs: -------------------------------------------------------------------------------- 1 | use syn::Variant; 2 | 3 | use { 4 | proc_macro2::{Span, TokenStream}, 5 | syn::{ 6 | parse::Error, parse::Result, parse_quote, spanned::Spanned, Fields, Index, Member, Path, 7 | }, 8 | }; 9 | 10 | use crate::attributes::FieldAttribute; 11 | use crate::trace; 12 | 13 | #[derive(Clone, Debug)] 14 | enum ConversionStrategy { 15 | FromPest, 16 | Outer(Span, Vec), 17 | Inner(Span, Vec, Option), 18 | } 19 | 20 | impl ConversionStrategy { 21 | fn from_attrs(attrs: Vec) -> Result { 22 | let mut attrs = attrs.into_iter(); 23 | Ok(match (attrs.next(), attrs.next()) { 24 | (Some(_), Some(attr)) => Err(Error::new( 25 | attr.span(), 26 | "only a single field attribute allowed", 27 | ))?, 28 | (None, None) => ConversionStrategy::FromPest, 29 | (Some(FieldAttribute::Outer(attr)), None) => ConversionStrategy::Outer( 30 | attr.span(), 31 | attr.with.into_iter().map(|attr| attr.path).collect(), 32 | ), 33 | (Some(FieldAttribute::Inner(attr)), None) => ConversionStrategy::Inner( 34 | attr.span(), 35 | attr.with.into_iter().map(|attr| attr.path).collect(), 36 | attr.rule.map(|attr| { 37 | let path = attr.path; 38 | let variant = attr.variant; 39 | parse_quote!(#path::#variant) 40 | }), 41 | ), 42 | _ => unreachable!(), 43 | }) 44 | } 45 | 46 | fn apply(self, member: Member) -> TokenStream { 47 | let conversion = match self { 48 | ConversionStrategy::FromPest => quote!(::from_pest::FromPest::from_pest(inner)?), 49 | ConversionStrategy::Outer(span, mods) => with_mods(quote_spanned!(span=>span), mods), 50 | ConversionStrategy::Inner(span, mods, rule) => { 51 | let pair = quote!(inner.next().ok_or(::from_pest::ConversionError::NoMatch)?); 52 | let get_span = if let Some(rule) = rule { 53 | let error_msg = trace(quote! { 54 | concat!( 55 | "in ", 56 | stringify!(#member), 57 | ", expected `", 58 | stringify!(#rule), 59 | "` but found `{:?}`" 60 | ), 61 | pair.as_rule(), 62 | }); 63 | quote_spanned! {span=>{ 64 | let pair = #pair; 65 | if pair.as_rule() == #rule { 66 | pair.as_span() 67 | } else { 68 | #error_msg 69 | return Err(::from_pest::ConversionError::NoMatch) 70 | // TODO: Should this be panicking instead? 71 | // panic!( 72 | // concat!( 73 | // "in ", 74 | // stringify!(#name), 75 | // ".", 76 | // stringify!(#member), 77 | // ", expected `", 78 | // stringify!(#rule), 79 | // "` but found `{:?}`" 80 | // ), 81 | // pair.as_rule(), 82 | // ) 83 | } 84 | }} 85 | } else { 86 | quote_spanned!(span=>#pair.as_span()) 87 | }; 88 | with_mods(get_span, mods) 89 | } 90 | }; 91 | if let Member::Named(name) = member { 92 | quote!(#name : #conversion) 93 | } else { 94 | conversion 95 | } 96 | } 97 | } 98 | 99 | fn with_mods(stream: TokenStream, mods: Vec) -> TokenStream { 100 | mods.into_iter() 101 | .fold(stream, |stream, path| quote!(#path(#stream))) 102 | } 103 | 104 | pub fn enum_convert(name: &Path, variant: &Variant) -> Result { 105 | let fields = variant.fields.clone(); 106 | Ok(match fields { 107 | Fields::Named(fields) => { 108 | let fields: Vec<_> = fields 109 | .named 110 | .into_iter() 111 | .map(|field| { 112 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 113 | Ok(ConversionStrategy::from_attrs(attrs)? 114 | .apply(Member::Named(field.ident.unwrap()))) 115 | }) 116 | .collect::>()?; 117 | quote!(#name{#(#fields,)*}) 118 | } 119 | Fields::Unnamed(fields) => { 120 | let fields: Vec<_> = fields 121 | .unnamed 122 | .into_iter() 123 | .enumerate() 124 | .map(|(i, field)| { 125 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 126 | Ok(ConversionStrategy::from_attrs(attrs)? 127 | .apply(Member::Unnamed(Index::from(i)))) 128 | }) 129 | .collect::>()?; 130 | quote!(#name(#(#fields),*)) 131 | } 132 | Fields::Unit => { 133 | let attrs = FieldAttribute::from_attributes(variant.attrs.clone())?; 134 | let real_name = 135 | ConversionStrategy::from_attrs(attrs)?.apply(Member::Unnamed(Index::from(0))); 136 | quote!(#real_name) 137 | } 138 | }) 139 | } 140 | 141 | pub fn struct_convert(name: &Path, fields: Fields) -> Result { 142 | Ok(match fields { 143 | Fields::Named(fields) => { 144 | let fields: Vec<_> = fields 145 | .named 146 | .into_iter() 147 | .map(|field| { 148 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 149 | Ok(ConversionStrategy::from_attrs(attrs)? 150 | .apply(Member::Named(field.ident.unwrap()))) 151 | }) 152 | .collect::>()?; 153 | quote!(#name{#(#fields,)*}) 154 | } 155 | Fields::Unnamed(fields) => { 156 | let fields: Vec<_> = fields 157 | .unnamed 158 | .into_iter() 159 | .enumerate() 160 | .map(|(i, field)| { 161 | let attrs = FieldAttribute::from_attributes(field.attrs)?; 162 | Ok(ConversionStrategy::from_attrs(attrs)? 163 | .apply(Member::Unnamed(Index::from(i)))) 164 | }) 165 | .collect::>()?; 166 | quote!(#name(#(#fields),*)) 167 | } 168 | Fields::Unit => { 169 | quote!(#name) 170 | } 171 | }) 172 | } 173 | -------------------------------------------------------------------------------- /derive/src/from_pest/mod.rs: -------------------------------------------------------------------------------- 1 | //! Machinery in charge of deriving `FromPest` for a type. 2 | //! 3 | //! Documentation in this module and submodules describes the requirement placed on _child_ 4 | //! functions. This is important as manipulation is done over untyped `TokenStream`. 5 | 6 | use { 7 | proc_macro2::TokenStream, 8 | std::path::PathBuf as FilePath, 9 | syn::{ 10 | parse::Error, parse::Result, spanned::Spanned, Data, DataEnum, DataStruct, DeriveInput, 11 | Ident, Path, 12 | }, 13 | }; 14 | 15 | use crate::attributes::DeriveAttribute; 16 | 17 | mod field; 18 | 19 | /// Creates implementation of `FromPest` for given derive input. 20 | /// 21 | /// For child functions, sets up an environment with: 22 | /// 23 | /// ```text 24 | /// type Self::Rule; 25 | /// type Self::FatalError = Void; 26 | /// let pest: &mut Pairs; 27 | /// ``` 28 | /// 29 | /// Child function is required to produce a number of statements that implement the semantics of 30 | /// `FromPest::from_pest`; that is: `Ok(Self)` => success, `pest` updated past the node; 31 | /// `Err(NoMatch)` => failure, `pest` is unchanged; `Err(Malformed)` impossible, panic instead. 32 | /// `?` and `return` may be used for early exit of failed matches. 33 | pub(crate) fn derive( 34 | DeriveInput { 35 | attrs, 36 | ident: name, 37 | generics, 38 | data, 39 | .. 40 | }: DeriveInput, 41 | ) -> Result { 42 | let attrs = DeriveAttribute::from_attributes(attrs)?; 43 | 44 | let grammar = { 45 | let mut grammar_attrs = attrs.iter().filter_map(|attr| match attr { 46 | DeriveAttribute::Grammar(attr) => Some(attr), 47 | _ => None, 48 | }); 49 | match (grammar_attrs.next(), grammar_attrs.next()) { 50 | (Some(_), Some(attr)) => Err(Error::new( 51 | attr.span(), 52 | "duplicate #[pest_ast(grammar)] not allowed", 53 | ))?, 54 | (None, None) => None, 55 | (Some(attr), None) => Some(FilePath::from(attr.lit.value())), 56 | _ => unreachable!(), 57 | } 58 | }; 59 | 60 | let (rule_enum, rule_variant) = { 61 | let mut rule_attrs = attrs.into_iter().filter_map(|attr| match attr { 62 | DeriveAttribute::Rule(attr) => Some(attr), 63 | _ => None, 64 | }); 65 | match (rule_attrs.next(), rule_attrs.next()) { 66 | (Some(_), Some(attr)) => Err(Error::new( 67 | attr.span(), 68 | "duplicate #[pest_ast(rule)] not allowed", 69 | ))?, 70 | (None, None) => Err(Error::new(name.span(), "#[pest_ast(rule)] required here"))?, 71 | (Some(attr), None) => (attr.path, attr.variant), 72 | _ => unreachable!(), 73 | } 74 | }; 75 | 76 | let (from_pest_lifetime, did_synthesize_lifetime) = generics 77 | .lifetimes() 78 | .next() 79 | .map(|def| (def.lifetime.clone(), false)) 80 | .unwrap_or_else(|| (parse_quote!('unique_lifetime_name), true)); 81 | 82 | let mut generics_ = generics.clone(); 83 | let (_, ty_generics, where_clause) = generics.split_for_impl(); 84 | if did_synthesize_lifetime { 85 | let lt = from_pest_lifetime.clone(); 86 | generics_.params.push(parse_quote!(#lt)); 87 | } 88 | let (impl_generics, _, _) = generics_.split_for_impl(); 89 | 90 | let implementation = match data { 91 | Data::Union(data) => Err(Error::new( 92 | data.union_token.span(), 93 | "Cannot derive FromPest for union", 94 | )), 95 | Data::Struct(data) => derive_for_struct(grammar, &name, &rule_enum, &rule_variant, data), 96 | Data::Enum(data) => derive_for_enum(grammar, &name, &rule_enum, &rule_variant, data), 97 | }?; 98 | 99 | Ok(quote! { 100 | impl #impl_generics ::from_pest::FromPest<#from_pest_lifetime> for #name #ty_generics #where_clause { 101 | type Rule = #rule_enum; 102 | type FatalError = ::from_pest::Void; 103 | 104 | fn from_pest( 105 | pest: &mut ::from_pest::pest::iterators::Pairs<#from_pest_lifetime, #rule_enum> 106 | ) -> ::std::result::Result> { 107 | #implementation 108 | } 109 | } 110 | }) 111 | } 112 | 113 | /// Implements `FromPest::from_pest` for some struct. 114 | /// 115 | /// For child functions, sets up an environment with: 116 | /// 117 | /// ```text 118 | /// let span: Span; // the span of this production 119 | /// let inner: &mut Pairs; // the contents of this production 120 | /// ``` 121 | /// 122 | /// Child function is required to produce an _expression_ which constructs an instance of `Self` 123 | /// from the `Pair`s in `inner` or early returns a `NoMatch`. `inner` and `span` are free working 124 | /// space, but `inner` should represent the point past all consumed productions afterwards. 125 | fn derive_for_struct( 126 | grammar: Option, 127 | name: &Ident, 128 | rule_enum: &Path, 129 | rule_variant: &Ident, 130 | DataStruct { fields, .. }: DataStruct, 131 | ) -> Result { 132 | if let Some(_path) = grammar { 133 | unimplemented!("Grammar introspection not implemented yet") 134 | } 135 | 136 | let construct = field::struct_convert(&parse_quote!(#name), fields)?; 137 | 138 | let extraneous = crate::trace( 139 | quote! { "when converting {}, found extraneous {:?}", stringify!(#name), inner}, 140 | ); 141 | 142 | Ok(quote! { 143 | let mut clone = pest.clone(); 144 | let pair = clone.next().ok_or(::from_pest::ConversionError::NoMatch)?; 145 | if pair.as_rule() == #rule_enum::#rule_variant { 146 | let span = pair.as_span(); 147 | let mut inner = pair.into_inner(); 148 | let inner = &mut inner; 149 | let this = #construct; 150 | if inner.clone().next().is_some() { 151 | #extraneous 152 | Err(::from_pest::ConversionError::Extraneous { 153 | current_node: stringify!(#name), 154 | })?; 155 | } 156 | *pest = clone; 157 | Ok(this) 158 | } else { 159 | Err(::from_pest::ConversionError::NoMatch) 160 | } 161 | }) 162 | } 163 | 164 | #[allow(unused)] 165 | #[allow(clippy::needless_pass_by_value)] 166 | fn derive_for_enum( 167 | grammar: Option, 168 | name: &Ident, 169 | rule_enum: &Path, 170 | rule_variant: &Ident, 171 | DataEnum { variants, .. }: DataEnum, 172 | ) -> Result { 173 | if let Some(_path) = grammar { 174 | unimplemented!("Grammar introspection not implemented yet") 175 | } 176 | 177 | let convert_variants: Vec = variants 178 | .into_iter() 179 | .map(|variant| { 180 | let variant_name = &variant.ident; 181 | let construct_variant = field::enum_convert(&parse_quote!(#name::#variant_name), &variant)?; 182 | let extraneous = crate::trace(quote! { 183 | "when converting {}, found extraneous {:?}", stringify!(#name), stringify!(#variant_name) 184 | }); 185 | 186 | Ok(quote! { 187 | let span = pair.as_span(); 188 | let mut inner = pair.clone().into_inner(); 189 | let inner = &mut inner; 190 | let this = #construct_variant; 191 | if inner.clone().next().is_some() { 192 | #extraneous 193 | Err(::from_pest::ConversionError::Extraneous { 194 | current_node: stringify!(#variant_name), 195 | })?; 196 | } 197 | Ok(this) 198 | }) 199 | }) 200 | .collect::>()?; 201 | 202 | Ok(quote! { 203 | let mut clone = pest.clone(); 204 | let pair = clone.next().ok_or(::from_pest::ConversionError::NoMatch)?; 205 | if pair.as_rule() == #rule_enum::#rule_variant { 206 | let this = Err(::from_pest::ConversionError::NoMatch) 207 | #(.or_else(|_: ::from_pest::ConversionError<::from_pest::Void>| { 208 | #convert_variants 209 | }))*?; 210 | *pest = clone; 211 | Ok(this) 212 | } else { 213 | Err(::from_pest::ConversionError::NoMatch) 214 | } 215 | }) 216 | } 217 | -------------------------------------------------------------------------------- /derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_snake_case)] 2 | #![recursion_limit = "150"] 3 | 4 | extern crate itertools; 5 | extern crate proc_macro; 6 | extern crate proc_macro2; 7 | #[macro_use] 8 | extern crate syn; 9 | #[macro_use] 10 | extern crate quote; 11 | 12 | #[allow(non_snake_case)] 13 | #[proc_macro_derive(FromPest, attributes(pest_ast))] 14 | pub fn derive_FromPest(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 15 | // let x = 16 | syn::parse(input) 17 | .and_then(from_pest::derive) 18 | .unwrap_or_else(|err| err.to_compile_error()) 19 | // .to_string(); 20 | // quote!(compile_error!(#x);) 21 | .into() 22 | } 23 | 24 | mod attributes; 25 | mod from_pest; 26 | 27 | #[cfg(feature = "trace")] 28 | fn trace(t: proc_macro2::TokenStream) -> proc_macro2::TokenStream { 29 | quote! { ::from_pest::log::trace!( #t ); } 30 | } 31 | 32 | #[cfg(not(feature = "trace"))] 33 | fn trace(_t: proc_macro2::TokenStream) -> proc_macro2::TokenStream { 34 | quote! {} 35 | } 36 | -------------------------------------------------------------------------------- /examples/csv.csv: -------------------------------------------------------------------------------- 1 | 65279,1179403647,1463895090 2 | 3.1415927,2.7182817,1.618034 3 | -40,-273.15 4 | 13,42 5 | 65537 6 | -------------------------------------------------------------------------------- /examples/csv.pest: -------------------------------------------------------------------------------- 1 | field = { (ASCII_DIGIT | "." | "-")+ } 2 | record = { field ~ ("," ~ field)* } 3 | file = { SOI ~ (record ~ ("\r\n" | "\n"))* ~ EOI } 4 | -------------------------------------------------------------------------------- /examples/csv.rs: -------------------------------------------------------------------------------- 1 | // Unfortunately, you currently have to import all four of these. 2 | // We're considering what it would look like to make this redundant, 3 | // and then you'd only need pest and pest-ast. 4 | 5 | #[macro_use] 6 | extern crate pest_derive; 7 | extern crate from_pest; 8 | #[macro_use] 9 | extern crate pest_ast; 10 | extern crate pest; 11 | 12 | mod csv { 13 | #[derive(Parser)] 14 | #[grammar = "../examples/csv.pest"] 15 | pub struct Parser; 16 | } 17 | 18 | mod ast { 19 | use super::csv::Rule; 20 | use pest::Span; 21 | 22 | fn span_into_str(span: Span) -> &str { 23 | span.as_str() 24 | } 25 | 26 | #[derive(Debug, FromPest)] 27 | #[pest_ast(rule(Rule::field))] 28 | pub struct Field { 29 | #[pest_ast(outer(with(span_into_str), with(str::parse), with(Result::unwrap)))] 30 | pub value: f64, 31 | } 32 | 33 | #[derive(Debug, FromPest)] 34 | #[pest_ast(rule(Rule::record))] 35 | pub struct Record { 36 | pub fields: Vec, 37 | } 38 | 39 | #[derive(Debug, FromPest)] 40 | #[pest_ast(rule(Rule::file))] 41 | pub struct File { 42 | pub records: Vec, 43 | _eoi: Eoi, 44 | } 45 | 46 | #[derive(Debug, FromPest)] 47 | #[pest_ast(rule(Rule::EOI))] 48 | struct Eoi; 49 | } 50 | 51 | fn main() -> Result<(), Box> { 52 | use crate::ast::File; 53 | use from_pest::FromPest; 54 | use pest::Parser; 55 | use std::fs; 56 | 57 | let source = String::from_utf8(fs::read("./examples/csv.csv")?)?; 58 | let mut parse_tree = csv::Parser::parse(csv::Rule::file, &source)?; 59 | println!("parse tree = {:#?}", parse_tree); 60 | let syntax_tree: File = File::from_pest(&mut parse_tree).expect("infallible"); 61 | println!("syntax tree = {:#?}", syntax_tree); 62 | println!(); 63 | 64 | let mut field_sum = 0.0; 65 | let mut record_count = 0; 66 | 67 | for record in syntax_tree.records { 68 | record_count += 1; 69 | for field in record.fields { 70 | field_sum += field.value; 71 | } 72 | } 73 | 74 | println!("Sum of fields: {}", field_sum); 75 | println!("Number of records: {}", record_count); 76 | 77 | Ok(()) 78 | } 79 | 80 | #[test] 81 | fn csv_example_runs() { 82 | main().unwrap() 83 | } 84 | -------------------------------------------------------------------------------- /pest-deconstruct.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The [`FromPest`] conversion framework to convert from pest trees into typed structure. 2 | 3 | #[doc(hidden)] 4 | pub extern crate log; 5 | #[doc(hidden)] 6 | pub extern crate pest; 7 | extern crate void; 8 | 9 | #[doc(inline)] 10 | pub use void::Void; 11 | 12 | use { 13 | pest::{ 14 | iterators::{Pair, Pairs}, 15 | RuleType, 16 | }, 17 | std::marker::PhantomData, 18 | }; 19 | 20 | /// An error that occurs during conversion. 21 | #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] 22 | pub enum ConversionError { 23 | /// No match occurred: this node is not present here 24 | NoMatch, 25 | /// Fatal error: this node is present but malformed 26 | Malformed(FatalError), 27 | /// Found unexpected tokens at the end 28 | Extraneous { current_node: &'static str }, 29 | } 30 | 31 | use std::fmt; 32 | 33 | impl fmt::Display for ConversionError 34 | where 35 | FatalError: fmt::Display, 36 | { 37 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 38 | match self { 39 | ConversionError::NoMatch => write!(f, "Rule did not match, failed to convert node"), 40 | ConversionError::Malformed(fatalerror) => write!(f, "Malformed node: {}", fatalerror), 41 | ConversionError::Extraneous { current_node, .. } => write!( 42 | f, 43 | "when converting {}, found extraneous tokens", 44 | current_node 45 | ), 46 | } 47 | } 48 | } 49 | 50 | use std::error; 51 | 52 | impl error::Error for ConversionError 53 | where 54 | FatalError: error::Error + 'static, 55 | { 56 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 57 | match self { 58 | ConversionError::NoMatch => None, 59 | ConversionError::Extraneous { .. } => None, 60 | ConversionError::Malformed(ref fatalerror) => Some(fatalerror), 61 | } 62 | } 63 | } 64 | 65 | /// Potentially borrowing conversion from a pest parse tree. 66 | pub trait FromPest<'pest>: Sized { 67 | /// The rule type for the parse tree this type corresponds to. 68 | type Rule: RuleType; 69 | /// A fatal error during conversion. 70 | type FatalError; 71 | /// Convert from a Pest parse tree. 72 | /// 73 | /// # Return type semantics 74 | /// 75 | /// - `Err(ConversionError::NoMatch)` => node not at head of the cursor, cursor unchanged 76 | /// - `Err(ConversionError::Malformed)` => fatal error; node at head of the cursor but malformed 77 | /// - `Ok` => success; the cursor has been updated past this node 78 | fn from_pest( 79 | pest: &mut Pairs<'pest, Self::Rule>, 80 | ) -> Result>; 81 | } 82 | 83 | /// Convert a production without storing it. 84 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for PhantomData { 85 | type Rule = Rule; 86 | type FatalError = T::FatalError; 87 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 88 | T::from_pest(pest).map(|_| PhantomData) 89 | } 90 | } 91 | 92 | /// For recursive grammars. 93 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for Box { 94 | type Rule = Rule; 95 | type FatalError = T::FatalError; 96 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 97 | T::from_pest(pest).map(Box::new) 98 | } 99 | } 100 | 101 | /// Convert an optional production. 102 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for Option { 103 | type Rule = Rule; 104 | type FatalError = T::FatalError; 105 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 106 | match T::from_pest(pest) { 107 | Err(ConversionError::NoMatch) => Ok(None), 108 | result => result.map(Some), 109 | } 110 | } 111 | } 112 | 113 | /// Convert many productions. (If `` is non-advancing, this will be non-terminating.) 114 | impl<'pest, Rule: RuleType, T: FromPest<'pest, Rule = Rule>> FromPest<'pest> for Vec { 115 | type Rule = Rule; 116 | type FatalError = T::FatalError; 117 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 118 | let mut acc = vec![]; 119 | loop { 120 | match T::from_pest(pest) { 121 | Ok(t) => acc.push(t), 122 | Err(ConversionError::NoMatch) => break, 123 | Err(error) => return Err(error), 124 | } 125 | } 126 | Ok(acc) 127 | } 128 | } 129 | 130 | /// Consume a production without doing any processing. 131 | impl<'pest, Rule: RuleType> FromPest<'pest> for Pair<'pest, Rule> { 132 | type Rule = Rule; 133 | type FatalError = Void; 134 | fn from_pest(pest: &mut Pairs<'pest, Rule>) -> Result> { 135 | pest.next().ok_or(ConversionError::NoMatch) 136 | } 137 | } 138 | 139 | macro_rules! impl_for_tuple { 140 | () => {}; 141 | ($ty1:ident $($ty:ident)*) => { 142 | impl<'pest, $ty1, $($ty,)* Rule: RuleType, FatalError> FromPest<'pest> for ($ty1, $($ty),*) 143 | where 144 | $ty1: FromPest<'pest, Rule=Rule, FatalError=FatalError>, 145 | $($ty: FromPest<'pest, Rule=Rule, FatalError=FatalError>,)* 146 | { 147 | type Rule = Rule; 148 | type FatalError = FatalError; 149 | fn from_pest(pest: &mut Pairs<'pest, Rule>) 150 | -> Result> 151 | { 152 | let mut clone = pest.clone(); 153 | let this = ( 154 | $ty1::from_pest(&mut clone)?, 155 | $($ty::from_pest(&mut clone)?),* 156 | ); 157 | *pest = clone; 158 | Ok(this) 159 | } 160 | } 161 | impl_for_tuple!($($ty)*); 162 | }; 163 | } 164 | 165 | impl_for_tuple!(A B C D); 166 | -------------------------------------------------------------------------------- /tests/csv.rs: -------------------------------------------------------------------------------- 1 | ../examples/csv.rs --------------------------------------------------------------------------------