├── .cargo └── config.toml ├── .clippy.toml ├── .git-blame-ignore-revs ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── config.yml │ └── feature_request.yml ├── PULL_REQUEST_TEMPLATE.md ├── renovate.json5 ├── settings.yml └── workflows │ ├── audit.yml │ ├── ci.yml │ ├── committed.yml │ ├── pre-commit.yml │ ├── rust-next.yml │ └── spelling.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── COPYRIGHT ├── Cargo.lock ├── Cargo.toml ├── LICENSE-MIT ├── README.md ├── _typos.toml ├── assets ├── bigbuckbunny.mp4 ├── links.txt ├── small.mp4 ├── testfile.txt └── trace.svg ├── benches ├── contains_token.rs ├── find_slice.rs ├── iter.rs ├── next_slice.rs └── number.rs ├── committed.toml ├── deny.toml ├── examples ├── arithmetic │ ├── bench.rs │ ├── main.rs │ ├── parser.rs │ ├── parser_ast.rs │ ├── parser_lexer.rs │ ├── test_parser.rs │ ├── test_parser_ast.rs │ └── test_parser_lexer.rs ├── css │ ├── main.rs │ └── parser.rs ├── custom_error.rs ├── http │ ├── bench.rs │ ├── main.rs │ ├── parser.rs │ └── parser_streaming.rs ├── ini │ ├── bench.rs │ ├── main.rs │ ├── parser.rs │ └── parser_str.rs ├── iterator.rs ├── json │ ├── bench.rs │ ├── json.rs │ ├── main.rs │ ├── parser_alt.rs │ ├── parser_dispatch.rs │ └── parser_partial.rs ├── json_iterator.rs ├── ndjson │ ├── example.ndjson │ ├── main.rs │ └── parser.rs ├── s_expression │ ├── main.rs │ └── parser.rs └── string │ ├── main.rs │ └── parser.rs ├── fuzz ├── .gitignore ├── Cargo.toml └── fuzz_targets │ └── fuzz_arithmetic.rs ├── proptest-regressions ├── character │ └── streaming.txt ├── number │ └── streaming.txt └── stream │ └── tests.txt ├── release.toml ├── src ├── _topic │ ├── arithmetic.rs │ ├── error.rs │ ├── fromstr.rs │ ├── http.rs │ ├── ini.rs │ ├── json.rs │ ├── language.rs │ ├── lexing.rs │ ├── mod.rs │ ├── nom.rs │ ├── partial.rs │ ├── performance.rs │ ├── s_expression.rs │ ├── stream.rs │ └── why.rs ├── _tutorial │ ├── chapter_0.rs │ ├── chapter_1.rs │ ├── chapter_2.rs │ ├── chapter_3.rs │ ├── chapter_4.rs │ ├── chapter_5.rs │ ├── chapter_6.rs │ ├── chapter_7.rs │ ├── chapter_8.rs │ └── mod.rs ├── ascii │ ├── mod.rs │ └── tests.rs ├── binary │ ├── bits │ │ ├── mod.rs │ │ └── tests.rs │ ├── mod.rs │ └── tests.rs ├── combinator │ ├── branch.rs │ ├── core.rs │ ├── debug │ │ ├── internals.rs │ │ └── mod.rs │ ├── impls.rs │ ├── mod.rs │ ├── multi.rs │ ├── sequence.rs │ └── tests.rs ├── error.rs ├── lib.rs ├── macros │ ├── dispatch.rs │ ├── mod.rs │ ├── seq.rs │ └── tests.rs ├── parser.rs ├── stream │ ├── bstr.rs │ ├── bytes.rs │ ├── locating.rs │ ├── mod.rs │ ├── partial.rs │ ├── range.rs │ ├── recoverable.rs │ ├── stateful.rs │ ├── tests.rs │ └── token.rs └── token │ ├── mod.rs │ └── tests.rs ├── tests └── testsuite │ ├── custom_errors.rs │ ├── float.rs │ ├── fnmut.rs │ ├── issues.rs │ ├── main.rs │ ├── multiline.rs │ ├── overflow.rs │ ├── reborrow_fold.rs │ └── utf8.rs └── third_party └── nativejson-benchmark ├── LINK └── data └── canada.json /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [resolver] 2 | incompatible-rust-versions = "fallback" 3 | -------------------------------------------------------------------------------- /.clippy.toml: -------------------------------------------------------------------------------- 1 | allow-print-in-tests = true 2 | allow-expect-in-tests = true 3 | allow-unwrap-in-tests = true 4 | allow-dbg-in-tests = true 5 | disallowed-methods = [ 6 | { path = "std::option::Option::map_or", reason = "prefer `map(..).unwrap_or(..)` for legibility" }, 7 | { path = "std::option::Option::map_or_else", reason = "prefer `map(..).unwrap_or_else(..)` for legibility" }, 8 | { path = "std::result::Result::map_or", reason = "prefer `map(..).unwrap_or(..)` for legibility" }, 9 | { path = "std::result::Result::map_or_else", reason = "prefer `map(..).unwrap_or_else(..)` for legibility" }, 10 | { path = "std::iter::Iterator::for_each", reason = "prefer `for` for side-effects" }, 11 | { path = "std::iter::Iterator::try_for_each", reason = "prefer `for` for side-effects" }, 12 | ] 13 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Switched to default rustfmt 2 | efe1b5073d5b744d8d2162007a935b96abe65f4d 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug report 2 | description: Things aren't working as expected 3 | labels: 'C-bug' 4 | body: 5 | - type: checkboxes 6 | attributes: 7 | label: "Please complete the following tasks" 8 | options: 9 | - label: "I have searched the [discussions](https://github.com/winnow-rs/winnow/discussions)" 10 | required: true 11 | - label: "I have searched the [open](https://github.com/winnow-rs/winnow/issues) and [rejected](https://github.com/winnow-rs/winnow/issues?q=is%3Aissue%20is%3Aclosed%20reason%3Anot-planned) issues" 12 | required: true 13 | - type: input 14 | attributes: 15 | label: rust version 16 | description: output of `rustc -V` 17 | validations: 18 | required: true 19 | - type: input 20 | attributes: 21 | label: winnow version 22 | description: Can be found in Cargo.lock or Cargo.toml of your project (i.e. `grep winnow Cargo.lock`). PLEASE DO NOT PUT "latest" HERE, use precise version. Put `master` (or other branch) if you're using the repo directly. 23 | validations: 24 | required: true 25 | - type: textarea 26 | attributes: 27 | label: Minimal reproducible code 28 | description: Please write a minimal complete program which has this bug. Do not point to an existing repository. 29 | value: | 30 | ```rust 31 | fn main() {} 32 | ``` 33 | validations: 34 | required: true 35 | - type: textarea 36 | attributes: 37 | label: Steps to reproduce the bug with the above code 38 | description: A command like `cargo run -- options...` or multiple commands. 39 | validations: 40 | required: true 41 | - type: textarea 42 | attributes: 43 | label: Actual Behaviour 44 | description: When I do like *this*, *that* is happening and I think it shouldn't. 45 | validations: 46 | required: true 47 | - type: textarea 48 | attributes: 49 | label: Expected Behaviour 50 | description: I think *this* should happen instead. 51 | validations: 52 | required: true 53 | - type: textarea 54 | attributes: 55 | label: Additional Context 56 | description: Add any other context about the problem here. 57 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Ask a question 4 | about: For support or brainstorming 5 | url: https://github.com/winnow-rs/winnow/discussions/new 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: Feature request 2 | description: Raise the bar on expectations 3 | labels: 'C-enhancement' 4 | body: 5 | - type: checkboxes 6 | attributes: 7 | label: Please complete the following tasks 8 | options: 9 | - label: "I have searched the [discussions](https://github.com/winnow-rs/winnow/discussions)" 10 | required: true 11 | - label: "I have searched the [open](https://github.com/winnow-rs/winnow/issues) and [rejected](https://github.com/winnow-rs/winnow/issues?q=is%3Aissue%20is%3Aclosed%20reason%3Anot-planned) issues" 12 | required: true 13 | - type: input 14 | attributes: 15 | label: winnow version 16 | description: Can be found in Cargo.lock or Cargo.toml of your project (i.e. `grep winnow Cargo.lock`). PLEASE DO NOT PUT "latest" HERE, use precise version. Put `master` (or other branch) if you're using the repo directly. 17 | validations: 18 | required: true 19 | - type: textarea 20 | attributes: 21 | label: Describe your use case 22 | description: Describe the problem you're trying to solve. This is not mandatory and we *do* consider features without a specific use case, but real problems have priority. 23 | validations: 24 | required: true 25 | - type: textarea 26 | attributes: 27 | label: Describe the solution you'd like 28 | description: Please explain what the wanted solution should look like. You are **strongly encouraged** to attach a snippet of (pseudo)code. 29 | validations: 30 | required: true 31 | - type: textarea 32 | attributes: 33 | label: Alternatives, if applicable 34 | description: A clear and concise description of any alternative solutions or features you've managed to come up with. 35 | - type: textarea 36 | attributes: 37 | label: Additional Context 38 | description: Add any other context about the feature request here. 39 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 10 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | schedule: [ 3 | 'before 5am on the first day of the month', 4 | ], 5 | semanticCommits: 'enabled', 6 | commitMessageLowerCase: 'never', 7 | configMigration: true, 8 | dependencyDashboard: true, 9 | customManagers: [ 10 | { 11 | customType: 'regex', 12 | fileMatch: [ 13 | '^rust-toolchain\\.toml$', 14 | 'Cargo.toml$', 15 | 'clippy.toml$', 16 | '\\.clippy.toml$', 17 | '^\\.github/workflows/ci.yml$', 18 | '^\\.github/workflows/rust-next.yml$', 19 | ], 20 | matchStrings: [ 21 | 'STABLE.*?(?\\d+\\.\\d+(\\.\\d+)?)', 22 | '(?\\d+\\.\\d+(\\.\\d+)?).*?STABLE', 23 | ], 24 | depNameTemplate: 'STABLE', 25 | packageNameTemplate: 'rust-lang/rust', 26 | datasourceTemplate: 'github-releases', 27 | }, 28 | ], 29 | packageRules: [ 30 | { 31 | commitMessageTopic: 'Rust Stable', 32 | matchManagers: [ 33 | 'custom.regex', 34 | ], 35 | matchDepNames: [ 36 | 'STABLE', 37 | ], 38 | extractVersion: '^(?\\d+\\.\\d+)', // Drop the patch version 39 | schedule: [ 40 | '* * * * *', 41 | ], 42 | automerge: true, 43 | }, 44 | // Goals: 45 | // - Keep version reqs low, ignoring compatible normal/build dependencies 46 | // - Take advantage of latest dev-dependencies 47 | // - Rollup safe upgrades to reduce CI runner load 48 | // - Help keep number of versions down by always using latest breaking change 49 | // - Have lockfile and manifest in-sync 50 | { 51 | matchManagers: [ 52 | 'cargo', 53 | ], 54 | matchDepTypes: [ 55 | 'build-dependencies', 56 | 'dependencies', 57 | ], 58 | matchCurrentVersion: '>=0.1.0', 59 | matchUpdateTypes: [ 60 | 'patch', 61 | ], 62 | enabled: false, 63 | }, 64 | { 65 | matchManagers: [ 66 | 'cargo', 67 | ], 68 | matchDepTypes: [ 69 | 'build-dependencies', 70 | 'dependencies', 71 | ], 72 | matchCurrentVersion: '>=1.0.0', 73 | matchUpdateTypes: [ 74 | 'minor', 75 | 'patch', 76 | ], 77 | enabled: false, 78 | }, 79 | { 80 | matchManagers: [ 81 | 'cargo', 82 | ], 83 | matchDepTypes: [ 84 | 'dev-dependencies', 85 | ], 86 | matchCurrentVersion: '>=0.1.0', 87 | matchUpdateTypes: [ 88 | 'patch', 89 | ], 90 | automerge: true, 91 | groupName: 'compatible (dev)', 92 | }, 93 | { 94 | matchManagers: [ 95 | 'cargo', 96 | ], 97 | matchDepTypes: [ 98 | 'dev-dependencies', 99 | ], 100 | matchCurrentVersion: '>=1.0.0', 101 | matchUpdateTypes: [ 102 | 'minor', 103 | 'patch', 104 | ], 105 | automerge: true, 106 | groupName: 'compatible (dev)', 107 | }, 108 | ], 109 | } 110 | -------------------------------------------------------------------------------- /.github/settings.yml: -------------------------------------------------------------------------------- 1 | # These settings are synced to GitHub by https://probot.github.io/apps/settings/ 2 | 3 | repository: 4 | description: "Making parsing a breeze" 5 | homepage: "https://docs.rs/winnow" 6 | topics: "rust parser grammar lexer" 7 | has_issues: true 8 | has_projects: false 9 | has_wiki: false 10 | has_downloads: true 11 | default_branch: main 12 | 13 | # Preference: people do clean commits 14 | allow_merge_commit: true 15 | # Backup in case we need to clean up commits 16 | allow_squash_merge: true 17 | # Not really needed 18 | allow_rebase_merge: false 19 | 20 | allow_auto_merge: true 21 | delete_branch_on_merge: true 22 | 23 | squash_merge_commit_title: "PR_TITLE" 24 | squash_merge_commit_message: "PR_BODY" 25 | merge_commit_message: "PR_BODY" 26 | 27 | labels: 28 | - name: "A-combinator" 29 | description: "Area: combinators" 30 | color: '#f7e101' 31 | - name: "A-input" 32 | description: "Area: input / parsing state" 33 | color: '#f7e101' 34 | - name: "A-error" 35 | description: "Area: error reporting" 36 | color: '#f7e101' 37 | - name: "A-docs" 38 | description: "Area: Documentation" 39 | color: '#f7e101' 40 | - name: "A-meta" 41 | description: "Area: Project-wide" 42 | color: '#f7e101' 43 | - name: "C-bug" 44 | description: "Category: Things not working as expected" 45 | color: '#f5f1fd' 46 | - name: "C-enhancement" 47 | description: "Category: Raise on the bar on expectations" 48 | color: '#f5f1fd' 49 | - name: "C-question" 50 | description: Uncertainty is involved 51 | color: '#f5f1fd' 52 | - name: "M-breaking-change" 53 | description: "Meta: Implementing or merging this will introduce a breaking change." 54 | color: '#E10C02' 55 | - name: "E-help-wanted" 56 | description: "Call for participation: Help is requested to fix this issue." 57 | color: '#02E10C' 58 | 59 | # This serves more as documentation. 60 | # Branch protection API was replaced by rulesets but settings isn't updated. 61 | # See https://github.com/repository-settings/app/issues/825 62 | # 63 | # branches: 64 | # - name: main 65 | # protection: 66 | # required_pull_request_reviews: null 67 | # required_conversation_resolution: true 68 | # required_status_checks: 69 | # # Required. Require branches to be up to date before merging. 70 | # strict: false 71 | # contexts: ["CI", "Spell Check with Typos"] 72 | # enforce_admins: false 73 | # restrictions: null 74 | -------------------------------------------------------------------------------- /.github/workflows/audit.yml: -------------------------------------------------------------------------------- 1 | name: Security audit 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | pull_request: 8 | paths: 9 | - '**/Cargo.toml' 10 | - '**/Cargo.lock' 11 | push: 12 | branches: 13 | - main 14 | - v*-main 15 | 16 | env: 17 | RUST_BACKTRACE: 1 18 | CARGO_TERM_COLOR: always 19 | CLICOLOR: 1 20 | 21 | concurrency: 22 | group: "${{ github.workflow }}-${{ github.ref }}" 23 | cancel-in-progress: true 24 | 25 | jobs: 26 | security_audit: 27 | permissions: 28 | issues: write # to create issues (actions-rs/audit-check) 29 | checks: write # to create check (actions-rs/audit-check) 30 | runs-on: ubuntu-latest 31 | # Prevent sudden announcement of a new advisory from failing ci: 32 | continue-on-error: true 33 | steps: 34 | - name: Checkout repository 35 | uses: actions/checkout@v4 36 | - uses: actions-rs/audit-check@v1 37 | with: 38 | token: ${{ secrets.GITHUB_TOKEN }} 39 | 40 | cargo_deny: 41 | permissions: 42 | issues: write # to create issues (actions-rs/audit-check) 43 | checks: write # to create check (actions-rs/audit-check) 44 | runs-on: ubuntu-latest 45 | strategy: 46 | matrix: 47 | checks: 48 | - bans licenses sources 49 | steps: 50 | - uses: actions/checkout@v4 51 | - uses: EmbarkStudios/cargo-deny-action@v2 52 | with: 53 | command: check ${{ matrix.checks }} 54 | rust-version: stable 55 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | pull_request: 8 | push: 9 | branches: 10 | - main 11 | - v*-main 12 | 13 | env: 14 | RUST_BACKTRACE: 1 15 | CARGO_TERM_COLOR: always 16 | CLICOLOR: 1 17 | 18 | concurrency: 19 | group: "${{ github.workflow }}-${{ github.ref }}" 20 | cancel-in-progress: true 21 | 22 | jobs: 23 | ci: 24 | permissions: 25 | contents: none 26 | name: CI 27 | needs: [test, msrv, lockfile, docs, rustfmt, clippy, minimal-versions] 28 | runs-on: ubuntu-latest 29 | if: "always()" 30 | steps: 31 | - name: Failed 32 | run: exit 1 33 | if: "contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped')" 34 | test: 35 | name: Test 36 | strategy: 37 | matrix: 38 | os: ["ubuntu-latest", "windows-latest", "macos-latest"] 39 | rust: ["stable"] 40 | continue-on-error: ${{ matrix.rust != 'stable' }} 41 | runs-on: ${{ matrix.os }} 42 | steps: 43 | - name: Checkout repository 44 | uses: actions/checkout@v4 45 | - name: Install Rust 46 | uses: dtolnay/rust-toolchain@stable 47 | with: 48 | toolchain: ${{ matrix.rust }} 49 | - uses: Swatinem/rust-cache@v2 50 | - uses: taiki-e/install-action@cargo-hack 51 | - name: Build 52 | run: cargo test --workspace --no-run 53 | - name: Test 54 | run: cargo hack test --each-feature --skip debug --workspace 55 | miri: 56 | name: Miri 57 | runs-on: ubuntu-latest 58 | env: 59 | MIRIFLAGS: "-Zmiri-disable-isolation" 60 | steps: 61 | - name: Checkout repository 62 | uses: actions/checkout@v3 63 | - name: Install Rust 64 | uses: dtolnay/rust-toolchain@stable 65 | with: 66 | toolchain: nightly 67 | components: miri 68 | - uses: Swatinem/rust-cache@v2 69 | # no-default features is a subset, not needed 70 | # all-features is `debug` which is extremely slow 71 | - name: Default features 72 | run: cargo miri test --workspace 73 | msrv: 74 | name: "Check MSRV" 75 | runs-on: ubuntu-latest 76 | steps: 77 | - name: Checkout repository 78 | uses: actions/checkout@v4 79 | - name: Install Rust 80 | uses: dtolnay/rust-toolchain@stable 81 | with: 82 | toolchain: stable 83 | - uses: Swatinem/rust-cache@v2 84 | - uses: taiki-e/install-action@cargo-hack 85 | - name: Default features 86 | run: cargo hack check --each-feature --locked --rust-version --ignore-private --workspace --all-targets --keep-going 87 | minimal-versions: 88 | name: Minimal versions 89 | runs-on: ubuntu-latest 90 | steps: 91 | - name: Checkout repository 92 | uses: actions/checkout@v4 93 | - name: Install stable Rust 94 | uses: dtolnay/rust-toolchain@stable 95 | with: 96 | toolchain: stable 97 | - name: Install nightly Rust 98 | uses: dtolnay/rust-toolchain@stable 99 | with: 100 | toolchain: nightly 101 | - name: Downgrade dependencies to minimal versions 102 | run: cargo +nightly generate-lockfile -Z minimal-versions 103 | - name: Compile with minimal versions 104 | run: cargo +stable check --workspace --all-features --locked --keep-going 105 | lockfile: 106 | runs-on: ubuntu-latest 107 | steps: 108 | - name: Checkout repository 109 | uses: actions/checkout@v4 110 | - name: Install Rust 111 | uses: dtolnay/rust-toolchain@stable 112 | with: 113 | toolchain: stable 114 | - uses: Swatinem/rust-cache@v2 115 | - name: "Is lockfile updated?" 116 | run: cargo update --workspace --locked 117 | docs: 118 | name: Docs 119 | runs-on: ubuntu-latest 120 | steps: 121 | - name: Checkout repository 122 | uses: actions/checkout@v4 123 | - name: Install Rust 124 | uses: dtolnay/rust-toolchain@stable 125 | with: 126 | toolchain: "1.80.0" # STABLE 127 | - uses: Swatinem/rust-cache@v2 128 | - name: Check documentation 129 | env: 130 | RUSTDOCFLAGS: -D warnings 131 | run: cargo doc --workspace --all-features --no-deps --document-private-items --keep-going 132 | rustfmt: 133 | name: rustfmt 134 | runs-on: ubuntu-latest 135 | steps: 136 | - name: Checkout repository 137 | uses: actions/checkout@v4 138 | - name: Install Rust 139 | uses: dtolnay/rust-toolchain@stable 140 | with: 141 | toolchain: "1.80.0" # STABLE 142 | components: rustfmt 143 | - uses: Swatinem/rust-cache@v2 144 | - name: Check formatting 145 | run: cargo fmt --all -- --check 146 | clippy: 147 | name: clippy 148 | runs-on: ubuntu-latest 149 | permissions: 150 | security-events: write # to upload sarif results 151 | steps: 152 | - name: Checkout repository 153 | uses: actions/checkout@v4 154 | - name: Install Rust 155 | uses: dtolnay/rust-toolchain@stable 156 | with: 157 | toolchain: "1.80.0" # STABLE 158 | components: clippy 159 | - uses: Swatinem/rust-cache@v2 160 | - name: Install SARIF tools 161 | run: cargo install clippy-sarif --locked 162 | - name: Install SARIF tools 163 | run: cargo install sarif-fmt --locked 164 | - name: Check 165 | run: > 166 | cargo clippy --workspace --all-features --all-targets --message-format=json 167 | | clippy-sarif 168 | | tee clippy-results.sarif 169 | | sarif-fmt 170 | continue-on-error: true 171 | - name: Upload 172 | uses: github/codeql-action/upload-sarif@v3 173 | with: 174 | sarif_file: clippy-results.sarif 175 | wait-for-processing: true 176 | - name: Report status 177 | run: cargo clippy --workspace --all-features --all-targets --keep-going -- -D warnings --allow deprecated 178 | coverage: 179 | name: Coverage 180 | runs-on: ubuntu-latest 181 | steps: 182 | - name: Checkout repository 183 | uses: actions/checkout@v4 184 | - name: Install Rust 185 | uses: dtolnay/rust-toolchain@stable 186 | with: 187 | toolchain: stable 188 | - uses: Swatinem/rust-cache@v2 189 | - name: Install cargo-tarpaulin 190 | run: cargo install cargo-tarpaulin 191 | - name: Gather coverage 192 | run: cargo tarpaulin --output-dir coverage --out lcov 193 | - name: Publish to Coveralls 194 | uses: coverallsapp/github-action@master 195 | with: 196 | github-token: ${{ secrets.GITHUB_TOKEN }} 197 | - name: Publish to Codecov 198 | uses: codecov/codecov-action@v3 199 | env: 200 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 201 | -------------------------------------------------------------------------------- /.github/workflows/committed.yml: -------------------------------------------------------------------------------- 1 | # Not run as part of pre-commit checks because they don't handle sending the correct commit 2 | # range to `committed` 3 | name: Lint Commits 4 | on: [pull_request] 5 | 6 | permissions: 7 | contents: read 8 | 9 | env: 10 | RUST_BACKTRACE: 1 11 | CARGO_TERM_COLOR: always 12 | CLICOLOR: 1 13 | 14 | concurrency: 15 | group: "${{ github.workflow }}-${{ github.ref }}" 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | committed: 20 | name: Lint Commits 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout Actions Repository 24 | uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | - name: Lint Commits 28 | uses: crate-ci/committed@master 29 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: pre-commit 2 | 3 | permissions: {} # none 4 | 5 | on: 6 | pull_request: 7 | push: 8 | branches: 9 | - main 10 | - v*-main 11 | 12 | env: 13 | RUST_BACKTRACE: 1 14 | CARGO_TERM_COLOR: always 15 | CLICOLOR: 1 16 | 17 | concurrency: 18 | group: "${{ github.workflow }}-${{ github.ref }}" 19 | cancel-in-progress: true 20 | 21 | jobs: 22 | pre-commit: 23 | permissions: 24 | contents: read 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v4 28 | - uses: actions/setup-python@v5 29 | with: 30 | python-version: '3.x' 31 | - uses: pre-commit/action@v3.0.1 32 | -------------------------------------------------------------------------------- /.github/workflows/rust-next.yml: -------------------------------------------------------------------------------- 1 | name: rust-next 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | schedule: 8 | - cron: '3 3 3 * *' 9 | 10 | env: 11 | RUST_BACKTRACE: 1 12 | CARGO_TERM_COLOR: always 13 | CLICOLOR: 1 14 | 15 | concurrency: 16 | group: "${{ github.workflow }}-${{ github.ref }}" 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | test: 21 | name: Test 22 | strategy: 23 | matrix: 24 | os: ["ubuntu-latest", "windows-latest", "macos-latest"] 25 | rust: ["stable", "beta"] 26 | include: 27 | - os: ubuntu-latest 28 | rust: "nightly" 29 | continue-on-error: ${{ matrix.rust != 'stable' }} 30 | runs-on: ${{ matrix.os }} 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v4 34 | - name: Install Rust 35 | uses: dtolnay/rust-toolchain@stable 36 | with: 37 | toolchain: ${{ matrix.rust }} 38 | - uses: Swatinem/rust-cache@v2 39 | - uses: taiki-e/install-action@cargo-hack 40 | - name: Build 41 | run: cargo test --workspace --no-run 42 | - name: Test 43 | run: cargo hack test --each-feature --skip debug --workspace 44 | latest: 45 | name: "Check latest dependencies" 46 | runs-on: ubuntu-latest 47 | steps: 48 | - name: Checkout repository 49 | uses: actions/checkout@v4 50 | - name: Install Rust 51 | uses: dtolnay/rust-toolchain@stable 52 | with: 53 | toolchain: stable 54 | - uses: Swatinem/rust-cache@v2 55 | - uses: taiki-e/install-action@cargo-hack 56 | - name: Update dependencies 57 | run: cargo update 58 | - name: Build 59 | run: cargo test --workspace --no-run 60 | - name: Test 61 | run: cargo hack test --each-feature --workspace 62 | -------------------------------------------------------------------------------- /.github/workflows/spelling.yml: -------------------------------------------------------------------------------- 1 | name: Spelling 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: [pull_request] 7 | 8 | env: 9 | RUST_BACKTRACE: 1 10 | CARGO_TERM_COLOR: always 11 | CLICOLOR: 1 12 | 13 | concurrency: 14 | group: "${{ github.workflow }}-${{ github.ref }}" 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | spelling: 19 | name: Spell Check with Typos 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Checkout Actions Repository 23 | uses: actions/checkout@v4 24 | - name: Spell Check Repo 25 | uses: crate-ci/typos@master 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: check-yaml 6 | stages: [commit] 7 | - id: check-json 8 | stages: [commit] 9 | - id: check-toml 10 | stages: [commit] 11 | - id: check-merge-conflict 12 | stages: [commit] 13 | - id: check-case-conflict 14 | stages: [commit] 15 | - id: detect-private-key 16 | stages: [commit] 17 | - repo: https://github.com/crate-ci/typos 18 | rev: v1.16.20 19 | hooks: 20 | - id: typos 21 | stages: [commit] 22 | - repo: https://github.com/crate-ci/committed 23 | rev: v1.0.20 24 | hooks: 25 | - id: committed 26 | stages: [commit-msg] 27 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to winnow 2 | Thanks for wanting to contribute! There are many ways to contribute and we 3 | appreciate any level you're willing to do. 4 | 5 | ## Feature Requests 6 | 7 | Need some new functionality to help? You can let us know by opening an 8 | [issue][new issue]. It's helpful to look through [all issues][all issues] in 9 | case it's already being talked about. 10 | 11 | ## Bug Reports 12 | 13 | Please let us know about what problems you run into, whether in behavior or 14 | ergonomics of API. You can do this by opening an [issue][new issue]. It's 15 | helpful to look through [all issues][all issues] in case it's already being 16 | talked about. 17 | 18 | ## Pull Requests 19 | 20 | Looking for an idea? Check our [issues][issues]. If the issue looks open ended, 21 | it is probably best to post on the issue how you are thinking of resolving the 22 | issue so you can get feedback early in the process. We want you to be 23 | successful and it can be discouraging to find out a lot of re-work is needed. 24 | 25 | Already have an idea? It might be good to first [create an issue][new issue] 26 | to propose it so we can make sure we are aligned and lower the risk of having 27 | to re-work some of it and the discouragement that goes along with that. 28 | 29 | ### `Parser`s 30 | 31 | Design guidelines 32 | - Generally grammar-level `Parser`s are free-functions and output/error 33 | conversion are inherent functions on `Parser`. `Parser::verify` is an 34 | example of some nuance as the logic is coupled to the `Parser` its applied 35 | to. 36 | - `Parser`s that directly process tokens must support complete vs streaming 37 | parsing. 38 | - `Parser`s that work with slices have `take` in their name. 39 | - When taking slices or repeatedly calling a `Parser`, control the number of 40 | times with a range, rather than hard coding it with the range in the name. 41 | - Where possible, write `Parser`s in a straight-forward manner, reusing other 42 | `Parser`s, so they may serve as examples for the user. 43 | 44 | ### Process 45 | 46 | As a heads up, we'll be running your PR through the following gauntlet: 47 | - warnings turned to compile errors 48 | - `cargo test` 49 | - `rustfmt` 50 | - `clippy` 51 | - `rustdoc` 52 | - [`committed`](https://github.com/crate-ci/committed) as we use [Conventional](https://www.conventionalcommits.org) commit style 53 | - [`typos`](https://github.com/crate-ci/typos) to check spelling 54 | 55 | Not everything can be checked automatically though. 56 | 57 | We request that the commit history gets cleaned up. 58 | 59 | We ask that commits are atomic, meaning they are complete and have a single responsibility. 60 | A complete commit should build, pass tests, update documentation and tests, and not have dead code. 61 | 62 | PRs should tell a cohesive story, with refactor and test commits that keep the 63 | fix or feature commits simple and clear. 64 | 65 | Specifically, we would encourage 66 | - File renames be isolated into their own commit 67 | - Add tests in a commit before their feature or fix, showing the current behavior (i.e. they should pass). 68 | The diff for the feature/fix commit will then show how the behavior changed, 69 | making the commit's intent clearer to reviewers and the community, and showing people that the 70 | test is verifying the expected state. 71 | - e.g. [clap#5520](https://github.com/clap-rs/clap/pull/5520) 72 | 73 | Note that we are talking about ideals. 74 | We understand having a clean history requires more advanced git skills; 75 | feel free to ask us for help! 76 | We might even suggest where it would work to be lax. 77 | We also understand that editing some early commits may cause a lot of churn 78 | with merge conflicts which can make it not worth editing all of the history. 79 | 80 | For code organization, we recommend 81 | - Grouping `impl` blocks next to their type (or trait) 82 | - Grouping private items after the `pub` item that uses them. 83 | - The intent is to help people quickly find the "relevant" details, allowing them to "dig deeper" as needed. Or put another way, the `pub` items serve as a table-of-contents. 84 | - The exact order is fuzzy; do what makes sense 85 | 86 | ## Releasing 87 | 88 | Pre-requisites 89 | - Running `cargo login` 90 | - A member of `winnow-rs:Maintainers` 91 | - Push permission to the repo 92 | - [`cargo-release`](https://github.com/crate-ci/cargo-release/) 93 | 94 | When we're ready to release, a project owner should do the following 95 | 1. Update the changelog (see `cargo release changes` for ideas) 96 | 2. Determine what the next version is, according to semver 97 | 3. Run [`cargo release -x `](https://github.com/crate-ci/cargo-release) 98 | 99 | [issues]: https://github.com/winnow-rs/winnow/issues 100 | [new issue]: https://github.com/winnow-rs/winnow/issues/new 101 | [all issues]: https://github.com/winnow-rs/winnow/issues?utf8=%E2%9C%93&q=is%3Aissue 102 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Short version for non-lawyers: 2 | 3 | Winnow is licensed under MIT terms. 4 | 5 | Longer version: 6 | 7 | Copyrights in winnow are retained by their contributors. No 8 | copyright assignment is required to contribute to winnow. 9 | 10 | Except as otherwise noted (below and/or in individual files), winnow is 11 | licensed under the MIT license 12 | or , at your option. 13 | 14 | Winnow is forked from nom and, similarly, Geal and other nom contributors 15 | retain copyright on their contributions. 16 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = ["fuzz"] 4 | 5 | [workspace.package] 6 | repository = "https://github.com/winnow-rs/winnow" 7 | license = "MIT" 8 | edition = "2021" 9 | rust-version = "1.65.0" # MSRV 10 | include = [ 11 | "build.rs", 12 | "src/**/*", 13 | "Cargo.toml", 14 | "Cargo.lock", 15 | "LICENSE*", 16 | "README.md", 17 | "benches/**/*", 18 | "examples/**/*" 19 | ] 20 | 21 | [workspace.lints.rust] 22 | rust_2018_idioms = { level = "warn", priority = -1 } 23 | unnameable_types = "warn" 24 | unreachable_pub = "warn" 25 | unsafe_op_in_unsafe_fn = "warn" 26 | unused_lifetimes = "warn" 27 | unused_macro_rules = "warn" 28 | 29 | [workspace.lints.clippy] 30 | bool_assert_comparison = "allow" 31 | branches_sharing_code = "allow" 32 | checked_conversions = "warn" 33 | collapsible_else_if = "allow" 34 | create_dir = "warn" 35 | dbg_macro = "warn" 36 | debug_assert_with_mut_call = "warn" 37 | doc_markdown = "warn" 38 | empty_enum = "warn" 39 | enum_glob_use = "warn" 40 | expl_impl_clone_on_copy = "warn" 41 | explicit_deref_methods = "warn" 42 | explicit_into_iter_loop = "warn" 43 | fallible_impl_from = "warn" 44 | filter_map_next = "warn" 45 | flat_map_option = "warn" 46 | float_cmp_const = "warn" 47 | fn_params_excessive_bools = "warn" 48 | from_iter_instead_of_collect = "warn" 49 | if_same_then_else = "allow" 50 | implicit_clone = "warn" 51 | imprecise_flops = "warn" 52 | inconsistent_struct_constructor = "warn" 53 | inefficient_to_string = "warn" 54 | infinite_loop = "warn" 55 | invalid_upcast_comparisons = "warn" 56 | large_digit_groups = "warn" 57 | large_stack_arrays = "warn" 58 | large_types_passed_by_value = "warn" 59 | let_and_return = "allow" # sometimes good to name what you are returning 60 | linkedlist = "warn" 61 | lossy_float_literal = "warn" 62 | macro_use_imports = "warn" 63 | mem_forget = "warn" 64 | mutex_integer = "warn" 65 | needless_continue = "allow" 66 | needless_for_each = "warn" 67 | negative_feature_names = "warn" 68 | path_buf_push_overwrite = "warn" 69 | ptr_as_ptr = "warn" 70 | rc_mutex = "warn" 71 | redundant_feature_names = "warn" 72 | ref_option_ref = "warn" 73 | rest_pat_in_fully_bound_structs = "warn" 74 | result_large_err = "allow" 75 | same_functions_in_if_condition = "warn" 76 | # self_named_module_files = "warn" # false-positives 77 | semicolon_if_nothing_returned = "warn" 78 | str_to_string = "warn" 79 | string_add = "warn" 80 | string_add_assign = "warn" 81 | string_lit_as_bytes = "warn" 82 | string_to_string = "warn" 83 | todo = "warn" 84 | trait_duplication_in_bounds = "warn" 85 | uninlined_format_args = "warn" 86 | verbose_file_reads = "warn" 87 | wildcard_imports = "allow" 88 | zero_sized_map_values = "warn" 89 | 90 | [profile.dev] 91 | panic = "abort" 92 | 93 | [profile.release] 94 | panic = "abort" 95 | codegen-units = 1 96 | lto = true 97 | 98 | [package] 99 | name = "winnow" 100 | version = "0.7.10" 101 | description = "A byte-oriented, zero-copy, parser combinators library" 102 | categories = ["parsing"] 103 | keywords = ["parser", "parser-combinators", "parsing", "streaming", "bit"] 104 | autoexamples = false 105 | repository.workspace = true 106 | license.workspace = true 107 | edition.workspace = true 108 | rust-version.workspace = true 109 | include.workspace = true 110 | 111 | [package.metadata.docs.rs] 112 | features = ["unstable-doc"] 113 | rustdoc-args = ["--cfg", "docsrs", "--generate-link-to-definition"] 114 | 115 | [package.metadata.release] 116 | pre-release-replacements = [ 117 | {file="CHANGELOG.md", search="Unreleased", replace="{{version}}", min=1}, 118 | {file="CHANGELOG.md", search="\\.\\.\\.HEAD", replace="...{{tag_name}}", exactly=1}, 119 | {file="CHANGELOG.md", search="ReleaseDate", replace="{{date}}", min=1}, 120 | {file="CHANGELOG.md", search="", replace="\n## [Unreleased] - ReleaseDate\n", exactly=1}, 121 | {file="CHANGELOG.md", search="", replace="\n[Unreleased]: https://github.com/winnow-rs/winnow/compare/{{tag_name}}...HEAD", exactly=1}, 122 | {file="src/lib.rs", search="blob/v.+\\..+\\..+/CHANGELOG.md", replace="blob/v{{version}}/CHANGELOG.md", exactly=1}, 123 | ] 124 | 125 | [features] 126 | default = ["std"] 127 | alloc = [] 128 | std = ["alloc", "memchr?/std"] 129 | simd = ["dep:memchr"] 130 | debug = ["std", "dep:anstream", "dep:anstyle", "dep:is_terminal_polyfill", "dep:terminal_size"] 131 | unstable-recover = [] 132 | 133 | unstable-doc = ["alloc", "std", "simd", "unstable-recover"] 134 | 135 | [dependencies] 136 | anstream = { version = "0.3.2", optional = true } 137 | anstyle = { version = "1.0.1", optional = true } 138 | is_terminal_polyfill = { version = "1.48.0", optional = true } 139 | memchr = { version = "2.5", optional = true, default-features = false } 140 | terminal_size = { version = "0.4.0", optional = true } 141 | 142 | [dev-dependencies] 143 | proptest = "1.2.0" 144 | criterion = "0.5.1" 145 | lexopt = "0.3.0" 146 | term-transcript = "0.2.0" 147 | snapbox = { version = "0.6.21", features = ["examples"] } 148 | circular = "0.3.0" 149 | rustc-hash = "1.1.0" 150 | automod = "1.0.14" 151 | annotate-snippets = "0.11.3" 152 | anyhow = "1.0.86" 153 | 154 | [profile.bench] 155 | debug = true 156 | lto = true 157 | codegen-units = 1 158 | 159 | [[example]] 160 | name = "arithmetic" 161 | test = true 162 | required-features = ["alloc"] 163 | 164 | [[example]] 165 | name = "css" 166 | test = true 167 | required-features = ["alloc"] 168 | 169 | [[example]] 170 | name = "custom_error" 171 | test = true 172 | required-features = ["alloc"] 173 | 174 | [[example]] 175 | name = "http" 176 | required-features = ["alloc"] 177 | 178 | [[example]] 179 | name = "ini" 180 | test = true 181 | required-features = ["std"] 182 | 183 | [[example]] 184 | name = "json" 185 | test = true 186 | required-features = ["std"] 187 | 188 | [[example]] 189 | name = "ndjson" 190 | test = true 191 | required-features = ["std"] 192 | 193 | [[example]] 194 | name = "json_iterator" 195 | required-features = ["std"] 196 | 197 | [[example]] 198 | name = "iterator" 199 | 200 | [[example]] 201 | name = "s_expression" 202 | required-features = ["alloc"] 203 | 204 | [[example]] 205 | name = "string" 206 | required-features = ["alloc"] 207 | 208 | [[bench]] 209 | name = "arithmetic" 210 | path = "examples/arithmetic/bench.rs" 211 | harness = false 212 | required-features = ["alloc"] 213 | 214 | [[bench]] 215 | name = "contains_token" 216 | harness = false 217 | 218 | [[bench]] 219 | name = "find_slice" 220 | harness = false 221 | 222 | [[bench]] 223 | name = "iter" 224 | harness = false 225 | 226 | [[bench]] 227 | name = "next_slice" 228 | harness = false 229 | 230 | [[bench]] 231 | name = "number" 232 | harness = false 233 | 234 | [[bench]] 235 | name = "http" 236 | path = "examples/http/bench.rs" 237 | harness = false 238 | required-features = ["alloc"] 239 | 240 | [[bench]] 241 | name = "ini" 242 | path = "examples/ini/bench.rs" 243 | harness = false 244 | required-features = ["std"] 245 | 246 | [[bench]] 247 | name = "json" 248 | path = "examples/json/bench.rs" 249 | harness = false 250 | required-features = ["std"] 251 | 252 | [lints] 253 | workspace = true 254 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining 2 | a copy of this software and associated documentation files (the 3 | "Software"), to deal in the Software without restriction, including 4 | without limitation the rights to use, copy, modify, merge, publish, 5 | distribute, sublicense, and/or sell copies of the Software, and to 6 | permit persons to whom the Software is furnished to do so, subject to 7 | the following conditions: 8 | 9 | The above copyright notice and this permission notice shall be 10 | included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 13 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 14 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 15 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 16 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 18 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # winnow, making parsing a breeze 2 | 3 | [![LICENSE](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) 4 | [![Build Status](https://github.com/winnow-rs/winnow/actions/workflows/ci.yml/badge.svg)](https://github.com/winnow-rs/winnow/actions/workflows/ci.yml) 5 | [![Coverage Status](https://coveralls.io/repos/github/winnow-rs/winnow/badge.svg?branch=main)](https://coveralls.io/github/winnow-rs/winnow?branch=main) 6 | [![Crates.io Version](https://img.shields.io/crates/v/winnow.svg)](https://crates.io/crates/winnow) 7 | 8 | ## About 9 | 10 | Build up a parser for your format of choice with the building blocks provided by `winnow`. 11 | 12 | For more details, see: 13 | - [Tutorial](https://docs.rs/winnow/latest/winnow/_tutorial/index.html) 14 | - [Special Topics](https://docs.rs/winnow/latest/winnow/_topic/index.html) 15 | - [Why winnow? How does it compare to ...?](https://docs.rs/winnow/latest/winnow/_topic/why/index.html) 16 | - [API Reference](https://docs.rs/winnow) 17 | - [List of combinators](https://docs.rs/winnow/latest/winnow/combinator/index.html) 18 | 19 | # Contributors 20 | 21 | winnow is the fruit of the work of many contributors over the years, many 22 | thanks for your help! In particular, thanks to [Geal](https://github.com/Geal) 23 | for the original [`nom` crate](https://crates.io/crates/nom). 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /_typos.toml: -------------------------------------------------------------------------------- 1 | [default.extend-words] 2 | # username 3 | dne = "dne" 4 | # test data 5 | hel = "hel" 6 | ba = "ba" 7 | nd = "nd" 8 | rror = "rror" 9 | -------------------------------------------------------------------------------- /assets/bigbuckbunny.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winnow-rs/winnow/73c6e05f4dc82bbdc4f7076c686a9bf6bebbdd47/assets/bigbuckbunny.mp4 -------------------------------------------------------------------------------- /assets/links.txt: -------------------------------------------------------------------------------- 1 | https://github.com/ekmett/machines 2 | https://www.fpcomplete.com/user/snoyberg/library-documentation/conduit-overview 3 | https://hackage.haskell.org/package/pipes 4 | 5 | http://en.wikipedia.org/wiki/Iteratee#cite_note-play-enumeratee-3 6 | http://okmij.org/ftp/Streams.html#design 7 | http://okmij.org/ftp/Haskell/Iteratee/Iteratee.hs 8 | http://okmij.org/ftp/Haskell/Iteratee/describe.pdf 9 | http://okmij.org/ftp/Haskell/Iteratee/IterDemo.hs 10 | http://okmij.org/ftp/Haskell/Iteratee/IterDemo1.hs 11 | http://mandubian.com/2012/08/27/understanding-play2-iteratees-for-normal-humans/ 12 | https://github.com/playframework/playframework/tree/master/framework/src/iteratees/src/main/scala/play/api/libs/Iteratee 13 | https://github.com/playframework/playframework/blob/master/framework/src/iteratees/src/main/scala/play/api/libs/iteratee/Iteratee.scala 14 | https://github.com/playframework/playframework/blob/master/framework/src/iteratees/src/main/scala/play/api/libs/iteratee/Enumerator.scala 15 | http://stackoverflow.com/questions/10177666/cant-understand-iteratee-enumerator-enumeratee-in-play-2-0 16 | http://stackoverflow.com/questions/10346592/how-to-write-an-enumeratee-to-chunk-an-enumerator-along-different-boundaries 17 | http://okmij.org/ftp/Haskell/Iteratee/Iteratee.hs 18 | http://www.mew.org/~kazu/proj/enumerator/ 19 | http://www.reddit.com/r/rust/comments/2aur8x/using_macros_to_parse_file_formats/ 20 | https://github.com/LeoTestard/rustlex 21 | https://github.com/rust-lang/rust/blob/master/src/test/run-pass/monad.rs 22 | www.reddit.com/r/programming/comments/z7lwn/rust_typeclasses_talk/ 23 | https://github.com/rust-lang/rfcs/pull/53 24 | http://apocalisp.wordpress.com/2011/10/26/tail-call-elimination-in-scala-monads/ 25 | http://www.cis.upenn.edu/~bcpierce/sf/current/toc.html 26 | http://www.reddit.com/r/rust/comments/1a57pw/haskeller_playing_with_rust_a_question_about/ 27 | 28 | -------------------------------------------------------------------------------- /assets/small.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winnow-rs/winnow/73c6e05f4dc82bbdc4f7076c686a9bf6bebbdd47/assets/small.mp4 -------------------------------------------------------------------------------- /assets/testfile.txt: -------------------------------------------------------------------------------- 1 | abcdabcdabcdabcdabcd 2 | efgh 3 | coin coin 4 | hello 5 | blah 6 | -------------------------------------------------------------------------------- /assets/trace.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 67 | 68 | 69 | 70 |
71 | 
$ ./string '"abc"'
72 | 
> delimited                                         | "\"abc\""∅
 73 |  > '"'                                              | "\"abc\""∅
 74 |  < '"'                                              | +1
 75 |  > repeat_fold                                      | "abc\""∅
 76 |   > alt                                             | "abc\""∅
 77 |    > take_till                                      | "abc\""∅
 78 |    < take_till                                      | +3
 79 |    | verify                                         | 
 80 |   < alt                                             | +3
 81 |   > alt                                             | "\""∅
 82 |    > take_till                                      | "\""∅
 83 |    < take_till                                      | backtrack
 84 |    > preceded                                       | "\""∅
 85 |     > '\\'                                          | "\""∅
 86 |     < '\\'                                          | backtrack
 87 |    < preceded                                       | backtrack
 88 |    > preceded                                       | "\""∅
 89 |     > '\\'                                          | "\""∅
 90 |     < '\\'                                          | backtrack
 91 |    < preceded                                       | backtrack
 92 |   < alt                                             | backtrack
 93 |  < repeat_fold                                      | +3
 94 |  > '"'                                              | "\""∅
 95 |  < '"'                                              | +1
 96 | < delimited                                         | +5
 97 | > eof                                               | ""∅
 98 | < eof                                               | +0
 99 | abc
100 |
101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /benches/contains_token.rs: -------------------------------------------------------------------------------- 1 | use criterion::black_box; 2 | 3 | use winnow::combinator::alt; 4 | use winnow::combinator::repeat; 5 | use winnow::prelude::*; 6 | use winnow::token::take_till; 7 | use winnow::token::take_while; 8 | 9 | fn contains_token(c: &mut criterion::Criterion) { 10 | let data = [ 11 | ("contiguous", CONTIGUOUS), 12 | ("interleaved", INTERLEAVED), 13 | ("canada", CANADA), 14 | ]; 15 | let mut group = c.benchmark_group("contains_token"); 16 | for (name, sample) in data { 17 | let len = sample.len(); 18 | group.throughput(criterion::Throughput::Bytes(len as u64)); 19 | 20 | group.bench_with_input(criterion::BenchmarkId::new("slice", name), &len, |b, _| { 21 | b.iter(|| black_box(parser_slice.parse_peek(black_box(sample)).unwrap())); 22 | }); 23 | group.bench_with_input(criterion::BenchmarkId::new("array", name), &len, |b, _| { 24 | b.iter(|| black_box(parser_array.parse_peek(black_box(sample)).unwrap())); 25 | }); 26 | group.bench_with_input(criterion::BenchmarkId::new("tuple", name), &len, |b, _| { 27 | b.iter(|| black_box(parser_tuple.parse_peek(black_box(sample)).unwrap())); 28 | }); 29 | group.bench_with_input( 30 | criterion::BenchmarkId::new("closure-or", name), 31 | &len, 32 | |b, _| { 33 | b.iter(|| black_box(parser_closure_or.parse_peek(black_box(sample)).unwrap())); 34 | }, 35 | ); 36 | group.bench_with_input( 37 | criterion::BenchmarkId::new("closure-matches", name), 38 | &len, 39 | |b, _| { 40 | b.iter(|| { 41 | black_box( 42 | parser_closure_matches 43 | .parse_peek(black_box(sample)) 44 | .unwrap(), 45 | ) 46 | }); 47 | }, 48 | ); 49 | } 50 | group.finish(); 51 | } 52 | 53 | fn parser_slice(input: &mut &str) -> ModalResult { 54 | let contains = &['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'][..]; 55 | repeat( 56 | 0.., 57 | alt((take_while(1.., contains), take_till(1.., contains))), 58 | ) 59 | .parse_next(input) 60 | } 61 | 62 | fn parser_array(input: &mut &str) -> ModalResult { 63 | let contains = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']; 64 | repeat( 65 | 0.., 66 | alt((take_while(1.., contains), take_till(1.., contains))), 67 | ) 68 | .parse_next(input) 69 | } 70 | 71 | fn parser_tuple(input: &mut &str) -> ModalResult { 72 | let contains = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'); 73 | repeat( 74 | 0.., 75 | alt((take_while(1.., contains), take_till(1.., contains))), 76 | ) 77 | .parse_next(input) 78 | } 79 | 80 | fn parser_closure_or(input: &mut &str) -> ModalResult { 81 | let contains = |c: char| { 82 | c == '0' 83 | || c == '1' 84 | || c == '2' 85 | || c == '3' 86 | || c == '4' 87 | || c == '5' 88 | || c == '6' 89 | || c == '7' 90 | || c == '8' 91 | || c == '9' 92 | }; 93 | repeat( 94 | 0.., 95 | alt((take_while(1.., contains), take_till(1.., contains))), 96 | ) 97 | .parse_next(input) 98 | } 99 | 100 | fn parser_closure_matches(input: &mut &str) -> ModalResult { 101 | let contains = |c: char| matches!(c, '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'); 102 | repeat( 103 | 0.., 104 | alt((take_while(1.., contains), take_till(1.., contains))), 105 | ) 106 | .parse_next(input) 107 | } 108 | 109 | const CONTIGUOUS: &str = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; 110 | const INTERLEAVED: &str = "0123456789abc0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab"; 111 | const CANADA: &str = include_str!("../third_party/nativejson-benchmark/data/canada.json"); 112 | 113 | criterion::criterion_group!(benches, contains_token); 114 | criterion::criterion_main!(benches); 115 | -------------------------------------------------------------------------------- /benches/find_slice.rs: -------------------------------------------------------------------------------- 1 | use criterion::black_box; 2 | 3 | use winnow::combinator::repeat; 4 | use winnow::prelude::*; 5 | use winnow::token::take_until; 6 | 7 | fn find_slice(c: &mut criterion::Criterion) { 8 | let empty = ""; 9 | let start_byte = "\r".repeat(100); 10 | let start_slice = "\r\n".repeat(100); 11 | let small = format!("{:>10}\r\n", "").repeat(100); 12 | let large = format!("{:>10000}\r\n", "").repeat(100); 13 | 14 | let data = [ 15 | ("empty", (empty, empty)), 16 | ("start", (&start_byte, &start_slice)), 17 | ("medium", (&small, &small)), 18 | ("large", (&large, &large)), 19 | ]; 20 | let mut group = c.benchmark_group("find_slice"); 21 | for (name, samples) in data { 22 | group.bench_with_input( 23 | criterion::BenchmarkId::new("byte", name), 24 | samples.0, 25 | |b, sample| { 26 | b.iter(|| black_box(parser_byte.parse_peek(black_box(sample)).unwrap())); 27 | }, 28 | ); 29 | 30 | group.bench_with_input( 31 | criterion::BenchmarkId::new("slice", name), 32 | samples.1, 33 | |b, sample| { 34 | b.iter(|| black_box(parser_slice.parse_peek(black_box(sample)).unwrap())); 35 | }, 36 | ); 37 | } 38 | group.finish(); 39 | } 40 | 41 | fn parser_byte(input: &mut &str) -> ModalResult { 42 | repeat(0.., (take_until(0.., "\r"), "\r")).parse_next(input) 43 | } 44 | 45 | fn parser_slice(input: &mut &str) -> ModalResult { 46 | repeat(0.., (take_until(0.., "\r\n"), "\r\n")).parse_next(input) 47 | } 48 | 49 | criterion::criterion_group!(benches, find_slice); 50 | criterion::criterion_main!(benches); 51 | -------------------------------------------------------------------------------- /benches/iter.rs: -------------------------------------------------------------------------------- 1 | use criterion::black_box; 2 | 3 | use winnow::combinator::opt; 4 | use winnow::prelude::*; 5 | use winnow::stream::AsChar; 6 | use winnow::token::one_of; 7 | 8 | fn iter(c: &mut criterion::Criterion) { 9 | let data = [ 10 | ("contiguous", CONTIGUOUS.as_bytes()), 11 | ("interleaved", INTERLEAVED.as_bytes()), 12 | ("canada", CANADA.as_bytes()), 13 | ]; 14 | let mut group = c.benchmark_group("iter"); 15 | for (name, sample) in data { 16 | let len = sample.len(); 17 | group.throughput(criterion::Throughput::Bytes(len as u64)); 18 | 19 | group.bench_with_input( 20 | criterion::BenchmarkId::new("iterate", name), 21 | &len, 22 | |b, _| { 23 | b.iter(|| black_box(iterate.parse_peek(black_box(sample)).unwrap())); 24 | }, 25 | ); 26 | group.bench_with_input( 27 | criterion::BenchmarkId::new("next_token", name), 28 | &len, 29 | |b, _| { 30 | b.iter(|| black_box(next_token.parse_peek(black_box(sample)).unwrap())); 31 | }, 32 | ); 33 | group.bench_with_input( 34 | criterion::BenchmarkId::new("opt(one_of)", name), 35 | &len, 36 | |b, _| { 37 | b.iter(|| black_box(opt_one_of.parse_peek(black_box(sample)).unwrap())); 38 | }, 39 | ); 40 | group.bench_with_input( 41 | criterion::BenchmarkId::new("take_while", name), 42 | &len, 43 | |b, _| { 44 | b.iter(|| black_box(take_while.parse_peek(black_box(sample)).unwrap())); 45 | }, 46 | ); 47 | group.bench_with_input(criterion::BenchmarkId::new("repeat", name), &len, |b, _| { 48 | b.iter(|| black_box(repeat.parse_peek(black_box(sample)).unwrap())); 49 | }); 50 | } 51 | group.finish(); 52 | } 53 | 54 | fn iterate(input: &mut &[u8]) -> ModalResult { 55 | let mut count = 0; 56 | for byte in input.iter() { 57 | if byte.is_dec_digit() { 58 | count += 1; 59 | } 60 | } 61 | input.finish(); 62 | Ok(count) 63 | } 64 | 65 | fn next_token(input: &mut &[u8]) -> ModalResult { 66 | let mut count = 0; 67 | while let Some(byte) = input.next_token() { 68 | if byte.is_dec_digit() { 69 | count += 1; 70 | } 71 | } 72 | Ok(count) 73 | } 74 | 75 | fn opt_one_of(input: &mut &[u8]) -> ModalResult { 76 | let mut count = 0; 77 | while !input.is_empty() { 78 | while opt(one_of(AsChar::is_dec_digit)) 79 | .parse_next(input)? 80 | .is_some() 81 | { 82 | count += 1; 83 | } 84 | while opt(one_of(|b: u8| !b.is_dec_digit())) 85 | .parse_next(input)? 86 | .is_some() 87 | {} 88 | } 89 | Ok(count) 90 | } 91 | 92 | fn take_while(input: &mut &[u8]) -> ModalResult { 93 | let mut count = 0; 94 | while !input.is_empty() { 95 | count += winnow::token::take_while(0.., AsChar::is_dec_digit) 96 | .parse_next(input)? 97 | .len(); 98 | let _ = winnow::token::take_while(0.., |b: u8| !b.is_dec_digit()).parse_next(input)?; 99 | } 100 | Ok(count) 101 | } 102 | 103 | fn repeat(input: &mut &[u8]) -> ModalResult { 104 | let mut count = 0; 105 | while !input.is_empty() { 106 | count += winnow::combinator::repeat(0.., one_of(AsChar::is_dec_digit)) 107 | .map(|count: usize| count) 108 | .parse_next(input)?; 109 | let () = 110 | winnow::combinator::repeat(0.., one_of(|b: u8| !b.is_dec_digit())).parse_next(input)?; 111 | } 112 | Ok(count) 113 | } 114 | 115 | const CONTIGUOUS: &str = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; 116 | const INTERLEAVED: &str = "0123456789abc0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab"; 117 | const CANADA: &str = include_str!("../third_party/nativejson-benchmark/data/canada.json"); 118 | 119 | criterion::criterion_group!(benches, iter); 120 | criterion::criterion_main!(benches); 121 | -------------------------------------------------------------------------------- /benches/next_slice.rs: -------------------------------------------------------------------------------- 1 | use criterion::black_box; 2 | 3 | use winnow::combinator::repeat; 4 | use winnow::prelude::*; 5 | use winnow::token::literal; 6 | use winnow::token::one_of; 7 | 8 | fn next_slice(c: &mut criterion::Criterion) { 9 | let mut group = c.benchmark_group("next_slice"); 10 | 11 | let name = "ascii"; 12 | let sample = "h".repeat(100); 13 | let sample = sample.as_str(); 14 | group.bench_with_input( 15 | criterion::BenchmarkId::new("char", name), 16 | sample, 17 | |b, sample| { 18 | b.iter(|| black_box(parser_ascii_char.parse_peek(black_box(sample)).unwrap())); 19 | }, 20 | ); 21 | group.bench_with_input( 22 | criterion::BenchmarkId::new("str", name), 23 | sample, 24 | |b, sample| { 25 | b.iter(|| black_box(parser_ascii_str.parse_peek(black_box(sample)).unwrap())); 26 | }, 27 | ); 28 | group.bench_with_input( 29 | criterion::BenchmarkId::new("one_of", name), 30 | sample, 31 | |b, sample| { 32 | b.iter(|| black_box(parser_ascii_one_of.parse_peek(black_box(sample)).unwrap())); 33 | }, 34 | ); 35 | group.bench_with_input( 36 | criterion::BenchmarkId::new("tag_char", name), 37 | sample, 38 | |b, sample| { 39 | b.iter(|| black_box(parser_ascii_tag_char.parse_peek(black_box(sample)).unwrap())); 40 | }, 41 | ); 42 | group.bench_with_input( 43 | criterion::BenchmarkId::new("tag_str", name), 44 | sample, 45 | |b, sample| { 46 | b.iter(|| black_box(parser_ascii_tag_str.parse_peek(black_box(sample)).unwrap())); 47 | }, 48 | ); 49 | 50 | let name = "utf8"; 51 | let sample = "🧑".repeat(100); 52 | let sample = sample.as_str(); 53 | group.bench_with_input( 54 | criterion::BenchmarkId::new("char", name), 55 | sample, 56 | |b, sample| { 57 | b.iter(|| black_box(parser_utf8_char.parse_peek(black_box(sample)).unwrap())); 58 | }, 59 | ); 60 | group.bench_with_input( 61 | criterion::BenchmarkId::new("str", name), 62 | sample, 63 | |b, sample| { 64 | b.iter(|| black_box(parser_utf8_str.parse_peek(black_box(sample)).unwrap())); 65 | }, 66 | ); 67 | group.bench_with_input( 68 | criterion::BenchmarkId::new("one_of", name), 69 | sample, 70 | |b, sample| { 71 | b.iter(|| black_box(parser_utf8_one_of.parse_peek(black_box(sample)).unwrap())); 72 | }, 73 | ); 74 | group.bench_with_input( 75 | criterion::BenchmarkId::new("tag_char", name), 76 | sample, 77 | |b, sample| { 78 | b.iter(|| black_box(parser_utf8_tag_char.parse_peek(black_box(sample)).unwrap())); 79 | }, 80 | ); 81 | group.bench_with_input( 82 | criterion::BenchmarkId::new("tag_str", name), 83 | sample, 84 | |b, sample| { 85 | b.iter(|| black_box(parser_utf8_tag_str.parse_peek(black_box(sample)).unwrap())); 86 | }, 87 | ); 88 | 89 | group.finish(); 90 | } 91 | 92 | fn parser_ascii_char(input: &mut &str) -> ModalResult { 93 | repeat(0.., 'h').parse_next(input) 94 | } 95 | 96 | fn parser_ascii_str(input: &mut &str) -> ModalResult { 97 | repeat(0.., "h").parse_next(input) 98 | } 99 | 100 | fn parser_ascii_one_of(input: &mut &str) -> ModalResult { 101 | repeat(0.., one_of('h')).parse_next(input) 102 | } 103 | 104 | fn parser_ascii_tag_char(input: &mut &str) -> ModalResult { 105 | repeat(0.., literal('h')).parse_next(input) 106 | } 107 | 108 | fn parser_ascii_tag_str(input: &mut &str) -> ModalResult { 109 | repeat(0.., literal("h")).parse_next(input) 110 | } 111 | 112 | fn parser_utf8_char(input: &mut &str) -> ModalResult { 113 | repeat(0.., '🧑').parse_next(input) 114 | } 115 | 116 | fn parser_utf8_str(input: &mut &str) -> ModalResult { 117 | repeat(0.., "🧑").parse_next(input) 118 | } 119 | 120 | fn parser_utf8_one_of(input: &mut &str) -> ModalResult { 121 | repeat(0.., one_of('🧑')).parse_next(input) 122 | } 123 | 124 | fn parser_utf8_tag_char(input: &mut &str) -> ModalResult { 125 | repeat(0.., literal('🧑')).parse_next(input) 126 | } 127 | 128 | fn parser_utf8_tag_str(input: &mut &str) -> ModalResult { 129 | repeat(0.., literal("🧑")).parse_next(input) 130 | } 131 | 132 | criterion::criterion_group!(benches, next_slice); 133 | criterion::criterion_main!(benches); 134 | -------------------------------------------------------------------------------- /benches/number.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate criterion; 3 | 4 | use criterion::Criterion; 5 | 6 | use winnow::ascii::float; 7 | use winnow::binary::be_u64; 8 | use winnow::error::InputError; 9 | use winnow::error::ParserError; 10 | use winnow::prelude::*; 11 | use winnow::stream::ParseSlice; 12 | 13 | type Stream<'i> = &'i [u8]; 14 | 15 | fn parser(i: &mut Stream<'_>) -> ModalResult { 16 | be_u64.parse_next(i) 17 | } 18 | 19 | fn number(c: &mut Criterion) { 20 | let data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; 21 | 22 | parser 23 | .parse_peek(&data[..]) 24 | .expect("should parse correctly"); 25 | c.bench_function("number", move |b| { 26 | b.iter(|| parser.parse_peek(&data[..]).unwrap()); 27 | }); 28 | } 29 | 30 | fn float_bytes(c: &mut Criterion) { 31 | println!( 32 | "float_bytes result: {:?}", 33 | float::<_, f64, InputError<_>>.parse_peek(&b"-1.234E-12"[..]) 34 | ); 35 | c.bench_function("float bytes", |b| { 36 | b.iter(|| float::<_, f64, InputError<_>>.parse_peek(&b"-1.234E-12"[..])); 37 | }); 38 | } 39 | 40 | fn float_str(c: &mut Criterion) { 41 | println!( 42 | "float_str result: {:?}", 43 | float::<_, f64, InputError<_>>.parse_peek("-1.234E-12") 44 | ); 45 | c.bench_function("float str", |b| { 46 | b.iter(|| float::<_, f64, InputError<_>>.parse_peek("-1.234E-12")); 47 | }); 48 | } 49 | 50 | fn std_float(input: &mut &[u8]) -> ModalResult { 51 | match input.parse_slice() { 52 | Some(n) => Ok(n), 53 | None => Err(ParserError::from_input(input)), 54 | } 55 | } 56 | 57 | fn std_float_bytes(c: &mut Criterion) { 58 | println!( 59 | "std_float_bytes result: {:?}", 60 | std_float.parse_peek(&b"-1.234E-12"[..]) 61 | ); 62 | c.bench_function("std_float bytes", |b| { 63 | b.iter(|| std_float.parse_peek(&b"-1.234E-12"[..])); 64 | }); 65 | } 66 | 67 | criterion_group!(benches, number, float_bytes, std_float_bytes, float_str); 68 | criterion_main!(benches); 69 | -------------------------------------------------------------------------------- /committed.toml: -------------------------------------------------------------------------------- 1 | style="conventional" 2 | ignore_author_re="(dependabot|renovate)" 3 | merge_commit = false 4 | -------------------------------------------------------------------------------- /examples/arithmetic/bench.rs: -------------------------------------------------------------------------------- 1 | mod parser; 2 | mod parser_ast; 3 | mod parser_lexer; 4 | 5 | use winnow::prelude::*; 6 | 7 | #[allow(clippy::eq_op, clippy::erasing_op)] 8 | fn arithmetic(c: &mut criterion::Criterion) { 9 | let data = " 2*2 / ( 5 - 1) + 3 / 4 * (2 - 7 + 567 *12 /2) + 3*(1+2*( 45 /2))"; 10 | let expected = 2 * 2 / (5 - 1) + 3 * (1 + 2 * (45 / 2)); 11 | 12 | assert_eq!(parser::expr.parse(data), Ok(expected)); 13 | assert_eq!( 14 | parser_ast::expr.parse(data).map(|ast| ast.eval()), 15 | Ok(expected) 16 | ); 17 | assert_eq!( 18 | parser_lexer::expr2.parse(data).map(|ast| ast.eval()), 19 | Ok(expected) 20 | ); 21 | c.bench_function("direct", |b| { 22 | b.iter(|| parser::expr.parse(data).unwrap()); 23 | }); 24 | c.bench_function("ast", |b| { 25 | b.iter(|| parser_ast::expr.parse(data).unwrap().eval()); 26 | }); 27 | c.bench_function("lexer", |b| { 28 | b.iter(|| parser_lexer::expr2.parse_peek(data).unwrap()); 29 | }); 30 | } 31 | 32 | criterion::criterion_group!(benches, arithmetic); 33 | criterion::criterion_main!(benches); 34 | -------------------------------------------------------------------------------- /examples/arithmetic/main.rs: -------------------------------------------------------------------------------- 1 | use winnow::prelude::*; 2 | 3 | mod parser; 4 | mod parser_ast; 5 | mod parser_lexer; 6 | #[cfg(test)] 7 | mod test_parser; 8 | #[cfg(test)] 9 | mod test_parser_ast; 10 | #[cfg(test)] 11 | mod test_parser_lexer; 12 | 13 | fn main() -> Result<(), lexopt::Error> { 14 | let args = Args::parse()?; 15 | 16 | let input = args.input.as_deref().unwrap_or("1 + 1"); 17 | if let Err(err) = calc(input, args.implementation) { 18 | println!("FAILED"); 19 | println!("{err}"); 20 | } 21 | 22 | Ok(()) 23 | } 24 | 25 | fn calc( 26 | input: &str, 27 | imp: Impl, 28 | ) -> Result<(), winnow::error::ParseError<&str, winnow::error::ContextError>> { 29 | println!("{input} ="); 30 | match imp { 31 | Impl::Eval => { 32 | let result = parser::expr.parse(input)?; 33 | println!(" {result}"); 34 | } 35 | Impl::Ast => { 36 | let result = parser_ast::expr.parse(input)?; 37 | println!(" {:#?}={}", result, result.eval()); 38 | } 39 | Impl::Lexer => { 40 | let tokens = parser_lexer::tokens.parse(input)?; 41 | println!(" {tokens:#?}"); 42 | let tokens = parser_lexer::Tokens::new(&tokens); 43 | let result = parser_lexer::expr.parse(tokens).unwrap(); 44 | println!(" {:#?}={}", result, result.eval()); 45 | } 46 | } 47 | Ok(()) 48 | } 49 | 50 | #[derive(Default)] 51 | struct Args { 52 | input: Option, 53 | implementation: Impl, 54 | } 55 | 56 | enum Impl { 57 | Eval, 58 | Ast, 59 | Lexer, 60 | } 61 | 62 | impl Default for Impl { 63 | fn default() -> Self { 64 | Self::Eval 65 | } 66 | } 67 | 68 | impl Args { 69 | fn parse() -> Result { 70 | use lexopt::prelude::*; 71 | 72 | let mut res = Args::default(); 73 | 74 | let mut args = lexopt::Parser::from_env(); 75 | while let Some(arg) = args.next()? { 76 | match arg { 77 | Long("impl") => { 78 | res.implementation = args.value()?.parse_with(|s| match s { 79 | "eval" => Ok(Impl::Eval), 80 | "ast" => Ok(Impl::Ast), 81 | "lexer" => Ok(Impl::Lexer), 82 | _ => Err("expected `eval`, `ast`"), 83 | })?; 84 | } 85 | Value(input) => { 86 | res.input = Some(input.string()?); 87 | } 88 | _ => return Err(arg.unexpected()), 89 | } 90 | } 91 | Ok(res) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /examples/arithmetic/parser.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use winnow::prelude::*; 4 | use winnow::Result; 5 | use winnow::{ 6 | ascii::{digit1 as digits, multispace0 as multispaces}, 7 | combinator::alt, 8 | combinator::delimited, 9 | combinator::repeat, 10 | token::one_of, 11 | }; 12 | 13 | // Parser definition 14 | 15 | pub(crate) fn expr(i: &mut &str) -> Result { 16 | let init = term.parse_next(i)?; 17 | 18 | repeat(0.., (one_of(['+', '-']), term)) 19 | .fold( 20 | move || init, 21 | |acc, (op, val): (char, i64)| { 22 | if op == '+' { 23 | acc + val 24 | } else { 25 | acc - val 26 | } 27 | }, 28 | ) 29 | .parse_next(i) 30 | } 31 | 32 | // We read an initial factor and for each time we find 33 | // a * or / operator followed by another factor, we do 34 | // the math by folding everything 35 | pub(crate) fn term(i: &mut &str) -> Result { 36 | let init = factor.parse_next(i)?; 37 | 38 | repeat(0.., (one_of(['*', '/']), factor)) 39 | .fold( 40 | move || init, 41 | |acc, (op, val): (char, i64)| { 42 | if op == '*' { 43 | acc * val 44 | } else { 45 | acc / val 46 | } 47 | }, 48 | ) 49 | .parse_next(i) 50 | } 51 | 52 | // We transform an integer string into a i64, ignoring surrounding whitespace 53 | // We look for a digit suite, and try to convert it. 54 | // If either str::from_utf8 or FromStr::from_str fail, 55 | // we fallback to the parens parser defined above 56 | pub(crate) fn factor(i: &mut &str) -> Result { 57 | delimited( 58 | multispaces, 59 | alt((digits.try_map(FromStr::from_str), parens)), 60 | multispaces, 61 | ) 62 | .parse_next(i) 63 | } 64 | 65 | // We parse any expr surrounded by parens, ignoring all whitespace around those 66 | fn parens(i: &mut &str) -> Result { 67 | delimited('(', expr, ')').parse_next(i) 68 | } 69 | -------------------------------------------------------------------------------- /examples/arithmetic/parser_ast.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::{Debug, Display, Formatter}; 3 | 4 | use std::str::FromStr; 5 | 6 | use winnow::prelude::*; 7 | use winnow::Result; 8 | use winnow::{ 9 | ascii::{digit1 as digits, multispace0 as multispaces}, 10 | combinator::alt, 11 | combinator::delimited, 12 | combinator::repeat, 13 | token::one_of, 14 | }; 15 | 16 | #[derive(Debug, Clone)] 17 | pub(crate) enum Expr { 18 | Value(i64), 19 | Add(Box, Box), 20 | Sub(Box, Box), 21 | Mul(Box, Box), 22 | Div(Box, Box), 23 | Paren(Box), 24 | } 25 | 26 | impl Expr { 27 | pub(crate) fn eval(&self) -> i64 { 28 | match self { 29 | Self::Value(v) => *v, 30 | Self::Add(lhs, rhs) => lhs.eval() + rhs.eval(), 31 | Self::Sub(lhs, rhs) => lhs.eval() - rhs.eval(), 32 | Self::Mul(lhs, rhs) => lhs.eval() * rhs.eval(), 33 | Self::Div(lhs, rhs) => lhs.eval() / rhs.eval(), 34 | Self::Paren(expr) => expr.eval(), 35 | } 36 | } 37 | } 38 | 39 | impl Display for Expr { 40 | fn fmt(&self, format: &mut Formatter<'_>) -> fmt::Result { 41 | use Expr::{Add, Div, Mul, Paren, Sub, Value}; 42 | match *self { 43 | Value(val) => write!(format, "{val}"), 44 | Add(ref left, ref right) => write!(format, "{left} + {right}"), 45 | Sub(ref left, ref right) => write!(format, "{left} - {right}"), 46 | Mul(ref left, ref right) => write!(format, "{left} * {right}"), 47 | Div(ref left, ref right) => write!(format, "{left} / {right}"), 48 | Paren(ref expr) => write!(format, "({expr})"), 49 | } 50 | } 51 | } 52 | 53 | pub(crate) fn expr(i: &mut &str) -> Result { 54 | let init = term.parse_next(i)?; 55 | 56 | repeat(0.., (one_of(['+', '-']), term)) 57 | .fold( 58 | move || init.clone(), 59 | |acc, (op, val): (char, Expr)| { 60 | if op == '+' { 61 | Expr::Add(Box::new(acc), Box::new(val)) 62 | } else { 63 | Expr::Sub(Box::new(acc), Box::new(val)) 64 | } 65 | }, 66 | ) 67 | .parse_next(i) 68 | } 69 | 70 | pub(crate) fn term(i: &mut &str) -> Result { 71 | let init = factor.parse_next(i)?; 72 | 73 | repeat(0.., (one_of(['*', '/']), factor)) 74 | .fold( 75 | move || init.clone(), 76 | |acc, (op, val): (char, Expr)| { 77 | if op == '*' { 78 | Expr::Mul(Box::new(acc), Box::new(val)) 79 | } else { 80 | Expr::Div(Box::new(acc), Box::new(val)) 81 | } 82 | }, 83 | ) 84 | .parse_next(i) 85 | } 86 | 87 | pub(crate) fn factor(i: &mut &str) -> Result { 88 | delimited( 89 | multispaces, 90 | alt((digits.try_map(FromStr::from_str).map(Expr::Value), parens)), 91 | multispaces, 92 | ) 93 | .parse_next(i) 94 | } 95 | 96 | fn parens(i: &mut &str) -> Result { 97 | delimited("(", expr, ")") 98 | .map(|e| Expr::Paren(Box::new(e))) 99 | .parse_next(i) 100 | } 101 | -------------------------------------------------------------------------------- /examples/arithmetic/test_parser.rs: -------------------------------------------------------------------------------- 1 | use snapbox::assert_data_eq; 2 | use snapbox::prelude::*; 3 | use snapbox::str; 4 | use winnow::prelude::*; 5 | 6 | use crate::parser::*; 7 | 8 | #[test] 9 | fn factor_test() { 10 | let input = "3"; 11 | let expected = str![[r#" 12 | Ok( 13 | ( 14 | "", 15 | 3, 16 | ), 17 | ) 18 | 19 | "#]]; 20 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 21 | 22 | let input = " 12"; 23 | let expected = str![[r#" 24 | Ok( 25 | ( 26 | "", 27 | 12, 28 | ), 29 | ) 30 | 31 | "#]]; 32 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 33 | 34 | let input = "537 "; 35 | let expected = str![[r#" 36 | Ok( 37 | ( 38 | "", 39 | 537, 40 | ), 41 | ) 42 | 43 | "#]]; 44 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 45 | 46 | let input = " 24 "; 47 | let expected = str![[r#" 48 | Ok( 49 | ( 50 | "", 51 | 24, 52 | ), 53 | ) 54 | 55 | "#]]; 56 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 57 | } 58 | 59 | #[test] 60 | fn term_test() { 61 | let input = " 12 *2 / 3"; 62 | let expected = str![[r#" 63 | Ok( 64 | ( 65 | "", 66 | 8, 67 | ), 68 | ) 69 | 70 | "#]]; 71 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 72 | 73 | let input = " 12 *2 / 3"; 74 | let expected = str![[r#" 75 | Ok( 76 | ( 77 | "", 78 | 8, 79 | ), 80 | ) 81 | 82 | "#]]; 83 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 84 | 85 | let input = " 2* 3 *2 *2 / 3"; 86 | let expected = str![[r#" 87 | Ok( 88 | ( 89 | "", 90 | 8, 91 | ), 92 | ) 93 | 94 | "#]]; 95 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 96 | 97 | let input = " 48 / 3/2"; 98 | let expected = str![[r#" 99 | Ok( 100 | ( 101 | "", 102 | 8, 103 | ), 104 | ) 105 | 106 | "#]]; 107 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 108 | } 109 | 110 | #[test] 111 | fn expr_test() { 112 | let input = " 1 + 2 "; 113 | let expected = str![[r#" 114 | Ok( 115 | ( 116 | "", 117 | 3, 118 | ), 119 | ) 120 | 121 | "#]]; 122 | assert_data_eq!(expr.parse_peek(input).to_debug(), expected); 123 | 124 | let input = " 12 + 6 - 4+ 3"; 125 | let expected = str![[r#" 126 | Ok( 127 | ( 128 | "", 129 | 17, 130 | ), 131 | ) 132 | 133 | "#]]; 134 | assert_data_eq!(expr.parse_peek(input).to_debug(), expected); 135 | 136 | let input = " 1 + 2*3 + 4"; 137 | let expected = str![[r#" 138 | Ok( 139 | ( 140 | "", 141 | 11, 142 | ), 143 | ) 144 | 145 | "#]]; 146 | assert_data_eq!(expr.parse_peek(input).to_debug(), expected); 147 | } 148 | 149 | #[test] 150 | fn parens_test() { 151 | let input = " ( 2 )"; 152 | let expected = str![[r#" 153 | Ok( 154 | ( 155 | "", 156 | 2, 157 | ), 158 | ) 159 | 160 | "#]]; 161 | assert_data_eq!(expr.parse_peek(input).to_debug(), expected); 162 | 163 | let input = " 2* ( 3 + 4 ) "; 164 | let expected = str![[r#" 165 | Ok( 166 | ( 167 | "", 168 | 14, 169 | ), 170 | ) 171 | 172 | "#]]; 173 | assert_data_eq!(expr.parse_peek(input).to_debug(), expected); 174 | 175 | let input = " 2*2 / ( 5 - 1) + 3"; 176 | let expected = str![[r#" 177 | Ok( 178 | ( 179 | "", 180 | 4, 181 | ), 182 | ) 183 | 184 | "#]]; 185 | assert_data_eq!(expr.parse_peek(input).to_debug(), expected); 186 | } 187 | -------------------------------------------------------------------------------- /examples/arithmetic/test_parser_ast.rs: -------------------------------------------------------------------------------- 1 | use snapbox::assert_data_eq; 2 | use snapbox::prelude::*; 3 | use snapbox::str; 4 | use winnow::prelude::*; 5 | 6 | use crate::parser_ast::*; 7 | 8 | #[test] 9 | fn factor_test() { 10 | let input = "3"; 11 | let expected = str![[r#" 12 | Ok( 13 | ( 14 | "", 15 | Value( 16 | 3, 17 | ), 18 | ), 19 | ) 20 | 21 | "#]]; 22 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 23 | 24 | let input = " 12"; 25 | let expected = str![[r#" 26 | Ok( 27 | ( 28 | "", 29 | Value( 30 | 12, 31 | ), 32 | ), 33 | ) 34 | 35 | "#]]; 36 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 37 | 38 | let input = "537 "; 39 | let expected = str![[r#" 40 | Ok( 41 | ( 42 | "", 43 | Value( 44 | 537, 45 | ), 46 | ), 47 | ) 48 | 49 | "#]]; 50 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 51 | 52 | let input = " 24 "; 53 | let expected = str![[r#" 54 | Ok( 55 | ( 56 | "", 57 | Value( 58 | 24, 59 | ), 60 | ), 61 | ) 62 | 63 | "#]]; 64 | assert_data_eq!(factor.parse_peek(input).to_debug(), expected); 65 | } 66 | 67 | #[test] 68 | fn term_test() { 69 | let input = " 12 *2 / 3"; 70 | let expected = str![[r#" 71 | Ok( 72 | ( 73 | "", 74 | Div( 75 | Mul( 76 | Value( 77 | 12, 78 | ), 79 | Value( 80 | 2, 81 | ), 82 | ), 83 | Value( 84 | 3, 85 | ), 86 | ), 87 | ), 88 | ) 89 | 90 | "#]]; 91 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 92 | 93 | let input = " 12 *2 / 3"; 94 | let expected = str![[r#" 95 | Ok( 96 | ( 97 | "", 98 | Div( 99 | Mul( 100 | Value( 101 | 12, 102 | ), 103 | Value( 104 | 2, 105 | ), 106 | ), 107 | Value( 108 | 3, 109 | ), 110 | ), 111 | ), 112 | ) 113 | 114 | "#]]; 115 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 116 | 117 | let input = " 2* 3 *2 *2 / 3"; 118 | let expected = str![[r#" 119 | Ok( 120 | ( 121 | "", 122 | Div( 123 | Mul( 124 | Mul( 125 | Mul( 126 | Value( 127 | 2, 128 | ), 129 | Value( 130 | 3, 131 | ), 132 | ), 133 | Value( 134 | 2, 135 | ), 136 | ), 137 | Value( 138 | 2, 139 | ), 140 | ), 141 | Value( 142 | 3, 143 | ), 144 | ), 145 | ), 146 | ) 147 | 148 | "#]]; 149 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 150 | 151 | let input = " 48 / 3/2"; 152 | let expected = str![[r#" 153 | Ok( 154 | ( 155 | "", 156 | Div( 157 | Div( 158 | Value( 159 | 48, 160 | ), 161 | Value( 162 | 3, 163 | ), 164 | ), 165 | Value( 166 | 2, 167 | ), 168 | ), 169 | ), 170 | ) 171 | 172 | "#]]; 173 | assert_data_eq!(term.parse_peek(input).to_debug(), expected); 174 | } 175 | 176 | #[test] 177 | fn expr_test() { 178 | let input = " 1 + 2 "; 179 | let expected = str![[r#" 180 | Ok( 181 | Add( 182 | Value( 183 | 1, 184 | ), 185 | Value( 186 | 2, 187 | ), 188 | ), 189 | ) 190 | 191 | "#]]; 192 | assert_data_eq!(expr.parse(input).to_debug(), expected); 193 | 194 | let input = " 12 + 6 - 4+ 3"; 195 | let expected = str![[r#" 196 | Ok( 197 | Add( 198 | Sub( 199 | Add( 200 | Value( 201 | 12, 202 | ), 203 | Value( 204 | 6, 205 | ), 206 | ), 207 | Value( 208 | 4, 209 | ), 210 | ), 211 | Value( 212 | 3, 213 | ), 214 | ), 215 | ) 216 | 217 | "#]]; 218 | assert_data_eq!(expr.parse(input).to_debug(), expected); 219 | 220 | let input = " 1 + 2*3 + 4"; 221 | let expected = str![[r#" 222 | Ok( 223 | Add( 224 | Add( 225 | Value( 226 | 1, 227 | ), 228 | Mul( 229 | Value( 230 | 2, 231 | ), 232 | Value( 233 | 3, 234 | ), 235 | ), 236 | ), 237 | Value( 238 | 4, 239 | ), 240 | ), 241 | ) 242 | 243 | "#]]; 244 | assert_data_eq!(expr.parse(input).to_debug(), expected); 245 | } 246 | 247 | #[test] 248 | fn parens_test() { 249 | let input = " ( 2 )"; 250 | let expected = str![[r#" 251 | Ok( 252 | Paren( 253 | Value( 254 | 2, 255 | ), 256 | ), 257 | ) 258 | 259 | "#]]; 260 | assert_data_eq!(expr.parse(input).to_debug(), expected); 261 | 262 | let input = " 2* ( 3 + 4 ) "; 263 | let expected = str![[r#" 264 | Ok( 265 | Mul( 266 | Value( 267 | 2, 268 | ), 269 | Paren( 270 | Add( 271 | Value( 272 | 3, 273 | ), 274 | Value( 275 | 4, 276 | ), 277 | ), 278 | ), 279 | ), 280 | ) 281 | 282 | "#]]; 283 | assert_data_eq!(expr.parse(input).to_debug(), expected); 284 | 285 | let input = " 2*2 / ( 5 - 1) + 3"; 286 | let expected = str![[r#" 287 | Ok( 288 | Add( 289 | Div( 290 | Mul( 291 | Value( 292 | 2, 293 | ), 294 | Value( 295 | 2, 296 | ), 297 | ), 298 | Paren( 299 | Sub( 300 | Value( 301 | 5, 302 | ), 303 | Value( 304 | 1, 305 | ), 306 | ), 307 | ), 308 | ), 309 | Value( 310 | 3, 311 | ), 312 | ), 313 | ) 314 | 315 | "#]]; 316 | assert_data_eq!(expr.parse(input).to_debug(), expected); 317 | } 318 | -------------------------------------------------------------------------------- /examples/css/main.rs: -------------------------------------------------------------------------------- 1 | use winnow::prelude::*; 2 | 3 | mod parser; 4 | 5 | use parser::hex_color; 6 | 7 | fn main() -> Result<(), lexopt::Error> { 8 | let args = Args::parse()?; 9 | 10 | let input = args.input.as_deref().unwrap_or("#AAAAAA"); 11 | 12 | println!("{input} ="); 13 | match hex_color.parse(input) { 14 | Ok(result) => { 15 | println!(" {result:?}"); 16 | } 17 | Err(err) => { 18 | println!(" {err}"); 19 | } 20 | } 21 | 22 | Ok(()) 23 | } 24 | 25 | #[derive(Default)] 26 | struct Args { 27 | input: Option, 28 | } 29 | 30 | impl Args { 31 | fn parse() -> Result { 32 | use lexopt::prelude::*; 33 | 34 | let mut res = Args::default(); 35 | 36 | let mut args = lexopt::Parser::from_env(); 37 | while let Some(arg) = args.next()? { 38 | match arg { 39 | Value(input) => { 40 | res.input = Some(input.string()?); 41 | } 42 | _ => return Err(arg.unexpected()), 43 | } 44 | } 45 | Ok(res) 46 | } 47 | } 48 | 49 | #[test] 50 | fn parse_color() { 51 | assert_eq!( 52 | hex_color.parse_peek("#2F14DF"), 53 | Ok(( 54 | "", 55 | parser::Color { 56 | red: 47, 57 | green: 20, 58 | blue: 223, 59 | } 60 | )) 61 | ); 62 | } 63 | -------------------------------------------------------------------------------- /examples/css/parser.rs: -------------------------------------------------------------------------------- 1 | use winnow::combinator::seq; 2 | use winnow::prelude::*; 3 | use winnow::token::take_while; 4 | use winnow::Result; 5 | 6 | #[derive(Debug, Eq, PartialEq)] 7 | pub(crate) struct Color { 8 | pub(crate) red: u8, 9 | pub(crate) green: u8, 10 | pub(crate) blue: u8, 11 | } 12 | 13 | impl std::str::FromStr for Color { 14 | // The error must be owned 15 | type Err = String; 16 | 17 | fn from_str(s: &str) -> Result { 18 | hex_color.parse(s).map_err(|e| e.to_string()) 19 | } 20 | } 21 | 22 | pub(crate) fn hex_color(input: &mut &str) -> Result { 23 | seq!(Color { 24 | _: '#', 25 | red: hex_primary, 26 | green: hex_primary, 27 | blue: hex_primary 28 | }) 29 | .parse_next(input) 30 | } 31 | 32 | fn hex_primary(input: &mut &str) -> Result { 33 | take_while(2, |c: char| c.is_ascii_hexdigit()) 34 | .try_map(|input| u8::from_str_radix(input, 16)) 35 | .parse_next(input) 36 | } 37 | -------------------------------------------------------------------------------- /examples/custom_error.rs: -------------------------------------------------------------------------------- 1 | use winnow::error::AddContext; 2 | use winnow::error::ErrMode; 3 | use winnow::error::FromExternalError; 4 | use winnow::error::ParserError; 5 | use winnow::prelude::*; 6 | use winnow::stream::Stream; 7 | 8 | #[derive(Debug)] 9 | pub enum CustomError { 10 | MyError, 11 | Winnow(I), 12 | External { 13 | cause: Box, 14 | input: I, 15 | }, 16 | } 17 | 18 | impl ParserError for CustomError { 19 | type Inner = Self; 20 | 21 | fn from_input(input: &I) -> Self { 22 | CustomError::Winnow(input.clone()) 23 | } 24 | 25 | fn into_inner(self) -> Result { 26 | Ok(self) 27 | } 28 | } 29 | 30 | impl AddContext for CustomError { 31 | #[inline] 32 | fn add_context( 33 | self, 34 | _input: &I, 35 | _token_start: &::Checkpoint, 36 | _context: C, 37 | ) -> Self { 38 | self 39 | } 40 | } 41 | 42 | impl FromExternalError 43 | for CustomError 44 | { 45 | #[inline] 46 | fn from_external_error(input: &I, e: E) -> Self { 47 | CustomError::External { 48 | cause: Box::new(e), 49 | input: input.clone(), 50 | } 51 | } 52 | } 53 | 54 | pub fn parse<'s>(_input: &mut &'s str) -> ModalResult<&'s str, CustomError<&'s str>> { 55 | Err(ErrMode::Backtrack(CustomError::MyError)) 56 | } 57 | 58 | fn main() {} 59 | 60 | #[cfg(test)] 61 | mod tests { 62 | use super::*; 63 | 64 | #[test] 65 | fn it_works() { 66 | let err = parse.parse_next(&mut "").unwrap_err(); 67 | assert!(matches!(err, ErrMode::Backtrack(CustomError::MyError))); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /examples/http/bench.rs: -------------------------------------------------------------------------------- 1 | mod parser; 2 | mod parser_streaming; 3 | 4 | fn one_test(c: &mut criterion::Criterion) { 5 | let data = &b"GET / HTTP/1.1 6 | Host: www.reddit.com 7 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1 8 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 9 | Accept-Language: en-us,en;q=0.5 10 | Accept-Encoding: gzip, deflate 11 | Connection: keep-alive 12 | 13 | "[..]; 14 | 15 | let mut http_group = c.benchmark_group("http"); 16 | http_group.throughput(criterion::Throughput::Bytes(data.len() as u64)); 17 | http_group.bench_with_input( 18 | criterion::BenchmarkId::new("complete", data.len()), 19 | data, 20 | |b, data| { 21 | b.iter(|| parser::parse(data).unwrap()); 22 | }, 23 | ); 24 | http_group.bench_with_input( 25 | criterion::BenchmarkId::new("streaming", data.len()), 26 | data, 27 | |b, data| { 28 | b.iter(|| parser_streaming::parse(data).unwrap()); 29 | }, 30 | ); 31 | 32 | http_group.finish(); 33 | } 34 | 35 | criterion::criterion_group!(http, one_test); 36 | criterion::criterion_main!(http); 37 | -------------------------------------------------------------------------------- /examples/http/main.rs: -------------------------------------------------------------------------------- 1 | mod parser; 2 | 3 | fn main() -> Result<(), lexopt::Error> { 4 | let args = Args::parse()?; 5 | 6 | let input = args.input.as_deref().unwrap_or( 7 | "GET / HTTP/1.1 8 | Host: www.reddit.com 9 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1 10 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 11 | Accept-Language: en-us,en;q=0.5 12 | Accept-Encoding: gzip, deflate 13 | Connection: keep-alive 14 | 15 | ", 16 | ); 17 | 18 | if let Some(result) = parser::parse(input.as_bytes()) { 19 | println!(" {result:#?}"); 20 | } 21 | 22 | Ok(()) 23 | } 24 | 25 | #[derive(Default)] 26 | struct Args { 27 | input: Option, 28 | } 29 | 30 | impl Args { 31 | fn parse() -> Result { 32 | use lexopt::prelude::*; 33 | 34 | let mut res = Args::default(); 35 | 36 | let mut args = lexopt::Parser::from_env(); 37 | while let Some(arg) = args.next()? { 38 | match arg { 39 | Value(input) => { 40 | res.input = Some(input.string()?); 41 | } 42 | _ => return Err(arg.unexpected()), 43 | } 44 | } 45 | Ok(res) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /examples/http/parser.rs: -------------------------------------------------------------------------------- 1 | use winnow::combinator::seq; 2 | use winnow::prelude::*; 3 | use winnow::{ascii::line_ending, combinator::repeat, token::take_while}; 4 | 5 | pub(crate) type Stream<'i> = &'i [u8]; 6 | 7 | #[rustfmt::skip] 8 | #[derive(Debug)] 9 | #[allow(dead_code)] 10 | pub(crate) struct Request<'a> { 11 | method: &'a [u8], 12 | uri: &'a [u8], 13 | version: &'a [u8], 14 | } 15 | 16 | #[derive(Debug)] 17 | #[allow(dead_code)] 18 | pub(crate) struct Header<'a> { 19 | name: &'a [u8], 20 | value: Vec<&'a [u8]>, 21 | } 22 | 23 | pub(crate) fn parse(data: &[u8]) -> Option, Vec>)>> { 24 | let mut buf = data; 25 | let mut v = Vec::new(); 26 | loop { 27 | match request(&mut buf) { 28 | Ok(r) => { 29 | v.push(r); 30 | 31 | if buf.is_empty() { 32 | //println!("{}", i); 33 | break; 34 | } 35 | } 36 | Err(e) => { 37 | println!("error: {e:?}"); 38 | return None; 39 | } 40 | } 41 | } 42 | 43 | Some(v) 44 | } 45 | 46 | fn request<'s>(input: &mut Stream<'s>) -> ModalResult<(Request<'s>, Vec>)> { 47 | let req = request_line(input)?; 48 | let h = repeat(1.., message_header).parse_next(input)?; 49 | let _ = line_ending.parse_next(input)?; 50 | 51 | Ok((req, h)) 52 | } 53 | 54 | fn request_line<'s>(input: &mut Stream<'s>) -> ModalResult> { 55 | seq!( Request { 56 | method: take_while(1.., is_token), 57 | _: take_while(1.., is_space), 58 | uri: take_while(1.., is_not_space), 59 | _: take_while(1.., is_space), 60 | version: http_version, 61 | _: line_ending, 62 | }) 63 | .parse_next(input) 64 | } 65 | 66 | fn http_version<'s>(input: &mut Stream<'s>) -> ModalResult<&'s [u8]> { 67 | let _ = "HTTP/".parse_next(input)?; 68 | let version = take_while(1.., is_version).parse_next(input)?; 69 | 70 | Ok(version) 71 | } 72 | 73 | fn message_header_value<'s>(input: &mut Stream<'s>) -> ModalResult<&'s [u8]> { 74 | let _ = take_while(1.., is_horizontal_space).parse_next(input)?; 75 | let data = take_while(1.., till_line_ending).parse_next(input)?; 76 | let _ = line_ending.parse_next(input)?; 77 | 78 | Ok(data) 79 | } 80 | 81 | fn message_header<'s>(input: &mut Stream<'s>) -> ModalResult> { 82 | seq!(Header { 83 | name: take_while(1.., is_token), 84 | _: ':', 85 | value: repeat(1.., message_header_value), 86 | }) 87 | .parse_next(input) 88 | } 89 | 90 | #[rustfmt::skip] 91 | #[allow(clippy::match_same_arms)] 92 | #[allow(clippy::match_like_matches_macro)] 93 | fn is_token(c: u8) -> bool { 94 | match c { 95 | 128..=255 => false, 96 | 0..=31 => false, 97 | b'(' => false, 98 | b')' => false, 99 | b'<' => false, 100 | b'>' => false, 101 | b'@' => false, 102 | b',' => false, 103 | b';' => false, 104 | b':' => false, 105 | b'\\' => false, 106 | b'"' => false, 107 | b'/' => false, 108 | b'[' => false, 109 | b']' => false, 110 | b'?' => false, 111 | b'=' => false, 112 | b'{' => false, 113 | b'}' => false, 114 | b' ' => false, 115 | _ => true, 116 | } 117 | } 118 | 119 | fn is_version(c: u8) -> bool { 120 | c.is_ascii_digit() || c == b'.' 121 | } 122 | 123 | fn till_line_ending(c: u8) -> bool { 124 | c != b'\r' && c != b'\n' 125 | } 126 | 127 | fn is_space(c: u8) -> bool { 128 | c == b' ' 129 | } 130 | 131 | fn is_not_space(c: u8) -> bool { 132 | c != b' ' 133 | } 134 | 135 | fn is_horizontal_space(c: u8) -> bool { 136 | c == b' ' || c == b'\t' 137 | } 138 | -------------------------------------------------------------------------------- /examples/http/parser_streaming.rs: -------------------------------------------------------------------------------- 1 | use winnow::combinator::seq; 2 | use winnow::{ 3 | ascii::line_ending, combinator::repeat, prelude::*, stream::Partial, token::take_while, 4 | }; 5 | 6 | pub(crate) type Stream<'i> = Partial<&'i [u8]>; 7 | 8 | #[rustfmt::skip] 9 | #[derive(Debug)] 10 | #[allow(dead_code)] 11 | pub(crate) struct Request<'a> { 12 | method: &'a [u8], 13 | uri: &'a [u8], 14 | version: &'a [u8], 15 | } 16 | 17 | #[derive(Debug)] 18 | #[allow(dead_code)] 19 | pub(crate) struct Header<'a> { 20 | name: &'a [u8], 21 | value: Vec<&'a [u8]>, 22 | } 23 | 24 | pub(crate) fn parse(data: &[u8]) -> Option, Vec>)>> { 25 | let mut buf = Partial::new(data); 26 | let mut v = Vec::new(); 27 | loop { 28 | match request(&mut buf) { 29 | Ok(r) => { 30 | v.push(r); 31 | 32 | if buf.is_empty() { 33 | //println!("{}", i); 34 | break; 35 | } 36 | } 37 | Err(e) => { 38 | println!("error: {e:?}"); 39 | return None; 40 | } 41 | } 42 | } 43 | 44 | Some(v) 45 | } 46 | 47 | fn request<'s>(input: &mut Stream<'s>) -> ModalResult<(Request<'s>, Vec>)> { 48 | let req = request_line(input)?; 49 | let h = repeat(1.., message_header).parse_next(input)?; 50 | let _ = line_ending.parse_next(input)?; 51 | 52 | Ok((req, h)) 53 | } 54 | 55 | fn request_line<'s>(input: &mut Stream<'s>) -> ModalResult> { 56 | seq!( Request { 57 | method: take_while(1.., is_token), 58 | _: take_while(1.., is_space), 59 | uri: take_while(1.., is_not_space), 60 | _: take_while(1.., is_space), 61 | version: http_version, 62 | _: line_ending, 63 | }) 64 | .parse_next(input) 65 | } 66 | 67 | fn http_version<'s>(input: &mut Stream<'s>) -> ModalResult<&'s [u8]> { 68 | let _ = "HTTP/".parse_next(input)?; 69 | let version = take_while(1.., is_version).parse_next(input)?; 70 | 71 | Ok(version) 72 | } 73 | 74 | fn message_header_value<'s>(input: &mut Stream<'s>) -> ModalResult<&'s [u8]> { 75 | let _ = take_while(1.., is_horizontal_space).parse_next(input)?; 76 | let data = take_while(1.., till_line_ending).parse_next(input)?; 77 | let _ = line_ending.parse_next(input)?; 78 | 79 | Ok(data) 80 | } 81 | 82 | fn message_header<'s>(input: &mut Stream<'s>) -> ModalResult> { 83 | seq!(Header { 84 | name: take_while(1.., is_token), 85 | _: ':', 86 | value: repeat(1.., message_header_value), 87 | }) 88 | .parse_next(input) 89 | } 90 | 91 | #[rustfmt::skip] 92 | #[allow(clippy::match_same_arms)] 93 | #[allow(clippy::match_like_matches_macro)] 94 | fn is_token(c: u8) -> bool { 95 | match c { 96 | 128..=255 => false, 97 | 0..=31 => false, 98 | b'(' => false, 99 | b')' => false, 100 | b'<' => false, 101 | b'>' => false, 102 | b'@' => false, 103 | b',' => false, 104 | b';' => false, 105 | b':' => false, 106 | b'\\' => false, 107 | b'"' => false, 108 | b'/' => false, 109 | b'[' => false, 110 | b']' => false, 111 | b'?' => false, 112 | b'=' => false, 113 | b'{' => false, 114 | b'}' => false, 115 | b' ' => false, 116 | _ => true, 117 | } 118 | } 119 | 120 | fn is_version(c: u8) -> bool { 121 | c.is_ascii_digit() || c == b'.' 122 | } 123 | 124 | fn till_line_ending(c: u8) -> bool { 125 | c != b'\r' && c != b'\n' 126 | } 127 | 128 | fn is_space(c: u8) -> bool { 129 | c == b' ' 130 | } 131 | 132 | fn is_not_space(c: u8) -> bool { 133 | c != b' ' 134 | } 135 | 136 | fn is_horizontal_space(c: u8) -> bool { 137 | c == b' ' || c == b'\t' 138 | } 139 | -------------------------------------------------------------------------------- /examples/ini/bench.rs: -------------------------------------------------------------------------------- 1 | use winnow::combinator::repeat; 2 | use winnow::prelude::*; 3 | use winnow::Result; 4 | 5 | mod parser; 6 | mod parser_str; 7 | 8 | fn bench_ini(c: &mut criterion::Criterion) { 9 | let str = "[owner] 10 | name=John Doe 11 | organization=Acme Widgets Inc. 12 | 13 | [database] 14 | server=192.0.2.62 15 | port=143 16 | file=payroll.dat 17 | \0"; 18 | 19 | let mut group = c.benchmark_group("ini"); 20 | group.throughput(criterion::Throughput::Bytes(str.len() as u64)); 21 | group.bench_function(criterion::BenchmarkId::new("bytes", str.len()), |b| { 22 | b.iter(|| parser::categories.parse_peek(str.as_bytes()).unwrap()); 23 | }); 24 | group.bench_function(criterion::BenchmarkId::new("str", str.len()), |b| { 25 | b.iter(|| parser_str::categories.parse_peek(str).unwrap()); 26 | }); 27 | } 28 | 29 | fn bench_ini_keys_and_values(c: &mut criterion::Criterion) { 30 | let str = "server=192.0.2.62 31 | port=143 32 | file=payroll.dat 33 | \0"; 34 | 35 | fn acc<'s>(i: &mut parser::Stream<'s>) -> Result> { 36 | repeat(0.., parser::key_value).parse_next(i) 37 | } 38 | 39 | let mut group = c.benchmark_group("ini keys and values"); 40 | group.throughput(criterion::Throughput::Bytes(str.len() as u64)); 41 | group.bench_function(criterion::BenchmarkId::new("bytes", str.len()), |b| { 42 | b.iter(|| acc.parse_peek(str.as_bytes()).unwrap()); 43 | }); 44 | } 45 | 46 | fn bench_ini_key_value(c: &mut criterion::Criterion) { 47 | let str = "server=192.0.2.62\n"; 48 | 49 | let mut group = c.benchmark_group("ini key value"); 50 | group.throughput(criterion::Throughput::Bytes(str.len() as u64)); 51 | group.bench_function(criterion::BenchmarkId::new("bytes", str.len()), |b| { 52 | b.iter(|| parser::key_value.parse_peek(str.as_bytes()).unwrap()); 53 | }); 54 | } 55 | 56 | criterion::criterion_group!( 57 | benches, 58 | bench_ini, 59 | bench_ini_keys_and_values, 60 | bench_ini_key_value 61 | ); 62 | criterion::criterion_main!(benches); 63 | -------------------------------------------------------------------------------- /examples/ini/main.rs: -------------------------------------------------------------------------------- 1 | use winnow::prelude::*; 2 | 3 | mod parser; 4 | mod parser_str; 5 | 6 | fn main() -> Result<(), lexopt::Error> { 7 | let args = Args::parse()?; 8 | 9 | let input = args.input.as_deref().unwrap_or("1 + 1"); 10 | 11 | if args.binary { 12 | match parser::categories.parse(input.as_bytes()) { 13 | Ok(result) => { 14 | println!(" {result:?}"); 15 | } 16 | Err(err) => { 17 | println!(" {err:?}"); 18 | } 19 | } 20 | } else { 21 | match parser_str::categories.parse(input) { 22 | Ok(result) => { 23 | println!(" {result:?}"); 24 | } 25 | Err(err) => { 26 | println!(" {err}"); 27 | } 28 | } 29 | } 30 | 31 | Ok(()) 32 | } 33 | 34 | #[derive(Default)] 35 | struct Args { 36 | input: Option, 37 | binary: bool, 38 | } 39 | 40 | impl Args { 41 | fn parse() -> Result { 42 | use lexopt::prelude::*; 43 | 44 | let mut res = Args::default(); 45 | 46 | let mut args = lexopt::Parser::from_env(); 47 | while let Some(arg) = args.next()? { 48 | match arg { 49 | Long("binary") => { 50 | res.binary = true; 51 | } 52 | Value(input) => { 53 | res.input = Some(input.string()?); 54 | } 55 | _ => return Err(arg.unexpected()), 56 | } 57 | } 58 | Ok(res) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /examples/ini/parser.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::str; 3 | 4 | use winnow::prelude::*; 5 | use winnow::Result; 6 | use winnow::{ 7 | ascii::{alphanumeric1 as alphanumeric, multispace0 as multispace, space0 as space}, 8 | combinator::opt, 9 | combinator::repeat, 10 | combinator::{delimited, separated_pair, terminated}, 11 | token::take_while, 12 | }; 13 | 14 | pub(crate) type Stream<'i> = &'i [u8]; 15 | 16 | pub(crate) fn categories<'s>( 17 | i: &mut Stream<'s>, 18 | ) -> Result>> { 19 | repeat( 20 | 0.., 21 | separated_pair( 22 | category, 23 | opt(multispace), 24 | repeat(0.., terminated(key_value, opt(multispace))), 25 | ), 26 | ) 27 | .parse_next(i) 28 | } 29 | 30 | fn category<'s>(i: &mut Stream<'s>) -> Result<&'s str> { 31 | delimited('[', take_while(0.., |c| c != b']'), ']') 32 | .try_map(str::from_utf8) 33 | .parse_next(i) 34 | } 35 | 36 | pub(crate) fn key_value<'s>(i: &mut Stream<'s>) -> Result<(&'s str, &'s str)> { 37 | let key = alphanumeric.try_map(str::from_utf8).parse_next(i)?; 38 | let _ = (opt(space), '=', opt(space)).parse_next(i)?; 39 | let val = take_while(0.., |c| c != b'\n' && c != b';') 40 | .try_map(str::from_utf8) 41 | .parse_next(i)?; 42 | let _ = opt((';', take_while(0.., |c| c != b'\n'))).parse_next(i)?; 43 | Ok((key, val)) 44 | } 45 | 46 | #[test] 47 | fn parse_category_test() { 48 | let ini_file = &b"[category] 49 | 50 | parameter=value 51 | key = value2"[..]; 52 | 53 | let ini_without_category = &b"\n\nparameter=value 54 | key = value2"[..]; 55 | 56 | let res = category.parse_peek(ini_file); 57 | println!("{res:?}"); 58 | match res { 59 | Ok((i, o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i), o), 60 | _ => println!("error"), 61 | } 62 | 63 | assert_eq!(res, Ok((ini_without_category, "category"))); 64 | } 65 | 66 | #[test] 67 | fn parse_key_value_test() { 68 | let ini_file = &b"parameter=value 69 | key = value2"[..]; 70 | 71 | let ini_without_key_value = &b"\nkey = value2"[..]; 72 | 73 | let res = key_value.parse_peek(ini_file); 74 | println!("{res:?}"); 75 | match res { 76 | Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i), o1, o2), 77 | _ => println!("error"), 78 | } 79 | 80 | assert_eq!(res, Ok((ini_without_key_value, ("parameter", "value")))); 81 | } 82 | 83 | #[test] 84 | fn parse_key_value_with_space_test() { 85 | let ini_file = &b"parameter = value 86 | key = value2"[..]; 87 | 88 | let ini_without_key_value = &b"\nkey = value2"[..]; 89 | 90 | let res = key_value.parse_peek(ini_file); 91 | println!("{res:?}"); 92 | match res { 93 | Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i), o1, o2), 94 | _ => println!("error"), 95 | } 96 | 97 | assert_eq!(res, Ok((ini_without_key_value, ("parameter", "value")))); 98 | } 99 | 100 | #[test] 101 | fn parse_key_value_with_comment_test() { 102 | let ini_file = &b"parameter=value;abc 103 | key = value2"[..]; 104 | 105 | let ini_without_key_value = &b"\nkey = value2"[..]; 106 | 107 | let res = key_value.parse_peek(ini_file); 108 | println!("{res:?}"); 109 | match res { 110 | Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i), o1, o2), 111 | _ => println!("error"), 112 | } 113 | 114 | assert_eq!(res, Ok((ini_without_key_value, ("parameter", "value")))); 115 | } 116 | 117 | #[test] 118 | fn parse_multiple_categories_test() { 119 | let ini_file = &b"[abcd] 120 | 121 | parameter=value;abc 122 | 123 | key = value2 124 | 125 | [category] 126 | parameter3=value3 127 | key4 = value4 128 | "[..]; 129 | 130 | let ini_after_parser = &b""[..]; 131 | 132 | let res = categories.parse_peek(ini_file); 133 | //println!("{:?}", res); 134 | match res { 135 | Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i), o), 136 | _ => println!("error"), 137 | } 138 | 139 | let mut expected_1: HashMap<&str, &str> = HashMap::new(); 140 | expected_1.insert("parameter", "value"); 141 | expected_1.insert("key", "value2"); 142 | let mut expected_2: HashMap<&str, &str> = HashMap::new(); 143 | expected_2.insert("parameter3", "value3"); 144 | expected_2.insert("key4", "value4"); 145 | let mut expected_h: HashMap<&str, HashMap<&str, &str>> = HashMap::new(); 146 | expected_h.insert("abcd", expected_1); 147 | expected_h.insert("category", expected_2); 148 | assert_eq!(res, Ok((ini_after_parser, expected_h))); 149 | } 150 | -------------------------------------------------------------------------------- /examples/ini/parser_str.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use winnow::prelude::*; 4 | use winnow::Result; 5 | use winnow::{ 6 | ascii::{alphanumeric1 as alphanumeric, space0 as space}, 7 | combinator::opt, 8 | combinator::repeat, 9 | combinator::{delimited, terminated}, 10 | token::{take_till, take_while}, 11 | }; 12 | 13 | pub(crate) type Stream<'i> = &'i str; 14 | 15 | pub(crate) fn categories<'s>( 16 | input: &mut Stream<'s>, 17 | ) -> Result>> { 18 | repeat(0.., category_and_keys).parse_next(input) 19 | } 20 | 21 | fn category_and_keys<'s>(i: &mut Stream<'s>) -> Result<(&'s str, HashMap<&'s str, &'s str>)> { 22 | (category, keys_and_values).parse_next(i) 23 | } 24 | 25 | fn category<'s>(i: &mut Stream<'s>) -> Result<&'s str> { 26 | terminated( 27 | delimited('[', take_while(0.., |c| c != ']'), ']'), 28 | opt(take_while(1.., [' ', '\r', '\n'])), 29 | ) 30 | .parse_next(i) 31 | } 32 | 33 | fn keys_and_values<'s>(input: &mut Stream<'s>) -> Result> { 34 | repeat(0.., key_value).parse_next(input) 35 | } 36 | 37 | fn key_value<'s>(i: &mut Stream<'s>) -> Result<(&'s str, &'s str)> { 38 | let key = alphanumeric.parse_next(i)?; 39 | let _ = (opt(space), "=", opt(space)).parse_next(i)?; 40 | let val = take_till(0.., is_line_ending_or_comment).parse_next(i)?; 41 | let _ = opt(space).parse_next(i)?; 42 | let _ = opt((";", till_line_ending)).parse_next(i)?; 43 | let _ = opt(space_or_line_ending).parse_next(i)?; 44 | 45 | Ok((key, val)) 46 | } 47 | 48 | fn is_line_ending_or_comment(chr: char) -> bool { 49 | chr == ';' || chr == '\n' 50 | } 51 | 52 | fn till_line_ending<'s>(i: &mut Stream<'s>) -> Result<&'s str> { 53 | take_while(0.., |c| c != '\r' && c != '\n').parse_next(i) 54 | } 55 | 56 | fn space_or_line_ending<'s>(i: &mut Stream<'s>) -> Result<&'s str> { 57 | take_while(1.., [' ', '\r', '\n']).parse_next(i) 58 | } 59 | 60 | #[test] 61 | fn parse_category_test() { 62 | let ini_file = "[category] 63 | 64 | parameter=value 65 | key = value2"; 66 | 67 | let ini_without_category = "parameter=value 68 | key = value2"; 69 | 70 | let res = category.parse_peek(ini_file); 71 | println!("{res:?}"); 72 | match res { 73 | Ok((i, o)) => println!("i: {i} | o: {o:?}"), 74 | _ => println!("error"), 75 | } 76 | 77 | assert_eq!(res, Ok((ini_without_category, "category"))); 78 | } 79 | 80 | #[test] 81 | fn parse_key_value_test() { 82 | let ini_file = "parameter=value 83 | key = value2"; 84 | 85 | let ini_without_key_value = "key = value2"; 86 | 87 | let res = key_value.parse_peek(ini_file); 88 | println!("{res:?}"); 89 | match res { 90 | Ok((i, (o1, o2))) => println!("i: {i} | o: ({o1:?},{o2:?})"), 91 | _ => println!("error"), 92 | } 93 | 94 | assert_eq!(res, Ok((ini_without_key_value, ("parameter", "value")))); 95 | } 96 | 97 | #[test] 98 | fn parse_key_value_with_space_test() { 99 | let ini_file = "parameter = value 100 | key = value2"; 101 | 102 | let ini_without_key_value = "key = value2"; 103 | 104 | let res = key_value.parse_peek(ini_file); 105 | println!("{res:?}"); 106 | match res { 107 | Ok((i, (o1, o2))) => println!("i: {i} | o: ({o1:?},{o2:?})"), 108 | _ => println!("error"), 109 | } 110 | 111 | assert_eq!(res, Ok((ini_without_key_value, ("parameter", "value")))); 112 | } 113 | 114 | #[test] 115 | fn parse_key_value_with_comment_test() { 116 | let ini_file = "parameter=value;abc 117 | key = value2"; 118 | 119 | let ini_without_key_value = "key = value2"; 120 | 121 | let res = key_value.parse_peek(ini_file); 122 | println!("{res:?}"); 123 | match res { 124 | Ok((i, (o1, o2))) => println!("i: {i} | o: ({o1:?},{o2:?})"), 125 | _ => println!("error"), 126 | } 127 | 128 | assert_eq!(res, Ok((ini_without_key_value, ("parameter", "value")))); 129 | } 130 | 131 | #[test] 132 | fn parse_multiple_keys_and_values_test() { 133 | let ini_file = "parameter=value;abc 134 | 135 | key = value2 136 | 137 | [category]"; 138 | 139 | let ini_without_key_value = "[category]"; 140 | 141 | let res = keys_and_values.parse_peek(ini_file); 142 | println!("{res:?}"); 143 | match res { 144 | Ok((i, ref o)) => println!("i: {i} | o: {o:?}"), 145 | _ => println!("error"), 146 | } 147 | 148 | let mut expected: HashMap<&str, &str> = HashMap::new(); 149 | expected.insert("parameter", "value"); 150 | expected.insert("key", "value2"); 151 | assert_eq!(res, Ok((ini_without_key_value, expected))); 152 | } 153 | 154 | #[test] 155 | fn parse_category_then_multiple_keys_and_values_test() { 156 | //FIXME: there can be an empty line or a comment line after a category 157 | let ini_file = "[abcd] 158 | parameter=value;abc 159 | 160 | key = value2 161 | 162 | [category]"; 163 | 164 | let ini_after_parser = "[category]"; 165 | 166 | let res = category_and_keys.parse_peek(ini_file); 167 | println!("{res:?}"); 168 | match res { 169 | Ok((i, ref o)) => println!("i: {i} | o: {o:?}"), 170 | _ => println!("error"), 171 | } 172 | 173 | let mut expected_h: HashMap<&str, &str> = HashMap::new(); 174 | expected_h.insert("parameter", "value"); 175 | expected_h.insert("key", "value2"); 176 | assert_eq!(res, Ok((ini_after_parser, ("abcd", expected_h)))); 177 | } 178 | 179 | #[test] 180 | fn parse_multiple_categories_test() { 181 | let ini_file = "[abcd] 182 | 183 | parameter=value;abc 184 | 185 | key = value2 186 | 187 | [category] 188 | parameter3=value3 189 | key4 = value4 190 | "; 191 | 192 | let res = categories.parse_peek(ini_file); 193 | //println!("{:?}", res); 194 | match res { 195 | Ok((i, ref o)) => println!("i: {i} | o: {o:?}"), 196 | _ => println!("error"), 197 | } 198 | 199 | let mut expected_1: HashMap<&str, &str> = HashMap::new(); 200 | expected_1.insert("parameter", "value"); 201 | expected_1.insert("key", "value2"); 202 | let mut expected_2: HashMap<&str, &str> = HashMap::new(); 203 | expected_2.insert("parameter3", "value3"); 204 | expected_2.insert("key4", "value4"); 205 | let mut expected_h: HashMap<&str, HashMap<&str, &str>> = HashMap::new(); 206 | expected_h.insert("abcd", expected_1); 207 | expected_h.insert("category", expected_2); 208 | assert_eq!(res, Ok(("", expected_h))); 209 | } 210 | -------------------------------------------------------------------------------- /examples/iterator.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::iter::Iterator; 3 | 4 | use winnow::ascii::alphanumeric1; 5 | use winnow::combinator::iterator; 6 | use winnow::combinator::{separated_pair, terminated}; 7 | use winnow::prelude::*; 8 | 9 | fn main() { 10 | let mut data = "abcabcabcabc"; 11 | 12 | fn parser<'s>(i: &mut &'s str) -> ModalResult<&'s str> { 13 | "abc".parse_next(i) 14 | } 15 | 16 | // `from_fn` (available from Rust 1.34) can create an iterator 17 | // from a closure 18 | let it = std::iter::from_fn(move || parser.parse_next(&mut data).ok()); 19 | 20 | for value in it { 21 | println!("parser returned: {value}"); 22 | } 23 | 24 | println!("\n********************\n"); 25 | 26 | let mut data = "abcabcabcabc"; 27 | 28 | // if `from_fn` is not available, it is possible to fold 29 | // over an iterator of functions 30 | let res = std::iter::repeat(parser) 31 | .take(3) 32 | .try_fold(Vec::new(), |mut acc, mut parser| { 33 | parser.parse_next(&mut data).map(|o| { 34 | acc.push(o); 35 | acc 36 | }) 37 | }); 38 | 39 | // will print "parser iterator returned: Ok(("abc", ["abc", "abc", "abc"]))" 40 | println!("\nparser iterator returned: {res:?}"); 41 | 42 | println!("\n********************\n"); 43 | 44 | let data = "key1:value1,key2:value2,key3:value3,;"; 45 | 46 | // `winnow::combinator::iterator` will return an iterator 47 | // producing the parsed values. Compared to the previous 48 | // solutions: 49 | // - we can work with a normal iterator like `from_fn` 50 | // - we can get the remaining input afterwards, like with the `try_fold` trick 51 | let mut winnow_it = iterator( 52 | data, 53 | terminated(separated_pair(alphanumeric1, ":", alphanumeric1), ","), 54 | ); 55 | 56 | let res = winnow_it 57 | .map(|(k, v)| (k.to_uppercase(), v)) 58 | .collect::>(); 59 | 60 | let parser_result: ModalResult<(_, _), ()> = winnow_it.finish(); 61 | let (remaining_input, ()) = parser_result.unwrap(); 62 | 63 | // will print "iterator returned {"key1": "value1", "key3": "value3", "key2": "value2"}, remaining input is ';'" 64 | println!("iterator returned {res:?}, remaining input is '{remaining_input}'"); 65 | } 66 | -------------------------------------------------------------------------------- /examples/json/bench.rs: -------------------------------------------------------------------------------- 1 | use winnow::prelude::*; 2 | use winnow::Partial; 3 | 4 | mod json; 5 | mod parser_alt; 6 | mod parser_dispatch; 7 | mod parser_partial; 8 | 9 | fn json_bench(c: &mut criterion::Criterion) { 10 | let data = [("small", SMALL), ("canada", CANADA)]; 11 | let mut group = c.benchmark_group("json"); 12 | for (name, sample) in data { 13 | let len = sample.len(); 14 | group.throughput(criterion::Throughput::Bytes(len as u64)); 15 | 16 | group.bench_with_input( 17 | criterion::BenchmarkId::new("dispatch", name), 18 | &len, 19 | |b, _| { 20 | type Error = winnow::error::ErrMode; 21 | 22 | b.iter(|| parser_dispatch::json::.parse_peek(sample).unwrap()); 23 | }, 24 | ); 25 | group.bench_with_input( 26 | criterion::BenchmarkId::new("modeless", name), 27 | &len, 28 | |b, _| { 29 | type Error = winnow::error::ContextError; 30 | 31 | b.iter(|| parser_dispatch::json::.parse_peek(sample).unwrap()); 32 | }, 33 | ); 34 | group.bench_with_input( 35 | criterion::BenchmarkId::new("empty-error", name), 36 | &len, 37 | |b, _| { 38 | type Error<'i> = winnow::error::EmptyError; 39 | 40 | b.iter(|| { 41 | parser_dispatch::json::> 42 | .parse_peek(sample) 43 | .unwrap() 44 | }); 45 | }, 46 | ); 47 | group.bench_with_input(criterion::BenchmarkId::new("alt", name), &len, |b, _| { 48 | type Error = winnow::error::ContextError; 49 | 50 | b.iter(|| parser_alt::json::.parse_peek(sample).unwrap()); 51 | }); 52 | group.bench_with_input( 53 | criterion::BenchmarkId::new("streaming", name), 54 | &len, 55 | |b, _| { 56 | type Error = winnow::error::ContextError; 57 | 58 | b.iter(|| { 59 | parser_partial::json:: 60 | .parse_peek(Partial::new(sample)) 61 | .unwrap() 62 | }); 63 | }, 64 | ); 65 | } 66 | group.finish(); 67 | } 68 | 69 | const SMALL: &str = " { \"a\"\t: 42, 70 | \"b\": [ \"x\", \"y\", 12 ,\"\\u2014\", \"\\uD83D\\uDE10\"] , 71 | \"c\": { \"hello\" : \"world\" 72 | } 73 | } "; 74 | 75 | const CANADA: &str = include_str!("../../third_party/nativejson-benchmark/data/canada.json"); 76 | 77 | criterion::criterion_group!(benches, json_bench,); 78 | criterion::criterion_main!(benches); 79 | -------------------------------------------------------------------------------- /examples/json/json.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | #[derive(Debug, PartialEq, Clone)] 4 | pub(crate) enum JsonValue { 5 | Null, 6 | Boolean(bool), 7 | Str(String), 8 | Num(f64), 9 | Array(Vec), 10 | Object(HashMap), 11 | } 12 | -------------------------------------------------------------------------------- /examples/json/main.rs: -------------------------------------------------------------------------------- 1 | mod json; 2 | mod parser_alt; 3 | mod parser_dispatch; 4 | #[allow(dead_code)] 5 | mod parser_partial; 6 | 7 | use winnow::error::EmptyError; 8 | use winnow::prelude::*; 9 | 10 | fn main() -> Result<(), lexopt::Error> { 11 | let args = Args::parse()?; 12 | 13 | let data = args.input.as_deref().unwrap_or(if args.invalid { 14 | " { \"a\"\t: 42, 15 | \"b\": [ \"x\", \"y\", 12 ] , 16 | \"c\": { 1\"hello\" : \"world\" 17 | } 18 | } " 19 | } else { 20 | " { \"a\"\t: 42, 21 | \"b\": [ \"x\", \"y\", 12 ] , 22 | \"c\": { \"hello\" : \"world\" 23 | } 24 | } " 25 | }); 26 | 27 | let result = match args.implementation { 28 | Impl::Naive => parser_alt::json::.parse(data), 29 | Impl::Dispatch => parser_dispatch::json::.parse(data), 30 | }; 31 | match result { 32 | Ok(json) => { 33 | println!("{json:#?}"); 34 | } 35 | Err(err) => { 36 | if args.pretty { 37 | println!("{err}"); 38 | } else { 39 | println!("{err:#?}"); 40 | } 41 | } 42 | } 43 | 44 | Ok(()) 45 | } 46 | 47 | #[derive(Default)] 48 | struct Args { 49 | input: Option, 50 | invalid: bool, 51 | pretty: bool, 52 | implementation: Impl, 53 | } 54 | 55 | enum Impl { 56 | Naive, 57 | Dispatch, 58 | } 59 | 60 | impl Default for Impl { 61 | fn default() -> Self { 62 | Self::Naive 63 | } 64 | } 65 | 66 | impl Args { 67 | fn parse() -> Result { 68 | use lexopt::prelude::*; 69 | 70 | let mut res = Args::default(); 71 | 72 | let mut args = lexopt::Parser::from_env(); 73 | while let Some(arg) = args.next()? { 74 | match arg { 75 | Long("invalid") => { 76 | res.invalid = true; 77 | } 78 | Long("pretty") => { 79 | // Only case where pretty matters 80 | res.pretty = true; 81 | res.invalid = true; 82 | } 83 | Long("impl") => { 84 | res.implementation = args.value()?.parse_with(|s| match s { 85 | "naive" => Ok(Impl::Naive), 86 | "dispatch" => Ok(Impl::Dispatch), 87 | _ => Err("expected `naive`, `dispatch`"), 88 | })?; 89 | } 90 | Value(input) => { 91 | res.input = Some(input.string()?); 92 | } 93 | _ => return Err(arg.unexpected()), 94 | } 95 | } 96 | Ok(res) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /examples/ndjson/main.rs: -------------------------------------------------------------------------------- 1 | mod parser; 2 | 3 | use std::io::Read; 4 | 5 | use winnow::error::ContextError; 6 | use winnow::error::ErrMode; 7 | use winnow::error::Needed; 8 | use winnow::prelude::*; 9 | use winnow::stream::Offset; 10 | 11 | fn main() -> Result<(), lexopt::Error> { 12 | let args = Args::parse()?; 13 | let input = args.input.ok_or_else(|| lexopt::Error::MissingValue { 14 | option: Some("".to_owned()), 15 | })?; 16 | 17 | let mut file = std::fs::File::open(input).map_err(to_lexopt)?; 18 | 19 | // Intentionally starting with a small buffer to make it easier to show `Incomplete` handling 20 | let buffer_size = 10; 21 | let min_buffer_growth = 100; 22 | let buffer_growth_factor = 2; 23 | let mut buffer = circular::Buffer::with_capacity(buffer_size); 24 | loop { 25 | let read = file.read(buffer.space()).map_err(to_lexopt)?; 26 | eprintln!("read {read}"); 27 | if read == 0 { 28 | // Should be EOF since we always make sure there is `available_space` 29 | assert_ne!(buffer.available_space(), 0); 30 | assert_eq!( 31 | buffer.available_data(), 32 | 0, 33 | "leftover data: {}", 34 | String::from_utf8_lossy(buffer.data()) 35 | ); 36 | break; 37 | } 38 | buffer.fill(read); 39 | 40 | loop { 41 | let mut input = 42 | parser::Stream::new(std::str::from_utf8(buffer.data()).map_err(to_lexopt)?); 43 | let start = input.checkpoint(); 44 | match parser::ndjson::.parse_next(&mut input) { 45 | Ok(value) => { 46 | println!("{value:?}"); 47 | println!(); 48 | // Tell the buffer how much we read 49 | let consumed = input.offset_from(&start); 50 | buffer.consume(consumed); 51 | } 52 | Err(ErrMode::Backtrack(e)) | Err(ErrMode::Cut(e)) => { 53 | return Err(fmt_lexopt(e.to_string())); 54 | } 55 | Err(ErrMode::Incomplete(Needed::Size(size))) => { 56 | // Without the format telling us how much space is required, we really should 57 | // treat this the same as `Unknown` but are doing this to demonstrate how to 58 | // handle `Size`. 59 | // 60 | // Even when the format has a header to tell us `Size`, we could hit incidental 61 | // `Size(1)`s, so make sure we buffer more space than that to avoid reading 62 | // one byte at a time 63 | let head_room = size.get().max(min_buffer_growth); 64 | let new_capacity = buffer.available_data() + head_room; 65 | eprintln!("growing buffer to {new_capacity}"); 66 | buffer.grow(new_capacity); 67 | if buffer.available_space() < head_room { 68 | eprintln!("buffer shift"); 69 | buffer.shift(); 70 | } 71 | break; 72 | } 73 | Err(ErrMode::Incomplete(Needed::Unknown)) => { 74 | let new_capacity = buffer_growth_factor * buffer.capacity(); 75 | eprintln!("growing buffer to {new_capacity}"); 76 | buffer.grow(new_capacity); 77 | break; 78 | } 79 | } 80 | } 81 | } 82 | 83 | Ok(()) 84 | } 85 | 86 | #[derive(Default)] 87 | struct Args { 88 | input: Option, 89 | } 90 | 91 | impl Args { 92 | fn parse() -> Result { 93 | use lexopt::prelude::*; 94 | 95 | let mut res = Args::default(); 96 | 97 | let mut args = lexopt::Parser::from_env(); 98 | while let Some(arg) = args.next()? { 99 | match arg { 100 | Value(input) => { 101 | res.input = Some(input.into()); 102 | } 103 | _ => return Err(arg.unexpected()), 104 | } 105 | } 106 | Ok(res) 107 | } 108 | } 109 | 110 | fn to_lexopt(e: impl std::error::Error + Send + Sync + 'static) -> lexopt::Error { 111 | lexopt::Error::Custom(Box::new(e)) 112 | } 113 | 114 | fn fmt_lexopt(e: String) -> lexopt::Error { 115 | lexopt::Error::Custom(e.into()) 116 | } 117 | -------------------------------------------------------------------------------- /examples/s_expression/main.rs: -------------------------------------------------------------------------------- 1 | //! In this example we build an [S-expression](https://en.wikipedia.org/wiki/S-expression) 2 | //! parser and tiny [lisp](https://en.wikipedia.org/wiki/Lisp_(programming_language)) interpreter. 3 | //! Lisp is a simple type of language made up of Atoms and Lists, forming easily parsable trees. 4 | 5 | #![cfg(feature = "alloc")] 6 | 7 | mod parser; 8 | 9 | fn main() { 10 | let expression_1 = "((if (= (+ 3 (/ 9 3)) 11 | (* 2 3)) 12 | * 13 | /) 14 | 456 123)"; 15 | println!( 16 | "\"{}\"\nevaled gives us: {:?}", 17 | expression_1, 18 | parser::eval_from_str(expression_1) 19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /examples/string/main.rs: -------------------------------------------------------------------------------- 1 | //! This example shows an example of how to parse an escaped string. The 2 | //! rules for the string are similar to JSON and rust. A string is: 3 | //! 4 | //! - Enclosed by double quotes 5 | //! - Can contain any raw unescaped code point besides \ and " 6 | //! - Matches the following escape sequences: \b, \f, \n, \r, \t, \", \\, \/ 7 | //! - Matches code points like Rust: \u{XXXX}, where XXXX can be up to 6 8 | //! hex characters 9 | //! - an escape followed by whitespace consumes all whitespace between the 10 | //! escape and the next non-whitespace character 11 | 12 | #![cfg(feature = "alloc")] 13 | 14 | mod parser; 15 | 16 | use winnow::prelude::*; 17 | 18 | fn main() -> Result<(), lexopt::Error> { 19 | let args = Args::parse()?; 20 | 21 | let data = args.input.as_deref().unwrap_or("\"abc\""); 22 | let result = parser::parse_string::<()>.parse(data); 23 | match result { 24 | Ok(data) => println!("{data}"), 25 | Err(err) => println!("{err:?}"), 26 | } 27 | 28 | Ok(()) 29 | } 30 | 31 | #[derive(Default)] 32 | struct Args { 33 | input: Option, 34 | } 35 | 36 | impl Args { 37 | fn parse() -> Result { 38 | use lexopt::prelude::*; 39 | 40 | let mut res = Args::default(); 41 | 42 | let mut args = lexopt::Parser::from_env(); 43 | while let Some(arg) = args.next()? { 44 | match arg { 45 | Value(input) => { 46 | res.input = Some(input.string()?); 47 | } 48 | _ => return Err(arg.unexpected()), 49 | } 50 | } 51 | Ok(res) 52 | } 53 | } 54 | 55 | #[test] 56 | fn simple() { 57 | let data = "\"abc\""; 58 | let result = parser::parse_string::<()>.parse(data); 59 | assert_eq!(result, Ok(String::from("abc"))); 60 | } 61 | 62 | #[test] 63 | fn escaped() { 64 | let data = "\"tab:\\tafter tab, newline:\\nnew line, quote: \\\", emoji: \\u{1F602}, newline:\\nescaped whitespace: \\ abc\""; 65 | let result = parser::parse_string::<()>.parse(data); 66 | assert_eq!( 67 | result, 68 | Ok(String::from("tab:\tafter tab, newline:\nnew line, quote: \", emoji: 😂, newline:\nescaped whitespace: abc")) 69 | ); 70 | } 71 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | artifacts 2 | corpus 3 | target 4 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | 2 | [package] 3 | name = "winnow-fuzz" 4 | version = "0.0.0" 5 | authors = ["David Korczynski "] 6 | publish = false 7 | edition = "2018" 8 | 9 | [package.metadata] 10 | cargo-fuzz = true 11 | 12 | [package.metadata.release] 13 | release = false 14 | 15 | [dependencies] 16 | libfuzzer-sys = "0.4.6" 17 | 18 | [dependencies.winnow] 19 | path = ".." 20 | 21 | [[bin]] 22 | name = "fuzz_arithmetic" 23 | path = "fuzz_targets/fuzz_arithmetic.rs" 24 | test = false 25 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/fuzz_arithmetic.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use libfuzzer_sys::fuzz_target; 3 | use std::str; 4 | 5 | use winnow::prelude::*; 6 | use winnow::{ 7 | ascii::{digit1 as digit, space0 as space}, 8 | combinator::alt, 9 | combinator::repeat, 10 | combinator::{delimited, terminated}, 11 | }; 12 | 13 | use std::cell::RefCell; 14 | 15 | thread_local! { 16 | pub static LEVEL: RefCell = const { RefCell::new(0) }; 17 | } 18 | 19 | fn reset() { 20 | LEVEL.with(|l| { 21 | *l.borrow_mut() = 0; 22 | }); 23 | } 24 | 25 | fn incr(i: &mut &str) -> ModalResult<()> { 26 | LEVEL.with(|l| { 27 | *l.borrow_mut() += 1; 28 | 29 | // limit the number of recursions, the fuzzer keeps running into them 30 | if *l.borrow() >= 8192 { 31 | Err(winnow::error::ParserError::from_input(i)) 32 | } else { 33 | Ok(()) 34 | } 35 | }) 36 | } 37 | 38 | fn decr() { 39 | LEVEL.with(|l| { 40 | *l.borrow_mut() -= 1; 41 | }); 42 | } 43 | 44 | fn parens(i: &mut &str) -> ModalResult { 45 | delimited( 46 | space, 47 | delimited(terminated("(", incr), expr, ")".map(|_| decr())), 48 | space, 49 | ) 50 | .parse_next(i) 51 | } 52 | 53 | fn factor(i: &mut &str) -> ModalResult { 54 | alt((delimited(space, digit, space).parse_to(), parens)).parse_next(i) 55 | } 56 | 57 | fn term(i: &mut &str) -> ModalResult { 58 | incr(i)?; 59 | let init = factor(i).inspect_err(|_e| { 60 | decr(); 61 | })?; 62 | 63 | let res = repeat(0.., alt((('*', factor), ('/', factor.verify(|i| *i != 0))))) 64 | .fold( 65 | || init, 66 | |acc, (op, val): (char, i64)| { 67 | if op == '*' { 68 | acc.saturating_mul(val) 69 | } else { 70 | match acc.checked_div(val) { 71 | Some(v) => v, 72 | // we get a division with overflow because we can get acc = i64::MIN and val = -1 73 | // the division by zero is already checked earlier by verify 74 | None => i64::MAX, 75 | } 76 | } 77 | }, 78 | ) 79 | .parse_next(i); 80 | 81 | decr(); 82 | res 83 | } 84 | 85 | fn expr(i: &mut &str) -> ModalResult { 86 | incr(i)?; 87 | let init = term(i).inspect_err(|_e| { 88 | decr(); 89 | })?; 90 | 91 | let res = repeat(0.., (alt(('+', '-')), term)) 92 | .fold( 93 | || init, 94 | |acc, (op, val): (char, i64)| { 95 | if op == '+' { 96 | acc.saturating_add(val) 97 | } else { 98 | acc.saturating_sub(val) 99 | } 100 | }, 101 | ) 102 | .parse_next(i); 103 | 104 | decr(); 105 | res 106 | } 107 | 108 | fuzz_target!(|data: &[u8]| { 109 | reset(); 110 | // fuzzed code goes here 111 | let _ = match str::from_utf8(data) { 112 | Ok(mut v) => { 113 | //println!("v: {}", v); 114 | factor(&mut v) 115 | } 116 | Err(_) => factor(&mut "2"), 117 | }; 118 | }); 119 | -------------------------------------------------------------------------------- /proptest-regressions/character/streaming.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 82a575ed1f031825e7474bff3702d0c42017471b5ac845bdbdc00c1534dbc4cb # shrinks to s = "" 8 | cc 155f8f4b052941ba58b8b90a5f8fa7da78c04c1a451083a4a89a348c86226904 # shrinks to s = "0" 9 | cc c35a5a751223822dd0a94416d5ca3cc53b4a61cdc4f9422251bc2c72712ed844 # shrinks to s = "-0" 10 | cc 478373182b684c42ce3746ea62d57a35a9c764ef75943e0bb1dc08f88b295581 # shrinks to s = "- " 11 | -------------------------------------------------------------------------------- /proptest-regressions/number/streaming.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 68154e0c90b20374781d3e3932bddb80e8c6a97901d0331bbd7e6daa75b794cb # shrinks to s = "0e" 8 | cc d31506b74ad24a80485adb176039e2fa82cf58798738288a2c810952c68d7600 # shrinks to s = "inf" 9 | -------------------------------------------------------------------------------- /proptest-regressions/stream/tests.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc dd80dcf3d84999d176d263e5de7ab8bd3c9b7d0b24fa74cb6004556677665b72 # shrinks to byte_len = 11, start = 0 8 | -------------------------------------------------------------------------------- /release.toml: -------------------------------------------------------------------------------- 1 | dependent-version = "fix" 2 | allow-branch = ["main", "v*-main"] 3 | -------------------------------------------------------------------------------- /src/_topic/arithmetic.rs: -------------------------------------------------------------------------------- 1 | //! # Arithmetic 2 | //! 3 | //! This parses arithmetic expressions and directly evaluates them. 4 | //! ```rust 5 | #![doc = include_str!("../../examples/arithmetic/parser.rs")] 6 | //! ``` 7 | -------------------------------------------------------------------------------- /src/_topic/error.rs: -------------------------------------------------------------------------------- 1 | //! # Custom Errors 2 | //! 3 | //! A lot can be accomplished with the built-in error tools, like: 4 | //! - [`ContextError`] 5 | //! - [`Parser::context`] 6 | //! - [`cut_err`] 7 | //! 8 | //! *(see [tutorial][chapter_7])* 9 | //! 10 | //! Most other needs can likely be met by using a custom context type with [`ContextError`] instead 11 | //! of [`StrContext`]. 12 | //! This will require implementing a custom renderer. 13 | //! 14 | //! ## `ParserError` Trait 15 | //! 16 | //! When needed, you can also create your own type that implements [`ParserError`]. 17 | //! 18 | //! Optional traits include: 19 | //! - [`AddContext`] 20 | //! - [`FromExternalError`] 21 | //! - [`ErrorConvert`] 22 | //! 23 | //! There are multiple strategies for implementing support for [`AddContext`] and [`FromExternalError`]: 24 | //! - Make your error type generic over the context or external error 25 | //! - Require a trait for the context or external error and `Box` it 26 | //! - Make the context an enum like [`StrContext`] 27 | //! - Implement the trait multiple times, one for each concrete context or external error type, 28 | //! allowing custom behavior per type 29 | //! 30 | //! Example: 31 | //!```rust 32 | #![doc = include_str!("../../examples/custom_error.rs")] 33 | //!``` 34 | 35 | #![allow(unused_imports)] 36 | use crate::combinator::cut_err; 37 | use crate::error::ContextError; 38 | use crate::error::ErrorConvert; 39 | use crate::error::StrContext; 40 | use crate::Parser; 41 | use crate::_tutorial::chapter_7; 42 | use crate::error::AddContext; 43 | use crate::error::FromExternalError; 44 | use crate::error::ParserError; 45 | -------------------------------------------------------------------------------- /src/_topic/fromstr.rs: -------------------------------------------------------------------------------- 1 | //! # Implementing `FromStr` 2 | //! 3 | //! The [`FromStr` trait][std::str::FromStr] provides 4 | //! a common interface to parse from a string. 5 | //! 6 | //! ```rust 7 | #![doc = include_str!("../../examples/css/parser.rs")] 8 | //! ``` 9 | -------------------------------------------------------------------------------- /src/_topic/http.rs: -------------------------------------------------------------------------------- 1 | //! # HTTP 2 | //! 3 | //! ```rust 4 | #![doc = include_str!("../../examples/http/parser.rs")] 5 | //! ``` 6 | -------------------------------------------------------------------------------- /src/_topic/ini.rs: -------------------------------------------------------------------------------- 1 | //! # INI 2 | //! 3 | //! ```rust 4 | #![doc = include_str!("../../examples/ini/parser.rs")] 5 | //! ``` 6 | -------------------------------------------------------------------------------- /src/_topic/json.rs: -------------------------------------------------------------------------------- 1 | //! # json 2 | //! 3 | //! ```rust,ignore 4 | #![doc = include_str!("../../examples/json/parser_dispatch.rs")] 5 | //! ``` 6 | -------------------------------------------------------------------------------- /src/_topic/lexing.rs: -------------------------------------------------------------------------------- 1 | //! # Lexing and Parsing 2 | //! 3 | //! ## Parse to AST 4 | //! 5 | //! The simplest way to write a parser is to parse directly to the AST. 6 | //! 7 | //! Example: 8 | //! ```rust 9 | #![doc = include_str!("../../examples/arithmetic/parser_ast.rs")] 10 | //! ``` 11 | //! 12 | //! ## Lexing 13 | //! 14 | //! However, there are times when you may want to separate lexing from parsing. 15 | //! Winnow provides [`TokenSlice`] to simplify this. 16 | //! 17 | //! Example: 18 | //! ```rust 19 | #![doc = include_str!("../../examples/arithmetic/parser_lexer.rs")] 20 | //! ``` 21 | 22 | #![allow(unused_imports)] 23 | use crate::stream::TokenSlice; 24 | -------------------------------------------------------------------------------- /src/_topic/mod.rs: -------------------------------------------------------------------------------- 1 | //! # Special Topics 2 | //! 3 | //! These are short recipes for accomplishing common tasks. 4 | //! 5 | //! - [Why `winnow`?][why] 6 | //! - [For `nom` users][nom] 7 | //! - Formats: 8 | //! - [Elements of Programming Languages][language] 9 | //! - [Arithmetic][arithmetic] 10 | //! - [s-expression][s_expression] 11 | //! - [json] 12 | //! - [INI][ini] 13 | //! - [HTTP][http] 14 | //! - Special Topics: 15 | //! - [Implementing `FromStr`][fromstr] 16 | //! - [Performance][performance] 17 | //! - [Parsing Partial Input][partial] 18 | //! - [Lexing and Parsing][lexing] 19 | //! - [Custom stream or token][stream] 20 | //! - [Custom errors][error] 21 | //! - [Debugging][crate::_tutorial::chapter_8] 22 | //! 23 | //! See also parsers written with `winnow`: 24 | //! 25 | //! - [`toml_edit`](https://crates.io/crates/toml_edit) 26 | //! - [`hcl-edit`](https://crates.io/crates/hcl-edit) 27 | #![allow(clippy::std_instead_of_core)] 28 | 29 | pub mod arithmetic; 30 | pub mod error; 31 | pub mod fromstr; 32 | pub mod http; 33 | pub mod ini; 34 | pub mod json; 35 | pub mod language; 36 | pub mod lexing; 37 | pub mod nom; 38 | pub mod partial; 39 | pub mod performance; 40 | pub mod s_expression; 41 | pub mod stream; 42 | pub mod why; 43 | -------------------------------------------------------------------------------- /src/_topic/partial.rs: -------------------------------------------------------------------------------- 1 | //! # Parsing Partial Input 2 | //! 3 | //! Typically, the input being parsed is all in-memory, or is complete. Some data sources are too 4 | //! large to fit into memory, only allowing parsing an incomplete or [`Partial`] subset of the 5 | //! data, requiring incrementally parsing. 6 | //! 7 | //! By wrapping a stream, like `&[u8]`, with [`Partial`], parsers will report when the data is 8 | //! [`Incomplete`] and more input is [`Needed`], allowing the caller to stream-in additional data 9 | //! to be parsed. The data is then parsed a chunk at a time. 10 | //! 11 | //! Chunks are typically defined by either: 12 | //! - A header reporting the number of bytes, like with [`length_and_then`] 13 | //! - [`Partial`] can explicitly be changed to being complete once the specified bytes are 14 | //! acquired via [`StreamIsPartial::complete`]. 15 | //! - A delimiter, like with [ndjson](https://github.com/ndjson/ndjson-spec/) 16 | //! - You can parse up-to the delimiter or do a `take_until(0.., delim).and_then(parser)` 17 | //! 18 | //! If the chunks are not homogeneous, a state machine will be needed to track what the expected 19 | //! parser is for the next chunk. 20 | //! 21 | //! Caveats: 22 | //! - `winnow` takes the approach of re-parsing from scratch. Chunks should be relatively small to 23 | //! prevent the re-parsing overhead from dominating. 24 | //! - Parsers like [`repeat`] do not know when an `eof` is from insufficient data or the end of the 25 | //! stream, causing them to always report [`Incomplete`]. 26 | //! 27 | //! # Example 28 | //! 29 | //! `main.rs`: 30 | //! ```rust,ignore 31 | #![doc = include_str!("../../examples/ndjson/main.rs")] 32 | //! ``` 33 | //! 34 | //! `parser.rs`: 35 | //! ```rust,ignore 36 | #![doc = include_str!("../../examples/ndjson/parser.rs")] 37 | //! ``` 38 | 39 | #![allow(unused_imports)] // Used for intra-doc links 40 | 41 | use crate::binary::length_and_then; 42 | use crate::combinator::repeat; 43 | use crate::error::ErrMode::Incomplete; 44 | use crate::error::Needed; 45 | use crate::stream::Partial; 46 | use crate::stream::StreamIsPartial; 47 | -------------------------------------------------------------------------------- /src/_topic/performance.rs: -------------------------------------------------------------------------------- 1 | //! # Performance 2 | //! 3 | //! ## Runtime Performance 4 | //! 5 | //! See also the general Rust [Performance Book](https://nnethercote.github.io/perf-book/) 6 | //! 7 | //! Tips 8 | //! - Try `cargo add winnow -F simd`. For some it offers significant performance improvements 9 | //! - When enough cases of an [`alt`] have unique prefixes, prefer [`dispatch`] 10 | //! - When parsing text, try to parse as bytes (`u8`) rather than `char`s ([`BStr`] can make 11 | //! debugging easier) 12 | //! - Find simplified subsets of the grammar to parse, falling back to the full grammar when it 13 | //! doesn't work. For example, when parsing json strings, parse them without support for escapes, 14 | //! falling back to escape support if it fails. 15 | //! - Watch for large return types. A surprising place these can show up is when chaining parsers 16 | //! with a tuple. 17 | //! 18 | //! ## Build-time Performance 19 | //! 20 | //! Returning complex types as `impl Trait` can negatively impact build times. This can hit in 21 | //! surprising cases like: 22 | //! ```rust 23 | //! # use winnow::prelude::*; 24 | //! fn foo() -> impl Parser 25 | //! # where 26 | //! # I: winnow::stream::Stream, 27 | //! # I: winnow::stream::StreamIsPartial, 28 | //! # E: winnow::error::ParserError, 29 | //! { 30 | //! // ...some chained combinators... 31 | //! # winnow::token::any 32 | //! } 33 | //! ``` 34 | //! 35 | //! Instead, wrap the combinators in a closure to simplify the type: 36 | //! ```rust 37 | //! # use winnow::prelude::*; 38 | //! fn foo() -> impl Parser 39 | //! # where 40 | //! # I: winnow::stream::Stream, 41 | //! # I: winnow::stream::StreamIsPartial, 42 | //! # E: winnow::error::ParserError, 43 | //! { 44 | //! move |input: &mut I| { 45 | //! // ...some chained combinators... 46 | //! # winnow::token::any 47 | //! .parse_next(input) 48 | //! } 49 | //! } 50 | //! ``` 51 | 52 | #![allow(unused_imports)] 53 | use crate::combinator::alt; 54 | use crate::combinator::dispatch; 55 | use crate::stream::BStr; 56 | -------------------------------------------------------------------------------- /src/_topic/s_expression.rs: -------------------------------------------------------------------------------- 1 | //! # s-expression 2 | //! 3 | //! ```rust 4 | #![doc = include_str!("../../examples/s_expression/parser.rs")] 5 | //! ``` 6 | -------------------------------------------------------------------------------- /src/_topic/stream.rs: -------------------------------------------------------------------------------- 1 | //! # Custom [`Stream`] 2 | //! 3 | //! `winnow` is batteries included with support for 4 | //! - Basic inputs like `&str`, newtypes with 5 | //! - Improved debug output like [`Bytes`] 6 | //! - [`Stateful`] for passing state through your parser, like tracking recursion 7 | //! depth 8 | //! - [`LocatingSlice`] for looking up the absolute position of a token 9 | //! 10 | //! ## Implementing a custom token 11 | //! 12 | //! The first level of customization is parsing [`&[MyItem]`][Stream#impl-Stream-for-%26%5BT%5D] 13 | //! or [`TokenSlice`]. 14 | //! 15 | //! The basic traits you may want for a custom token type are: 16 | //! 17 | //! | trait | usage | 18 | //! |---|---| 19 | //! | [`AsChar`] |Transforms common types to a char for basic token parsing| 20 | //! | [`ContainsToken`] |Look for the token in the given set| 21 | //! 22 | //! See also [`TokenSlice`], [lexing]. 23 | //! 24 | //! ## Implementing a custom stream 25 | //! 26 | //! Let's assume we have an input type we'll call `MyStream`. 27 | //! `MyStream` is a sequence of `MyItem` tokens. 28 | //! 29 | //! The goal is to define parsers with this signature: `&mut MyStream -> ModalResult`. 30 | //! ```rust 31 | //! # use winnow::prelude::*; 32 | //! # type MyStream<'i> = &'i str; 33 | //! # type Output<'i> = &'i str; 34 | //! fn parser<'s>(i: &mut MyStream<'s>) -> ModalResult> { 35 | //! "test".parse_next(i) 36 | //! } 37 | //! ``` 38 | //! 39 | //! Like above, you'll need to implement the related token traits for `MyItem`. 40 | //! 41 | //! The traits you may want to implement for `MyStream` include: 42 | //! 43 | //! | trait | usage | 44 | //! |---|---| 45 | //! | [`Stream`] |Core trait for driving parsing| 46 | //! | [`StreamIsPartial`] | Marks the input as being the complete buffer or a partial buffer for streaming input | 47 | //! | [`AsBytes`] |Casts the input type to a byte slice| 48 | //! | [`AsBStr`] |Casts the input type to a slice of ASCII / UTF-8-like bytes| 49 | //! | [`Compare`] |Character comparison operations| 50 | //! | [`FindSlice`] |Look for a substring in self| 51 | //! | [`Location`] |Calculate location within initial input| 52 | //! | [`Offset`] |Calculate the offset between slices| 53 | //! 54 | //! And for `&[MyItem]` (slices returned by [`Stream`]): 55 | //! 56 | //! | trait | usage | 57 | //! |---|---| 58 | //! | [`SliceLen`] |Calculate the input length| 59 | //! | [`ParseSlice`] |Used to integrate `&str`'s `parse()` method| 60 | 61 | #![allow(unused_imports)] // Here for intra-dock links 62 | use super::lexing; 63 | use crate::stream::*; 64 | -------------------------------------------------------------------------------- /src/_topic/why.rs: -------------------------------------------------------------------------------- 1 | //! # Why `winnow`? 2 | //! 3 | //! To answer this question, it will be useful to contrast this with other approaches to parsing. 4 | //! 5 | //!
6 | //! 7 | //! **Note:** This will focus on principles and priorities. For a deeper and wider wider 8 | //! comparison with other Rust parser libraries, see 9 | //! [parse-rosetta-rs](https://github.com/rosetta-rs/parse-rosetta-rs). 10 | //! 11 | //!
12 | //! 13 | //! ## Hand-written parsers 14 | //! 15 | //! Typically, a hand-written parser gives you the flexibility to get 16 | //! - Fast parse performance 17 | //! - Fast compile-time 18 | //! - Small binary sizes 19 | //! - High quality error message 20 | //! - Fewer dependencies to audit 21 | //! 22 | //! However, this comes at the cost of doing it all yourself, including 23 | //! - Optimizing for each of the above characteristics you care about 24 | //! - Ensuring the safety of any `unsafe` code (buffer overflows being a common bug with parsers) 25 | //! - Being aware of, familiar with, and correctly implement the relevant algorithms. 26 | //! matklad, who has written two rust compile frontends, commented 27 | //! ["I’ve implemented a production-grade Pratt parser once, but I no longer immediately understand that code :-)"](https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html) 28 | //! 29 | //! This approach works well if: 30 | //! - Your format is small and is unlikely to change 31 | //! - Your format is large but you have people who can focus solely on parsing, like with large 32 | //! programming languages 33 | //! 34 | //! ## `winnow` 35 | //! 36 | //! Unlike traditional programming language parsers that use 37 | //! [lex](https://en.wikipedia.org/wiki/Lex_(software)) or 38 | //! [yacc](https://en.wikipedia.org/wiki/Yacc), you can think of `winnow` as a general version of 39 | //! the helpers you would create along the way to writing a hand-written parser. 40 | //! 41 | //! `winnow` includes support for: 42 | //! - Zero-copy parsing 43 | //! - [Parse traces][trace] for easier debugging 44 | //! - [Streaming parsing][Partial] for network communication or large file 45 | //! - [Stateful] parsers 46 | //! 47 | //! For binary formats, `winnow` includes: 48 | //! - [A hexadecimal view][crate::Bytes] in [trace] 49 | //! - [TLV](https://en.wikipedia.org/wiki/Type-length-value) (e.g. [`length_take`]) 50 | //! - Some common parsers to help get started, like numbers 51 | //! 52 | //! For text formats, `winnow` includes: 53 | //! - [Tracking of spans][crate::LocatingSlice] 54 | //! - [A textual view when parsing as bytes][crate::BStr] in [trace] 55 | //! - Ability to evaluate directly, parse to an AST, or lex and parse the format 56 | //! 57 | //! This works well for: 58 | //! - Prototyping for what will be a hand-written parser 59 | //! - When you want to minimize the work to evolve your format 60 | //! - When you don't have contributors focused solely on parsing and your grammar is large enough 61 | //! to be unwieldy to hand write. 62 | //! 63 | //! ## `nom` 64 | //! 65 | //! `winnow` is a fork of the venerable [`nom`](https://crates.io/crates/nom). The difference 66 | //! between them is largely in priorities. `nom` prioritizes: 67 | //! - Lower churn for existing users while `winnow` is trying to find ways to make things better 68 | //! for the parsers yet to be written. 69 | //! - Having a small core, relying on external crates like 70 | //! [`nom-locate`](https://crates.io/crates/nom_locate) and 71 | //! [`nom-supreme`](https://crates.io/crates/nom-supreme), encouraging flexibility among users 72 | //! and to not block users on new features being merged while `winnow` aims to include all the 73 | //! fundamentals for parsing to ensure the experience is cohesive and high quality. 74 | //! 75 | //! For more details, see the [design differences][super::nom#api-differences]. 76 | //! 77 | //! See also our [nom migration guide][super::nom#migrating-from-nom]. 78 | //! 79 | //! ## `chumsky` 80 | //! 81 | //! [`chumsky`](https://crates.io/crates/chumsky) is an up and coming parser-combinator library 82 | //! that includes advanced features like error recovery. 83 | //! 84 | //! Probably the biggest diverging philosophy is `chumsky`s stance: 85 | //! 86 | //! > "If you need to implement either `Parser` or `Strategy` by hand, that's a problem that needs fixing". 87 | //! 88 | //! This is under "batteries included" but it also ties into the feeling that `chumsky` acts more like 89 | //! a framework. Instead of composing together helpers, you are expected to do everything through 90 | //! their system to the point that it is non-trivial to implement their `Parser` trait and are 91 | //! encouraged to use the 92 | //! [`custom`](https://docs.rs/chumsky/0.9.0/chumsky/primitive/fn.custom.html) helper. This 93 | //! requires re-framing everything to fit within their model and makes the code harder to understand 94 | //! and debug as you are working with abstract operations that will eventually be applied 95 | //! rather than directly with the parsers. 96 | //! 97 | //! In contrast, `winnow` is an introspectable toolbox that can easily be customized at any level. 98 | //! Probably the biggest thing that `winnow` loses out on is optimizations from ["parse modes" via 99 | //! GATs](https://github.com/zesterer/chumsky/pull/82) which allows downstream parsers to tell 100 | //! upstream parsers when information will be discarded, allowing bypassing expensive operations, 101 | //! like allocations. This requires a lot more complex interaction with parsers that isn't as 102 | //! trivial to do with bare functions which would lose out on any of that side-band information. 103 | //! Instead, we work around this with things like the [`Accumulate`] trait. 104 | 105 | #![allow(unused_imports)] 106 | use crate::binary::length_take; 107 | use crate::combinator::trace; 108 | use crate::stream::Accumulate; 109 | use crate::stream::Partial; 110 | use crate::stream::Stateful; 111 | -------------------------------------------------------------------------------- /src/_tutorial/chapter_0.rs: -------------------------------------------------------------------------------- 1 | //! # Chapter 0: Introduction 2 | //! 3 | //! This tutorial assumes that you are: 4 | //! - Already familiar with Rust 5 | //! - Using `winnow` for the first time 6 | //! 7 | //! The focus will be on parsing in-memory strings (`&str`). Once done, you might want to check the 8 | //! [Special Topics][_topic] for more specialized topics or examples. 9 | //! 10 | //! ## About 11 | //! 12 | //! `winnow` is a parser-combinator library. In other words, it gives you tools to define: 13 | //! - "parsers", or functions that take an input and give back an output 14 | //! - "combinators", or functions that take parsers and _combine_ them together! 15 | //! 16 | //! While "combinator" might be an unfamiliar word, you are likely using them in your rust code 17 | //! today, like with the [`Iterator`] trait: 18 | //! ```rust 19 | //! let data = vec![1, 2, 3, 4, 5]; 20 | //! let even_count = data.iter() 21 | //! .copied() // combinator 22 | //! .filter(|d| d % 2 == 0) // combinator 23 | //! .count(); // combinator 24 | //! ``` 25 | //! 26 | //! Parser combinators are great because: 27 | //! 28 | //! - Individual parser functions are small, focused on one thing, ignoring the rest 29 | //! - You can write tests focused on individual parsers (unit tests and property-based tests) 30 | //! in addition to testing the top-level parser as a whole. 31 | //! - Top-level parsing code looks close to the grammar you would have written 32 | 33 | #![allow(unused_imports)] 34 | use crate::_topic; 35 | use std::iter::Iterator; 36 | 37 | pub use super::chapter_1 as next; 38 | pub use crate::_tutorial as table_of_contents; 39 | -------------------------------------------------------------------------------- /src/_tutorial/chapter_1.rs: -------------------------------------------------------------------------------- 1 | //! # Chapter 1: The Winnow Way 2 | //! 3 | //! First of all, we need to understand the way that winnow thinks about parsing. 4 | //! As discussed in the introduction, winnow lets us compose more complex parsers from more simple 5 | //! ones (using "combinators"). 6 | //! 7 | //! Let's discuss what a "parser" actually does. A parser takes an input and advances it until it returns 8 | //! a result, where: 9 | //! - `Ok` indicates the parser successfully found what it was looking for; or 10 | //! - `Err` indicates the parser could not find what it was looking for. 11 | //! 12 | //! Parsers do more than just return a binary "success"/"failure" code. 13 | //! - On success, the parser will return the processed data. The input will be advanced to the end of 14 | //! what was processed, pointing to what will be parsed next. 15 | //! - If the parser failed, then there are multiple errors that could be returned. 16 | //! We'll explore this further in [`chapter_7`]. 17 | //! 18 | //! ```text 19 | //! ┌─► Ok(what matched the parser) 20 | //! ┌────────┐ │ 21 | //! my input───►│a parser├──►either──┤ 22 | //! └────────┘ └─► Err(...) 23 | //! ``` 24 | //! 25 | //! 26 | //! To represent this model of the world, winnow uses the [`Result`] type. 27 | //! The `Ok` variant has `output: O`; 28 | //! whereas the `Err` variant stores an error. 29 | //! 30 | //! You can import that from: 31 | //! 32 | //! ```rust 33 | //! use winnow::Result; 34 | //! ``` 35 | //! 36 | //! To combine parsers, we need a common way to refer to them which is where the [`Parser`] 37 | //! trait comes in with [`Parser::parse_next`] being the primary way to drive 38 | //! parsing forward. 39 | //! In [`chapter_6`], we'll cover how to integrate these into your application, particularly with 40 | //! [`Parser::parse`]. 41 | //! 42 | //! You'll note that `I` and `O` are parameterized -- while most of the examples in this book 43 | //! will be with `&str` (i.e. parsing a string); [they do not have to be strings][stream]; nor do they 44 | //! have to be the same type (consider the simple example where `I = &str`, and `O = u64` -- this 45 | //! parses a string into an unsigned integer.) 46 | //! 47 | //! # Let's write our first parser! 48 | //! 49 | //! The simplest parser we can write is one which successfully does nothing. 50 | //! 51 | //! To make it easier to implement a [`Parser`], the trait is implemented for 52 | //! functions of the form `Fn(&mut I) -> Result`. 53 | //! 54 | //! This parser function should take in a `&str`: 55 | //! 56 | //! - Since it is supposed to succeed, we know it will return the `Ok` variant. 57 | //! - Since it does nothing to our input, the input will be left where it started. 58 | //! - Since it doesn't parse anything, it also should just return an empty string. 59 | //! 60 | //! ```rust 61 | //! use winnow::Result; 62 | //! use winnow::Parser; 63 | //! 64 | //! pub fn do_nothing_parser<'s>(input: &mut &'s str) -> Result<&'s str> { 65 | //! Ok("") 66 | //! } 67 | //! 68 | //! fn main() { 69 | //! let mut input = "0x1a2b Hello"; 70 | //! 71 | //! let output = do_nothing_parser.parse_next(&mut input).unwrap(); 72 | //! // Same as: 73 | //! // let output = do_nothing_parser(&mut input).unwrap(); 74 | //! 75 | //! assert_eq!(input, "0x1a2b Hello"); 76 | //! assert_eq!(output, ""); 77 | //! } 78 | //! ``` 79 | 80 | #![allow(unused_imports)] 81 | use super::chapter_6; 82 | use super::chapter_7; 83 | use crate::Parser; 84 | use crate::_topic::stream; 85 | 86 | pub use super::chapter_0 as previous; 87 | pub use super::chapter_2 as next; 88 | pub use crate::_tutorial as table_of_contents; 89 | -------------------------------------------------------------------------------- /src/_tutorial/chapter_4.rs: -------------------------------------------------------------------------------- 1 | //! # Chapter 4: Parsers With Custom Return Types 2 | //! 3 | //! So far, we have seen mostly functions that take an `&str`, and return a 4 | //! [`Result<&str>`]. Splitting strings into smaller strings and characters is certainly 5 | //! useful, but it's not the only thing winnow is capable of! 6 | //! 7 | //! A useful operation when parsing is to convert between types; for example 8 | //! parsing from `&str` to another primitive, like [`usize`]. 9 | //! 10 | //! All we need to do for our parser to return a different type is to change 11 | //! the type parameter of [`Result`] to the desired return type. 12 | //! For example, to return a `usize`, return a `Result`. 13 | //! 14 | //! One winnow-native way of doing a type conversion is to use the 15 | //! [`Parser::parse_to`] combinator 16 | //! to convert from a successful parse to a particular type using [`FromStr`]. 17 | //! 18 | //! The following code converts from a string containing a number to `usize`: 19 | //! ```rust 20 | //! # use winnow::prelude::*; 21 | //! # use winnow::Result; 22 | //! # use winnow::ascii::digit1; 23 | //! # 24 | //! fn parse_digits(input: &mut &str) -> Result { 25 | //! digit1 26 | //! .parse_to() 27 | //! .parse_next(input) 28 | //! } 29 | //! 30 | //! fn main() { 31 | //! let mut input = "1024 Hello"; 32 | //! 33 | //! let output = parse_digits.parse_next(&mut input).unwrap(); 34 | //! assert_eq!(input, " Hello"); 35 | //! assert_eq!(output, 1024); 36 | //! 37 | //! assert!(parse_digits(&mut "Z").is_err()); 38 | //! } 39 | //! ``` 40 | //! 41 | //! `Parser::parse_to` is just a convenient form of [`Parser::try_map`] which we can use to handle 42 | //! all radices of numbers: 43 | //! ```rust 44 | //! # use winnow::prelude::*; 45 | //! # use winnow::Result; 46 | //! # use winnow::token::take_while; 47 | //! use winnow::combinator::dispatch; 48 | //! use winnow::token::take; 49 | //! use winnow::combinator::fail; 50 | //! 51 | //! fn parse_digits(input: &mut &str) -> Result { 52 | //! dispatch!(take(2usize); 53 | //! "0b" => parse_bin_digits.try_map(|s| usize::from_str_radix(s, 2)), 54 | //! "0o" => parse_oct_digits.try_map(|s| usize::from_str_radix(s, 8)), 55 | //! "0d" => parse_dec_digits.try_map(|s| usize::from_str_radix(s, 10)), 56 | //! "0x" => parse_hex_digits.try_map(|s| usize::from_str_radix(s, 16)), 57 | //! _ => fail, 58 | //! ).parse_next(input) 59 | //! } 60 | //! 61 | //! // ... 62 | //! # fn parse_bin_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 63 | //! # take_while(1.., ( 64 | //! # ('0'..='1'), 65 | //! # )).parse_next(input) 66 | //! # } 67 | //! # 68 | //! # fn parse_oct_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 69 | //! # take_while(1.., ( 70 | //! # ('0'..='7'), 71 | //! # )).parse_next(input) 72 | //! # } 73 | //! # 74 | //! # fn parse_dec_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 75 | //! # take_while(1.., ( 76 | //! # ('0'..='9'), 77 | //! # )).parse_next(input) 78 | //! # } 79 | //! # 80 | //! # fn parse_hex_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 81 | //! # take_while(1.., ( 82 | //! # ('0'..='9'), 83 | //! # ('A'..='F'), 84 | //! # ('a'..='f'), 85 | //! # )).parse_next(input) 86 | //! # } 87 | //! 88 | //! fn main() { 89 | //! let mut input = "0x1a2b Hello"; 90 | //! 91 | //! let digits = parse_digits.parse_next(&mut input).unwrap(); 92 | //! 93 | //! assert_eq!(input, " Hello"); 94 | //! assert_eq!(digits, 0x1a2b); 95 | //! 96 | //! assert!(parse_digits(&mut "ghiWorld").is_err()); 97 | //! } 98 | //! ``` 99 | //! 100 | //! See also [`Parser`] for more output-modifying parsers. 101 | 102 | #![allow(unused_imports)] 103 | use crate::Parser; 104 | use crate::Result; 105 | use std::str::FromStr; 106 | 107 | pub use super::chapter_3 as previous; 108 | pub use super::chapter_5 as next; 109 | pub use crate::_tutorial as table_of_contents; 110 | -------------------------------------------------------------------------------- /src/_tutorial/chapter_6.rs: -------------------------------------------------------------------------------- 1 | //! # Chapter 6: Integrating the Parser 2 | //! 3 | //! So far, we've highlighted how to incrementally parse, but how do we bring this all together 4 | //! into our application? 5 | //! 6 | //! Parsers we've been working with look like: 7 | //! ```rust 8 | //! # use winnow::error::ContextError; 9 | //! # use winnow::Parser; 10 | //! use winnow::Result; 11 | //! 12 | //! pub fn parser<'s>(input: &mut &'s str) -> Result<&'s str> { 13 | //! // ... 14 | //! # Ok("") 15 | //! } 16 | //! ``` 17 | //! 1. We have to decide what to do about the "remainder" of the `input`. 18 | //! 2. The [`Result`] may not be compatible with the rest of the Rust ecosystem. 19 | //! Normally, Rust applications want errors that are `std::error::Error + Send + Sync + 'static` 20 | //! meaning: 21 | //! - They implement the [`std::error::Error`] trait 22 | //! - They can be sent across threads 23 | //! - They are safe to be referenced across threads 24 | //! - They do not borrow 25 | //! 26 | //! winnow provides [`Parser::parse`] to help with this: 27 | //! - Ensures we hit [`eof`] 28 | //! - Wraps the error in [`ParseError`] 29 | //! - For simple cases, [`ParseError`] provides a [`std::fmt::Display`] impl to render the error. 30 | //! - For more involved cases, [`ParseError`] provides the original [`input`][ParseError::input] and the 31 | //! [`offset`][ParseError::offset] of where it failed so you can capture this information in 32 | //! your error, [rendering it as you wish][chapter_7#error-adaptation-and-rendering]. 33 | //! - Converts from [`ModalResult`] to [`Result`] (if used, more on this in [`chapter_7`]) 34 | //! 35 | //! However, [`ParseError`] will still need some level of adaptation to integrate with your 36 | //! application's error type (like with `?`). 37 | //! 38 | //! ```rust 39 | //! # use winnow::prelude::*; 40 | //! # use winnow::Result; 41 | //! # use winnow::token::take_while; 42 | //! # use winnow::combinator::dispatch; 43 | //! # use winnow::token::take; 44 | //! # use winnow::combinator::fail; 45 | //! use winnow::Parser; 46 | //! 47 | //! #[derive(Debug, PartialEq, Eq)] 48 | //! pub struct Hex(usize); 49 | //! 50 | //! impl std::str::FromStr for Hex { 51 | //! type Err = anyhow::Error; 52 | //! 53 | //! fn from_str(input: &str) -> Result { 54 | //! parse_digits 55 | //! .map(Hex) 56 | //! .parse(input) 57 | //! .map_err(|e| anyhow::format_err!("{e}")) 58 | //! } 59 | //! } 60 | //! 61 | //! // ... 62 | //! # fn parse_digits<'s>(input: &mut &'s str) -> Result { 63 | //! # dispatch!(take(2usize); 64 | //! # "0b" => parse_bin_digits.try_map(|s| usize::from_str_radix(s, 2)), 65 | //! # "0o" => parse_oct_digits.try_map(|s| usize::from_str_radix(s, 8)), 66 | //! # "0d" => parse_dec_digits.try_map(|s| usize::from_str_radix(s, 10)), 67 | //! # "0x" => parse_hex_digits.try_map(|s| usize::from_str_radix(s, 16)), 68 | //! # _ => fail, 69 | //! # ).parse_next(input) 70 | //! # } 71 | //! # 72 | //! # fn parse_bin_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 73 | //! # take_while(1.., ( 74 | //! # ('0'..='1'), 75 | //! # )).parse_next(input) 76 | //! # } 77 | //! # 78 | //! # fn parse_oct_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 79 | //! # take_while(1.., ( 80 | //! # ('0'..='7'), 81 | //! # )).parse_next(input) 82 | //! # } 83 | //! # 84 | //! # fn parse_dec_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 85 | //! # take_while(1.., ( 86 | //! # ('0'..='9'), 87 | //! # )).parse_next(input) 88 | //! # } 89 | //! # 90 | //! # fn parse_hex_digits<'s>(input: &mut &'s str) -> Result<&'s str> { 91 | //! # take_while(1.., ( 92 | //! # ('0'..='9'), 93 | //! # ('A'..='F'), 94 | //! # ('a'..='f'), 95 | //! # )).parse_next(input) 96 | //! # } 97 | //! 98 | //! fn main() { 99 | //! let input = "0x1a2b"; 100 | //! assert_eq!(input.parse::().unwrap(), Hex(0x1a2b)); 101 | //! 102 | //! let input = "0x1a2b Hello"; 103 | //! assert!(input.parse::().is_err()); 104 | //! let input = "ghiHello"; 105 | //! assert!(input.parse::().is_err()); 106 | //! } 107 | //! ``` 108 | 109 | #![allow(unused_imports)] 110 | use super::chapter_1; 111 | use super::chapter_7; 112 | use crate::combinator::eof; 113 | use crate::error::ErrMode; 114 | use crate::error::ParseError; 115 | use crate::ModalResult; 116 | use crate::Parser; 117 | 118 | pub use super::chapter_5 as previous; 119 | pub use super::chapter_7 as next; 120 | pub use crate::_tutorial as table_of_contents; 121 | -------------------------------------------------------------------------------- /src/_tutorial/chapter_8.rs: -------------------------------------------------------------------------------- 1 | //! # Chapter 8: Debugging 2 | //! 3 | //! When things inevitably go wrong, you can introspect the parsing state by running your test case 4 | //! with `--features winnow/debug`. 5 | //! 6 | //! For example, the trace output of an [escaped string parser][crate::_topic::language#escaped-strings]: 7 | //! ![Trace output from string example](https://raw.githubusercontent.com/winnow-rs/winnow/main/assets/trace.svg "Example output") 8 | //! 9 | //! You can extend your own parsers to show up by wrapping their body with 10 | //! [`trace`][crate::combinator::trace]. Going back to [`do_nothing_parser`][super::chapter_1]. 11 | //! ```rust 12 | //! # use winnow::ModalResult; 13 | //! # use winnow::Parser; 14 | //! use winnow::combinator::trace; 15 | //! 16 | //! pub fn do_nothing_parser<'s>(input: &mut &'s str) -> ModalResult<&'s str> { 17 | //! trace( 18 | //! "do_nothing_parser", 19 | //! |i: &mut _| Ok("") 20 | //! ).parse_next(input) 21 | //! } 22 | //! # 23 | //! # fn main() { 24 | //! # let mut input = "0x1a2b Hello"; 25 | //! # 26 | //! # let output = do_nothing_parser.parse_next(&mut input).unwrap(); 27 | //! # // Same as: 28 | //! # // let output = do_nothing_parser(&mut input).unwrap(); 29 | //! # 30 | //! # assert_eq!(input, "0x1a2b Hello"); 31 | //! # assert_eq!(output, ""); 32 | //! # } 33 | //! ``` 34 | 35 | pub use super::chapter_7 as previous; 36 | pub use crate::_topic as next; 37 | pub use crate::_tutorial as table_of_contents; 38 | -------------------------------------------------------------------------------- /src/_tutorial/mod.rs: -------------------------------------------------------------------------------- 1 | //! # Tutorial 2 | //! 3 | //! Table of Contents 4 | #![allow(clippy::std_instead_of_core)] 5 | 6 | pub mod chapter_0; 7 | pub mod chapter_1; 8 | pub mod chapter_2; 9 | pub mod chapter_3; 10 | pub mod chapter_4; 11 | pub mod chapter_5; 12 | pub mod chapter_6; 13 | pub mod chapter_7; 14 | pub mod chapter_8; 15 | -------------------------------------------------------------------------------- /src/binary/bits/tests.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::error::ErrMode; 3 | use crate::error::InputError; 4 | use crate::prelude::*; 5 | use crate::Partial; 6 | 7 | #[test] 8 | /// Take the `bits` function and assert that remaining bytes are correctly returned, if the 9 | /// previous bytes are fully consumed 10 | fn test_complete_byte_consumption_bits() { 11 | let input = &[0x12, 0x34, 0x56, 0x78][..]; 12 | 13 | // Take 3 bit slices with sizes [4, 8, 4]. 14 | #[allow(clippy::type_complexity)] 15 | let result: ModalResult<(&[u8], (u8, u8, u8)), InputError<_>> = 16 | bits::<_, _, ErrMode>, _, _>(( 17 | take(4usize), 18 | take(8usize), 19 | take(4usize), 20 | )) 21 | .parse_peek(input); 22 | 23 | let output = result.expect("We take 2 bytes and the input is longer than 2 bytes"); 24 | 25 | let remaining = output.0; 26 | assert_eq!(remaining, [0x56, 0x78]); 27 | 28 | let parsed = output.1; 29 | assert_eq!(parsed.0, 0x01); 30 | assert_eq!(parsed.1, 0x23); 31 | assert_eq!(parsed.2, 0x04); 32 | } 33 | 34 | #[test] 35 | /// Take the `bits` function and assert that remaining bytes are correctly returned, if the 36 | /// previous bytes are NOT fully consumed. Partially consumed bytes are supposed to be dropped. 37 | /// I.e. if we consume 1.5 bytes of 4 bytes, 2 bytes will be returned, bits 13-16 will be 38 | /// dropped. 39 | fn test_partial_byte_consumption_bits() { 40 | let input = &[0x12, 0x34, 0x56, 0x78][..]; 41 | 42 | // Take bit slices with sizes [4, 8]. 43 | let result: ModalResult<(&[u8], (u8, u8)), InputError<_>> = 44 | bits::<_, _, ErrMode>, _, _>((take(4usize), take(8usize))) 45 | .parse_peek(input); 46 | 47 | let output = result.expect("We take 1.5 bytes and the input is longer than 2 bytes"); 48 | 49 | let remaining = output.0; 50 | assert_eq!(remaining, [0x56, 0x78]); 51 | 52 | let parsed = output.1; 53 | assert_eq!(parsed.0, 0x01); 54 | assert_eq!(parsed.1, 0x23); 55 | } 56 | 57 | #[test] 58 | #[cfg(feature = "std")] 59 | /// Ensure that in Incomplete error is thrown, if too few bytes are passed for a given parser. 60 | fn test_incomplete_bits() { 61 | let input = Partial::new(&[0x12][..]); 62 | 63 | // Take bit slices with sizes [4, 8]. 64 | let result: ModalResult<(_, (u8, u8)), InputError<_>> = 65 | bits::<_, _, ErrMode>, _, _>((take(4usize), take(8usize))) 66 | .parse_peek(input); 67 | 68 | assert!(result.is_err()); 69 | let error = result.err().unwrap(); 70 | assert_eq!("Parsing requires 2 more data", error.to_string()); 71 | } 72 | 73 | #[test] 74 | fn test_take_complete_0() { 75 | let input = &[0b00010010][..]; 76 | let count = 0usize; 77 | assert_eq!(count, 0usize); 78 | let offset = 0usize; 79 | 80 | let result: ModalResult<((&[u8], usize), usize), InputError<_>> = 81 | take(count).parse_peek((input, offset)); 82 | 83 | assert_eq!(result, Ok(((input, offset), 0))); 84 | } 85 | 86 | #[test] 87 | fn test_take_complete_eof() { 88 | let input = &[0b00010010][..]; 89 | 90 | let result: ModalResult<((&[u8], usize), usize), InputError<_>> = 91 | take(1usize).parse_peek((input, 8)); 92 | 93 | assert_eq!( 94 | result, 95 | Err(crate::error::ErrMode::Backtrack( 96 | InputError::at((input, 8),) 97 | )) 98 | ); 99 | } 100 | 101 | #[test] 102 | fn test_take_complete_span_over_multiple_bytes() { 103 | let input = &[0b00010010, 0b00110100, 0b11111111, 0b11111111][..]; 104 | 105 | let result: ModalResult<((&[u8], usize), usize), InputError<_>> = 106 | take(24usize).parse_peek((input, 4)); 107 | 108 | assert_eq!( 109 | result, 110 | Ok((([0b11111111].as_ref(), 4), 0b1000110100111111111111)) 111 | ); 112 | } 113 | 114 | #[test] 115 | fn test_take_partial_0() { 116 | let input = Partial::new(&[][..]); 117 | let count = 0usize; 118 | assert_eq!(count, 0usize); 119 | let offset = 0usize; 120 | 121 | let result: ModalResult<((_, usize), usize), InputError<_>> = 122 | take(count).parse_peek((input, offset)); 123 | 124 | assert_eq!(result, Ok(((input, offset), 0))); 125 | } 126 | 127 | #[test] 128 | fn test_pattern_partial_ok() { 129 | let input = Partial::new(&[0b00011111][..]); 130 | let offset = 0usize; 131 | let bits_to_take = 4usize; 132 | let value_to_pattern = 0b0001; 133 | 134 | let result: ModalResult<((_, usize), usize), InputError<_>> = 135 | pattern(value_to_pattern, bits_to_take).parse_peek((input, offset)); 136 | 137 | assert_eq!(result, Ok(((input, bits_to_take), value_to_pattern))); 138 | } 139 | 140 | #[test] 141 | fn test_pattern_partial_err() { 142 | let input = Partial::new(&[0b00011111][..]); 143 | let offset = 0usize; 144 | let bits_to_take = 4usize; 145 | let value_to_pattern = 0b1111; 146 | 147 | let result: ModalResult<((_, usize), usize), InputError<_>> = 148 | pattern(value_to_pattern, bits_to_take).parse_peek((input, offset)); 149 | 150 | assert_eq!( 151 | result, 152 | Err(crate::error::ErrMode::Backtrack(InputError::at(( 153 | input, offset 154 | ),))) 155 | ); 156 | } 157 | 158 | #[test] 159 | fn test_bool_0_complete() { 160 | let input = [0b10000000].as_ref(); 161 | 162 | let result: ModalResult<((&[u8], usize), bool), InputError<_>> = bool.parse_peek((input, 0)); 163 | 164 | assert_eq!(result, Ok(((input, 1), true))); 165 | } 166 | 167 | #[test] 168 | fn test_bool_eof_complete() { 169 | let input = [0b10000000].as_ref(); 170 | 171 | let result: ModalResult<((&[u8], usize), bool), InputError<_>> = bool.parse_peek((input, 8)); 172 | 173 | assert_eq!( 174 | result, 175 | Err(crate::error::ErrMode::Backtrack( 176 | InputError::at((input, 8),) 177 | )) 178 | ); 179 | } 180 | 181 | #[test] 182 | fn test_bool_0_partial() { 183 | let input = Partial::new([0b10000000].as_ref()); 184 | 185 | #[allow(clippy::type_complexity)] 186 | let result: ModalResult<((Partial<&[u8]>, usize), bool), InputError<_>> = 187 | bool.parse_peek((input, 0)); 188 | 189 | assert_eq!(result, Ok(((input, 1), true))); 190 | } 191 | 192 | #[test] 193 | fn test_bool_eof_partial() { 194 | let input = Partial::new([0b10000000].as_ref()); 195 | 196 | #[allow(clippy::type_complexity)] 197 | let result: ModalResult<((Partial<&[u8]>, usize), bool), InputError<_>> = 198 | bool.parse_peek((input, 8)); 199 | 200 | assert_eq!( 201 | result, 202 | Err(crate::error::ErrMode::Incomplete(Needed::new(1))) 203 | ); 204 | } 205 | -------------------------------------------------------------------------------- /src/combinator/debug/mod.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "debug", allow(clippy::std_instead_of_core))] 2 | 3 | #[cfg(feature = "debug")] 4 | mod internals; 5 | 6 | use crate::error::ParserError; 7 | use crate::stream::Stream; 8 | use crate::Parser; 9 | 10 | /// Trace the execution of the parser 11 | /// 12 | /// Note that [`Parser::context`] also provides high level trace information. 13 | /// 14 | /// See [tutorial][crate::_tutorial::chapter_8] for more details. 15 | /// 16 | /// # Example 17 | /// 18 | /// ```rust 19 | /// # use winnow::{error::ErrMode, error::Needed}; 20 | /// # use winnow::token::take_while; 21 | /// # use winnow::stream::AsChar; 22 | /// # use winnow::prelude::*; 23 | /// use winnow::combinator::trace; 24 | /// 25 | /// fn short_alpha<'s>(s: &mut &'s [u8]) -> ModalResult<&'s [u8]> { 26 | /// trace("short_alpha", 27 | /// take_while(3..=6, AsChar::is_alpha) 28 | /// ).parse_next(s) 29 | /// } 30 | /// 31 | /// assert_eq!(short_alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..]))); 32 | /// assert_eq!(short_alpha.parse_peek(b"lengthy"), Ok((&b"y"[..], &b"length"[..]))); 33 | /// assert_eq!(short_alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..]))); 34 | /// assert!(short_alpha.parse_peek(b"ed").is_err()); 35 | /// assert!(short_alpha.parse_peek(b"12345").is_err()); 36 | /// ``` 37 | #[cfg_attr(not(feature = "debug"), allow(unused_variables))] 38 | #[cfg_attr(not(feature = "debug"), allow(unused_mut))] 39 | #[cfg_attr(not(feature = "debug"), inline(always))] 40 | pub fn trace>( 41 | name: impl crate::lib::std::fmt::Display, 42 | parser: impl Parser, 43 | ) -> impl Parser { 44 | #[cfg(feature = "debug")] 45 | { 46 | internals::Trace::new(parser, name) 47 | } 48 | #[cfg(not(feature = "debug"))] 49 | { 50 | parser 51 | } 52 | } 53 | 54 | #[cfg_attr(not(feature = "debug"), allow(unused_variables))] 55 | pub(crate) fn trace_result>( 56 | name: impl crate::lib::std::fmt::Display, 57 | res: &Result, 58 | ) { 59 | #[cfg(feature = "debug")] 60 | { 61 | let depth = internals::Depth::existing(); 62 | let severity = internals::Severity::with_result(res); 63 | internals::result(*depth, &name, severity); 64 | } 65 | } 66 | 67 | pub(crate) struct DisplayDebug(pub(crate) D); 68 | 69 | impl crate::lib::std::fmt::Display for DisplayDebug { 70 | fn fmt(&self, f: &mut crate::lib::std::fmt::Formatter<'_>) -> crate::lib::std::fmt::Result { 71 | write!(f, "{:?}", self.0) 72 | } 73 | } 74 | 75 | #[test] 76 | #[cfg(feature = "std")] 77 | #[cfg_attr(miri, ignore)] 78 | #[cfg(unix)] 79 | #[cfg(feature = "debug")] 80 | fn example() { 81 | use term_transcript::{test::TestConfig, ShellOptions}; 82 | 83 | let path = snapbox::cmd::compile_example("string", ["--features=debug"]).unwrap(); 84 | 85 | let current_dir = path.parent().unwrap(); 86 | let cmd = path.file_name().unwrap(); 87 | // HACK: term_transcript doesn't allow non-UTF8 paths 88 | let cmd = format!("./{}", cmd.to_string_lossy()); 89 | 90 | TestConfig::new( 91 | ShellOptions::default() 92 | .with_current_dir(current_dir) 93 | .with_env("CLICOLOR_FORCE", "1"), 94 | ) 95 | .test("assets/trace.svg", [format!(r#"{cmd} '"abc"'"#).as_str()]); 96 | } 97 | -------------------------------------------------------------------------------- /src/combinator/sequence.rs: -------------------------------------------------------------------------------- 1 | use crate::combinator::trace; 2 | use crate::error::ParserError; 3 | use crate::stream::Stream; 4 | use crate::*; 5 | 6 | #[doc(inline)] 7 | pub use crate::seq; 8 | 9 | /// Sequence two parsers, only returning the output from the second. 10 | /// 11 | /// See also [`seq`] to generalize this across any number of fields. 12 | /// 13 | /// # Example 14 | /// 15 | /// ```rust 16 | /// # use winnow::{error::ErrMode, error::Needed}; 17 | /// # use winnow::prelude::*; 18 | /// # use winnow::error::Needed::Size; 19 | /// use winnow::combinator::preceded; 20 | /// 21 | /// fn parser<'i>(input: &mut &'i str) -> ModalResult<&'i str> { 22 | /// preceded("abc", "efg").parse_next(input) 23 | /// } 24 | /// 25 | /// assert_eq!(parser.parse_peek("abcefg"), Ok(("", "efg"))); 26 | /// assert_eq!(parser.parse_peek("abcefghij"), Ok(("hij", "efg"))); 27 | /// assert!(parser.parse_peek("").is_err()); 28 | /// assert!(parser.parse_peek("123").is_err()); 29 | /// ``` 30 | #[doc(alias = "ignore_then")] 31 | pub fn preceded( 32 | mut ignored: IgnoredParser, 33 | mut parser: ParseNext, 34 | ) -> impl Parser 35 | where 36 | Input: Stream, 37 | Error: ParserError, 38 | IgnoredParser: Parser, 39 | ParseNext: Parser, 40 | { 41 | trace("preceded", move |input: &mut Input| { 42 | let _ = ignored.parse_next(input)?; 43 | parser.parse_next(input) 44 | }) 45 | } 46 | 47 | /// Sequence two parsers, only returning the output of the first. 48 | /// 49 | /// See also [`seq`] to generalize this across any number of fields. 50 | /// 51 | /// # Example 52 | /// 53 | /// ```rust 54 | /// # use winnow::{error::ErrMode, error::Needed}; 55 | /// # use winnow::prelude::*; 56 | /// # use winnow::error::Needed::Size; 57 | /// use winnow::combinator::terminated; 58 | /// 59 | /// fn parser<'i>(input: &mut &'i str) -> ModalResult<&'i str> { 60 | /// terminated("abc", "efg").parse_next(input) 61 | /// } 62 | /// 63 | /// assert_eq!(parser.parse_peek("abcefg"), Ok(("", "abc"))); 64 | /// assert_eq!(parser.parse_peek("abcefghij"), Ok(("hij", "abc"))); 65 | /// assert!(parser.parse_peek("").is_err()); 66 | /// assert!(parser.parse_peek("123").is_err()); 67 | /// ``` 68 | #[doc(alias = "then_ignore")] 69 | pub fn terminated( 70 | mut parser: ParseNext, 71 | mut ignored: IgnoredParser, 72 | ) -> impl Parser 73 | where 74 | Input: Stream, 75 | Error: ParserError, 76 | ParseNext: Parser, 77 | IgnoredParser: Parser, 78 | { 79 | trace("terminated", move |input: &mut Input| { 80 | let o = parser.parse_next(input)?; 81 | ignored.parse_next(input).map(|_| o) 82 | }) 83 | } 84 | 85 | /// Sequence three parsers, only returning the values of the first and third. 86 | /// 87 | /// See also [`seq`] to generalize this across any number of fields. 88 | /// 89 | /// # Example 90 | /// 91 | /// ```rust 92 | /// # use winnow::{error::ErrMode, error::Needed}; 93 | /// # use winnow::error::Needed::Size; 94 | /// # use winnow::prelude::*; 95 | /// use winnow::combinator::separated_pair; 96 | /// 97 | /// fn parser<'i>(input: &mut &'i str) -> ModalResult<(&'i str, &'i str)> { 98 | /// separated_pair("abc", "|", "efg").parse_next(input) 99 | /// } 100 | /// 101 | /// assert_eq!(parser.parse_peek("abc|efg"), Ok(("", ("abc", "efg")))); 102 | /// assert_eq!(parser.parse_peek("abc|efghij"), Ok(("hij", ("abc", "efg")))); 103 | /// assert!(parser.parse_peek("").is_err()); 104 | /// assert!(parser.parse_peek("123").is_err()); 105 | /// ``` 106 | pub fn separated_pair( 107 | mut first: P1, 108 | mut sep: SepParser, 109 | mut second: P2, 110 | ) -> impl Parser 111 | where 112 | Input: Stream, 113 | Error: ParserError, 114 | P1: Parser, 115 | SepParser: Parser, 116 | P2: Parser, 117 | { 118 | trace("separated_pair", move |input: &mut Input| { 119 | let o1 = first.parse_next(input)?; 120 | let _ = sep.parse_next(input)?; 121 | second.parse_next(input).map(|o2| (o1, o2)) 122 | }) 123 | } 124 | 125 | /// Sequence three parsers, only returning the output of the second. 126 | /// 127 | /// See also [`seq`] to generalize this across any number of fields. 128 | /// 129 | /// # Example 130 | /// 131 | /// ```rust 132 | /// # use winnow::{error::ErrMode, error::Needed}; 133 | /// # use winnow::error::Needed::Size; 134 | /// # use winnow::prelude::*; 135 | /// use winnow::combinator::delimited; 136 | /// 137 | /// fn parser<'i>(input: &mut &'i str) -> ModalResult<&'i str> { 138 | /// delimited("(", "abc", ")").parse_next(input) 139 | /// } 140 | /// 141 | /// assert_eq!(parser.parse_peek("(abc)"), Ok(("", "abc"))); 142 | /// assert_eq!(parser.parse_peek("(abc)def"), Ok(("def", "abc"))); 143 | /// assert!(parser.parse_peek("").is_err()); 144 | /// assert!(parser.parse_peek("123").is_err()); 145 | /// ``` 146 | #[doc(alias = "between")] 147 | #[doc(alias = "padded")] 148 | pub fn delimited< 149 | Input, 150 | Ignored1, 151 | Output, 152 | Ignored2, 153 | Error, 154 | IgnoredParser1, 155 | ParseNext, 156 | IgnoredParser2, 157 | >( 158 | mut ignored1: IgnoredParser1, 159 | mut parser: ParseNext, 160 | mut ignored2: IgnoredParser2, 161 | ) -> impl Parser 162 | where 163 | Input: Stream, 164 | Error: ParserError, 165 | IgnoredParser1: Parser, 166 | ParseNext: Parser, 167 | IgnoredParser2: Parser, 168 | { 169 | trace("delimited", move |input: &mut Input| { 170 | let _ = ignored1.parse_next(input)?; 171 | let o2 = parser.parse_next(input)?; 172 | ignored2.parse_next(input).map(|_| o2) 173 | }) 174 | } 175 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! > winnow, making parsing a breeze 2 | //! 3 | //! `winnow` is a parser combinator library 4 | //! 5 | //! Quick links: 6 | //! - [List of combinators][crate::combinator] 7 | //! - [Tutorial][_tutorial::chapter_0] 8 | //! - [Special Topics][_topic] 9 | //! - [Discussions](https://github.com/winnow-rs/winnow/discussions) 10 | //! - [CHANGELOG](https://github.com/winnow-rs/winnow/blob/v0.7.10/CHANGELOG.md) (includes major version migration 11 | //! guides) 12 | //! 13 | //! ## Aspirations 14 | //! 15 | //! `winnow` aims to be your "do everything" parser, much like people treat regular expressions. 16 | //! 17 | //! In roughly priority order: 18 | //! 1. Support writing parser declaratively while not getting in the way of imperative-style 19 | //! parsing when needed, working as an open-ended toolbox rather than a close-ended framework. 20 | //! 2. Flexible enough to be used for any application, including parsing strings, binary data, 21 | //! or separate [lexing and parsing phases][_topic::lexing] 22 | //! 3. Zero-cost abstractions, making it easy to write high performance parsers 23 | //! 4. Easy to use, making it trivial for one-off uses 24 | //! 25 | //! In addition: 26 | //! - Resilient maintainership, including 27 | //! - Willing to break compatibility rather than batching up breaking changes in large releases 28 | //! - Leverage feature flags to keep one active branch 29 | //! - We will support the last 6 months of rust releases (MSRV, currently 1.64.0) 30 | //! 31 | //! See also [Special Topic: Why winnow?][crate::_topic::why] 32 | //! 33 | //! ## Example 34 | //! 35 | //! Run 36 | //! ```console 37 | //! $ cargo add winnow 38 | //! ``` 39 | //! 40 | //! Then use it to parse: 41 | //! ```rust 42 | //! # #[cfg(feature = "alloc")] { 43 | #![doc = include_str!("../examples/css/parser.rs")] 44 | //! # } 45 | //! ``` 46 | //! 47 | //! See also the [Tutorial][_tutorial::chapter_0] and [Special Topics][_topic] 48 | 49 | #![cfg_attr(docsrs, feature(doc_auto_cfg))] 50 | #![cfg_attr(docsrs, feature(doc_cfg))] 51 | #![cfg_attr(docsrs, feature(extended_key_value_attributes))] 52 | #![cfg_attr(all(not(feature = "std"), not(test)), no_std)] 53 | #![warn(missing_docs)] 54 | #![warn(clippy::std_instead_of_core)] 55 | #![warn(clippy::std_instead_of_alloc)] 56 | #![warn(clippy::print_stderr)] 57 | #![warn(clippy::print_stdout)] 58 | 59 | #[cfg(feature = "alloc")] 60 | #[cfg_attr(test, macro_use)] 61 | #[allow(unused_extern_crates)] 62 | extern crate alloc; 63 | 64 | #[doc = include_str!("../README.md")] 65 | #[cfg(doctest)] 66 | pub struct ReadmeDoctests; 67 | 68 | /// Lib module to re-export everything needed from `std` or `core`/`alloc`. This is how `serde` does 69 | /// it, albeit there it is not public. 70 | #[doc(hidden)] 71 | pub(crate) mod lib { 72 | #![allow(unused_imports)] 73 | 74 | /// `std` facade allowing `std`/`core` to be interchangeable. Reexports `alloc` crate optionally, 75 | /// as well as `core` or `std` 76 | #[cfg(not(feature = "std"))] 77 | /// internal std exports for no_std compatibility 78 | pub(crate) mod std { 79 | #[doc(hidden)] 80 | #[cfg(not(feature = "alloc"))] 81 | pub(crate) use core::borrow; 82 | 83 | #[cfg(feature = "alloc")] 84 | #[doc(hidden)] 85 | pub(crate) use alloc::{borrow, boxed, collections, string, vec}; 86 | 87 | #[doc(hidden)] 88 | pub(crate) use core::{ 89 | cmp, convert, fmt, hash, iter, mem, ops, option, result, slice, str, 90 | }; 91 | } 92 | 93 | #[cfg(feature = "std")] 94 | /// internal std exports for `no_std` compatibility 95 | pub(crate) mod std { 96 | #![allow(clippy::std_instead_of_core)] 97 | #![allow(clippy::std_instead_of_alloc)] 98 | #[doc(hidden)] 99 | pub(crate) use std::{ 100 | borrow, boxed, cmp, collections, convert, fmt, hash, iter, mem, ops, result, slice, 101 | str, string, vec, 102 | }; 103 | } 104 | } 105 | 106 | pub(crate) mod util { 107 | #[allow(dead_code)] 108 | pub(crate) fn from_fn) -> core::fmt::Result>( 109 | f: F, 110 | ) -> FromFn { 111 | FromFn(f) 112 | } 113 | 114 | pub(crate) struct FromFn(F) 115 | where 116 | F: Fn(&mut core::fmt::Formatter<'_>) -> core::fmt::Result; 117 | 118 | impl core::fmt::Debug for FromFn 119 | where 120 | F: Fn(&mut core::fmt::Formatter<'_>) -> core::fmt::Result, 121 | { 122 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 123 | (self.0)(f) 124 | } 125 | } 126 | 127 | impl core::fmt::Display for FromFn 128 | where 129 | F: Fn(&mut core::fmt::Formatter<'_>) -> core::fmt::Result, 130 | { 131 | fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 132 | (self.0)(f) 133 | } 134 | } 135 | } 136 | 137 | #[macro_use] 138 | mod macros; 139 | 140 | #[macro_use] 141 | pub mod error; 142 | 143 | mod parser; 144 | 145 | pub mod stream; 146 | 147 | pub mod ascii; 148 | pub mod binary; 149 | pub mod combinator; 150 | pub mod token; 151 | 152 | #[cfg(feature = "unstable-doc")] 153 | pub mod _topic; 154 | #[cfg(feature = "unstable-doc")] 155 | pub mod _tutorial; 156 | 157 | /// Core concepts available for glob import 158 | /// 159 | /// Including 160 | /// - [`StreamIsPartial`][crate::stream::StreamIsPartial] 161 | /// - [`Parser`] 162 | /// 163 | /// ## Example 164 | /// 165 | /// ```rust 166 | /// use winnow::prelude::*; 167 | /// 168 | /// fn parse_data(input: &mut &str) -> ModalResult { 169 | /// // ... 170 | /// # winnow::ascii::dec_uint(input) 171 | /// } 172 | /// 173 | /// fn main() { 174 | /// let result = parse_data.parse("100"); 175 | /// assert_eq!(result, Ok(100)); 176 | /// } 177 | /// ``` 178 | pub mod prelude { 179 | pub use crate::error::ModalError as _; 180 | pub use crate::error::ParserError as _; 181 | pub use crate::stream::AsChar as _; 182 | pub use crate::stream::ContainsToken as _; 183 | pub use crate::stream::Stream as _; 184 | pub use crate::stream::StreamIsPartial as _; 185 | pub use crate::ModalParser; 186 | pub use crate::ModalResult; 187 | pub use crate::Parser; 188 | #[cfg(feature = "unstable-recover")] 189 | #[cfg(feature = "std")] 190 | pub use crate::RecoverableParser as _; 191 | 192 | #[cfg(test)] 193 | pub(crate) use crate::TestResult; 194 | } 195 | 196 | pub use error::ModalResult; 197 | pub use error::Result; 198 | pub use parser::*; 199 | pub use stream::BStr; 200 | pub use stream::Bytes; 201 | pub use stream::LocatingSlice; 202 | pub use stream::Partial; 203 | pub use stream::Stateful; 204 | pub use stream::Str; 205 | 206 | #[cfg(test)] 207 | pub(crate) use error::TestResult; 208 | -------------------------------------------------------------------------------- /src/macros/dispatch.rs: -------------------------------------------------------------------------------- 1 | /// `match` for parsers 2 | /// 3 | /// While `match` works by accepting a value and returning values: 4 | /// ```rust,ignore 5 | /// let result_value = match scrutinee_value { 6 | /// ArmPattern => arm_value, 7 | /// }; 8 | /// ``` 9 | /// `dispatch!` composes parsers: 10 | /// ```rust,ignore 11 | /// let result_parser = dispatch!{scrutinee_parser; 12 | /// ArmPattern => arm_parser, 13 | /// }; 14 | /// ``` 15 | /// 16 | /// This is useful when parsers have unique prefixes to test for. 17 | /// This offers better performance over 18 | /// [`alt`][crate::combinator::alt] though it might be at the cost of duplicating parts of your grammar 19 | /// if you needed to [`peek(input_parser)`][crate::combinator::peek] the scrutinee. 20 | /// 21 | /// For tight control over the error in a catch-all case, use [`fail`][crate::combinator::fail]. 22 | /// 23 | /// # Example 24 | /// 25 | /// ```rust 26 | /// use winnow::prelude::*; 27 | /// use winnow::combinator::dispatch; 28 | /// # use winnow::token::take; 29 | /// # use winnow::token::take_while; 30 | /// # use winnow::combinator::fail; 31 | /// 32 | /// fn integer(input: &mut &str) -> ModalResult { 33 | /// dispatch! {take(2usize); 34 | /// "0b" => take_while(1.., '0'..='1').try_map(|s| u64::from_str_radix(s, 2)), 35 | /// "0o" => take_while(1.., '0'..='7').try_map(|s| u64::from_str_radix(s, 8)), 36 | /// "0d" => take_while(1.., '0'..='9').try_map(|s| u64::from_str_radix(s, 10)), 37 | /// "0x" => take_while(1.., ('0'..='9', 'a'..='f', 'A'..='F')).try_map(|s| u64::from_str_radix(s, 16)), 38 | /// _ => fail::<_, u64, _>, 39 | /// } 40 | /// .parse_next(input) 41 | /// } 42 | /// 43 | /// assert_eq!(integer.parse_peek("0x100 Hello"), Ok((" Hello", 0x100))); 44 | /// ``` 45 | /// 46 | /// ```rust 47 | /// use winnow::prelude::*; 48 | /// use winnow::combinator::dispatch; 49 | /// # use winnow::token::any; 50 | /// # use winnow::combinator::preceded; 51 | /// # use winnow::combinator::empty; 52 | /// # use winnow::combinator::fail; 53 | /// 54 | /// fn escaped(input: &mut &str) -> ModalResult { 55 | /// preceded('\\', escape_seq_char).parse_next(input) 56 | /// } 57 | /// 58 | /// fn escape_seq_char(input: &mut &str) -> ModalResult { 59 | /// dispatch! {any; 60 | /// 'b' => empty.value('\u{8}'), 61 | /// 'f' => empty.value('\u{c}'), 62 | /// 'n' => empty.value('\n'), 63 | /// 'r' => empty.value('\r'), 64 | /// 't' => empty.value('\t'), 65 | /// '\\' => empty.value('\\'), 66 | /// '"' => empty.value('"'), 67 | /// _ => fail::<_, char, _>, 68 | /// } 69 | /// .parse_next(input) 70 | /// } 71 | /// 72 | /// assert_eq!(escaped.parse_peek("\\nHello"), Ok(("Hello", '\n'))); 73 | /// ``` 74 | #[macro_export] 75 | #[doc(hidden)] // forced to be visible in intended location 76 | macro_rules! dispatch { 77 | ( 78 | $scrutinee_parser:expr; 79 | $( $arm_pat:pat $(if $arm_pred:expr)? => $arm_parser: expr ),+ $(,)? 80 | ) => { 81 | $crate::combinator::trace("dispatch", move |i: &mut _| 82 | { 83 | use $crate::Parser; 84 | let initial = $scrutinee_parser.parse_next(i)?; 85 | match initial { 86 | $( 87 | $arm_pat $(if $arm_pred)? => $arm_parser.parse_next(i), 88 | )* 89 | } 90 | }) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/macros/mod.rs: -------------------------------------------------------------------------------- 1 | mod dispatch; 2 | mod seq; 3 | 4 | #[cfg(test)] 5 | macro_rules! assert_parse( 6 | ($left: expr, $right: expr) => { 7 | let res: $crate::error::ModalResult<_, $crate::error::InputError<_>> = $left; 8 | snapbox::assert_data_eq!(snapbox::data::ToDebug::to_debug(&res), $right); 9 | }; 10 | ); 11 | 12 | macro_rules! impl_partial_eq { 13 | ($lhs:ty, $rhs:ty) => { 14 | #[allow(unused_lifetimes)] 15 | impl<'a> PartialEq<$rhs> for $lhs { 16 | #[inline] 17 | fn eq(&self, other: &$rhs) -> bool { 18 | let l = self.as_ref(); 19 | let r: &Self = other.as_ref(); 20 | PartialEq::eq(l, r) 21 | } 22 | } 23 | 24 | #[allow(unused_lifetimes)] 25 | impl<'a> PartialEq<$lhs> for $rhs { 26 | #[inline] 27 | fn eq(&self, other: &$lhs) -> bool { 28 | PartialEq::eq(other, self) 29 | } 30 | } 31 | }; 32 | } 33 | 34 | macro_rules! impl_partial_ord { 35 | ($lhs:ty, $rhs:ty) => { 36 | #[allow(unused_lifetimes)] 37 | impl<'a> PartialOrd<$rhs> for $lhs { 38 | #[inline] 39 | fn partial_cmp(&self, other: &$rhs) -> Option { 40 | let l = self.as_ref(); 41 | let r: &Self = other.as_ref(); 42 | PartialOrd::partial_cmp(l, r) 43 | } 44 | } 45 | 46 | #[allow(unused_lifetimes)] 47 | impl<'a> PartialOrd<$lhs> for $rhs { 48 | #[inline] 49 | fn partial_cmp(&self, other: &$lhs) -> Option { 50 | PartialOrd::partial_cmp(other, self) 51 | } 52 | } 53 | }; 54 | } 55 | 56 | #[cfg(test)] 57 | mod tests; 58 | -------------------------------------------------------------------------------- /src/stream/range.rs: -------------------------------------------------------------------------------- 1 | /// A range bounded inclusively for counting parses performed 2 | /// 3 | /// This is flexible in what can be converted to a [Range]: 4 | /// ```rust 5 | /// # #[cfg(feature = "std")] { 6 | /// # use winnow::prelude::*; 7 | /// # use winnow::token::any; 8 | /// # use winnow::combinator::repeat; 9 | /// # fn inner(input: &mut &str) -> ModalResult { 10 | /// # any.parse_next(input) 11 | /// # } 12 | /// # let mut input = "0123456789012345678901234567890123456789"; 13 | /// # let input = &mut input; 14 | /// let parser: Vec<_> = repeat(5, inner).parse_next(input).unwrap(); 15 | /// # let mut input = "0123456789012345678901234567890123456789"; 16 | /// # let input = &mut input; 17 | /// let parser: Vec<_> = repeat(.., inner).parse_next(input).unwrap(); 18 | /// # let mut input = "0123456789012345678901234567890123456789"; 19 | /// # let input = &mut input; 20 | /// let parser: Vec<_> = repeat(1.., inner).parse_next(input).unwrap(); 21 | /// # let mut input = "0123456789012345678901234567890123456789"; 22 | /// # let input = &mut input; 23 | /// let parser: Vec<_> = repeat(5..8, inner).parse_next(input).unwrap(); 24 | /// # let mut input = "0123456789012345678901234567890123456789"; 25 | /// # let input = &mut input; 26 | /// let parser: Vec<_> = repeat(5..=8, inner).parse_next(input).unwrap(); 27 | /// # } 28 | /// ``` 29 | #[derive(PartialEq, Eq, Copy, Clone)] 30 | pub struct Range { 31 | pub(crate) start_inclusive: usize, 32 | pub(crate) end_inclusive: Option, 33 | } 34 | 35 | impl Range { 36 | #[inline(always)] 37 | fn raw(start_inclusive: usize, end_inclusive: Option) -> Self { 38 | Self { 39 | start_inclusive, 40 | end_inclusive, 41 | } 42 | } 43 | } 44 | 45 | impl crate::lib::std::ops::RangeBounds for Range { 46 | #[inline(always)] 47 | fn start_bound(&self) -> crate::lib::std::ops::Bound<&usize> { 48 | crate::lib::std::ops::Bound::Included(&self.start_inclusive) 49 | } 50 | 51 | #[inline(always)] 52 | fn end_bound(&self) -> crate::lib::std::ops::Bound<&usize> { 53 | if let Some(end_inclusive) = &self.end_inclusive { 54 | crate::lib::std::ops::Bound::Included(end_inclusive) 55 | } else { 56 | crate::lib::std::ops::Bound::Unbounded 57 | } 58 | } 59 | } 60 | 61 | impl From for Range { 62 | #[inline(always)] 63 | fn from(fixed: usize) -> Self { 64 | (fixed..=fixed).into() 65 | } 66 | } 67 | 68 | impl From> for Range { 69 | #[inline(always)] 70 | fn from(range: crate::lib::std::ops::Range) -> Self { 71 | let start_inclusive = range.start; 72 | let end_inclusive = Some(range.end.saturating_sub(1)); 73 | Self::raw(start_inclusive, end_inclusive) 74 | } 75 | } 76 | 77 | impl From for Range { 78 | #[inline(always)] 79 | fn from(_: crate::lib::std::ops::RangeFull) -> Self { 80 | let start_inclusive = 0; 81 | let end_inclusive = None; 82 | Self::raw(start_inclusive, end_inclusive) 83 | } 84 | } 85 | 86 | impl From> for Range { 87 | #[inline(always)] 88 | fn from(range: crate::lib::std::ops::RangeFrom) -> Self { 89 | let start_inclusive = range.start; 90 | let end_inclusive = None; 91 | Self::raw(start_inclusive, end_inclusive) 92 | } 93 | } 94 | 95 | impl From> for Range { 96 | #[inline(always)] 97 | fn from(range: crate::lib::std::ops::RangeTo) -> Self { 98 | let start_inclusive = 0; 99 | let end_inclusive = Some(range.end.saturating_sub(1)); 100 | Self::raw(start_inclusive, end_inclusive) 101 | } 102 | } 103 | 104 | impl From> for Range { 105 | #[inline(always)] 106 | fn from(range: crate::lib::std::ops::RangeInclusive) -> Self { 107 | let start_inclusive = *range.start(); 108 | let end_inclusive = Some(*range.end()); 109 | Self::raw(start_inclusive, end_inclusive) 110 | } 111 | } 112 | 113 | impl From> for Range { 114 | #[inline(always)] 115 | fn from(range: crate::lib::std::ops::RangeToInclusive) -> Self { 116 | let start_inclusive = 0; 117 | let end_inclusive = Some(range.end); 118 | Self::raw(start_inclusive, end_inclusive) 119 | } 120 | } 121 | 122 | impl crate::lib::std::fmt::Display for Range { 123 | fn fmt(&self, f: &mut crate::lib::std::fmt::Formatter<'_>) -> crate::lib::std::fmt::Result { 124 | self.start_inclusive.fmt(f)?; 125 | match self.end_inclusive { 126 | Some(e) if e == self.start_inclusive => {} 127 | Some(e) => { 128 | "..=".fmt(f)?; 129 | e.fmt(f)?; 130 | } 131 | None => { 132 | "..".fmt(f)?; 133 | } 134 | } 135 | Ok(()) 136 | } 137 | } 138 | 139 | impl crate::lib::std::fmt::Debug for Range { 140 | fn fmt(&self, f: &mut crate::lib::std::fmt::Formatter<'_>) -> crate::lib::std::fmt::Result { 141 | write!(f, "{self}") 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /tests/testsuite/custom_errors.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use winnow::ascii::digit1 as digit; 4 | #[cfg(feature = "alloc")] 5 | use winnow::combinator::repeat; 6 | use winnow::combinator::terminated; 7 | use winnow::error::ParserError; 8 | use winnow::prelude::*; 9 | use winnow::stream::Stream; 10 | use winnow::Partial; 11 | 12 | #[derive(Debug)] 13 | pub(crate) struct CustomError(String); 14 | 15 | impl<'a> ParserError> for CustomError { 16 | type Inner = Self; 17 | 18 | fn from_input(_: &Partial<&'a str>) -> Self { 19 | CustomError("error".to_owned()) 20 | } 21 | 22 | fn append(self, _: &Partial<&'a str>, _: & as Stream>::Checkpoint) -> Self { 23 | self 24 | } 25 | 26 | fn into_inner(self) -> Result { 27 | Ok(self) 28 | } 29 | } 30 | 31 | fn test1<'i>(input: &mut Partial<&'i str>) -> ModalResult<&'i str, CustomError> { 32 | //fix_error!(input, CustomError, tag!("abcd")) 33 | "abcd".parse_next(input) 34 | } 35 | 36 | fn test2<'i>(input: &mut Partial<&'i str>) -> ModalResult<&'i str, CustomError> { 37 | //terminated!(input, test1, fix_error!(CustomError, digit)) 38 | terminated(test1, digit).parse_next(input) 39 | } 40 | 41 | fn test3<'i>(input: &mut Partial<&'i str>) -> ModalResult<&'i str, CustomError> { 42 | test1 43 | .verify(|s: &str| s.starts_with("abcd")) 44 | .parse_next(input) 45 | } 46 | 47 | #[cfg(feature = "alloc")] 48 | fn test4<'i>(input: &mut Partial<&'i str>) -> ModalResult, CustomError> { 49 | repeat(4, test1).parse_next(input) 50 | } 51 | -------------------------------------------------------------------------------- /tests/testsuite/float.rs: -------------------------------------------------------------------------------- 1 | use std::str; 2 | use std::str::FromStr; 3 | 4 | use snapbox::prelude::*; 5 | use snapbox::str; 6 | 7 | use winnow::ascii::digit1 as digit; 8 | use winnow::combinator::alt; 9 | use winnow::combinator::delimited; 10 | use winnow::combinator::opt; 11 | use winnow::prelude::*; 12 | 13 | use crate::TestResult; 14 | 15 | fn unsigned_float<'i>(i: &mut &'i [u8]) -> TestResult<&'i [u8], f32> { 16 | let float_bytes = alt(( 17 | delimited(digit, ".", opt(digit)), 18 | delimited(opt(digit), ".", digit), 19 | )) 20 | .take(); 21 | let float_str = float_bytes.try_map(str::from_utf8); 22 | float_str.try_map(FromStr::from_str).parse_next(i) 23 | } 24 | 25 | fn float<'i>(i: &mut &'i [u8]) -> TestResult<&'i [u8], f32> { 26 | (opt(alt(("+", "-"))), unsigned_float) 27 | .map(|(sign, value)| { 28 | sign.and_then(|s| if s[0] == b'-' { Some(-1f32) } else { None }) 29 | .unwrap_or(1f32) 30 | * value 31 | }) 32 | .parse_next(i) 33 | } 34 | 35 | #[test] 36 | fn unsigned_float_test() { 37 | assert_parse!( 38 | unsigned_float.parse_peek(&b"123.456;"[..]), 39 | str![[r#" 40 | Ok( 41 | ( 42 | [ 43 | 59, 44 | ], 45 | 123.456, 46 | ), 47 | ) 48 | 49 | "#]] 50 | .raw() 51 | ); 52 | assert_parse!( 53 | unsigned_float.parse_peek(&b"0.123;"[..]), 54 | str![[r#" 55 | Ok( 56 | ( 57 | [ 58 | 59, 59 | ], 60 | 0.123, 61 | ), 62 | ) 63 | 64 | "#]] 65 | .raw() 66 | ); 67 | assert_parse!( 68 | unsigned_float.parse_peek(&b"123.0;"[..]), 69 | str![[r#" 70 | Ok( 71 | ( 72 | [ 73 | 59, 74 | ], 75 | 123.0, 76 | ), 77 | ) 78 | 79 | "#]] 80 | .raw() 81 | ); 82 | assert_parse!( 83 | unsigned_float.parse_peek(&b"123.;"[..]), 84 | str![[r#" 85 | Ok( 86 | ( 87 | [ 88 | 59, 89 | ], 90 | 123.0, 91 | ), 92 | ) 93 | 94 | "#]] 95 | .raw() 96 | ); 97 | assert_parse!( 98 | unsigned_float.parse_peek(&b".123;"[..]), 99 | str![[r#" 100 | Ok( 101 | ( 102 | [ 103 | 59, 104 | ], 105 | 0.123, 106 | ), 107 | ) 108 | 109 | "#]] 110 | .raw() 111 | ); 112 | } 113 | 114 | #[test] 115 | fn float_test() { 116 | assert_parse!( 117 | float.parse_peek(&b"123.456;"[..]), 118 | str![[r#" 119 | Ok( 120 | ( 121 | [ 122 | 59, 123 | ], 124 | 123.456, 125 | ), 126 | ) 127 | 128 | "#]] 129 | .raw() 130 | ); 131 | assert_parse!( 132 | float.parse_peek(&b"+123.456;"[..]), 133 | str![[r#" 134 | Ok( 135 | ( 136 | [ 137 | 59, 138 | ], 139 | 123.456, 140 | ), 141 | ) 142 | 143 | "#]] 144 | .raw() 145 | ); 146 | assert_parse!( 147 | float.parse_peek(&b"-123.456;"[..]), 148 | str![[r#" 149 | Ok( 150 | ( 151 | [ 152 | 59, 153 | ], 154 | -123.456, 155 | ), 156 | ) 157 | 158 | "#]] 159 | .raw() 160 | ); 161 | } 162 | -------------------------------------------------------------------------------- /tests/testsuite/fnmut.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "alloc")] 2 | 3 | use winnow::combinator::repeat; 4 | use winnow::Parser; 5 | 6 | #[test] 7 | #[cfg(feature = "std")] 8 | fn parse() { 9 | let mut counter = 0; 10 | 11 | let res = { 12 | let mut parser = repeat::<_, _, Vec<_>, (), _>(0.., |i: &mut _| { 13 | counter += 1; 14 | "abc".parse_next(i) 15 | }); 16 | 17 | parser.parse_peek("abcabcabcabc").unwrap() 18 | }; 19 | 20 | println!("res: {res:?}"); 21 | assert_eq!(counter, 5); 22 | } 23 | 24 | #[test] 25 | fn accumulate() { 26 | let mut v = Vec::new(); 27 | 28 | let (_, count) = { 29 | let mut parser = repeat::<_, _, usize, (), _>(0.., |i: &mut _| { 30 | let o = "abc".parse_next(i)?; 31 | v.push(o); 32 | Ok(()) 33 | }); 34 | parser.parse_peek("abcabcabcabc").unwrap() 35 | }; 36 | 37 | println!("v: {v:?}"); 38 | assert_eq!(count, 4); 39 | assert_eq!(v.len(), 4); 40 | } 41 | -------------------------------------------------------------------------------- /tests/testsuite/main.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | macro_rules! assert_parse( 3 | ($left: expr, $right: expr) => { 4 | let res: winnow::error::ModalResult<_, winnow::error::InputError<_>> = $left; 5 | snapbox::assert_data_eq!(snapbox::data::ToDebug::to_debug(&res), $right); 6 | }; 7 | ); 8 | 9 | type TestResult = winnow::ModalResult>; 10 | 11 | automod::dir!("tests/testsuite"); 12 | -------------------------------------------------------------------------------- /tests/testsuite/multiline.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "alloc")] 2 | 3 | use winnow::{ 4 | ascii::{alphanumeric1 as alphanumeric, line_ending as eol}, 5 | combinator::repeat, 6 | combinator::terminated, 7 | prelude::*, 8 | }; 9 | 10 | pub(crate) fn end_of_line<'i>(input: &mut &'i str) -> ModalResult<&'i str> { 11 | if input.is_empty() { 12 | Ok(*input) 13 | } else { 14 | eol.parse_next(input) 15 | } 16 | } 17 | 18 | pub(crate) fn read_line<'i>(input: &mut &'i str) -> ModalResult<&'i str> { 19 | terminated(alphanumeric, end_of_line).parse_next(input) 20 | } 21 | 22 | pub(crate) fn read_lines<'i>(input: &mut &'i str) -> ModalResult> { 23 | repeat(0.., read_line).parse_next(input) 24 | } 25 | 26 | #[cfg(feature = "alloc")] 27 | #[test] 28 | fn read_lines_test() { 29 | let res = Ok(("", vec!["Duck", "Dog", "Cow"])); 30 | 31 | assert_eq!(read_lines.parse_peek("Duck\nDog\nCow\n"), res); 32 | assert_eq!(read_lines.parse_peek("Duck\nDog\nCow"), res); 33 | } 34 | -------------------------------------------------------------------------------- /tests/testsuite/overflow.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::unreadable_literal)] 2 | #![cfg(target_pointer_width = "64")] 3 | 4 | use snapbox::prelude::*; 5 | use snapbox::str; 6 | 7 | #[cfg(feature = "alloc")] 8 | use winnow::binary::be_u64; 9 | #[cfg(feature = "alloc")] 10 | use winnow::binary::length_take; 11 | #[cfg(feature = "alloc")] 12 | use winnow::combinator::repeat; 13 | use winnow::prelude::*; 14 | use winnow::token::take; 15 | use winnow::Partial; 16 | 17 | use crate::TestResult; 18 | 19 | // Parser definition 20 | 21 | #[test] 22 | fn overflow_incomplete_tuple() { 23 | // We request a length that would trigger an overflow if computing consumed + requested 24 | #[allow(clippy::type_complexity)] 25 | fn parser02<'i>( 26 | i: &mut Partial<&'i [u8]>, 27 | ) -> TestResult, (&'i [u8], &'i [u8])> { 28 | (take(1_usize), take(18446744073709551615_usize)).parse_next(i) 29 | } 30 | 31 | assert_parse!( 32 | parser02.parse_peek(Partial::new(&b"3"[..])), 33 | str![[r#" 34 | Err( 35 | Incomplete( 36 | Size( 37 | 18446744073709551615, 38 | ), 39 | ), 40 | ) 41 | 42 | "#]] 43 | .raw() 44 | ); 45 | } 46 | 47 | #[test] 48 | #[cfg(feature = "alloc")] 49 | fn overflow_incomplete_length_bytes() { 50 | fn multi<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 51 | repeat(0.., length_take(be_u64)).parse_next(i) 52 | } 53 | 54 | // Trigger an overflow in length_take 55 | assert_parse!( 56 | multi.parse_peek(Partial::new( 57 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xff"[..] 58 | )), 59 | str![[r#" 60 | Err( 61 | Incomplete( 62 | Size( 63 | 18446744073709551615, 64 | ), 65 | ), 66 | ) 67 | 68 | "#]] 69 | .raw() 70 | ); 71 | } 72 | 73 | #[test] 74 | #[cfg(feature = "alloc")] 75 | fn overflow_incomplete_many0() { 76 | fn multi<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 77 | repeat(0.., length_take(be_u64)).parse_next(i) 78 | } 79 | 80 | // Trigger an overflow in repeat 81 | assert_parse!( 82 | multi.parse_peek(Partial::new( 83 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xef"[..] 84 | )), 85 | str![[r#" 86 | Err( 87 | Incomplete( 88 | Size( 89 | 18446744073709551599, 90 | ), 91 | ), 92 | ) 93 | 94 | "#]] 95 | .raw() 96 | ); 97 | } 98 | 99 | #[test] 100 | #[cfg(feature = "alloc")] 101 | fn overflow_incomplete_many1() { 102 | use winnow::combinator::repeat; 103 | 104 | fn multi<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 105 | repeat(1.., length_take(be_u64)).parse_next(i) 106 | } 107 | 108 | // Trigger an overflow in repeat 109 | assert_parse!( 110 | multi.parse_peek(Partial::new( 111 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xef"[..] 112 | )), 113 | str![[r#" 114 | Err( 115 | Incomplete( 116 | Size( 117 | 18446744073709551599, 118 | ), 119 | ), 120 | ) 121 | 122 | "#]] 123 | .raw() 124 | ); 125 | } 126 | 127 | #[test] 128 | #[cfg(feature = "alloc")] 129 | fn overflow_incomplete_many_till0() { 130 | use winnow::combinator::repeat_till; 131 | 132 | #[allow(clippy::type_complexity)] 133 | fn multi<'i>( 134 | i: &mut Partial<&'i [u8]>, 135 | ) -> TestResult, (Vec<&'i [u8]>, &'i [u8])> { 136 | repeat_till(0.., length_take(be_u64), "abc").parse_next(i) 137 | } 138 | 139 | // Trigger an overflow in repeat_till 140 | assert_parse!( 141 | multi.parse_peek(Partial::new( 142 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xef"[..] 143 | )), 144 | str![[r#" 145 | Err( 146 | Incomplete( 147 | Size( 148 | 18446744073709551599, 149 | ), 150 | ), 151 | ) 152 | 153 | "#]] 154 | .raw() 155 | ); 156 | } 157 | 158 | #[test] 159 | #[cfg(feature = "alloc")] 160 | fn overflow_incomplete_many_m_n() { 161 | use winnow::combinator::repeat; 162 | 163 | fn multi<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 164 | repeat(2..=4, length_take(be_u64)).parse_next(i) 165 | } 166 | 167 | // Trigger an overflow in repeat 168 | assert_parse!( 169 | multi.parse_peek(Partial::new( 170 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xef"[..] 171 | )), 172 | str![[r#" 173 | Err( 174 | Incomplete( 175 | Size( 176 | 18446744073709551599, 177 | ), 178 | ), 179 | ) 180 | 181 | "#]] 182 | .raw() 183 | ); 184 | } 185 | 186 | #[test] 187 | #[cfg(feature = "alloc")] 188 | fn overflow_incomplete_count() { 189 | fn counter<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 190 | repeat(2, length_take(be_u64)).parse_next(i) 191 | } 192 | 193 | assert_parse!( 194 | counter.parse_peek(Partial::new( 195 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xef"[..] 196 | )), 197 | str![[r#" 198 | Err( 199 | Incomplete( 200 | Size( 201 | 18446744073709551599, 202 | ), 203 | ), 204 | ) 205 | 206 | "#]] 207 | .raw() 208 | ); 209 | } 210 | 211 | #[test] 212 | #[cfg(feature = "alloc")] 213 | fn overflow_incomplete_length_repeat() { 214 | use winnow::binary::be_u8; 215 | use winnow::binary::length_repeat; 216 | 217 | fn multi<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 218 | length_repeat(be_u8, length_take(be_u64)).parse_next(i) 219 | } 220 | 221 | assert_parse!( 222 | multi.parse_peek(Partial::new( 223 | &b"\x04\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xee"[..] 224 | )), 225 | str![[r#" 226 | Err( 227 | Incomplete( 228 | Size( 229 | 18446744073709551598, 230 | ), 231 | ), 232 | ) 233 | 234 | "#]] 235 | .raw() 236 | ); 237 | } 238 | 239 | #[test] 240 | #[cfg(feature = "alloc")] 241 | fn overflow_incomplete_length_take() { 242 | fn multi<'i>(i: &mut Partial<&'i [u8]>) -> TestResult, Vec<&'i [u8]>> { 243 | repeat(0.., length_take(be_u64)).parse_next(i) 244 | } 245 | 246 | assert_parse!( 247 | multi.parse_peek(Partial::new( 248 | &b"\x00\x00\x00\x00\x00\x00\x00\x01\xaa\xff\xff\xff\xff\xff\xff\xff\xff"[..] 249 | )), 250 | str![[r#" 251 | Err( 252 | Incomplete( 253 | Size( 254 | 18446744073709551615, 255 | ), 256 | ), 257 | ) 258 | 259 | "#]] 260 | .raw() 261 | ); 262 | } 263 | -------------------------------------------------------------------------------- /tests/testsuite/reborrow_fold.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | // #![allow(unused_variables)] 3 | 4 | use std::str; 5 | 6 | use winnow::combinator::delimited; 7 | use winnow::combinator::repeat; 8 | use winnow::error::InputError; 9 | use winnow::prelude::*; 10 | use winnow::token::take_till; 11 | 12 | use crate::TestResult; 13 | 14 | fn atom<'a>(_tomb: &mut ()) -> impl ModalParser<&'a [u8], String, InputError<&'a [u8]>> { 15 | take_till(1.., [' ', '\t', '\r', '\n']) 16 | .try_map(str::from_utf8) 17 | .map(ToString::to_string) 18 | } 19 | 20 | // FIXME: should we support the use case of borrowing data mutably in a parser? 21 | fn list<'a>(i: &mut &'a [u8], tomb: &mut ()) -> TestResult<&'a [u8], String> { 22 | delimited( 23 | '(', 24 | repeat(0.., atom(tomb)).fold(String::new, |mut acc: String, next: String| { 25 | acc.push_str(next.as_str()); 26 | acc 27 | }), 28 | ')', 29 | ) 30 | .parse_next(i) 31 | } 32 | -------------------------------------------------------------------------------- /third_party/nativejson-benchmark/LINK: -------------------------------------------------------------------------------- 1 | https://github.com/miloyip/nativejson-benchmark.git 2 | --------------------------------------------------------------------------------