├── .github └── workflows │ ├── check.yml │ ├── msrv.yml │ ├── playground.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE-MIT ├── README.md ├── bench.sh ├── examples ├── ack.jq ├── add.jq ├── benches.json ├── bf.jq ├── cumsum-xy.jq ├── cumsum.jq ├── ex-implode.jq ├── fib.bf ├── from.jq ├── group-by.jq ├── hello.bf ├── kv-entries.jq ├── kv-update.jq ├── kv.jq ├── last.jq ├── min-max.jq ├── pyramid.jq ├── range-prop.jq ├── reduce-update.jq ├── reduce.jq ├── repeat.jq ├── reverse.jq ├── sort.jq ├── to-fromjson.jq ├── tree-contains.jq ├── tree-flatten.jq ├── tree-paths.jq ├── tree-update.jq ├── try-catch.jq └── upto.jq ├── jaq-core ├── Cargo.toml ├── fuzz │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── fuzz_targets │ │ ├── data.rs │ │ ├── load_and_compile.rs │ │ └── parse_tokens.rs │ └── init_corpus.sh ├── src │ ├── box_iter.rs │ ├── compile.rs │ ├── exn.rs │ ├── filter.rs │ ├── fold.rs │ ├── into_iter.rs │ ├── lib.rs │ ├── load │ │ ├── arbitrary.rs │ │ ├── lex.rs │ │ ├── mod.rs │ │ ├── parse.rs │ │ ├── prec_climb.rs │ │ └── test.rs │ ├── ops.rs │ ├── path.rs │ ├── rc_iter.rs │ ├── rc_lazy_list.rs │ ├── rc_list.rs │ ├── stack.rs │ └── val.rs └── tests │ ├── common │ └── mod.rs │ ├── path.rs │ └── tests.rs ├── jaq-json ├── Cargo.toml ├── src │ ├── defs.jq │ └── lib.rs └── tests │ ├── common │ └── mod.rs │ ├── defs.rs │ └── funs.rs ├── jaq-play ├── .gitignore ├── Cargo.toml ├── README.md ├── github-mark-white.svg ├── index.html ├── jaq.svg └── src │ ├── lib.rs │ ├── main.js │ ├── style.css │ └── worker.js ├── jaq-std ├── Cargo.toml ├── src │ ├── defs.jq │ ├── lib.rs │ ├── math.rs │ ├── regex.rs │ └── time.rs └── tests │ ├── common │ └── mod.rs │ ├── defs.rs │ └── funs.rs └── jaq ├── Cargo.toml ├── src ├── cli.rs ├── help.txt └── main.rs └── tests ├── a.jq ├── b.jq ├── data.json ├── golden.rs └── mods ├── c.jq └── d.jq /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: Build and check 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | paths: 7 | - '**.rs' 8 | - '**/Cargo.*' 9 | pull_request: 10 | branches: [ main ] 11 | 12 | env: 13 | CARGO_TERM_COLOR: always 14 | 15 | jobs: 16 | check: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - uses: actions/cache/restore@v4 22 | with: 23 | path: | 24 | ~/.cargo/registry/index/ 25 | ~/.cargo/registry/cache/ 26 | key: cargo-${{ hashFiles('**/Cargo.lock') }} 27 | 28 | - name: Build 29 | run: cargo build --verbose 30 | 31 | - name: Check jaq-core without default features 32 | working-directory: jaq-core 33 | run: cargo check --no-default-features 34 | 35 | - name: Check jaq-std without default features 36 | working-directory: jaq-std 37 | run: cargo check --no-default-features 38 | 39 | - name: Check jaq-json without default features 40 | working-directory: jaq-json 41 | run: cargo check --no-default-features 42 | 43 | - name: Check jaq-core fuzzing target compilation 44 | working-directory: jaq-core/fuzz 45 | run: cargo check 46 | 47 | - name: Clippy 48 | run: cargo clippy -- -Dwarnings 49 | -------------------------------------------------------------------------------- /.github/workflows/msrv.yml: -------------------------------------------------------------------------------- 1 | name: Build with MSRV (minimal supported Rust version) 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | paths: 7 | - 'Cargo.lock' 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - uses: actions/cache@v4 16 | with: 17 | path: | 18 | ~/.cargo/registry/index/ 19 | ~/.cargo/registry/cache/ 20 | key: cargo-${{ hashFiles('**/Cargo.lock') }} 21 | restore-keys: cargo- 22 | 23 | - uses: dtolnay/rust-toolchain@1.63 24 | - name: Check jaq-core 25 | working-directory: jaq-core 26 | run: cargo check 27 | 28 | - uses: dtolnay/rust-toolchain@1.65 29 | - name: Check jaq 30 | working-directory: jaq 31 | run: cargo check 32 | - name: Build 33 | run: cargo build --verbose 34 | -------------------------------------------------------------------------------- /.github/workflows/playground.yml: -------------------------------------------------------------------------------- 1 | name: Deploy jaq playground 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 7 | permissions: 8 | contents: read 9 | pages: write 10 | id-token: write 11 | 12 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 13 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 14 | concurrency: 15 | group: "pages" 16 | cancel-in-progress: false 17 | 18 | jobs: 19 | deploy: 20 | environment: 21 | name: github-pages 22 | url: ${{ steps.deployment.outputs.page_url }} 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v4 27 | 28 | - uses: actions/cache/restore@v4 29 | with: 30 | path: | 31 | ~/.cargo/registry/index/ 32 | ~/.cargo/registry/cache/ 33 | key: cargo-${{ hashFiles('**/Cargo.lock') }} 34 | 35 | - name: Install wasm-pack 36 | shell: bash 37 | run: | 38 | VERSION="0.12.1" 39 | DIR=wasm-pack-v${VERSION}-x86_64-unknown-linux-musl 40 | wget https://github.com/rustwasm/wasm-pack/releases/download/v${VERSION}/${DIR}.tar.gz 41 | tar xzf ${DIR}.tar.gz 42 | mv ${DIR}/wasm-pack ~/.cargo/bin 43 | 44 | - name: Compile 45 | run: wasm-pack build --target web --no-typescript --no-pack --release 46 | working-directory: jaq-play 47 | 48 | - name: Upload artifact 49 | uses: actions/upload-pages-artifact@v3 50 | with: 51 | path: 'jaq-play' 52 | 53 | - name: Deploy to GitHub Pages 54 | id: deployment 55 | uses: actions/deploy-pages@v4 56 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # Inspiration: 2 | # 3 | # https://github.com/sharkdp/hyperfine/blob/d449ebd7c18246b7c3f6ee19a97a4cf24e34e106/.github/workflows/CICD.yml 4 | # https://github.com/BurntSushi/ripgrep/blob/61733f6378b62fa2dc2e7f3eff2f2e7182069ca9/.github/workflows/release.yml 5 | # https://github.com/XAMPPRocky/tokei/blob/ae77e1945631fd9457f7d455f2f0f2f889356f58/.github/workflows/mean_bean_deploy.yml 6 | 7 | name: Release 8 | on: 9 | workflow_dispatch: 10 | push: 11 | tags: 12 | - "v[0-9]*" 13 | 14 | defaults: 15 | run: 16 | shell: bash 17 | 18 | env: 19 | NAME: ${{ github.event.repository.name }} 20 | VERSION: ${{ github.ref_name }} 21 | 22 | # for gh release upload 23 | permissions: 24 | contents: write 25 | 26 | jobs: 27 | create-release: 28 | name: create-release 29 | runs-on: ubuntu-latest 30 | steps: 31 | - uses: actions/checkout@v4 32 | - name: Create GitHub release 33 | env: 34 | GH_TOKEN: ${{ github.token }} 35 | run: gh release create --draft ${VERSION} 36 | 37 | build: 38 | name: ${{ matrix.target }} (${{ matrix.os }}) 39 | runs-on: ${{ matrix.os }} 40 | needs: create-release 41 | strategy: 42 | fail-fast: false 43 | matrix: 44 | include: 45 | - { target: aarch64-unknown-linux-gnu , os: ubuntu-22.04 } 46 | - { target: armv7-unknown-linux-gnueabihf, os: ubuntu-22.04 } 47 | - { target: arm-unknown-linux-gnueabi , os: ubuntu-22.04 } 48 | - { target: arm-unknown-linux-musleabihf , os: ubuntu-22.04 } 49 | - { target: i686-pc-windows-msvc , os: windows-2019 } 50 | - { target: i686-unknown-linux-gnu , os: ubuntu-22.04 } 51 | - { target: i686-unknown-linux-musl , os: ubuntu-22.04 } 52 | - { target: x86_64-apple-darwin , os: macos-13 } 53 | - { target: aarch64-apple-darwin , os: macos-13 } 54 | # fails in mimalloc, possibly mingw related, see: 55 | # https://github.com/purpleprotocol/mimalloc_rust/issues/125 56 | # - { target: x86_64-pc-windows-gnu , os: windows-2019 } 57 | - { target: x86_64-pc-windows-msvc , os: windows-2019 } 58 | - { target: x86_64-unknown-linux-gnu , os: ubuntu-22.04 } 59 | - { target: x86_64-unknown-linux-musl , os: ubuntu-22.04 } 60 | steps: 61 | - uses: actions/checkout@v4 62 | 63 | - name: Restore cached Cargo registry 64 | if: ${{ !contains(matrix.target, '-pc-windows-') }} 65 | uses: actions/cache/restore@v4 66 | with: 67 | path: | 68 | ~/.cargo/registry/index/ 69 | ~/.cargo/registry/cache/ 70 | key: cargo-${{ hashFiles('**/Cargo.lock') }} 71 | 72 | - name: Install Rust toolchain 73 | uses: actions-rust-lang/setup-rust-toolchain@v1 74 | with: 75 | target: ${{ matrix.target }} 76 | 77 | - name: Install cross-compilation tools 78 | uses: taiki-e/setup-cross-toolchain-action@v1 79 | with: 80 | target: ${{ matrix.target }} 81 | 82 | - name: Build 83 | run: cargo build --release 84 | 85 | - name: Determine paths 86 | id: paths 87 | run: | 88 | EXE_suffix="" ; case ${{ matrix.target }} in *-pc-windows-*) EXE_suffix=".exe" ;; esac 89 | BIN_PATH="target/${{ matrix.target }}/release/${NAME}${EXE_suffix}" 90 | PKG_NAME=${NAME}-${{ matrix.target }}${EXE_suffix} 91 | cp ${BIN_PATH} ${PKG_NAME} 92 | echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_OUTPUT 93 | 94 | # - name: Compress binary 95 | # if: startsWith( matrix.os, 'ubuntu' ) 96 | # run: upx ${{ steps.paths.outputs.PKG_NAME }} 97 | 98 | - name: Upload release archive 99 | env: 100 | GH_TOKEN: ${{ github.token }} 101 | run: gh release upload ${VERSION} ${{ steps.paths.outputs.PKG_NAME }} 102 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | paths: 7 | - '**.rs' 8 | - '**/Cargo.*' 9 | pull_request: 10 | branches: [ main ] 11 | 12 | env: 13 | CARGO_TERM_COLOR: always 14 | 15 | jobs: 16 | build: 17 | name: build ${{ matrix.target }} 18 | runs-on: ubuntu-latest 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | include: 23 | - { target: i686-unknown-linux-gnu } 24 | - { target: x86_64-unknown-linux-gnu } 25 | steps: 26 | - uses: actions/checkout@v4 27 | 28 | - uses: actions/cache/restore@v4 29 | with: 30 | path: | 31 | ~/.cargo/registry/index/ 32 | ~/.cargo/registry/cache/ 33 | key: cargo-${{ hashFiles('**/Cargo.lock') }} 34 | 35 | - name: Install Rust toolchain 36 | uses: actions-rust-lang/setup-rust-toolchain@v1 37 | with: 38 | target: ${{ matrix.target }} 39 | 40 | - name: Install cross-compilation tools 41 | if: matrix.target != 'x86_64-unknown-linux-gnu' 42 | uses: taiki-e/setup-cross-toolchain-action@v1 43 | with: 44 | target: ${{ matrix.target }} 45 | 46 | - name: Run tests 47 | run: cargo test --verbose 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "jaq-core", 4 | "jaq-std", 5 | "jaq-json", 6 | "jaq", 7 | "jaq-play", 8 | ] 9 | 10 | resolver = "2" 11 | 12 | [profile.release] 13 | strip = true 14 | codegen-units = 1 15 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any 2 | person obtaining a copy of this software and associated 3 | documentation files (the "Software"), to deal in the 4 | Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, 6 | publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software 8 | is furnished to do so, subject to the following 9 | conditions: 10 | 11 | The above copyright notice and this permission notice 12 | shall be included in all copies or substantial portions 13 | of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Benchmark jq implementations passed as arguments 4 | # 5 | # Example usage: 6 | # 7 | # ./bench.sh target/release/jaq jq 8 | 9 | TIME='timeout 10 /usr/bin/time -f %U' 10 | 11 | echo -n '{"name": "empty", "n": 512, "time": {' 12 | for j in $@; do 13 | t=$($TIME bash -c "for n in {1..512}; do $j -n 'empty'; done" 2>&1) 14 | [ $j != $1 ] && echo -n ', ' 15 | echo -n '"'$j'": ['$t']' 16 | done 17 | echo '}}' 18 | 19 | echo -n '{"name": "bf-fib", "n": 13, "time": {' 20 | for j in $@; do 21 | t=$($TIME $j -sRrf examples/bf.jq examples/fib.bf 2>&1 > /dev/null) 22 | [ $j != $1 ] && echo -n ', ' 23 | echo -n '"'$j'": ['$t']' 24 | done 25 | echo '}}' 26 | 27 | echo -n '{"name": "defs", "n": 100000, "time": {' 28 | for j in $@; do 29 | t=$($TIME $j -n -f <(for i in `seq 100000`; do echo "def a: 0;"; done; echo empty) 2>&1 > /dev/null) 30 | [ $? != 0 ] && t="" # on error 31 | [ $j != $1 ] && echo -n ', ' 32 | echo -n '"'$j'": ['$t']' 33 | done 34 | echo '}}' 35 | 36 | while read -r line; do 37 | b=`echo $line | $1 -r .name` 38 | n=`echo $line | $1 .n` 39 | echo -n '{"name": "'$b'", "n": '$n', "time": {' 40 | for j in $@; do 41 | [ $j != $1 ] && echo -n ', ' 42 | echo -n '"'$j'": [' 43 | for i in `seq 3`; do 44 | t=$(echo $n | $TIME $j "$(cat examples/$b.jq) | length" 2>&1 > /dev/null) 45 | [ -z "$t" ] && break # terminate on timeout 46 | [ $i -ne 1 ] && echo -n ', ' 47 | echo -n $t 48 | done 49 | echo -n ']' 50 | done 51 | echo '}}' 52 | done 11 | # Adapted for jaq by Michael Färber 12 | 13 | def skip_loop: last(recurse( 14 | .input[.cursor:.cursor+1] as $c | 15 | .cursor += 1 | 16 | if $c == "[" then .depth += 1 17 | elif $c == "]" then .depth -= 1 | select(.saved_depth <= .depth) 18 | elif $c == "" then error("unmatching loop") 19 | else . 20 | end)) | .cursor += 1 | .depth -= 1; 21 | 22 | def backward_loop: last(recurse( 23 | .input[.cursor:.cursor+1] as $c | 24 | .cursor -= 1 | 25 | if $c == "[" then .depth -= 1 | select(.saved_depth < .depth) 26 | elif $c == "]" then .depth += 1 27 | elif .cursor < 0 then error("unmatching loop") 28 | else . 29 | end)) | .depth -= 1; 30 | 31 | # Given an array, assure that it has at least length i+1 by filling it up with `null`, 32 | # then update the i-th position with f 33 | def assign(i; f): 34 | if i < length then .[i] |= f 35 | # this yields different output in jq due to its implementation of `limit`, 36 | # where `limit(0; f)` yields the first element of `f` (instead of no elements) 37 | else . + [limit(i - length; repeat(null)), (null | f)] end; 38 | 39 | { input: ., cursor: 0, memory: [], pointer: 0, depth: 0, output: [] } | 40 | until( 41 | .cursor >= (.input | length); 42 | .input[.cursor:.cursor+1] as $c | 43 | .cursor += 1 | 44 | if $c == ">" then .pointer += 1 45 | elif $c == "<" then .pointer -= 1 | if .pointer < 0 then error("negative pointer") else . end 46 | elif $c == "+" then .pointer as $p | .memory |= assign($p; (. + 1) % 256) 47 | elif $c == "-" then .pointer as $p | .memory |= assign($p; (. + 255) % 256) 48 | elif $c == "." then .output += [.memory[.pointer]] 49 | elif $c == "," then error(", is not implemented") 50 | elif $c == "[" then .depth += 1 | if .memory[.pointer] > 0 then . else .saved_depth = .depth | skip_loop end 51 | elif $c == "]" then .depth -= 1 | .cursor -= 1 | .saved_depth = .depth | backward_loop 52 | else . 53 | end 54 | ) | .output | implode 55 | -------------------------------------------------------------------------------- /examples/cumsum-xy.jq: -------------------------------------------------------------------------------- 1 | [foreach range(.) as $x (0; . + $x; $x, .)] 2 | -------------------------------------------------------------------------------- /examples/cumsum.jq: -------------------------------------------------------------------------------- 1 | [foreach range(.) as $x (0; . + $x)] 2 | -------------------------------------------------------------------------------- /examples/ex-implode.jq: -------------------------------------------------------------------------------- 1 | [limit(.; repeat("a"))] | add | explode | implode 2 | -------------------------------------------------------------------------------- /examples/fib.bf: -------------------------------------------------------------------------------- 1 | >>+++++++++++[-<<++++>+++>>+<]>>+<++<<->>[>>>++++++++++<<[->+>-[>+>>]>[+[-<+>]>+ 2 | >>]<<<<<<]>>[-]>>>++++++++++<[->-[>+>>]>[+[-<+>]>+>>]<<<<<]>[-]>>[>++++++[-<++++ 3 | ++++>]<.<<+>+>[-]]<[<[->-<]++++++[->++++++++<]>.[-]]<<++++++[-<++++++++>]<.[-]<< 4 | <<[->+<]>-[<<<<.>.>>>[-<+>]]<<[->>+<<]>>>[-<+<<+>>>]<<] 5 | -------------------------------------------------------------------------------- /examples/from.jq: -------------------------------------------------------------------------------- 1 | [limit(.; 0 | recurse(.+1))] 2 | -------------------------------------------------------------------------------- /examples/group-by.jq: -------------------------------------------------------------------------------- 1 | [range(0; .)] | group_by(. % 2) 2 | -------------------------------------------------------------------------------- /examples/hello.bf: -------------------------------------------------------------------------------- 1 | +++++++++[>++++++++>+++++++++++>+++++<<<-]>.>++.+++++++..+++.>-.------------.<++++++++.--------.+++.------.--------.>+. 2 | -------------------------------------------------------------------------------- /examples/kv-entries.jq: -------------------------------------------------------------------------------- 1 | [range(.) | {(tostring): .}] | add | with_entries(.value += 1) 2 | -------------------------------------------------------------------------------- /examples/kv-update.jq: -------------------------------------------------------------------------------- 1 | [range(.) | {(tostring): .}] | add | .[] += 1 2 | -------------------------------------------------------------------------------- /examples/kv.jq: -------------------------------------------------------------------------------- 1 | [range(.) | {(tostring): .}] | add 2 | -------------------------------------------------------------------------------- /examples/last.jq: -------------------------------------------------------------------------------- 1 | last(range(.)) -------------------------------------------------------------------------------- /examples/min-max.jq: -------------------------------------------------------------------------------- 1 | [range(.)] | min, max 2 | -------------------------------------------------------------------------------- /examples/pyramid.jq: -------------------------------------------------------------------------------- 1 | def pyramid($max): def rec: if . < $max then ., (.+1 | rec), . end; rec; . as $max | 0 | [pyramid($max)] | length 2 | -------------------------------------------------------------------------------- /examples/range-prop.jq: -------------------------------------------------------------------------------- 1 | # verifies that for a range of inputs, 2 | # range($from; $upto; $by) yields 3 | # max(0, ⌈($upto - $from) / $by⌉) outputs 4 | # (if $by is not 0 and $upto, $from, and $by are integer) 5 | [ 6 | { from: 1, upto: range(-.; .), by: range(-.; .) | select(. != 0) } | 7 | ([range(.from; .upto; .by)] | length) == 8 | ([(.upto - .from) / .by | ceil, 0] | max) 9 | ] 10 | -------------------------------------------------------------------------------- /examples/reduce-update.jq: -------------------------------------------------------------------------------- 1 | reduce range(.) as $x ([[]]; .[0] += [$x]) 2 | -------------------------------------------------------------------------------- /examples/reduce.jq: -------------------------------------------------------------------------------- 1 | reduce range(.) as $x ([]; . + [$x + .[-1]]) 2 | -------------------------------------------------------------------------------- /examples/repeat.jq: -------------------------------------------------------------------------------- 1 | [limit(.; repeat(1))] 2 | -------------------------------------------------------------------------------- /examples/reverse.jq: -------------------------------------------------------------------------------- 1 | [range(.)] | reverse 2 | -------------------------------------------------------------------------------- /examples/sort.jq: -------------------------------------------------------------------------------- 1 | [range(.) | -.] | sort 2 | -------------------------------------------------------------------------------- /examples/to-fromjson.jq: -------------------------------------------------------------------------------- 1 | [range(.) | tojson] | join(",") | "[" + . + "]" | fromjson 2 | -------------------------------------------------------------------------------- /examples/tree-contains.jq: -------------------------------------------------------------------------------- 1 | nth(.; 0 | recurse([., .])) | [contains(.)] 2 | -------------------------------------------------------------------------------- /examples/tree-flatten.jq: -------------------------------------------------------------------------------- 1 | nth(.; 0 | recurse([., .])) | flatten 2 | -------------------------------------------------------------------------------- /examples/tree-paths.jq: -------------------------------------------------------------------------------- 1 | nth(.; 0 | recurse([., .])) | [paths] 2 | -------------------------------------------------------------------------------- /examples/tree-update.jq: -------------------------------------------------------------------------------- 1 | nth(.; 0 | recurse([., .])) | (.. | scalars) |= .+1 2 | -------------------------------------------------------------------------------- /examples/try-catch.jq: -------------------------------------------------------------------------------- 1 | [range(.) | try error catch .] 2 | -------------------------------------------------------------------------------- /examples/upto.jq: -------------------------------------------------------------------------------- 1 | def upto($max): if . < $max then ., (.+1 | upto($max)) end; . as $max | 0 | [upto($max)] -------------------------------------------------------------------------------- /jaq-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jaq-core" 3 | version = "2.2.0" 4 | authors = ["Michael Färber "] 5 | edition = "2021" 6 | license = "MIT" 7 | readme = "../README.md" 8 | description = "Interpreter for the jaq language" 9 | repository = "https://github.com/01mf02/jaq" 10 | keywords = ["json", "query", "jq"] 11 | categories = ["parser-implementations", "compilers"] 12 | rust-version = "1.63" 13 | 14 | [features] 15 | default = ["std"] 16 | std = [] 17 | 18 | [dependencies] 19 | arbitrary = { version = "1.4", optional = true } 20 | dyn-clone = "1.0" 21 | once_cell = "1.16.0" 22 | typed-arena = "2.0.2" 23 | 24 | [dev-dependencies] 25 | jaq-std = { path = "../jaq-std" } 26 | jaq-json = { path = "../jaq-json", features = ["serde_json"] } 27 | serde_json = "1.0" 28 | -------------------------------------------------------------------------------- /jaq-core/fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage 5 | Cargo.lock 6 | -------------------------------------------------------------------------------- /jaq-core/fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jaq-core-fuzz" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | 7 | [package.metadata] 8 | cargo-fuzz = true 9 | 10 | [dependencies] 11 | libfuzzer-sys = "0.4" 12 | 13 | jaq-core = { path = "..", features = ["arbitrary"] } 14 | jaq-std = { path = "../../jaq-std" } 15 | jaq-json = { path = "../../jaq-json" } 16 | 17 | # Prevent this from interfering with workspaces 18 | [workspace] 19 | members = ["."] 20 | 21 | [profile.release] 22 | debug = 1 23 | 24 | [[bin]] 25 | name = "load_and_compile" 26 | path = "fuzz_targets/load_and_compile.rs" 27 | test = false 28 | doc = false 29 | 30 | [[bin]] 31 | name = "data" 32 | path = "fuzz_targets/data.rs" 33 | test = false 34 | doc = false 35 | 36 | [[bin]] 37 | name = "parse_tokens" 38 | path = "fuzz_targets/parse_tokens.rs" 39 | test = false 40 | doc = false 41 | -------------------------------------------------------------------------------- /jaq-core/fuzz/README.md: -------------------------------------------------------------------------------- 1 | # Fuzz targets for jaq 2 | 3 | This directory contains fuzz targets laid out for easy use with cargo-fuzz. 4 | These fuzz targets fit the libFuzzer harness interface and 5 | can therefore be utilized by a variety of fuzzing engines if desired. 6 | 7 | ## Quick start to fuzzing 8 | 9 | 1. `rustup toolchain install nightly` 10 | 2. `cargo +nightly install cargo-fuzz` 11 | 3. `cargo +nightly fuzz run load_and_compile` 12 | 13 | (Instead of `cargo +nightly`, you can also use `RUSTC_BOOTSTRAP=1 cargo`. 14 | That way, you also do not have to install a nightly toolchain.) 15 | 16 | Congratulations, you are now fuzzing jaq's core with libFuzzer! 17 | 18 | This fuzzing process will keep going until it 19 | hits a bug, runs out of memory, or the process is terminated. 20 | The challenge is now to figure out ways to 21 | get to bugs quicker and define more buggy conditions for the fuzzer to find. 22 | In the `corpus` directory you will find 23 | interesting inputs the fuzzer has generated, and saved as useful, 24 | because they reach a new part of the program. 25 | You can add your own inputs here (e.g. valid jaq programs) as a starting corpus 26 | to bootstrap your fuzzing efforts deeper in jaq's core. 27 | 28 | 29 | ## Targets 30 | 31 | ### load_and_compile 32 | 33 | This target loads and compiles fuzzer generated jaq programs, 34 | but does not execute the programs. 35 | The target is interested in 36 | whether the program compiles, not whether it is correct when it runs. 37 | The target becomes slower as it gets deeper into jaq. 38 | This has been productive for finding shallower bugs. 39 | 40 | To get an interesting initial input corpus, run: 41 | 42 | jaq-core/fuzz/init_corpus.sh 43 | 44 | ### parse_tokens 45 | 46 | This target bypasses the lexer by directly creating 47 | tokens to be parsed as terms or definitions. 48 | Skipping the lexer makes this target relatively fast. 49 | -------------------------------------------------------------------------------- /jaq-core/fuzz/fuzz_targets/data.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use jaq_core::load::{Arena, File, Loader}; 4 | use jaq_core::{Compiler, Ctx, Native, RcIter}; 5 | 6 | use libfuzzer_sys::fuzz_target; 7 | 8 | fuzz_target!(|data: String| { 9 | let program = File { 10 | code: ".[]", 11 | path: (), 12 | }; 13 | 14 | let loader = Loader::new([]); 15 | let arena = Arena::default(); 16 | 17 | let modules = loader.load(&arena, program).unwrap(); 18 | 19 | let filter = Compiler::<_, Native<_>>::default() 20 | .compile(modules) 21 | .unwrap(); 22 | 23 | let inputs = RcIter::new(core::iter::empty()); 24 | let _ = filter.run((Ctx::new([], &inputs), jaq_json::Val::from(data))); 25 | }); 26 | -------------------------------------------------------------------------------- /jaq-core/fuzz/fuzz_targets/load_and_compile.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use jaq_core::load::{Arena, File, Loader}; 4 | use jaq_core::Compiler; 5 | 6 | use libfuzzer_sys::fuzz_target; 7 | 8 | fuzz_target!(|code: &str| { 9 | if code.contains("\"") || code.contains("main") { 10 | return; 11 | } 12 | 13 | let program = File { code, path: () }; 14 | let loader = Loader::new(jaq_std::defs()); 15 | let arena = Arena::default(); 16 | 17 | if let Ok(modules) = loader.load(&arena, program) { 18 | let _ = Compiler::default() 19 | .with_funs(jaq_std::funs::()) 20 | .compile(modules); 21 | } 22 | }); 23 | -------------------------------------------------------------------------------- /jaq-core/fuzz/fuzz_targets/parse_tokens.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | libfuzzer_sys::fuzz_target!(|tokens: Vec>| { 4 | //println!("{tokens:?}"); 5 | let parser = || jaq_core::load::Parser::new(tokens.as_slice()); 6 | let _ = parser().parse(|p| p.defs()); 7 | let _ = parser().parse(|p| p.term()); 8 | }); 9 | -------------------------------------------------------------------------------- /jaq-core/fuzz/init_corpus.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Generate initial input corpus for fuzzing. 3 | 4 | if [ ! -d "jaq-core" ]; then 5 | echo Error: run this script from the main jaq directory, such as: 6 | echo jaq-core/fuzz/init_corpus.sh 7 | exit 1 8 | fi 9 | 10 | cargo test -- --nocapture 2> programs 11 | sed -i '/^ /d' programs 12 | 13 | CORPUS=jaq-core/fuzz/corpus/load_and_compile 14 | mkdir -p $CORPUS 15 | cp examples/*.jq $CORPUS/ 16 | 17 | i=0 18 | while IFS= read -r line 19 | do 20 | echo "$line" > $CORPUS/test$i 21 | i=$((i+1)) 22 | done < programs 23 | 24 | -------------------------------------------------------------------------------- /jaq-core/src/box_iter.rs: -------------------------------------------------------------------------------- 1 | //! Boxed iterators. 2 | 3 | use alloc::boxed::Box; 4 | 5 | /// A boxed iterator. 6 | pub type BoxIter<'a, T> = Box + 'a>; 7 | 8 | /// A boxed iterator over `Result`s. 9 | pub type Results<'a, T, E> = BoxIter<'a, Result>; 10 | 11 | /// Return a boxed iterator that yields a single element. 12 | pub fn box_once<'a, T: 'a>(x: T) -> BoxIter<'a, T> { 13 | Box::new(core::iter::once(x)) 14 | } 15 | 16 | /// If `x` is an `Err`, return it as iterator, else apply `f` to `x` and return its output. 17 | pub fn then<'a, T, U: 'a, E: 'a>( 18 | x: Result, 19 | f: impl FnOnce(T) -> Results<'a, U, E>, 20 | ) -> Results<'a, U, E> { 21 | x.map_or_else(|e| box_once(Err(e)), f) 22 | } 23 | 24 | /// Return next element if iterator returns at most one element, else `None`. 25 | /// 26 | /// This is one of the most important functions for performance in jaq. 27 | /// It enables optimisations for the case when a filter yields exactly one output, 28 | /// which is very common in typical jq programs. 29 | /// 30 | /// For example, for the filter `f | g`, this function is called on the outputs of `f`. 31 | /// If `f` yields a single output `y`, then the output of `f | g` is `y` applied to `g`. 32 | /// This has two beneficial consequences: 33 | /// 34 | /// 1. We can estimate the number of outputs of `f | g` by estimating the outputs of `g`. 35 | /// (In general, we cannot do this even when we know the number of outputs of `f`, 36 | /// because `g` might yield a different number of elements for each output of `f`.) 37 | /// 2. We do not need to clone the context when passing it to `g`, 38 | /// because we know that `g` is only called once. 39 | /// 40 | /// This optimisation applies to many other filters as well. 41 | /// 42 | /// To see the impact of this function, you can replace its implementation with just `None`. 43 | /// This preserves correctness, but can result in severely degraded performance. 44 | fn next_if_one(iter: &mut impl Iterator) -> Option { 45 | if iter.size_hint().1 == Some(1) { 46 | let ly = iter.next()?; 47 | // the Rust documentation states that 48 | // "a buggy iterator may yield [..] more than the upper bound of elements", 49 | // but so far, it seems that all iterators here are not buggy :) 50 | debug_assert!(iter.next().is_none()); 51 | return Some(ly); 52 | } 53 | None 54 | } 55 | 56 | /// For every element `y` returned by `l`, return the output of `r(y, x)`. 57 | /// 58 | /// In case that `l` returns only a single element, this does not clone `x`. 59 | pub fn map_with<'a, T: Clone + 'a, U: 'a, V: 'a>( 60 | mut l: impl Iterator + 'a, 61 | x: T, 62 | r: impl Fn(U, T) -> V + 'a, 63 | ) -> BoxIter<'a, V> { 64 | match next_if_one(&mut l) { 65 | Some(ly) => box_once(r(ly, x)), 66 | None => Box::new(l.map(move |ly| r(ly, x.clone()))), 67 | } 68 | } 69 | 70 | /// For every element `y` returned by `l`, return the outputs of `r(y, x)`. 71 | /// 72 | /// In case that `l` returns only a single element, this does not clone `x`. 73 | pub fn flat_map_with<'a, T: Clone + 'a, U: 'a, V: 'a>( 74 | mut l: impl Iterator + 'a, 75 | x: T, 76 | r: impl Fn(U, T) -> BoxIter<'a, V> + 'a, 77 | ) -> BoxIter<'a, V> { 78 | match next_if_one(&mut l) { 79 | Some(ly) => Box::new(r(ly, x)), 80 | None => Box::new(l.flat_map(move |ly| r(ly, x.clone()))), 81 | } 82 | } 83 | 84 | /// Combination of [`Iterator::flat_map`] and [`then`]. 85 | pub fn flat_map_then<'a, T: 'a, U: 'a, E: 'a>( 86 | mut l: impl Iterator> + 'a, 87 | r: impl Fn(T) -> Results<'a, U, E> + 'a, 88 | ) -> Results<'a, U, E> { 89 | match next_if_one(&mut l) { 90 | Some(ly) => then(ly, r), 91 | None => Box::new(l.flat_map(move |y| then(y, |y| r(y)))), 92 | } 93 | } 94 | 95 | /// Combination of [`flat_map_with`] and [`then`]. 96 | pub fn flat_map_then_with<'a, T: Clone + 'a, U: 'a, V: 'a, E: 'a>( 97 | l: impl Iterator> + 'a, 98 | x: T, 99 | r: impl Fn(U, T) -> Results<'a, V, E> + 'a, 100 | ) -> Results<'a, V, E> { 101 | flat_map_with(l, x, move |y, x| then(y, |y| r(y, x))) 102 | } 103 | -------------------------------------------------------------------------------- /jaq-core/src/exn.rs: -------------------------------------------------------------------------------- 1 | //! Exceptions and errors. 2 | 3 | use alloc::{string::String, string::ToString, vec::Vec}; 4 | use core::fmt::{self, Display}; 5 | 6 | /// Exception. 7 | /// 8 | /// This is either an error or control flow data internal to jaq. 9 | /// Users should only be able to observe errors. 10 | #[derive(Clone, Debug)] 11 | pub struct Exn<'a, V>(pub(crate) Inner<'a, V>); 12 | 13 | #[derive(Clone, Debug)] 14 | pub(crate) enum Inner<'a, V> { 15 | Err(Error), 16 | /// Tail-recursive call. 17 | /// 18 | /// This is used internally to execute tail-recursive filters. 19 | /// If this can be observed by users, then this is a bug. 20 | TailCall(&'a crate::compile::TermId, crate::filter::Vars<'a, V>, V), 21 | Break(usize), 22 | } 23 | 24 | impl Exn<'_, V> { 25 | /// If the exception is an error, yield it, else yield the exception. 26 | pub(crate) fn get_err(self) -> Result, Self> { 27 | match self.0 { 28 | Inner::Err(e) => Ok(e), 29 | _ => Err(self), 30 | } 31 | } 32 | } 33 | 34 | impl From> for Exn<'_, V> { 35 | fn from(e: Error) -> Self { 36 | Exn(Inner::Err(e)) 37 | } 38 | } 39 | 40 | #[derive(Clone, Debug, PartialEq, Eq)] 41 | enum Part { 42 | Val(V), 43 | Str(S), 44 | } 45 | 46 | /// Error that occurred during filter execution. 47 | #[derive(Clone, Debug, PartialEq, Eq)] 48 | pub struct Error(Part>>); 49 | 50 | impl Error { 51 | /// Create a new error from a value. 52 | pub fn new(v: V) -> Self { 53 | Self(Part::Val(v)) 54 | } 55 | 56 | /// Create a path expression error. 57 | pub fn path_expr() -> Self { 58 | Self(Part::Str(Vec::from([Part::Str("invalid path expression")]))) 59 | } 60 | 61 | /// Create a type error. 62 | pub fn typ(v: V, typ: &'static str) -> Self { 63 | use Part::{Str, Val}; 64 | [Str("cannot use "), Val(v), Str(" as "), Str(typ)] 65 | .into_iter() 66 | .collect() 67 | } 68 | 69 | /// Create a math error. 70 | pub fn math(l: V, op: crate::ops::Math, r: V) -> Self { 71 | use Part::{Str, Val}; 72 | [ 73 | Str("cannot calculate "), 74 | Val(l), 75 | Str(" "), 76 | Str(op.as_str()), 77 | Str(" "), 78 | Val(r), 79 | ] 80 | .into_iter() 81 | .collect() 82 | } 83 | 84 | /// Create an indexing error. 85 | pub fn index(l: V, r: V) -> Self { 86 | use Part::{Str, Val}; 87 | [Str("cannot index "), Val(l), Str(" with "), Val(r)] 88 | .into_iter() 89 | .collect() 90 | } 91 | } 92 | 93 | impl> Error { 94 | /// Build an error from something that can be converted to a string. 95 | pub fn str(s: impl ToString) -> Self { 96 | Self(Part::Val(V::from(s.to_string()))) 97 | } 98 | } 99 | 100 | impl FromIterator> for Error { 101 | fn from_iter>>(iter: T) -> Self { 102 | Self(Part::Str(iter.into_iter().collect())) 103 | } 104 | } 105 | 106 | impl + Display> Error { 107 | /// Convert the error into a value to be used by `catch` filters. 108 | pub fn into_val(self) -> V { 109 | if let Part::Val(v) = self.0 { 110 | v 111 | } else { 112 | V::from(self.to_string()) 113 | } 114 | } 115 | } 116 | 117 | impl Display for Error { 118 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 119 | match &self.0 { 120 | Part::Val(v) => v.fmt(f), 121 | Part::Str(parts) => parts.iter().try_for_each(|part| match part { 122 | Part::Val(v) => v.fmt(f), 123 | Part::Str(s) => s.fmt(f), 124 | }), 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /jaq-core/src/fold.rs: -------------------------------------------------------------------------------- 1 | //! Functions on iterators over results. 2 | 3 | use crate::box_iter::Results; 4 | use alloc::vec::Vec; 5 | 6 | enum Fold<'a, X, Y, E> { 7 | /// things to be processed 8 | Input(Y), 9 | /// things to be output, then to be input 10 | Output(X, Results<'a, Y, E>), 11 | } 12 | 13 | pub(crate) fn fold<'a, T: 'a, TC: Clone + 'a, U: 'a, UC: 'a, E: 'a>( 14 | xs: impl Iterator> + Clone + 'a, 15 | init: U, 16 | f: impl Fn(T, U) -> Results<'a, U, E> + 'a, 17 | tc: impl Fn(&T) -> TC + 'a, 18 | inner: impl Fn(TC, &U) -> Option + 'a, 19 | outer: impl Fn(U) -> Option + 'a, 20 | ) -> impl Iterator> + 'a { 21 | let mut stack = Vec::from([(xs, Fold::::Input(init))]); 22 | core::iter::from_fn(move || loop { 23 | let (mut xs, fold) = stack.pop()?; 24 | match fold { 25 | Fold::Output(x, mut ys) => match ys.next() { 26 | None => continue, 27 | Some(y) => { 28 | // do not grow the stack if the output is empty 29 | if ys.size_hint() != (0, Some(0)) { 30 | stack.push((xs.clone(), Fold::Output(x.clone(), ys))); 31 | } 32 | match y { 33 | Ok(y) => { 34 | let inner = inner(x, &y); 35 | stack.push((xs, Fold::Input(y))); 36 | if let Some(inner) = inner { 37 | return Some(Ok(inner)); 38 | } 39 | } 40 | Err(e) => return Some(Err(e)), 41 | } 42 | } 43 | }, 44 | Fold::Input(y) => match xs.next() { 45 | None => { 46 | if let Some(outer) = outer(y) { 47 | return Some(Ok(outer)); 48 | } 49 | } 50 | Some(Ok(x)) => stack.push((xs, Fold::Output(tc(&x), f(x, y)))), 51 | Some(Err(e)) => return Some(Err(e)), 52 | }, 53 | } 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /jaq-core/src/into_iter.rs: -------------------------------------------------------------------------------- 1 | //! Functions and types for `IntoIterator` and `FnOnce() -> Iterator`. 2 | 3 | #[derive(Clone)] 4 | pub struct Delay(F); 5 | 6 | impl I> IntoIterator for Delay { 7 | type Item = I::Item; 8 | type IntoIter = I; 9 | fn into_iter(self) -> Self::IntoIter { 10 | self.0() 11 | } 12 | } 13 | 14 | #[derive(Clone)] 15 | pub enum Either { 16 | L(L), 17 | R(R), 18 | } 19 | 20 | pub struct EitherIter(Either); 21 | 22 | impl> Iterator for EitherIter { 23 | type Item = L::Item; 24 | fn next(&mut self) -> Option { 25 | match &mut self.0 { 26 | Either::L(l) => l.next(), 27 | Either::R(r) => r.next(), 28 | } 29 | } 30 | fn size_hint(&self) -> (usize, Option) { 31 | match &self.0 { 32 | Either::L(l) => l.size_hint(), 33 | Either::R(r) => r.size_hint(), 34 | } 35 | } 36 | } 37 | 38 | impl> IntoIterator for Either { 39 | type Item = L::Item; 40 | type IntoIter = EitherIter; 41 | fn into_iter(self) -> Self::IntoIter { 42 | EitherIter(match self { 43 | Self::L(l) => Either::L(l.into_iter()), 44 | Self::R(r) => Either::R(r.into_iter()), 45 | }) 46 | } 47 | } 48 | 49 | pub fn collect_if_once I + Clone>( 50 | f: F, 51 | ) -> Either, Delay> { 52 | let mut iter = f.clone()(); 53 | if iter.size_hint().1 == Some(1) { 54 | if let Some(x) = iter.next() { 55 | assert!(iter.next().is_none()); 56 | return Either::L(core::iter::once(x)); 57 | } 58 | } 59 | Either::R(Delay(f)) 60 | } 61 | -------------------------------------------------------------------------------- /jaq-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! JSON query language interpreter. 2 | //! 3 | //! This crate allows you to execute jq-like filters. 4 | //! 5 | //! The example below demonstrates how to use this crate. 6 | //! See the implementation in the `jaq` crate if you are interested in how to: 7 | //! 8 | //! * enable usage of the standard library, 9 | //! * load JSON files lazily, 10 | //! * handle errors etc. 11 | //! 12 | //! (This example requires enabling the `serde_json` feature for `jaq-json`.) 13 | //! 14 | //! ~~~ 15 | //! use jaq_core::{load, Compiler, Ctx, Error, FilterT, RcIter}; 16 | //! use jaq_json::Val; 17 | //! use serde_json::{json, Value}; 18 | //! 19 | //! let input = json!(["Hello", "world"]); 20 | //! let program = File { code: ".[]", path: () }; 21 | //! 22 | //! use load::{Arena, File, Loader}; 23 | //! 24 | //! let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); 25 | //! let arena = Arena::default(); 26 | //! 27 | //! // parse the filter 28 | //! let modules = loader.load(&arena, program).unwrap(); 29 | //! 30 | //! // compile the filter 31 | //! let filter = jaq_core::Compiler::default() 32 | //! .with_funs(jaq_std::funs().chain(jaq_json::funs())) 33 | //! .compile(modules) 34 | //! .unwrap(); 35 | //! 36 | //! let inputs = RcIter::new(core::iter::empty()); 37 | //! 38 | //! // iterator over the output values 39 | //! let mut out = filter.run((Ctx::new([], &inputs), Val::from(input))); 40 | //! 41 | //! assert_eq!(out.next(), Some(Ok(Val::from(json!("Hello")))));; 42 | //! assert_eq!(out.next(), Some(Ok(Val::from(json!("world")))));; 43 | //! assert_eq!(out.next(), None);; 44 | //! ~~~ 45 | #![no_std] 46 | #![forbid(unsafe_code)] 47 | #![warn(missing_docs)] 48 | 49 | extern crate alloc; 50 | #[cfg(feature = "std")] 51 | extern crate std; 52 | 53 | pub mod box_iter; 54 | pub mod compile; 55 | mod exn; 56 | mod filter; 57 | mod fold; 58 | mod into_iter; 59 | pub mod load; 60 | pub mod ops; 61 | pub mod path; 62 | mod rc_iter; 63 | mod rc_lazy_list; 64 | mod rc_list; 65 | mod stack; 66 | pub mod val; 67 | 68 | pub use compile::Compiler; 69 | pub use exn::{Error, Exn}; 70 | pub use filter::{Ctx, Cv, FilterT, Native, RunPtr, UpdatePtr}; 71 | pub use rc_iter::RcIter; 72 | pub use val::{ValR, ValT, ValX, ValXs}; 73 | 74 | use alloc::string::String; 75 | use rc_list::List as RcList; 76 | use stack::Stack; 77 | 78 | type Inputs<'i, V> = RcIter> + 'i>; 79 | 80 | /// Argument of a definition, such as `$v` or `f` in `def foo($v; f): ...`. 81 | /// 82 | /// In jq, we can bind filters in three different ways: 83 | /// 84 | /// 1. `f as $x | ...` 85 | /// 2. `def g($x): ...; g(f)` 86 | /// 3. `def g(fx): ...; g(f)` 87 | /// 88 | /// In the first two cases, we bind the outputs of `f` to a variable `$x`. 89 | /// In the third case, we bind `f` to a filter `fx` 90 | /// 91 | /// When writing a native filter, this is used to declare its arguments. 92 | /// It is passed to [`compile::Compiler::with_funs`]. 93 | #[derive(Debug, Clone, PartialEq, Eq)] 94 | pub enum Bind { 95 | /// binding to a variable 96 | Var(V), 97 | /// binding to a filter 98 | Fun(F), 99 | } 100 | 101 | impl Bind { 102 | /// Move references inward. 103 | pub(crate) fn as_ref(&self) -> Bind<&V, &F> { 104 | match self { 105 | Self::Var(x) => Bind::Var(x), 106 | Self::Fun(x) => Bind::Fun(x), 107 | } 108 | } 109 | } 110 | 111 | impl Bind { 112 | /// Apply a function to both binding types. 113 | pub(crate) fn map(self, f: impl FnOnce(T) -> U) -> Bind { 114 | match self { 115 | Self::Var(x) => Bind::Var(f(x)), 116 | Self::Fun(x) => Bind::Fun(f(x)), 117 | } 118 | } 119 | } 120 | 121 | /// Function from a value to a stream of value results. 122 | #[derive(Debug, Clone)] 123 | pub struct Filter(compile::TermId, compile::Lut); 124 | 125 | impl Filter { 126 | /// Run a filter on given input, yielding output values. 127 | pub fn run<'a>(&'a self, cv: Cv<'a, F::V>) -> impl Iterator> + 'a { 128 | self.0 129 | .run(&self.1, cv) 130 | .map(|v| v.map_err(|e| e.get_err().ok().unwrap())) 131 | } 132 | 133 | /// Run a filter on given input, panic if it does not yield the given output. 134 | /// 135 | /// This is for testing purposes. 136 | pub fn yields(&self, x: F::V, ys: impl Iterator>) { 137 | let inputs = RcIter::new(core::iter::empty()); 138 | let out = self.run((Ctx::new([], &inputs), x)); 139 | assert!(out.eq(ys)); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /jaq-core/src/load/arbitrary.rs: -------------------------------------------------------------------------------- 1 | //! Generate arbitrary instances of central lexer types. 2 | //! 3 | //! This is useful for fuzzing the parser. 4 | //! 5 | //! This code can generate any tokens that might ever be output by the lexer. 6 | //! However, it also outputs many tokens that would never be output by the lexer. 7 | //! It is hard to avoid this because we cannot construct 8 | //! specific values of type `&str` at runtime other than by `arbitrary()`, 9 | //! which does not allow us to provide side conditions for the generated strings. 10 | //! For example, we cannot generate random `&str`s that can be parsed to numbers. 11 | //! (If we would use `String`, then this would be no problem. 12 | //! But `String` cannot encode the position of tokens, unlike `&str`.) 13 | //! 14 | //! We are aiming at creating tokens that satisfy just enough constraints 15 | //! such that the parser and the compiler do not panic. 16 | 17 | use super::lex::{StrPart, Tok, Token}; 18 | use arbitrary::{Arbitrary, Result, Unstructured}; 19 | 20 | impl<'a> Arbitrary<'a> for StrPart<&'a str, Token<&'a str>> { 21 | fn arbitrary(u: &mut Unstructured<'a>) -> Result { 22 | match u.choose_index(3)? { 23 | 0 => Ok(StrPart::Str(u.arbitrary()?)), 24 | 1 => Ok(StrPart::Term(Token("(", Tok::Block(u.arbitrary()?)))), 25 | _ => Ok(StrPart::Char(u.arbitrary()?)), 26 | } 27 | } 28 | } 29 | 30 | impl<'a> Arbitrary<'a> for Token<&'a str> { 31 | fn arbitrary(u: &mut Unstructured<'a>) -> Result { 32 | let s: &str = u.arbitrary::<&str>()?.trim(); 33 | let mut chars = s.chars(); 34 | let tok = match chars.next() { 35 | Some('a'..='z' | 'A'..='Z' | '_') => Tok::Word, 36 | Some('$') => Tok::Var, 37 | Some('0'..='9') => Tok::Num, 38 | Some('-') if matches!(chars.next(), Some('0'..='9')) => Tok::Num, 39 | Some('"') => Tok::Str(u.arbitrary()?), 40 | Some('(' | '[' | '{') => Tok::Block(u.arbitrary()?), 41 | Some(_) => Tok::Sym, 42 | None => u.choose_iter([Tok::Word, Tok::Num])?, 43 | }; 44 | Ok(Self(s, tok)) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /jaq-core/src/load/lex.rs: -------------------------------------------------------------------------------- 1 | //! Lexing. 2 | 3 | use alloc::vec::Vec; 4 | 5 | /// Component of a string potentially containing escape sequences. 6 | /// 7 | /// `S` is a type of strings (without escape sequences), and 8 | /// `F` is a type of interpolated filters. 9 | #[derive(Debug)] 10 | pub enum StrPart { 11 | /// string without escape sequences 12 | Str(S), 13 | /// interpolated term (`\(...)`) 14 | /// 15 | /// Here, the contained term `T` must be of the shape 16 | /// `Token(s, Tok::Block(...))` such that the first character of `s` is '('. 17 | Term(T), 18 | /// escaped character (e.g. `\n`, `t`, `\u0041`) 19 | Char(char), 20 | } 21 | 22 | /// Token (tree) generic over string type `S`. 23 | /// 24 | /// If the contained `Tok` is of the shape: 25 | /// * `Tok::Block(...)`, then `S` must start with `'('`, `'['`, or `'{'`. 26 | /// * `Tok::Var`, then `S` must start with `'$'`. 27 | #[derive(Debug)] 28 | pub struct Token(pub S, pub Tok); 29 | 30 | /// Type of token, generic over string type `S`. 31 | /// 32 | /// This data structure should normally not be constructed manually. 33 | /// It is exposed mostly for fuzzing. 34 | #[derive(Debug)] 35 | pub enum Tok { 36 | /// keywords such as `def`, but also identifiers such as `map`, `f::g` 37 | Word, 38 | /// variables such as `$x` 39 | Var, 40 | /// formatters such as `@csv` 41 | Fmt, 42 | /// number 43 | Num, 44 | /// (interpolated) string, surrounded by opening and closing '"' 45 | Str(Vec>>), 46 | /// symbol such as `.`, `;`, `-`, `|`, or `+=` 47 | Sym, 48 | /// delimited tokens, e.g. `(...)` or `[...]` 49 | Block(Vec>), 50 | } 51 | 52 | /// Type of character that we expected. 53 | /// 54 | /// Each variant is annoted with jq programs that trigger it. 55 | #[derive(Clone, Debug)] 56 | #[non_exhaustive] 57 | pub enum Expect { 58 | /// `0e`, `0.` 59 | Digit, 60 | /// `$`, `@` 61 | Ident, 62 | /// `(`, `[`, `{` 63 | Delim(S), 64 | /// `"\a"` 65 | Escape, 66 | /// `"\ux"`, `"\uD800"` 67 | Unicode, 68 | /// `&`, `§`, `💣` 69 | Token, 70 | } 71 | 72 | impl Expect<&str> { 73 | /// Return human-readable description of what we expected. 74 | pub fn as_str(&self) -> &'static str { 75 | match self { 76 | Self::Digit => "digit", 77 | Self::Ident => "identifier", 78 | Self::Delim("(") => "closing parenthesis", 79 | Self::Delim("[") => "closing bracket", 80 | Self::Delim("{") => "closing brace", 81 | Self::Delim("\"") => "closing quote", 82 | Self::Delim(_) => panic!(), 83 | Self::Escape => "string escape sequence", 84 | Self::Unicode => "4-digit hexadecimal UTF-8 code point", 85 | Self::Token => "token", 86 | } 87 | } 88 | } 89 | 90 | /// Lexer error, storing what we expected and what we got instead. 91 | pub type Error = (Expect, S); 92 | 93 | /// Lexer for jq files. 94 | pub struct Lexer { 95 | i: S, 96 | e: Vec>, 97 | } 98 | 99 | impl<'a> Lexer<&'a str> { 100 | /// Initialise a new lexer for the given input. 101 | #[must_use] 102 | pub fn new(i: &'a str) -> Self { 103 | let e = Vec::new(); 104 | Self { i, e } 105 | } 106 | 107 | /// Lex, returning the resulting tokens and errors. 108 | pub fn lex(mut self) -> Result>, Vec>> { 109 | let tokens = self.tokens(); 110 | self.space(); 111 | if !self.i.is_empty() { 112 | self.e.push((Expect::Token, self.i)); 113 | } 114 | 115 | if self.e.is_empty() { 116 | Ok(tokens) 117 | } else { 118 | Err(self.e) 119 | } 120 | } 121 | 122 | fn next(&mut self) -> Option { 123 | let mut chars = self.i.chars(); 124 | let c = chars.next()?; 125 | self.i = chars.as_str(); 126 | Some(c) 127 | } 128 | 129 | fn take(&mut self, len: usize) -> &'a str { 130 | let (head, tail) = self.i.split_at(len); 131 | self.i = tail; 132 | head 133 | } 134 | 135 | fn trim(&mut self, f: impl FnMut(char) -> bool) { 136 | self.i = self.i.trim_start_matches(f); 137 | } 138 | 139 | fn consumed(&mut self, skip: usize, f: impl FnOnce(&mut Self)) -> &'a str { 140 | self.with_consumed(|l| { 141 | l.i = &l.i[skip..]; 142 | f(l); 143 | }) 144 | .0 145 | } 146 | 147 | fn with_consumed(&mut self, f: impl FnOnce(&mut Self) -> T) -> (&'a str, T) { 148 | let start = self.i; 149 | let y = f(self); 150 | (&start[..start.len() - self.i.len()], y) 151 | } 152 | 153 | /// Whitespace and comments. 154 | fn space(&mut self) { 155 | loop { 156 | self.i = self.i.trim_start(); 157 | match self.i.strip_prefix('#') { 158 | Some(comment) => self.i = comment, 159 | None => break, 160 | } 161 | // ignore all lines that end with an odd number of backslashes 162 | loop { 163 | let (before, after) = self.i.split_once('\n').unwrap_or((self.i, "")); 164 | let before = before.strip_suffix('\r').unwrap_or(before); 165 | self.i = after; 166 | // does the line end with an even number of backslashes? 167 | if before.chars().rev().take_while(|c| *c == '\\').count() % 2 == 0 { 168 | break; 169 | } 170 | } 171 | } 172 | } 173 | 174 | fn mod_then_ident(&mut self) { 175 | self.ident0(); 176 | if let Some(rest) = self.i.strip_prefix("::") { 177 | self.i = rest.strip_prefix(['@', '$']).unwrap_or(rest); 178 | self.ident1(); 179 | } 180 | } 181 | 182 | /// Lex a sequence matching `[a-zA-Z0-9_]*`. 183 | fn ident0(&mut self) { 184 | self.trim(|c: char| c.is_ascii_alphanumeric() || c == '_'); 185 | } 186 | 187 | /// Lex a sequence matching `[a-zA-Z_][a-zA-Z0-9_]*`. 188 | fn ident1(&mut self) { 189 | let first = |c: char| c.is_ascii_alphabetic() || c == '_'; 190 | if let Some(rest) = self.i.strip_prefix(first) { 191 | self.i = rest; 192 | self.ident0(); 193 | } else { 194 | self.e.push((Expect::Ident, self.i)); 195 | } 196 | } 197 | 198 | /// Lex a non-empty digit sequence. 199 | fn digits1(&mut self) { 200 | if let Some(rest) = self.i.strip_prefix(|c: char| c.is_ascii_digit()) { 201 | self.i = rest.trim_start_matches(|c: char| c.is_ascii_digit()); 202 | } else { 203 | self.e.push((Expect::Digit, self.i)); 204 | } 205 | } 206 | 207 | /// Decimal with optional exponent. 208 | fn num(&mut self) { 209 | self.trim(|c| c.is_ascii_digit()); 210 | if let Some(i) = self.i.strip_prefix('.') { 211 | self.i = i; 212 | self.digits1(); 213 | } 214 | if let Some(i) = self.i.strip_prefix(['e', 'E']) { 215 | self.i = i.strip_prefix(['+', '-']).unwrap_or(i); 216 | self.digits1(); 217 | } 218 | } 219 | 220 | fn escape(&mut self) -> Option>> { 221 | let mut chars = self.i.chars(); 222 | let part = match chars.next() { 223 | Some(c @ ('\\' | '/' | '"')) => StrPart::Char(c), 224 | Some('b') => StrPart::Char('\x08'), 225 | Some('f') => StrPart::Char('\x0C'), 226 | Some('n') => StrPart::Char('\n'), 227 | Some('r') => StrPart::Char('\r'), 228 | Some('t') => StrPart::Char('\t'), 229 | Some('u') => { 230 | let err_at = |lex: &mut Self, pos| { 231 | lex.i = pos; 232 | lex.e.push((Expect::Unicode, lex.i)); 233 | None 234 | }; 235 | let mut hex = 0; 236 | let start_i = chars.as_str(); 237 | for _ in 0..4 { 238 | let cur_i = chars.as_str(); 239 | if let Some(digit) = chars.next().and_then(|c| c.to_digit(16)) { 240 | hex = (hex << 4) + digit; 241 | } else { 242 | return err_at(self, cur_i); 243 | } 244 | } 245 | match char::from_u32(hex) { 246 | None => return err_at(self, start_i), 247 | Some(c) => StrPart::Char(c), 248 | } 249 | } 250 | Some('(') => { 251 | let (full, block) = self.with_consumed(Self::block); 252 | return Some(StrPart::Term(Token(full, block))); 253 | } 254 | Some(_) | None => { 255 | self.e.push((Expect::Escape, self.i)); 256 | return None; 257 | } 258 | }; 259 | 260 | self.i = chars.as_str(); 261 | Some(part) 262 | } 263 | 264 | /// Lex a (possibly interpolated) string. 265 | /// 266 | /// The input string has to start with '"'. 267 | fn str(&mut self) -> Tok<&'a str> { 268 | let start = self.take(1); 269 | assert_eq!(start, "\""); 270 | let mut parts = Vec::new(); 271 | 272 | loop { 273 | let s = self.consumed(0, |lex| lex.trim(|c| c != '\\' && c != '"')); 274 | if !s.is_empty() { 275 | parts.push(StrPart::Str(s)); 276 | } 277 | match self.next() { 278 | Some('"') => break, 279 | Some('\\') => self.escape().map(|part| parts.push(part)), 280 | // SAFETY: due to `lex.trim()` 281 | Some(_) => unreachable!(), 282 | None => { 283 | self.e.push((Expect::Delim(start), self.i)); 284 | break; 285 | } 286 | }; 287 | } 288 | Tok::Str(parts) 289 | } 290 | 291 | fn token(&mut self) -> Option> { 292 | self.space(); 293 | 294 | let is_op = |c| "|=!<>+-*/%".contains(c); 295 | 296 | let mut chars = self.i.chars(); 297 | let (s, tok) = match chars.next()? { 298 | 'a'..='z' | 'A'..='Z' | '_' => (self.consumed(1, Self::mod_then_ident), Tok::Word), 299 | '$' => (self.consumed(1, Self::ident1), Tok::Var), 300 | '@' => (self.consumed(1, Self::ident1), Tok::Fmt), 301 | '0'..='9' => (self.consumed(1, Self::num), Tok::Num), 302 | c if is_op(c) => (self.consumed(1, |lex| lex.trim(is_op)), Tok::Sym), 303 | '.' => match chars.next() { 304 | Some('.') => (self.take(2), Tok::Sym), 305 | Some('a'..='z' | 'A'..='Z' | '_') => (self.consumed(2, Self::ident0), Tok::Sym), 306 | _ => (self.take(1), Tok::Sym), 307 | }, 308 | ':' | ';' | ',' | '?' => (self.take(1), Tok::Sym), 309 | '"' => self.with_consumed(Self::str), 310 | '(' | '[' | '{' => self.with_consumed(Self::block), 311 | _ => return None, 312 | }; 313 | Some(Token(s, tok)) 314 | } 315 | 316 | fn tokens(&mut self) -> Vec> { 317 | core::iter::from_fn(|| self.token()).collect() 318 | } 319 | 320 | /// Lex a sequence of tokens that is surrounded by parentheses, curly braces, or brackets. 321 | /// 322 | /// The input string has to start with either '(', '[', or '{'. 323 | fn block(&mut self) -> Tok<&'a str> { 324 | let open = self.take(1); 325 | let close = match open { 326 | "(" => ')', 327 | "[" => ']', 328 | "{" => '}', 329 | _ => panic!(), 330 | }; 331 | let mut tokens = self.tokens(); 332 | 333 | self.space(); 334 | if let Some(rest) = self.i.strip_prefix(close) { 335 | tokens.push(Token(&self.i[..1], Tok::Sym)); 336 | self.i = rest; 337 | } else { 338 | self.e.push((Expect::Delim(open), self.i)); 339 | } 340 | Tok::Block(tokens) 341 | } 342 | } 343 | 344 | impl<'a> Token<&'a str> { 345 | /// Return the string slice corresponding to an optional token. 346 | /// 347 | /// If the token is not present, return an empty string slice starting at the end of `code`. 348 | pub fn opt_as_str(found: Option<&Self>, code: &'a str) -> &'a str { 349 | found.map_or(&code[code.len()..], |found| found.as_str()) 350 | } 351 | 352 | /// Return the string slice corresponding to the token. 353 | pub fn as_str(&self) -> &'a str { 354 | self.0 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /jaq-core/src/load/mod.rs: -------------------------------------------------------------------------------- 1 | //! Combined file loading, lexing, and parsing for multiple modules. 2 | 3 | #[cfg(feature = "arbitrary")] 4 | mod arbitrary; 5 | pub mod lex; 6 | pub mod parse; 7 | mod prec_climb; 8 | pub mod test; 9 | 10 | use crate::{ops, path}; 11 | #[cfg(feature = "std")] 12 | use alloc::boxed::Box; 13 | use alloc::{string::String, vec::Vec}; 14 | pub use lex::Lexer; 15 | use lex::Token; 16 | pub use parse::Parser; 17 | use parse::{Def, Term}; 18 | #[cfg(feature = "std")] 19 | use std::path::{Path, PathBuf}; 20 | 21 | #[cfg(feature = "std")] 22 | extern crate std; 23 | 24 | /// Storage for loaded modules. 25 | /// 26 | /// Once Rust has [internal references](https://smallcultfollowing.com/babysteps/blog/2024/06/02/the-borrow-checker-within/#step-4-internal-references), 27 | /// this should become unnecessary. 28 | /// I can't wait for it to happen! 29 | #[derive(Default)] 30 | pub struct Arena(typed_arena::Arena); 31 | 32 | /// Combined file loader, lexer, and parser for multiple modules. 33 | pub struct Loader { 34 | #[allow(clippy::type_complexity)] 35 | mods: Vec<(File, Result, Error>)>, 36 | /// function to read module file contents from a path 37 | read: R, 38 | /// currently processed modules 39 | /// 40 | /// This is used to detect circular dependencies between modules. 41 | open: Vec

, 42 | } 43 | 44 | /// Contents `C` and path `P` of a (module) file. 45 | /// 46 | /// This is useful for creating precise error messages. 47 | #[derive(Clone, Debug, Default)] 48 | pub struct File { 49 | /// contents of the file 50 | pub code: C, 51 | /// path of the file 52 | pub path: P, 53 | } 54 | 55 | /// Information to resolve module/data imports. 56 | pub struct Import<'a, S, P> { 57 | /// absolute path of the module where the import/include directive appears 58 | /// 59 | /// This is a path `P`, not a string `S`, because it usually does not appear in the source. 60 | pub parent: &'a P, 61 | /// relative path of the imported/included module, as given in the source 62 | pub path: &'a S, 63 | /// metadata attached to the import/include directive 64 | pub meta: &'a Option>, 65 | } 66 | 67 | impl File { 68 | /// Apply a function to the contents of a file. 69 | /// 70 | /// This is useful to go from a reference `&str` to an owned `String`, 71 | /// in order to save the `File` without its corresponding [`Arena`]. 72 | pub fn map_code(self, f: impl Fn(C) -> C2) -> File { 73 | File { 74 | code: f(self.code), 75 | path: self.path, 76 | } 77 | } 78 | } 79 | 80 | /// Error occurring during loading of a single module. 81 | #[derive(Debug)] 82 | pub enum Error { 83 | /// input/output errors, for example when trying to load a module that does not exist 84 | Io(Vec<(S, String)>), 85 | /// lex errors, for example when loading a module `($) (` 86 | Lex(Vec>), 87 | /// parse errors, for example when loading a module `(+) *` 88 | Parse(Vec>), 89 | } 90 | 91 | /// Module containing strings `S` and a body `B`. 92 | #[derive(Default)] 93 | pub struct Module>> { 94 | /// metadata (optional) 95 | pub(crate) meta: Option>, 96 | /// included and imported modules 97 | /// 98 | /// Suppose that we have [`Modules`] `mods` and the current [`Module`] is `mods[id]`. 99 | /// Then for every `(id_, name)` in `mods[id].1.mods`, we have that 100 | /// the included/imported module is stored in `mods[id_]` (`id_ < id`), and 101 | /// the module is included if `name` is `None` and imported if `name` is `Some(name)`. 102 | pub(crate) mods: Vec<(usize, Option)>, 103 | /// imported variables, storing path and name (always starts with `$`) 104 | pub(crate) vars: Vec<(S, S, Option>)>, 105 | /// everything that comes after metadata and includes/imports 106 | pub(crate) body: B, 107 | } 108 | 109 | /// Tree of modules containing definitions. 110 | /// 111 | /// By convention, the last module contains a single definition that is the `main` filter. 112 | pub type Modules = Vec<(File, Module)>; 113 | 114 | /// Errors occurring during loading of multiple modules. 115 | /// 116 | /// For example, suppose that we have 117 | /// a file `l.jq` that yields a lex error, 118 | /// a file `p.jq` that yields a parse error, and 119 | /// a file `i.jq` that includes a non-existing module. 120 | /// If we then include all these files in our main program, 121 | /// [`Errors`] will contain each file with a different [`Error`]. 122 | pub type Errors> = Vec<(File, E)>; 123 | 124 | impl, B> parse::Module { 125 | fn map( 126 | self, 127 | mut f: impl FnMut(&S, Option>) -> Result, 128 | ) -> Result, Error> { 129 | // the prelude module is included implicitly in every module (except itself) 130 | let mut mods = Vec::from([(0, None)]); 131 | let mut vars = Vec::new(); 132 | let mut errs = Vec::new(); 133 | for (path, as_, meta) in self.deps { 134 | match as_ { 135 | Some(x) if x.starts_with('$') => vars.push((path, x, meta)), 136 | as_ => match f(&path, meta) { 137 | Ok(mid) => mods.push((mid, as_)), 138 | Err(e) => errs.push((path, e)), 139 | }, 140 | } 141 | } 142 | if errs.is_empty() { 143 | Ok(Module { 144 | meta: self.meta, 145 | mods, 146 | vars, 147 | body: self.body, 148 | }) 149 | } else { 150 | Err(Error::Io(errs)) 151 | } 152 | } 153 | } 154 | 155 | impl Module { 156 | fn map_body(self, f: impl FnOnce(B) -> B2) -> Module { 157 | Module { 158 | meta: self.meta, 159 | mods: self.mods, 160 | vars: self.vars, 161 | body: f(self.body), 162 | } 163 | } 164 | } 165 | 166 | type ReadResult

= Result, String>; 167 | type ReadFn

= fn(Import<&str, P>) -> ReadResult

; 168 | 169 | impl<'s, P: Default> Loader<&'s str, P, ReadFn

> { 170 | /// Initialise the loader with prelude definitions. 171 | /// 172 | /// The prelude is a special module that is implicitly included by all other modules 173 | /// (including the main module). 174 | /// That means that all filters defined in the prelude can be called from any module. 175 | /// 176 | /// The prelude is normally initialised with filters like `map` or `true`. 177 | pub fn new(prelude: impl IntoIterator>) -> Self { 178 | let defs = [ 179 | Def::new("!recurse", Vec::new(), Term::recurse("!recurse")), 180 | Def::new("!empty", Vec::new(), Term::empty()), 181 | ]; 182 | 183 | let prelude = Module { 184 | body: defs.into_iter().chain(prelude).collect(), 185 | ..Module::default() 186 | }; 187 | 188 | Self { 189 | // the first module is reserved for the prelude 190 | mods: Vec::from([(File::default(), Ok(prelude))]), 191 | read: |_path| Err("module loading not supported".into()), 192 | open: Vec::new(), 193 | } 194 | } 195 | } 196 | 197 | #[cfg(feature = "std")] 198 | impl Term { 199 | fn obj_key(&self, key: S) -> Option<&Self> { 200 | if let Term::Obj(kvs) = self { 201 | kvs.iter().find_map(|(k, v)| { 202 | if *k.as_str()? == key { 203 | v.as_ref() 204 | } else { 205 | None 206 | } 207 | }) 208 | } else { 209 | None 210 | } 211 | } 212 | 213 | fn unconcat(&self) -> Box + '_> { 214 | match self { 215 | Self::BinOp(l, parse::BinaryOp::Comma, r) => Box::new(l.unconcat().chain(r.unconcat())), 216 | _ => Box::new(core::iter::once(self)), 217 | } 218 | } 219 | } 220 | 221 | #[cfg(feature = "std")] 222 | fn expand_prefix(path: &Path, pre: &str, f: impl FnOnce() -> Option) -> Option { 223 | let rest = path.strip_prefix(pre).ok()?; 224 | let mut replace = f()?; 225 | replace.push(rest); 226 | Some(replace) 227 | } 228 | 229 | #[cfg(feature = "std")] 230 | impl<'a> Import<'a, &'a str, PathBuf> { 231 | fn meta_paths(&self) -> impl Iterator + '_ { 232 | let paths = self.meta.as_ref().and_then(|meta| { 233 | let v = meta.obj_key("search")?; 234 | let iter = if let Term::Arr(Some(a)) = v { 235 | Box::new(a.unconcat().filter_map(|v| v.as_str())) 236 | } else if let Some(s) = v.as_str() { 237 | Box::new(core::iter::once(s)) 238 | } else { 239 | Box::new(core::iter::empty()) as Box> 240 | }; 241 | Some(iter.map(|s| Path::new(*s).to_path_buf())) 242 | }); 243 | paths.into_iter().flatten() 244 | } 245 | 246 | /// Try to find a file with given extension in the given search paths. 247 | pub fn find(self, paths: &[PathBuf], ext: &str) -> Result { 248 | let parent = Path::new(self.parent).parent().unwrap_or(Path::new(".")); 249 | 250 | let mut rel = Path::new(self.path).to_path_buf(); 251 | if !rel.is_relative() { 252 | Err("non-relative path")? 253 | } 254 | rel.set_extension(ext); 255 | 256 | #[cfg(target_os = "windows")] 257 | let home = "USERPROFILE"; 258 | #[cfg(not(target_os = "windows"))] 259 | let home = "HOME"; 260 | 261 | use std::env; 262 | let home = || env::var_os(home).map(PathBuf::from); 263 | let origin = || env::current_exe().ok()?.parent().map(PathBuf::from); 264 | let expand = |path: &PathBuf| { 265 | let home = expand_prefix(path, "~", home); 266 | let orig = expand_prefix(path, "$ORIGIN", origin); 267 | home.or(orig).unwrap_or_else(|| path.clone()) 268 | }; 269 | 270 | // search paths given in the metadata are relative to the parent file, whereas 271 | // search paths given on the command-line (`paths`, via `-L`) are not 272 | let meta = self.meta_paths().map(|p| parent.join(expand(&p))); 273 | meta.chain(paths.iter().map(expand)) 274 | .map(|path| path.join(&rel)) 275 | .filter_map(|path| path.canonicalize().ok()) 276 | .find(|path| path.is_file()) 277 | .ok_or_else(|| "file not found".into()) 278 | } 279 | 280 | fn read(self, paths: &[PathBuf], ext: &str) -> ReadResult { 281 | use alloc::string::ToString; 282 | let path = self.find(paths, ext)?; 283 | let code = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; 284 | Ok(File { code, path }) 285 | } 286 | } 287 | 288 | /// Apply function to path of every imported data file, accumulating errors. 289 | pub fn import( 290 | mods: &Modules, 291 | mut f: impl FnMut(Import) -> Result<(), String>, 292 | ) -> Result<(), Errors> { 293 | let mut errs = Vec::new(); 294 | let mut vals = Vec::new(); 295 | for (mod_file, module) in mods { 296 | let mut mod_errs = Vec::new(); 297 | for (path, _name, meta) in &module.vars { 298 | let parent = &mod_file.path; 299 | match f(Import { parent, path, meta }) { 300 | Ok(v) => vals.push(v), 301 | Err(e) => mod_errs.push((*path, e)), 302 | } 303 | } 304 | if !mod_errs.is_empty() { 305 | errs.push((mod_file.clone(), Error::Io(mod_errs))); 306 | } 307 | } 308 | if errs.is_empty() { 309 | Ok(()) 310 | } else { 311 | Err(errs) 312 | } 313 | } 314 | 315 | impl Loader { 316 | /// Provide a function to return the contents of included/imported module files. 317 | /// 318 | /// For every included/imported module, the loader will call this function to 319 | /// obtain the contents of the module. 320 | /// For example, if we have `include "foo"`, the loader calls `read("foo")`. 321 | pub fn with_read(self, read: R2) -> Loader { 322 | let Self { mods, open, .. } = self; 323 | Loader { mods, read, open } 324 | } 325 | } 326 | 327 | #[cfg(feature = "std")] 328 | impl Loader { 329 | /// Read the contents of included/imported module files by performing file I/O. 330 | pub fn with_std_read( 331 | self, 332 | paths: &[PathBuf], 333 | ) -> Loader) -> ReadResult + '_> { 334 | self.with_read(|import: Import<&str, PathBuf>| import.read(paths, "jq")) 335 | } 336 | } 337 | 338 | impl<'s, P: Clone + Eq, R: FnMut(Import<&'s str, P>) -> ReadResult

> Loader<&'s str, P, R> { 339 | /// Load a set of modules, starting from a given file. 340 | pub fn load( 341 | mut self, 342 | arena: &'s Arena, 343 | file: File<&'s str, P>, 344 | ) -> Result, Errors<&'s str, P>> { 345 | let result = parse_main(file.code) 346 | .and_then(|m| { 347 | m.map(|path, meta| { 348 | let (parent, meta) = (&file.path, &meta); 349 | self.find(arena, Import { parent, path, meta }) 350 | }) 351 | }) 352 | .map(|m| m.map_body(|body| Vec::from([Def::new("main", Vec::new(), body)]))); 353 | self.mods.push((file, result)); 354 | 355 | let mut mods = Vec::new(); 356 | let mut errs = Vec::new(); 357 | for (file, result) in self.mods { 358 | match result { 359 | Ok(m) => mods.push((file, m)), 360 | Err(e) => errs.push((file, e)), 361 | } 362 | } 363 | if errs.is_empty() { 364 | Ok(mods) 365 | } else { 366 | Err(errs) 367 | } 368 | } 369 | 370 | fn find(&mut self, arena: &'s Arena, import: Import<&'s str, P>) -> Result { 371 | let file = (self.read)(import)?; 372 | 373 | let mut mods = self.mods.iter(); 374 | if let Some(id) = mods.position(|(file_, _)| file.path == file_.path) { 375 | return Ok(id); 376 | }; 377 | if self.open.contains(&file.path) { 378 | return Err("circular include/import".into()); 379 | } 380 | 381 | let code = &**arena.0.alloc(file.code); 382 | self.open.push(file.path.clone()); 383 | let defs = parse_defs(code).and_then(|m| { 384 | m.map(|path, meta| { 385 | let (parent, meta) = (&file.path, &meta); 386 | self.find(arena, Import { parent, path, meta }) 387 | }) 388 | }); 389 | assert!(self.open.pop().as_ref() == Some(&file.path)); 390 | 391 | let id = self.mods.len(); 392 | let path = file.path; 393 | self.mods.push((File { path, code }, defs)); 394 | Ok(id) 395 | } 396 | } 397 | 398 | fn parse_main(code: &str) -> Result>, Error<&str>> { 399 | let tokens = lex::Lexer::new(code).lex().map_err(Error::Lex)?; 400 | let conv_err = |(expected, found)| (expected, Token::opt_as_str(found, code)); 401 | parse::Parser::new(&tokens) 402 | .parse(|p| p.module(|p| p.term())) 403 | .map_err(|e| Error::Parse(e.into_iter().map(conv_err).collect())) 404 | } 405 | 406 | fn parse_defs(code: &str) -> Result>>, Error<&str>> { 407 | let tokens = lex::Lexer::new(code).lex().map_err(Error::Lex)?; 408 | let conv_err = |(expected, found)| (expected, Token::opt_as_str(found, code)); 409 | parse::Parser::new(&tokens) 410 | .parse(|p| p.module(|p| p.defs())) 411 | .map_err(|e| Error::Parse(e.into_iter().map(conv_err).collect())) 412 | } 413 | 414 | /// Lex a string and parse resulting tokens, returning [`None`] if any error occurred. 415 | /// 416 | /// Example: 417 | /// 418 | /// ~~~ 419 | /// # use jaq_core::load::parse; 420 | /// let t = parse("[] | .[]", |p| p.term()); 421 | /// ~~~ 422 | pub fn parse<'s, T: Default, F>(s: &'s str, f: F) -> Option 423 | where 424 | F: for<'t> FnOnce(&mut Parser<'s, 't>) -> parse::Result<'s, 't, T>, 425 | { 426 | Parser::new(&Lexer::new(s).lex().ok()?).parse(f).ok() 427 | } 428 | 429 | /// Return the span of a string slice `part` relative to a string slice `whole`. 430 | /// 431 | /// The caller must ensure that `part` is fully contained inside `whole`. 432 | pub fn span(whole: &str, part: &str) -> core::ops::Range { 433 | let start = part.as_ptr() as usize - whole.as_ptr() as usize; 434 | start..start + part.len() 435 | } 436 | -------------------------------------------------------------------------------- /jaq-core/src/load/prec_climb.rs: -------------------------------------------------------------------------------- 1 | //! Precedence climbing for parsing expressions with binary operators. 2 | //! 3 | //! This allows you to parse expressions that are 4 | //! separated by binary operators with precedence and associativity. 5 | //! For example, in the expression `1 + 2 * 3`, we usually want to 6 | //! parse this into `1 + (2 * 3)`, not `(1 + 2) * 3`. 7 | //! This is handled by saying that `*` has higher *precedence* than `+`. 8 | //! Also, when we have a power operator `^`, we want 9 | //! `2 ^ 3 ^ 4` to mean `(2 ^ 3) ^ 4`, not `2 ^ (3 ^ 4)`. 10 | //! This is handled by saying that `^` is *left-associative*. 11 | //! 12 | //! This was adapted from 13 | //! . 14 | 15 | use core::iter::Peekable; 16 | 17 | /// Associativity of an operator. 18 | pub enum Associativity { 19 | /// `(x + y) + z` 20 | Left, 21 | /// `x + (y + z)` 22 | Right, 23 | } 24 | 25 | /// Binary operator. 26 | pub trait Op { 27 | /// "Stickiness" of the operator 28 | fn precedence(&self) -> usize; 29 | /// Is the operator left- or right-associative? 30 | fn associativity(&self) -> Associativity; 31 | } 32 | 33 | /// An expression that can be built from other expressions with some operator. 34 | pub trait Expr { 35 | /// Combine two expressions with an operator. 36 | fn from_op(lhs: Self, op: O, rhs: Self) -> Self; 37 | } 38 | 39 | /// Perform precedence climbing. 40 | pub fn climb>(head: T, iter: impl IntoIterator) -> T { 41 | climb1(head, &mut iter.into_iter().peekable(), 0) 42 | } 43 | 44 | fn climb1, I>(mut x: T, iter: &mut Peekable, min_prec: usize) -> T 45 | where 46 | I: Iterator, 47 | { 48 | while let Some((op, mut rhs)) = iter.next_if(|(op, _)| op.precedence() >= min_prec) { 49 | let right_assoc = matches!(op.associativity(), Associativity::Right); 50 | let this_prec = op.precedence(); 51 | 52 | while let Some(next) = iter.peek() { 53 | let next_prec = next.0.precedence(); 54 | 55 | if next_prec > this_prec || (right_assoc && next_prec == this_prec) { 56 | rhs = climb1(rhs, iter, next_prec); 57 | } else { 58 | break; 59 | } 60 | } 61 | x = T::from_op(x, op, rhs); 62 | } 63 | x 64 | } 65 | 66 | /// Simple arithmetic expressions 67 | #[test] 68 | fn test() { 69 | enum Arith { 70 | Add, 71 | Sub, 72 | Mul, 73 | Div, 74 | } 75 | 76 | impl Op for Arith { 77 | fn precedence(&self) -> usize { 78 | match self { 79 | Arith::Add | Arith::Sub => 0, 80 | Arith::Mul | Arith::Div => 1, 81 | } 82 | } 83 | 84 | fn associativity(&self) -> Associativity { 85 | Associativity::Right 86 | } 87 | } 88 | 89 | impl Expr for isize { 90 | fn from_op(lhs: Self, op: Arith, rhs: Self) -> Self { 91 | match op { 92 | Arith::Add => lhs + rhs, 93 | Arith::Sub => lhs - rhs, 94 | Arith::Mul => lhs * rhs, 95 | Arith::Div => lhs / rhs, 96 | } 97 | } 98 | } 99 | 100 | use Arith::{Add, Div, Mul, Sub}; 101 | // 1 + 2 * 3 - 6 / 2 = 102 | // 1 + 6 - 3 = 4 103 | let head: isize = 1; 104 | let tail = [(Add, 2), (Mul, 3), (Sub, 6), (Div, 2)]; 105 | let out = climb(head, tail); 106 | assert_eq!(out, 4); 107 | } 108 | -------------------------------------------------------------------------------- /jaq-core/src/load/test.rs: -------------------------------------------------------------------------------- 1 | //! Unit tests. 2 | 3 | use alloc::vec::Vec; 4 | 5 | /// A single jq unit test. 6 | pub struct Test { 7 | /// jq filter 8 | pub filter: S, 9 | /// input value in JSON format 10 | pub input: S, 11 | /// output values in JSON format 12 | pub output: Vec, 13 | } 14 | 15 | /// Parser for a jq unit test. 16 | pub struct Parser(I); 17 | 18 | impl Parser { 19 | /// Create a parser from an iterator over lines. 20 | pub fn new(lines: I) -> Self { 21 | Self(lines) 22 | } 23 | } 24 | 25 | impl, I: Iterator> Iterator for Parser { 26 | type Item = Test; 27 | 28 | fn next(&mut self) -> Option { 29 | let lines = &mut self.0; 30 | Some(Test { 31 | filter: lines.find(|l| !(l.is_empty() || l.starts_with('#')))?, 32 | input: lines.next()?, 33 | output: lines.take_while(|l| !l.is_empty()).collect(), 34 | }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /jaq-core/src/ops.rs: -------------------------------------------------------------------------------- 1 | //! Binary operations. 2 | 3 | use core::ops::{Add, Div, Mul, Rem, Sub}; 4 | 5 | /// Arithmetic operation, such as `+`, `-`, `*`, `/`, `%`. 6 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 7 | pub enum Math { 8 | /// Addition 9 | Add, 10 | /// Subtraction 11 | Sub, 12 | /// Multiplication 13 | Mul, 14 | /// Division 15 | Div, 16 | /// Remainder 17 | Rem, 18 | } 19 | 20 | impl Math { 21 | /// Perform the arithmetic operation on the given inputs. 22 | pub fn run(&self, l: I, r: I) -> O 23 | where 24 | I: Add + Sub + Mul + Div + Rem, 25 | { 26 | match self { 27 | Self::Add => l + r, 28 | Self::Sub => l - r, 29 | Self::Mul => l * r, 30 | Self::Div => l / r, 31 | Self::Rem => l % r, 32 | } 33 | } 34 | } 35 | 36 | impl Math { 37 | /// String representation of an arithmetic operation. 38 | pub fn as_str(&self) -> &'static str { 39 | match self { 40 | Self::Add => "+", 41 | Self::Sub => "-", 42 | Self::Mul => "*", 43 | Self::Div => "/", 44 | Self::Rem => "%", 45 | } 46 | } 47 | } 48 | 49 | /// An operation that orders two values, such as `<`, `<=`, `>`, `>=`, `==`, `!=`. 50 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 51 | pub enum Cmp { 52 | /// Less-than (<). 53 | Lt, 54 | /// Less-than or equal (<=). 55 | Le, 56 | /// Greater-than (>). 57 | Gt, 58 | /// Greater-than or equal (>=). 59 | Ge, 60 | /// Equals (=). 61 | Eq, 62 | /// Not equals (!=). 63 | Ne, 64 | } 65 | 66 | impl Cmp { 67 | /// Perform the ordering operation on the given inputs. 68 | pub fn run(&self, l: &I, r: &I) -> bool { 69 | match self { 70 | Self::Gt => l > r, 71 | Self::Ge => l >= r, 72 | Self::Lt => l < r, 73 | Self::Le => l <= r, 74 | Self::Eq => l == r, 75 | Self::Ne => l != r, 76 | } 77 | } 78 | } 79 | 80 | impl Cmp { 81 | /// String representation of a comparison operation. 82 | pub fn as_str(&self) -> &'static str { 83 | match self { 84 | Self::Lt => "<", 85 | Self::Gt => ">", 86 | Self::Le => "<=", 87 | Self::Ge => ">=", 88 | Self::Eq => "==", 89 | Self::Ne => "!=", 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /jaq-core/src/path.rs: -------------------------------------------------------------------------------- 1 | //! Paths and their parts. 2 | 3 | use crate::box_iter::{box_once, flat_map_with, map_with, then, BoxIter}; 4 | use crate::val::{ValR, ValT, ValX, ValXs}; 5 | use alloc::{boxed::Box, vec::Vec}; 6 | 7 | /// Path such as `.[].a?[1:]`. 8 | #[derive(Clone, Debug)] 9 | pub struct Path(pub Vec<(Part, Opt)>); 10 | 11 | /// Part of a path, such as `[]`, `a`, and `[1:]` in `.[].a?[1:]`. 12 | #[derive(Clone, Debug)] 13 | pub enum Part { 14 | /// Access arrays with integer and objects with string indices 15 | Index(I), 16 | /// Iterate over arrays with optional range bounds and over objects without bounds 17 | /// If both are `None`, return iterator over whole array/object 18 | Range(Option, Option), 19 | } 20 | 21 | /// Optionality of a path part, i.e. whether `?` is present. 22 | /// 23 | /// For example, `[] | .a` fails with an error, while `[] | .a?` returns nothing. 24 | /// By default, path parts are *essential*, meaning that they fail. 25 | /// Annotating them with `?` makes them *optional*. 26 | #[derive(Copy, Clone, Debug)] 27 | pub enum Opt { 28 | /// Return nothing if the input cannot be accessed with the path 29 | Optional, 30 | /// Fail if the input cannot be accessed with the path 31 | Essential, 32 | } 33 | 34 | impl Default for Part { 35 | fn default() -> Self { 36 | Self::Range(None, None) 37 | } 38 | } 39 | 40 | impl Opt { 41 | /// If `self` is optional, return `x`, else fail with `f(x)`. 42 | pub fn fail(self, x: T, f: impl FnOnce(T) -> E) -> Result { 43 | match self { 44 | Self::Optional => Ok(x), 45 | Self::Essential => Err(f(x)), 46 | } 47 | } 48 | } 49 | 50 | impl<'a, U: Clone + 'a, E: Clone + 'a, T: Clone + IntoIterator> + 'a> Path { 51 | pub(crate) fn explode(self) -> impl Iterator, E>> + 'a { 52 | Path(Vec::new()) 53 | .combinations(self.0.into_iter()) 54 | .map(Path::transpose) 55 | } 56 | } 57 | 58 | impl<'a, U: Clone + 'a> Path { 59 | fn combinations(self, mut iter: I) -> BoxIter<'a, Self> 60 | where 61 | I: Iterator, Opt)> + Clone + 'a, 62 | F: IntoIterator + Clone + 'a, 63 | { 64 | if let Some((part, opt)) = iter.next() { 65 | let parts = part.into_iter(); 66 | flat_map_with(parts, (self, iter), move |part, (mut prev, iter)| { 67 | prev.0.push((part, opt)); 68 | prev.combinations(iter) 69 | }) 70 | } else { 71 | box_once(self) 72 | } 73 | } 74 | } 75 | 76 | impl<'a, V: ValT + 'a> Path { 77 | pub(crate) fn run(self, v: V) -> BoxIter<'a, ValR> { 78 | run(self.0.into_iter(), v) 79 | } 80 | 81 | pub(crate) fn update(mut self, v: V, f: F) -> ValX<'a, V> 82 | where 83 | F: Fn(V) -> ValXs<'a, V>, 84 | { 85 | if let Some(last) = self.0.pop() { 86 | update(self.0.into_iter(), last, v, &f) 87 | } else { 88 | // should be unreachable 89 | Ok(v) 90 | } 91 | } 92 | } 93 | 94 | fn run<'a, V: ValT + 'a, I>(mut iter: I, val: V) -> BoxIter<'a, ValR> 95 | where 96 | I: Iterator, Opt)> + Clone + 'a, 97 | { 98 | if let Some((part, opt)) = iter.next() { 99 | let essential = matches!(opt, Opt::Essential); 100 | let ys = part.run(val).filter(move |v| essential || v.is_ok()); 101 | flat_map_with(ys, iter, move |v, iter| then(v, |v| run(iter, v))) 102 | } else { 103 | box_once(Ok(val)) 104 | } 105 | } 106 | 107 | fn update<'a, V: ValT + 'a, P, F>(mut iter: P, last: (Part, Opt), v: V, f: &F) -> ValX<'a, V> 108 | where 109 | P: Iterator, Opt)> + Clone, 110 | F: Fn(V) -> ValXs<'a, V>, 111 | { 112 | if let Some((part, opt)) = iter.next() { 113 | use core::iter::once; 114 | part.update(v, opt, |v| once(update(iter.clone(), last.clone(), v, f))) 115 | } else { 116 | last.0.update(v, last.1, f) 117 | } 118 | } 119 | 120 | impl<'a, V: ValT + 'a> Part { 121 | fn run(&self, v: V) -> impl Iterator> + 'a { 122 | match self { 123 | Self::Index(idx) => box_once(v.index(idx)), 124 | Self::Range(None, None) => Box::new(v.values()), 125 | Self::Range(from, upto) => box_once(v.range(from.as_ref()..upto.as_ref())), 126 | } 127 | } 128 | 129 | fn update(&self, v: V, opt: Opt, f: F) -> ValX<'a, V> 130 | where 131 | F: Fn(V) -> I, 132 | I: Iterator>, 133 | { 134 | match self { 135 | Self::Index(idx) => v.map_index(idx, opt, f), 136 | Self::Range(None, None) => v.map_values(opt, f), 137 | Self::Range(from, upto) => v.map_range(from.as_ref()..upto.as_ref(), opt, f), 138 | } 139 | } 140 | } 141 | 142 | impl<'a, U: Clone + 'a, F: IntoIterator + Clone + 'a> Part { 143 | fn into_iter(self) -> BoxIter<'a, Part> { 144 | use Part::{Index, Range}; 145 | match self { 146 | Index(i) => Box::new(i.into_iter().map(Index)), 147 | Range(None, None) => box_once(Range(None, None)), 148 | Range(Some(from), None) => { 149 | Box::new(from.into_iter().map(|from| Range(Some(from), None))) 150 | } 151 | Range(None, Some(upto)) => { 152 | Box::new(upto.into_iter().map(|upto| Range(None, Some(upto)))) 153 | } 154 | Range(Some(from), Some(upto)) => { 155 | Box::new(flat_map_with(from.into_iter(), upto, move |from, upto| { 156 | map_with(upto.into_iter(), from, move |upto, from| { 157 | Range(Some(from), Some(upto)) 158 | }) 159 | })) 160 | } 161 | } 162 | } 163 | } 164 | 165 | impl Path { 166 | pub(crate) fn map_ref<'a, U>(&'a self, mut f: impl FnMut(&'a T) -> U) -> Path { 167 | let path = self.0.iter(); 168 | let path = path.map(move |(part, opt)| (part.as_ref().map(&mut f), *opt)); 169 | Path(path.collect()) 170 | } 171 | } 172 | 173 | impl Part { 174 | /// Apply a function to the contained indices. 175 | pub(crate) fn map U>(self, mut f: F) -> Part { 176 | use Part::{Index, Range}; 177 | match self { 178 | Index(i) => Index(f(i)), 179 | Range(from, upto) => Range(from.map(&mut f), upto.map(&mut f)), 180 | } 181 | } 182 | } 183 | 184 | impl Path> { 185 | fn transpose(self) -> Result, E> { 186 | self.0 187 | .into_iter() 188 | .map(|(part, opt)| Ok((part.transpose()?, opt))) 189 | .collect::>() 190 | .map(Path) 191 | } 192 | } 193 | 194 | impl Part> { 195 | fn transpose(self) -> Result, E> { 196 | match self { 197 | Self::Index(i) => Ok(Part::Index(i?)), 198 | Self::Range(from, upto) => Ok(Part::Range(from.transpose()?, upto.transpose()?)), 199 | } 200 | } 201 | } 202 | 203 | impl Part { 204 | fn as_ref(&self) -> Part<&F> { 205 | match self { 206 | Self::Index(i) => Part::Index(i), 207 | Self::Range(from, upto) => Part::Range(from.as_ref(), upto.as_ref()), 208 | } 209 | } 210 | } 211 | 212 | impl From> for Path { 213 | fn from(p: Part) -> Self { 214 | Self(Vec::from([(p, Opt::Essential)])) 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /jaq-core/src/rc_iter.rs: -------------------------------------------------------------------------------- 1 | /// A more flexible version of `&mut impl Iterator`. 2 | pub struct RcIter(core::cell::RefCell); 3 | 4 | impl + ?Sized> Iterator for &RcIter { 5 | type Item = T; 6 | fn next(&mut self) -> Option { 7 | self.0.borrow_mut().next() 8 | } 9 | } 10 | 11 | impl RcIter { 12 | /// Construct a new mutable iterator. 13 | pub const fn new(iter: I) -> Self { 14 | Self(core::cell::RefCell::new(iter)) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /jaq-core/src/rc_lazy_list.rs: -------------------------------------------------------------------------------- 1 | use alloc::boxed::Box; 2 | use alloc::rc::Rc; 3 | use once_cell::unsync::Lazy; 4 | 5 | #[derive(Clone)] 6 | pub struct List<'a, T>(Rc, Eval<'a, T>>>); 7 | struct Node<'a, T>(Option<(T, List<'a, T>)>); 8 | type Eval<'a, T> = Box Node<'a, T> + 'a>; 9 | 10 | impl Drop for List<'_, T> { 11 | fn drop(&mut self) { 12 | while let Some((_head, tail)) = Rc::get_mut(&mut self.0) 13 | .and_then(Lazy::get_mut) 14 | .and_then(|node| node.0.take()) 15 | { 16 | *self = tail; 17 | } 18 | } 19 | } 20 | 21 | impl<'a, T> Node<'a, T> { 22 | fn from_iter(mut iter: impl Iterator + 'a) -> Self { 23 | Self(iter.next().map(|x| (x, List::from_iter(iter)))) 24 | } 25 | } 26 | 27 | impl<'a, T> List<'a, T> { 28 | pub fn from_iter(iter: impl Iterator + 'a) -> Self { 29 | Self(Rc::new(Lazy::new(Box::new(|| Node::from_iter(iter))))) 30 | } 31 | } 32 | 33 | impl<'a, T: Clone + 'a> Iterator for List<'a, T> { 34 | type Item = T; 35 | 36 | fn next(&mut self) -> Option { 37 | match &Lazy::force(&self.0).0 { 38 | None => None, 39 | Some((x, xs)) => { 40 | let x = x.clone(); 41 | *self = xs.clone(); 42 | Some(x) 43 | } 44 | } 45 | } 46 | } 47 | 48 | #[test] 49 | fn drop() { 50 | let list = List::from_iter(0..100_000); 51 | // clone() ensures that we keep a copy of the whole list around 52 | // sum() then evaluates the whole list 53 | assert_eq!(list.clone().sum::(), 4999950000); 54 | // at the end, a long, fully evaluated list is dropped, 55 | // which would result in a stack overflow without the custom `Drop` impl 56 | std::mem::drop(list); 57 | } 58 | -------------------------------------------------------------------------------- /jaq-core/src/rc_list.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, PartialEq, Eq)] 2 | pub struct List(alloc::rc::Rc>); 3 | 4 | #[derive(Clone, Debug, PartialEq, Eq)] 5 | enum Node { 6 | Nil, 7 | Cons(T, List), 8 | } 9 | 10 | impl Clone for List { 11 | fn clone(&self) -> Self { 12 | Self(self.0.clone()) 13 | } 14 | } 15 | 16 | impl Default for List { 17 | fn default() -> Self { 18 | Self::new() 19 | } 20 | } 21 | 22 | impl List { 23 | pub fn pop(self) -> Option<(T, Self)> { 24 | match alloc::rc::Rc::try_unwrap(self.0).unwrap_or_else(|rc| (*rc).clone()) { 25 | Node::Nil => None, 26 | Node::Cons(head, tail) => Some((head, tail)), 27 | } 28 | } 29 | } 30 | 31 | impl List { 32 | /// Return an empty list. 33 | pub fn new() -> Self { 34 | Self(Node::Nil.into()) 35 | } 36 | 37 | /// Append a new element to the list. 38 | pub fn cons(self, x: T) -> Self { 39 | Self(Node::Cons(x, self).into()) 40 | } 41 | 42 | /// Repeatedly add elements to the list. 43 | pub fn extend(self, iter: impl IntoIterator) -> Self { 44 | iter.into_iter().fold(self, Self::cons) 45 | } 46 | 47 | /// Return the element most recently added to the list. 48 | pub fn head(&self) -> Option<&T> { 49 | match &*self.0 { 50 | Node::Nil => None, 51 | Node::Cons(x, _) => Some(x), 52 | } 53 | } 54 | 55 | /// Get the `n`-th element from the list, starting from the most recently added. 56 | pub fn get(&self, n: usize) -> Option<&T> { 57 | self.skip(n).head() 58 | } 59 | 60 | /// Remove the `n` top values from the list. 61 | /// 62 | /// If `n` is greater than the number of list elements, return nil. 63 | pub fn skip(&self, n: usize) -> &Self { 64 | let mut cur = self; 65 | for _ in 0..n { 66 | match &*cur.0 { 67 | Node::Cons(_, xs) => cur = xs, 68 | Node::Nil => break, 69 | } 70 | } 71 | cur 72 | } 73 | 74 | #[cfg(test)] 75 | fn iter(&self) -> impl Iterator { 76 | use alloc::boxed::Box; 77 | match &*self.0 { 78 | Node::Cons(x, xs) => Box::new(core::iter::once(x).chain(xs.iter())), 79 | Node::Nil => Box::new(core::iter::empty()) as Box>, 80 | } 81 | } 82 | } 83 | 84 | #[test] 85 | fn test() { 86 | use alloc::{vec, vec::Vec}; 87 | let eq = |l: &List<_>, a| assert_eq!(l.iter().cloned().collect::>(), a); 88 | 89 | let l = List::new().cons(2).cons(1).cons(0); 90 | eq(&l, vec![0, 1, 2]); 91 | 92 | eq(l.skip(0), vec![0, 1, 2]); 93 | eq(l.skip(1), vec![1, 2]); 94 | eq(l.skip(2), vec![2]); 95 | eq(l.skip(3), vec![]); 96 | 97 | assert_eq!(l.get(0), Some(&0)); 98 | assert_eq!(l.get(1), Some(&1)); 99 | assert_eq!(l.get(2), Some(&2)); 100 | assert_eq!(l.get(3), None); 101 | } 102 | -------------------------------------------------------------------------------- /jaq-core/src/stack.rs: -------------------------------------------------------------------------------- 1 | use alloc::vec::Vec; 2 | use core::ops::ControlFlow; 3 | 4 | pub struct Stack(Vec, F); 5 | 6 | impl Stack { 7 | pub fn new(v: Vec, f: F) -> Self { 8 | Self(v, f) 9 | } 10 | } 11 | 12 | /// If `F` returns `Break(x)`, then `x` is returned 13 | /// If `F` returns `Continue(iter)`, then the iterator is pushed onto the stack 14 | impl ControlFlow> Iterator for Stack { 15 | type Item = I::Item; 16 | 17 | fn next(&mut self) -> Option { 18 | // uncomment this to verify that the stack does not grow 19 | //println!("stack size: {}", self.0.len()); 20 | loop { 21 | let mut top = self.0.pop()?; 22 | if let Some(next) = top.next() { 23 | // try not to grow the stack with empty iterators left behind 24 | if top.size_hint() != (0, Some(0)) { 25 | self.0.push(top); 26 | } 27 | match self.1(next) { 28 | ControlFlow::Break(next) => return Some(next), 29 | ControlFlow::Continue(iter) => self.0.push(iter), 30 | } 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /jaq-core/src/val.rs: -------------------------------------------------------------------------------- 1 | //! Values that can be processed by jaq. 2 | //! 3 | //! To process your own value type with jaq, 4 | //! you need to implement the [`ValT`] trait. 5 | 6 | use crate::box_iter::BoxIter; 7 | use crate::path::Opt; 8 | use core::fmt::Display; 9 | use core::ops::{Add, Div, Mul, Neg, Rem, Sub}; 10 | 11 | // Makes `f64::from_str` accessible as intra-doc link. 12 | #[cfg(doc)] 13 | use core::str::FromStr; 14 | 15 | /// Value or eRror. 16 | pub type ValR = Result>; 17 | /// Value or eXception. 18 | pub type ValX<'a, V> = Result>; 19 | /// Stream of values and eXceptions. 20 | pub type ValXs<'a, V> = BoxIter<'a, ValX<'a, V>>; 21 | 22 | /// Range of options, used for iteration operations. 23 | pub type Range = core::ops::Range>; 24 | 25 | /// Values that can be processed by jaq. 26 | /// 27 | /// Implement this trait if you want jaq to process your own type of values. 28 | pub trait ValT: 29 | Clone 30 | + Display 31 | + From 32 | + From 33 | + From 34 | + FromIterator 35 | + PartialEq 36 | + PartialOrd 37 | + Add> 38 | + Sub> 39 | + Mul> 40 | + Div> 41 | + Rem> 42 | + Neg> 43 | { 44 | /// Create a number from a string. 45 | /// 46 | /// The number should adhere to the format accepted by [`f64::from_str`]. 47 | fn from_num(n: &str) -> ValR; 48 | 49 | /// Create an associative map (or object) from a sequence of key-value pairs. 50 | /// 51 | /// This is used when creating values with the syntax `{k: v}`. 52 | fn from_map>(iter: I) -> ValR; 53 | 54 | /// Yield the children of a value. 55 | /// 56 | /// This is used by `.[]`. 57 | fn values(self) -> alloc::boxed::Box>>; 58 | 59 | /// Yield the child of a value at the given index. 60 | /// 61 | /// This is used by `.[k]`. 62 | /// 63 | /// If `v.index(k)` is `Ok(_)`, then it is contained in `v.values()`. 64 | fn index(self, index: &Self) -> ValR; 65 | 66 | /// Yield a slice of the value with the given range. 67 | /// 68 | /// This is used by `.[s:e]`, `.[s:]`, and `.[:e]`. 69 | fn range(self, range: Range<&Self>) -> ValR; 70 | 71 | /// Map a function over the children of the value. 72 | /// 73 | /// This is used by 74 | /// - `.[] |= f` (`opt` = [`Opt::Essential`]) and 75 | /// - `.[]? |= f` (`opt` = [`Opt::Optional`]). 76 | /// 77 | /// If the children of the value are undefined, then: 78 | /// 79 | /// - If `opt` is [`Opt::Essential`], return an error. 80 | /// - If `opt` is [`Opt::Optional`] , return the input value. 81 | fn map_values<'a, I: Iterator>>( 82 | self, 83 | opt: Opt, 84 | f: impl Fn(Self) -> I, 85 | ) -> ValX<'a, Self>; 86 | 87 | /// Map a function over the child of the value at the given index. 88 | /// 89 | /// This is used by `.[k] |= f`. 90 | /// 91 | /// See [`Self::map_values`] for the behaviour of `opt`. 92 | fn map_index<'a, I: Iterator>>( 93 | self, 94 | index: &Self, 95 | opt: Opt, 96 | f: impl Fn(Self) -> I, 97 | ) -> ValX<'a, Self>; 98 | 99 | /// Map a function over the slice of the value with the given range. 100 | /// 101 | /// This is used by `.[s:e] |= f`, `.[s:] |= f`, and `.[:e] |= f`. 102 | /// 103 | /// See [`Self::map_values`] for the behaviour of `opt`. 104 | fn map_range<'a, I: Iterator>>( 105 | self, 106 | range: Range<&Self>, 107 | opt: Opt, 108 | f: impl Fn(Self) -> I, 109 | ) -> ValX<'a, Self>; 110 | 111 | /// Return a boolean representation of the value. 112 | /// 113 | /// This is used by `if v then ...`. 114 | fn as_bool(&self) -> bool; 115 | 116 | /// If the value is a string, return it. 117 | /// 118 | /// If `v.as_str()` yields `Some(s)`, then 119 | /// `"\(v)"` yields `s`, otherwise it yields `v.to_string()` 120 | /// (provided by [`Display`]). 121 | fn as_str(&self) -> Option<&str>; 122 | } 123 | -------------------------------------------------------------------------------- /jaq-core/tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | use jaq_json::{Error, Val, ValR}; 2 | use serde_json::Value; 3 | 4 | fn yields(x: Val, code: &str, ys: impl Iterator) { 5 | use jaq_core::load::{Arena, File, Loader}; 6 | use jaq_core::{Compiler, Native}; 7 | eprintln!("{}", code.replace('\n', " ")); 8 | 9 | let arena = Arena::default(); 10 | let loader = Loader::new([]); 11 | let modules = loader.load(&arena, File { path: (), code }).unwrap(); 12 | let filter = Compiler::<_, Native<_>>::default() 13 | .compile(modules) 14 | .unwrap(); 15 | filter.yields(x, ys) 16 | } 17 | 18 | pub fn fail(x: Value, f: &str, err: Error) { 19 | yields(x.into(), f, core::iter::once(Err(err))) 20 | } 21 | 22 | pub fn give(x: Value, f: &str, y: Value) { 23 | yields(x.into(), f, core::iter::once(Ok(y.into()))) 24 | } 25 | 26 | pub fn gives(x: Value, f: &str, ys: [Value; N]) { 27 | yields(x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) 28 | } 29 | 30 | #[macro_export] 31 | macro_rules! yields { 32 | ($func_name:ident, $filter:expr, $output:expr) => { 33 | #[test] 34 | fn $func_name() { 35 | give(json!(null), $filter, json!($output)) 36 | } 37 | }; 38 | } 39 | -------------------------------------------------------------------------------- /jaq-core/tests/path.rs: -------------------------------------------------------------------------------- 1 | pub mod common; 2 | 3 | use common::{fail, give, gives}; 4 | use jaq_core::Error; 5 | use serde_json::json; 6 | 7 | #[test] 8 | fn index_access() { 9 | give(json!([0, 1, 2]), ".[-4]", json!(null)); 10 | give(json!([0, 1, 2]), ".[-3]", json!(0)); 11 | give(json!([0, 1, 2]), ".[-1]", json!(2)); 12 | give(json!([0, 1, 2]), ".[ 0]", json!(0)); 13 | give(json!([0, 1, 2]), ".[ 2]", json!(2)); 14 | give(json!([0, 1, 2]), ".[ 3]", json!(null)); 15 | 16 | give(json!({"a": 1}), ".a", json!(1)); 17 | give(json!({"a": 1}), ".a?", json!(1)); 18 | give(json!({"a": 1}), ".a ?", json!(1)); 19 | give(json!({"a": 1}), r#"."a""#, json!(1)); 20 | give(json!({"a": 1}), r#". "a""#, json!(1)); 21 | give(json!({"a": 1}), r#".["a"]"#, json!(1)); 22 | give(json!({"a": 1}), r#". ["a"]"#, json!(1)); 23 | give(json!({"a_": 1}), ".a_", json!(1)); 24 | give(json!({"_a": 1}), "._a", json!(1)); 25 | give(json!({"_0": 1}), "._0", json!(1)); 26 | 27 | give(json!({"a": 1}), r#".[0, "a", 0 == 0]?"#, json!(1)); 28 | give(json!([0, 1, 2]), r#".["a", 0, 0 == 0]?"#, json!(0)); 29 | give(json!([0, 1, 2]), r#".[3]?"#, json!(null)); 30 | gives(json!("asdf"), ".[0]?", []); 31 | 32 | give(json!(1), "[1, 2, 3][.]", json!(2)); 33 | 34 | gives( 35 | json!({"a": 1, "b": 2}), 36 | r#".["b", "a"]"#, 37 | [json!(2), json!(1)], 38 | ); 39 | } 40 | 41 | #[test] 42 | fn iter_access() { 43 | gives(json!([0, 1, 2]), ".[]", [json!(0), json!(1), json!(2)]); 44 | gives(json!({"a": [1, 2]}), ".a[]", [json!(1), json!(2)]); 45 | gives(json!({"a": 1, "b": 2}), ".[]", [json!(1), json!(2)]); 46 | // TODO: correct this 47 | //gives(json!({"b": 2, "a": 1}), ".[]", [json!(2), json!(1)]); 48 | gives(json!("asdf"), ".[]?", []); 49 | } 50 | 51 | #[test] 52 | fn range_access() { 53 | give(json!("Möwe"), ".[1:-1]", json!("öw")); 54 | give(json!("नमस्ते"), ".[1:5]", json!("मस्त")); 55 | 56 | give(json!([0, 1, 2]), ".[-4:4]", json!([0, 1, 2])); 57 | give(json!([0, 1, 2]), ".[0:3]", json!([0, 1, 2])); 58 | give(json!([0, 1, 2]), ".[1:]", json!([1, 2])); 59 | give(json!([0, 1, 2]), ".[:-1]", json!([0, 1])); 60 | give(json!([0, 1, 2]), ".[1:0]", json!([])); 61 | give(json!([0, 1, 2]), ".[4:5]", json!([])); 62 | 63 | give(json!([0, 1, 2]), ".[0:2,3.14]?", json!([0, 1])); 64 | } 65 | 66 | #[test] 67 | fn iter_assign() { 68 | give(json!([1, 2]), ".[] = .", json!([[1, 2], [1, 2]])); 69 | give( 70 | json!({"a": [1,2], "b": 3}), 71 | ".a[] = .b+.b", 72 | json!({"a": [6,6], "b": 3}), 73 | ); 74 | } 75 | 76 | yields!(index_keyword, r#"{"if": 0} | .if"#, 0); 77 | yields!(obj_keyword, "{if: 0} | .if", 0); 78 | 79 | yields!(key_update1, "{} | .a |= .+1", json!({"a": 1})); 80 | yields!(key_update2, "{} | .a? |= .+1", json!({"a": 1})); 81 | 82 | // `.[f]?` is *not* the same as `(.[f])?` 83 | // (we're writing `0[]` to simulate `error` here) 84 | yields!(index_opt_inner, "try .[0[]]? catch 1", 1); 85 | yields!(index_opt_outer, "1, (.[0[]])?", 1); 86 | 87 | #[test] 88 | fn index_update() { 89 | give(json!({"a": 1}), ".b |= .", json!({"a": 1, "b": null})); 90 | give(json!({"a": 1}), ".b |= 1", json!({"a": 1, "b": 1})); 91 | give(json!({"a": 1}), ".b |= .+1", json!({"a": 1, "b": 1})); 92 | give(json!({"a": 1, "b": 2}), ".b |= {}[]", json!({"a": 1})); 93 | give(json!({"a": 1, "b": 2}), ".a += 1", json!({"a": 2, "b": 2})); 94 | 95 | give(json!([0, 1, 2]), ".[1] |= .+2", json!([0, 3, 2])); 96 | give(json!([0, 1, 2]), ".[-1,-1] |= {}[]", json!([0])); 97 | give(json!([0, 1, 2]), ".[ 0, 0] |= {}[]", json!([2])); 98 | 99 | /* TODO: reenable these tests 100 | use Error::IndexOutOfBounds as Oob; 101 | fail(json!([0, 1, 2]), ".[ 3] |= 3", Oob(3)); 102 | fail(json!([0, 1, 2]), ".[-4] |= -1", Oob(-4)); 103 | */ 104 | 105 | give(json!({"a": 1}), r#".[0, "a"]? |= .+1"#, json!({"a": 2})); 106 | give(json!([0, 1, 2]), r#".["a", 0]? |= .+1"#, json!([1, 1, 2])); 107 | give(json!([0, 1, 2]), r#".[3]? |= .+1"#, json!([0, 1, 2])); 108 | give(json!("asdf"), ".[0]? |= .+1", json!("asdf")); 109 | } 110 | 111 | #[test] 112 | fn iter_update() { 113 | // precedence tests 114 | give(json!([]), ".[] |= . or 0", json!([])); 115 | gives(json!([]), ".[] |= .,.", [json!([]), json!([])]); 116 | give(json!([]), ".[] |= (.,.)", json!([])); 117 | give(json!([0]), ".[] |= .+1 | .+[2]", json!([1, 2])); 118 | // this yields a syntax error in jq, but it is consistent to permit this 119 | give(json!([[1]]), ".[] |= .[] |= .+1", json!([[2]])); 120 | // ditto 121 | give(json!([[1]]), ".[] |= .[] += 1", json!([[2]])); 122 | 123 | give(json!([1]), ".[] |= .+1", json!([2])); 124 | give(json!([[1]]), ".[][] |= .+1", json!([[2]])); 125 | 126 | give( 127 | json!({"a": 1, "b": 2}), 128 | ".[] |= ((if .>1 then . else {}[] end) | .+1)", 129 | json!({"b": 3}), 130 | ); 131 | 132 | give(json!([[0, 1], "a"]), ".[][]? |= .+1", json!([[1, 2], "a"])); 133 | } 134 | 135 | #[test] 136 | fn range_update() { 137 | give(json!([0, 1, 2]), ".[:2] |= [.[] | .+5]", json!([5, 6, 2])); 138 | give(json!([0, 1, 2]), ".[-2:-1] |= [5]+.", json!([0, 5, 1, 2])); 139 | give( 140 | json!([0, 1, 2]), 141 | ".[-2:-1,-1] |= [5,6]+.", 142 | json!([0, 5, 6, 5, 6, 1, 2]), 143 | ); 144 | 145 | give( 146 | json!([0, 1, 2]), 147 | ".[:2,3.0]? |= [.[] | .+1]", 148 | json!([1, 2, 2]), 149 | ); 150 | } 151 | 152 | // Test what happens when update filter returns multiple values. 153 | // Watch out: here, jaq diverges frequently from jq; 154 | // jq considers only the first value of the filter regardless of the updated value, 155 | // whereas jaq may consider multiple values depending on the updated value. 156 | // This behaviour is easier to implement and more flexible. 157 | #[test] 158 | fn update_mult() { 159 | // first the cases where jaq and jq agree 160 | give(json!({"a": 1}), ".a |= (.,.+1)", json!({"a": 1})); 161 | 162 | // jq returns null here 163 | gives(json!(1), ". |= {}[]", []); 164 | // jq returns just 1 here 165 | gives(json!(1), ". |= (.,.)", [json!(1), json!(1)]); 166 | // jq returns just [1] here 167 | give(json!([1]), ".[] |= (., .+1)", json!([1, 2])); 168 | // jq returns just [1,2] here 169 | give(json!([1, 3]), ".[] |= (., .+1)", json!([1, 2, 3, 4])); 170 | // here comes a huge WTF: jq returns [2,4] here 171 | // this is a known bug: 172 | give(json!([1, 2, 3, 4, 5]), ".[] |= {}[]", json!([])); 173 | } 174 | 175 | #[test] 176 | fn update_complex() { 177 | // jq returns 1 here, which looks like a bug 178 | // in general, `a | a |= .` 179 | // works in jq when `a` is either null, a number, or a boolean --- it 180 | // does *not* work when `a` is a string, an array, or an object! 181 | fail(json!(0), "0 |= .+1", Error::path_expr()); 182 | } 183 | 184 | yields!(alt_update_l, "[1, 2] | .[] // . |= 3", [3, 3]); 185 | yields!(alt_update_r, "[] | .[] // . |= 3", 3); 186 | -------------------------------------------------------------------------------- /jaq-json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jaq-json" 3 | version = "1.1.2" 4 | authors = ["Michael Färber "] 5 | edition = "2021" 6 | license = "MIT" 7 | readme = "../README.md" 8 | description = "JSON values for jaq" 9 | repository = "https://github.com/01mf02/jaq" 10 | keywords = ["json", "query", "jq"] 11 | categories = ["data-structures"] 12 | rust-version = "1.65" 13 | 14 | [features] 15 | default = ["parse"] 16 | parse = ["hifijson"] 17 | 18 | [dependencies] 19 | jaq-core = { version = "2.1.0", path = "../jaq-core" } 20 | jaq-std = { version = "2.1.0", path = "../jaq-std" } 21 | 22 | foldhash = { version = "0.1", default-features = false } 23 | hifijson = { version = "0.2.0", default-features = false, features = ["alloc"], optional = true } 24 | indexmap = { version = "2.0", default-features = false } 25 | serde_json = { version = "1.0.81", default-features = false, optional = true } 26 | -------------------------------------------------------------------------------- /jaq-json/src/defs.jq: -------------------------------------------------------------------------------- 1 | # Conversion 2 | def tonumber: if isnumber then . else fromjson end; 3 | 4 | # Arrays 5 | def transpose: [range([.[] | length] | max) as $i | [.[][$i]]]; 6 | 7 | # Objects <-> Arrays 8 | def keys: keys_unsorted | sort; 9 | def to_entries: [keys_unsorted[] as $k | { key: $k, value: .[$k] }]; 10 | def from_entries: map({ (.key): .value }) | add + {}; 11 | def with_entries(f): to_entries | map(f) | from_entries; 12 | 13 | # Paths 14 | def paths(f): path_values | if .[1] | f then .[0] else empty end; 15 | 16 | # Indexing 17 | def in(xs) : . as $x | xs | has ($x); 18 | def inside(xs): . as $x | xs | contains($x); 19 | def index($i): indices($i)[ 0]; 20 | def rindex($i): indices($i)[-1]; 21 | 22 | # Formatting 23 | def @json: tojson; 24 | 25 | -------------------------------------------------------------------------------- /jaq-json/tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | use jaq_json::{Error, Val, ValR}; 2 | use serde_json::Value; 3 | 4 | fn yields(x: Val, code: &str, ys: impl Iterator) { 5 | use jaq_core::load::{Arena, File, Loader}; 6 | eprintln!("{}", code.replace('\n', " ")); 7 | 8 | let arena = Arena::default(); 9 | let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); 10 | let modules = loader.load(&arena, File { path: (), code }).unwrap(); 11 | let filter = jaq_core::Compiler::default() 12 | .with_funs(jaq_std::funs().chain(jaq_json::funs())) 13 | .compile(modules) 14 | .unwrap(); 15 | filter.yields(x, ys) 16 | } 17 | 18 | pub fn fail(x: Value, f: &str, err: Error) { 19 | yields(x.into(), f, core::iter::once(Err(err))) 20 | } 21 | 22 | pub fn give(x: Value, f: &str, y: Value) { 23 | yields(x.into(), f, core::iter::once(Ok(y.into()))) 24 | } 25 | 26 | pub fn gives(x: Value, f: &str, ys: [Value; N]) { 27 | yields(x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) 28 | } 29 | 30 | #[macro_export] 31 | macro_rules! yields { 32 | ($func_name:ident, $filter:expr, $output: expr) => { 33 | #[test] 34 | fn $func_name() { 35 | give(json!(null), $filter, json!($output)) 36 | } 37 | }; 38 | } 39 | -------------------------------------------------------------------------------- /jaq-json/tests/defs.rs: -------------------------------------------------------------------------------- 1 | //! Tests for filters in the standard library, sorted by name. 2 | 3 | pub mod common; 4 | 5 | use common::give; 6 | use serde_json::json; 7 | 8 | #[test] 9 | fn entries() { 10 | let obj = json!({"a": 1, "b": 2}); 11 | let entries = json!([{"key": "a", "value": 1}, {"key": "b", "value": 2}]); 12 | let objk = json!({"ak": 1, "bk": 2}); 13 | 14 | give(obj.clone(), "to_entries", entries.clone()); 15 | give(entries, "from_entries", obj.clone()); 16 | give(obj, r#"with_entries(.key += "k")"#, objk); 17 | 18 | let arr = json!([null, 0]); 19 | let entries = json!([{"key": 0, "value": null}, {"key": 1, "value": 0}]); 20 | give(arr, "to_entries", entries); 21 | 22 | give(json!([]), "from_entries", json!({})); 23 | } 24 | 25 | #[test] 26 | fn inside() { 27 | give( 28 | json!(["foo", "bar"]), 29 | r#"map(in({"foo": 42}))"#, 30 | json!([true, false]), 31 | ); 32 | give(json!([2, 0]), r#"map(in([0,1]))"#, json!([false, true])); 33 | 34 | give(json!("bar"), r#"inside("foobar")"#, json!(true)); 35 | 36 | let f = r#"inside(["foobar", "foobaz", "blarp"])"#; 37 | give(json!(["baz", "bar"]), f, json!(true)); 38 | give(json!(["bazzzz", "bar"]), f, json!(false)); 39 | 40 | let f = r#"inside({"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]})"#; 41 | give(json!({"foo": 12, "bar": [{"barp": 12}]}), f, json!(true)); 42 | give(json!({"foo": 12, "bar": [{"barp": 15}]}), f, json!(false)); 43 | } 44 | 45 | yields!( 46 | keys, 47 | r#"{"foo":null,"abc":null,"fax":null,"az":null} | keys"#, 48 | ["abc", "az", "fax", "foo"] 49 | ); 50 | 51 | yields!(paths_num, "1 | [paths]", json!([])); 52 | yields!(paths_null, "null | [paths]", json!([])); 53 | yields!(paths_arr, "[1, 2] | [paths]", [[0], [1]]); 54 | yields!( 55 | paths_arr_obj, 56 | "{a: [1, [2]], b: {c: 3}} | [paths]", 57 | json!([["a"], ["a", 0], ["a", 1], ["a", 1, 0], ["b"], ["b", "c"]]) 58 | ); 59 | 60 | yields!( 61 | paths_filter, 62 | "[{a: 1, b: [2, 3]} | paths(. < [])]", 63 | json!([["a"], ["b", 0], ["b", 1]]) 64 | ); 65 | 66 | const RECURSE_PATHS: &str = "def paths: 67 | { x: ., p: [] } | 68 | recurse((.x | keys_unsorted?)[] as $k | .x |= .[$k] | .p += [$k]) | 69 | .p | if . == [] then empty else . end;"; 70 | 71 | yields!( 72 | recurse_paths, 73 | &(RECURSE_PATHS.to_owned() + "{a: [1, [2]], b: {c: 3}} | [paths]"), 74 | json!([["a"], ["a", 0], ["a", 1], ["a", 1, 0], ["b"], ["b", "c"]]) 75 | ); 76 | 77 | #[test] 78 | fn transpose() { 79 | let y = json!([[1, 2], [3, null]]); 80 | give(json!([[1, 3], [2]]), "transpose", y); 81 | 82 | let y = json!([[1, 2], [3, 4]]); 83 | give(json!([[1, 3], [2, 4]]), "transpose", y); 84 | } 85 | 86 | #[test] 87 | fn walk() { 88 | give( 89 | json!([[4, 1, 7], [8, 5, 2], [3, 6, 9]]), 90 | r#"walk(if . < [] then . else sort end)"#, 91 | json!([[1, 4, 7], [2, 5, 8], [3, 6, 9]]), 92 | ); 93 | 94 | give( 95 | json!({"a": {"b": 1, "c": 2}}), 96 | r#"walk(if . < {} then . + 1 else . + {"l": length} end)"#, 97 | json!({"a": {"b": 2, "c": 3, "l": 2}, "l": 1}), 98 | ); 99 | } 100 | 101 | #[test] 102 | fn while_until() { 103 | give( 104 | json!(1), 105 | "[while(. < 100; . * 2)]", 106 | json!([1, 2, 4, 8, 16, 32, 64]), 107 | ); 108 | give( 109 | json!("a"), 110 | "[while(length < 4; . + \"a\")]", 111 | json!(["a", "aa", "aaa"]), 112 | ); 113 | give( 114 | json!([1, 2, 3]), 115 | "[while(length > 0; .[1:])]", 116 | json!([[1, 2, 3], [2, 3], [3]]), 117 | ); 118 | 119 | give(json!(50), "until(. > 100; . * 2)", json!(200)); 120 | give( 121 | json!([1, 2, 3]), 122 | "until(length == 1; .[1:]) | .[0]", 123 | json!(3), 124 | ); 125 | give( 126 | json!(5), 127 | "[.,1] | until(.[0] < 1; [.[0] - 1, .[1] * .[0]]) | .[1]", 128 | json!(120), 129 | ); 130 | } 131 | 132 | yields!( 133 | format_json, 134 | r#"[0, 0 == 0, {}.a, "hello", {}, [] | @json]"#, 135 | ["0", "true", "null", "\"hello\"", "{}", "[]"] 136 | ); 137 | -------------------------------------------------------------------------------- /jaq-json/tests/funs.rs: -------------------------------------------------------------------------------- 1 | //! Tests for named core filters, sorted by name. 2 | 3 | pub mod common; 4 | 5 | use common::give; 6 | use serde_json::json; 7 | 8 | #[test] 9 | fn has() { 10 | /* TODO: reenable these tests 11 | let err = Error::Index(Val::Null, Val::Int(0)); 12 | fail(json!(null), "has(0)", err); 13 | let err = Error::Index(Val::Int(0), Val::Null); 14 | fail(json!(0), "has([][0])", err); 15 | let err = Error::Index(Val::Int(0), Val::Int(1)); 16 | fail(json!(0), "has(1)", err); 17 | let err = Error::Index(Val::Str("a".to_string().into()), Val::Int(0)); 18 | fail(json!("a"), "has(0)", err); 19 | */ 20 | 21 | give(json!([0, null]), "has(0)", json!(true)); 22 | give(json!([0, null]), "has(1)", json!(true)); 23 | give(json!([0, null]), "has(2)", json!(false)); 24 | 25 | give(json!({"a": 1, "b": null}), r#"has("a")"#, json!(true)); 26 | give(json!({"a": 1, "b": null}), r#"has("b")"#, json!(true)); 27 | give(json!({"a": 1, "b": null}), r#"has("c")"#, json!(false)); 28 | } 29 | 30 | yields!(indices_str, r#""a,b, cd, efg" | indices(", ")"#, [3, 7]); 31 | yields!( 32 | indices_arr_num, 33 | "[0, 1, 2, 1, 3, 1, 4] | indices(1)", 34 | [1, 3, 5] 35 | ); 36 | yields!( 37 | indices_arr_arr, 38 | "[0, 1, 2, 3, 1, 4, 2, 5, 1, 2, 6, 7] | indices([1, 2])", 39 | [1, 8] 40 | ); 41 | yields!(indices_arr_str, r#"["a", "b", "c"] | indices("b")"#, [1]); 42 | 43 | yields!(indices_arr_empty, "[0, 1] | indices([])", json!([])); 44 | yields!(indices_arr_larger, "[1, 2] | indices([1, 2, 3])", json!([])); 45 | 46 | yields!(indices_arr_overlap, "[0, 0, 0] | indices([0, 0])", [0, 1]); 47 | yields!(indices_str_overlap, r#""aaa" | indices("aa")"#, [0, 1]); 48 | yields!(indices_str_gb1, r#""🇬🇧!" | indices("!")"#, [2]); 49 | yields!(indices_str_gb2, r#""🇬🇧🇬🇧" | indices("🇬🇧")"#, [0, 2]); 50 | 51 | #[test] 52 | fn keys_unsorted() { 53 | give(json!([0, null, "a"]), "keys_unsorted", json!([0, 1, 2])); 54 | give(json!({"a": 1, "b": 2}), "keys_unsorted", json!(["a", "b"])); 55 | 56 | /* TODO: reenable these tests 57 | let err = |v| Error::Type(v, Type::Iter); 58 | fail(json!(0), "keys_unsorted", err(Val::Int(0))); 59 | fail(json!(null), "keys_unsorted", err(Val::Null)); 60 | */ 61 | } 62 | 63 | yields!(length_str_foo, r#""ƒoo" | length"#, 3); 64 | yields!(length_str_namaste, r#""नमस्ते" | length"#, 6); 65 | yields!(length_obj, r#"{"a": 5, "b": 3} | length"#, 2); 66 | yields!(length_int_pos, " 2 | length", 2); 67 | yields!(length_int_neg, "-2 | length", 2); 68 | yields!(length_float_pos, " 2.5 | length", 2.5); 69 | yields!(length_float_neg, "-2.5 | length", 2.5); 70 | 71 | #[test] 72 | fn tojson() { 73 | // TODO: correct this 74 | give(json!(1.0), "tojson", json!("1.0")); 75 | give(json!(0), "1.0 | tojson", json!("1.0")); 76 | give(json!(0), "1.1 | tojson", json!("1.1")); 77 | give(json!(0), "0.0 / 0.0 | tojson", json!("null")); 78 | give(json!(0), "1.0 / 0.0 | tojson", json!("null")); 79 | } 80 | 81 | #[test] 82 | fn math_rem() { 83 | // generated with this command with modification for errors and float rounding 84 | // cargo run -- -rn 'def f: -2, -1, 0, 2.1, 3, 2000000001; f as $a | f as $b | "give!(json!(null), \"\($a) / \($b)\", \(try ($a % $b) catch tojson));"' 85 | // TODO: use fail!()? 86 | give(json!(null), "-2 % -2", json!(0)); 87 | give(json!(null), "-2 % -1", json!(0)); 88 | give( 89 | json!(null), 90 | "try (-2 % 0) catch .", 91 | json!("cannot calculate -2 % 0"), 92 | ); 93 | give(json!(null), "-2 % 2.1", json!(-2.0)); 94 | give(json!(null), "-2 % 3", json!(-2)); 95 | give(json!(null), "-2 % 2000000001", json!(-2)); 96 | give(json!(null), "-1 % -2", json!(-1)); 97 | give(json!(null), "-1 % -1", json!(0)); 98 | give( 99 | json!(null), 100 | "try (-1 % 0) catch .", 101 | json!("cannot calculate -1 % 0"), 102 | ); 103 | give(json!(null), "-1 % 2.1", json!(-1.0)); 104 | give(json!(null), "-1 % 3", json!(-1)); 105 | give(json!(null), "-1 % 2000000001", json!(-1)); 106 | give(json!(null), "0 % -2", json!(0)); 107 | give(json!(null), "0 % -1", json!(0)); 108 | give( 109 | json!(null), 110 | "try (0 % 0) catch .", 111 | json!("cannot calculate 0 % 0"), 112 | ); 113 | give(json!(null), "0 % 2.1", json!(0.0)); 114 | give(json!(null), "0 % 3", json!(0)); 115 | give(json!(null), "0 % 2000000001", json!(0)); 116 | give(json!(null), "2.1 % -2 | . * 1000 | round", json!(100)); 117 | give(json!(null), "2.1 % -1 | . * 1000 | round", json!(100)); 118 | give(json!(null), "2.1 % 0 | isnan", json!(true)); 119 | give(json!(null), "2.1 % 2.1", json!(0.0)); 120 | give(json!(null), "2.1 % 3", json!(2.1)); 121 | give(json!(null), "2.1 % 2000000001", json!(2.1)); 122 | give(json!(null), "3 % -2", json!(1)); 123 | give(json!(null), "3 % -1", json!(0)); 124 | give( 125 | json!(null), 126 | "try (3 % 0) catch .", 127 | json!("cannot calculate 3 % 0"), 128 | ); 129 | give(json!(null), "3 % 2.1 | . * 1000 | round", json!(900)); 130 | give(json!(null), "3 % 3", json!(0)); 131 | give(json!(null), "3 % 2000000001", json!(3)); 132 | give(json!(null), "2000000001 % -2", json!(1)); 133 | give(json!(null), "2000000001 % -1", json!(0)); 134 | give( 135 | json!(null), 136 | "try (2000000001 % 0) catch .", 137 | json!("cannot calculate 2000000001 % 0"), 138 | ); 139 | give( 140 | json!(null), 141 | "2000000001 % 2.1 | . * 1000 | round", 142 | json!(1800), // 1000 in jq 143 | ); 144 | give(json!(null), "2000000001 % 3", json!(0)); 145 | give(json!(null), "2000000001 % 2000000001", json!(0)); 146 | } 147 | 148 | yields!( 149 | path_values, 150 | "[{a: 1, b: [2, 3]} | path_values]", 151 | json!([[["a"], 1], [["b"], [2, 3]], [["b", 0], 2], [["b", 1], 3]]) 152 | ); 153 | -------------------------------------------------------------------------------- /jaq-play/.gitignore: -------------------------------------------------------------------------------- 1 | pkg 2 | -------------------------------------------------------------------------------- /jaq-play/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jaq-play" 3 | description = "Web interface for jaq" 4 | version = "0.0.0" 5 | authors = ["Michael Färber "] 6 | categories = ["wasm"] 7 | readme = "README.md" 8 | repository = "https://github.com/01mf02/jaq" 9 | license = "MIT" 10 | edition = "2021" 11 | 12 | [package.metadata.wasm-pack.profile.dev] 13 | wasm-opt = false 14 | 15 | [lib] 16 | crate-type = ["cdylib", "rlib"] 17 | 18 | [dependencies] 19 | jaq-core = { path = "../jaq-core" } 20 | jaq-std = { path = "../jaq-std" } 21 | jaq-json = { path = "../jaq-json" } 22 | 23 | aho-corasick = "1.1.2" 24 | codesnake = { version = "0.2" } 25 | hifijson = "0.2" 26 | log = "0.4.17" 27 | unicode-width = "0.1.13" 28 | 29 | console_log = { version = "1.0", features = ["color"] } 30 | getrandom = { version = "0.2", features = ["js"] } 31 | wasm-bindgen = { version = "0.2" } 32 | web-sys = { version = "0.3", features = ["DedicatedWorkerGlobalScope"] } 33 | js-sys = { version = "0.3" } 34 | -------------------------------------------------------------------------------- /jaq-play/README.md: -------------------------------------------------------------------------------- 1 | jaq playground 2 | ============== 3 | 4 | The [jaq playground](https://gedenkt.at/jaq/) 5 | allows executing jq programs with the jq interpreter _jaq_. 6 | The jaq playground processes all data on the machine of its user. 7 | This has a few advantages: 8 | 9 | - *Privacy*: The jaq playground sends no data to the Internet. 10 | - *Speed*: Because your own computer does the processing, 11 | you get answers in milliseconds, not seconds. 12 | - *Offline*: You can use the jaq playground even without an internet connection. 13 | - *Streaming*: 14 | The jaq playground shows you outputs as soon as they arrive, whereas 15 | the jq playground shows you outputs only once there are no more new outputs. 16 | This means that you can process infinite data streams with the jaq playground. 17 | 18 | 19 | ## Usage 20 | 21 | The playground is divided into three main sections: 22 | 23 | - Filter: the jq program that should be executed 24 | - Input: the input that is given to the jq program (by default in JSON) 25 | - Output: the output of the jq program 26 | 27 | When clicking on the "Run" button, 28 | the jq program is executed with the given input, and its output is shown. 29 | 30 | To try it, you can use the filter `.[]` 31 | (which destructures a value into its components) and the following input: 32 | 33 | ~~~ json 34 | ["Hello", "World"] 35 | {"x": 1, "y": 2} 36 | ~~~ 37 | 38 | Running this should yield the following outputs: 39 | 40 | ~~~ json 41 | "Hello" 42 | "World" 43 | 1 44 | 2 45 | ~~~ 46 | 47 | To show that you can yield an infinite output stream, you can use the filter 48 | `range(0; infinite)` which yields the stream of natural numbers (0, 1, 2, ...). 49 | Because there are infinitely many of those, the execution will never terminate. 50 | For that reason, during program execution, 51 | the "Run" button turns into a "Stop" button, 52 | which you can use to terminate execution. 53 | 54 | 55 | ## Settings 56 | 57 | You can configure the execution of jq programs with the "Settings" dialogue. 58 | It has several settings, which influence how inputs are processed and 59 | how the outputs should be displayed. 60 | 61 | ### Input Settings 62 | 63 | - raw: Reads every line as string instead of as JSON value. 64 | - slurp: Collects all input values into an array. 65 | When combined with "raw", reads the whole input as a single string. 66 | - null: Yields only the `null` value as input. 67 | This is particularly useful in conjunction with the 68 | [`input`](https://jqlang.github.io/jq/manual/#input) filter. 69 | 70 | For example, with the filter `.` and the input `[1, 2] "x"`, 71 | no option yields `[1, 2] "x"`, 72 | "raw" yields `"[1, 2] \"x\""`, 73 | "slurp" yields `[[1, 2], "x"]`, and 74 | "null" yields just `null`. 75 | 76 | ### Output Settings 77 | 78 | - raw: Prints output strings without quotes and escaping. 79 | - compact: Prints output values without intermediate spaces. 80 | - tab: Indent with tab characters. This has precedence over "indent". 81 | - indent: Indent with the given amount of white-space characters. 82 | 83 | For example, with the filter `.` and the input `[1, 2] "x"`, 84 | "raw" yields `"[ 1, 2 ] x"`, 85 | "compact" yields `[1,2] "x"`, 86 | "tab" yields `[ 1, 2 ] "x"`, and 87 | "indent" set to 2 yields `[ 1, 2 ] "x"`. 88 | (To make the output more compact, I replaced newline with space.) 89 | 90 | 91 | ## Building 92 | 93 | First install `wasm-pack`: 94 | 95 | cargo install wasm-pack 96 | 97 | Compile the WASM binaries (use `--release` instead of `--dev` for better performance): 98 | 99 | wasm-pack build --target web --no-typescript --no-pack --dev 100 | 101 | To serve: 102 | 103 | python3 -m http.server --bind 127.0.0.1 104 | -------------------------------------------------------------------------------- /jaq-play/github-mark-white.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaq-play/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | jaq playground 10 | 11 | 12 | 13 |

14 | 15 | 16 | 17 |
18 | 19 |
20 |
21 | Input 22 |
23 |
    24 |
  • 25 | 26 | 27 |
    • 28 |
  • 29 | 30 | 31 |
    • 32 |
  • 33 | 34 | 35 |
    • 36 |
37 |
38 |
39 | Output 40 |
41 |
    42 |
  • 43 | 44 | 45 |
    • 46 |
  • 47 | 48 | 49 |
    • 50 | 56 |
  • 57 | 58 | 59 |
    • 60 |
  • 61 | 62 | 63 |
    • 64 |
65 |
66 |
67 |
68 |
69 | 70 | 71 | 72 |
73 |
74 | 75 |
76 |

Filter

77 | 78 |
79 | 80 |
81 |
82 |

Input

83 | 84 |
85 | 86 |
87 |

Output

88 | 

 89 |     
 90 |   
 91 |   
 94 |   
 98 |   
 99 | 
100 | 
101 | 
102 | 


--------------------------------------------------------------------------------
/jaq-play/jaq.svg:
--------------------------------------------------------------------------------
1 | 
2 | 
3 |   
4 |   
5 | 
6 | 


--------------------------------------------------------------------------------
/jaq-play/src/lib.rs:
--------------------------------------------------------------------------------
  1 | #![no_std]
  2 | extern crate alloc;
  3 | 
  4 | use alloc::{borrow::ToOwned, format, string::ToString};
  5 | use alloc::{boxed::Box, string::String, vec::Vec};
  6 | use core::fmt::{self, Debug, Display, Formatter};
  7 | use jaq_core::{compile, load, Ctx, Native, RcIter};
  8 | use jaq_json::{fmt_str, Val};
  9 | use wasm_bindgen::prelude::*;
 10 | 
 11 | type Filter = jaq_core::Filter>;
 12 | 
 13 | struct FormatterFn(F);
 14 | 
 15 | impl fmt::Result> Display for FormatterFn {
 16 |     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
 17 |         self.0(f)
 18 |     }
 19 | }
 20 | 
 21 | struct PpOpts {
 22 |     compact: bool,
 23 |     indent: String,
 24 | }
 25 | 
 26 | impl PpOpts {
 27 |     fn indent(&self, f: &mut Formatter, level: usize) -> fmt::Result {
 28 |         if !self.compact {
 29 |             write!(f, "{}", self.indent.repeat(level))?;
 30 |         }
 31 |         Ok(())
 32 |     }
 33 | 
 34 |     fn newline(&self, f: &mut Formatter) -> fmt::Result {
 35 |         if !self.compact {
 36 |             writeln!(f)?;
 37 |         }
 38 |         Ok(())
 39 |     }
 40 | }
 41 | 
 42 | fn fmt_seq(fmt: &mut Formatter, opts: &PpOpts, level: usize, xs: I, f: F) -> fmt::Result
 43 | where
 44 |     I: IntoIterator,
 45 |     F: Fn(&mut Formatter, T) -> fmt::Result,
 46 | {
 47 |     opts.newline(fmt)?;
 48 |     let mut iter = xs.into_iter().peekable();
 49 |     while let Some(x) = iter.next() {
 50 |         opts.indent(fmt, level + 1)?;
 51 |         f(fmt, x)?;
 52 |         if iter.peek().is_some() {
 53 |             write!(fmt, ",")?;
 54 |         }
 55 |         opts.newline(fmt)?;
 56 |     }
 57 |     opts.indent(fmt, level)
 58 | }
 59 | 
 60 | fn fmt_val(f: &mut Formatter, opts: &PpOpts, level: usize, v: &Val) -> fmt::Result {
 61 |     let display = |s| FormatterFn(move |f: &mut Formatter| fmt_str(f, &escape(s)));
 62 |     match v {
 63 |         Val::Null => span(f, "null", "null"),
 64 |         Val::Bool(b) => span(f, "boolean", b),
 65 |         Val::Int(i) => span(f, "number", i),
 66 |         Val::Float(x) if x.is_finite() => span_dbg(f, "number", x),
 67 |         Val::Float(_) => span(f, "null", "null"),
 68 |         Val::Num(n) => span(f, "number", n),
 69 |         Val::Str(s) => span(f, "string", display(s)),
 70 |         Val::Arr(a) if a.is_empty() => write!(f, "[]"),
 71 |         Val::Arr(a) => {
 72 |             write!(f, "[")?;
 73 |             fmt_seq(f, opts, level, &**a, |f, x| fmt_val(f, opts, level + 1, x))?;
 74 |             write!(f, "]")
 75 |         }
 76 |         Val::Obj(o) if o.is_empty() => write!(f, "{{}}"),
 77 |         Val::Obj(o) => {
 78 |             write!(f, "{{")?;
 79 |             fmt_seq(f, opts, level, &**o, |f, (k, val)| {
 80 |                 span(f, "key", display(k))?;
 81 |                 write!(f, ":")?;
 82 |                 if !opts.compact {
 83 |                     write!(f, " ")?;
 84 |                 }
 85 |                 fmt_val(f, opts, level + 1, val)
 86 |             })?;
 87 |             write!(f, "}}")
 88 |         }
 89 |     }
 90 | }
 91 | 
 92 | fn span(f: &mut Formatter, cls: &str, el: impl Display) -> fmt::Result {
 93 |     write!(f, "{el}")
 94 | }
 95 | 
 96 | fn span_dbg(f: &mut Formatter, cls: &str, el: impl Debug) -> fmt::Result {
 97 |     write!(f, "{el:?}")
 98 | }
 99 | 
100 | fn escape(s: &str) -> String {
101 |     use aho_corasick::AhoCorasick;
102 |     let patterns = &["&", "<", ">"];
103 |     let replaces = &["&", "<", ">"];
104 |     let ac = AhoCorasick::new(patterns).unwrap();
105 |     ac.replace_all(s, replaces)
106 | }
107 | 
108 | #[allow(dead_code)]
109 | #[derive(Debug)]
110 | struct Settings {
111 |     raw_input: bool,
112 |     slurp: bool,
113 |     null_input: bool,
114 |     raw_output: bool,
115 |     compact: bool,
116 |     //join_output: bool,
117 |     indent: usize,
118 |     tab: bool,
119 | }
120 | 
121 | impl Settings {
122 |     fn try_from(v: &JsValue) -> Option {
123 |         let get = |key: &str| js_sys::Reflect::get(v, &key.into()).ok();
124 |         let get_bool = |key: &str| get(key).and_then(|v| v.as_bool());
125 |         let as_usize = |v: JsValue| v.as_string()?.parse().ok();
126 |         Some(Self {
127 |             raw_input: get_bool("raw-input")?,
128 |             slurp: get_bool("slurp")?,
129 |             null_input: get_bool("null-input")?,
130 |             raw_output: get_bool("raw-output")?,
131 |             compact: get_bool("compact")?,
132 |             //join_output: get_bool("join-output")?,
133 |             indent: get("indent").and_then(as_usize)?,
134 |             tab: get_bool("tab")?,
135 |         })
136 |     }
137 | }
138 | 
139 | use web_sys::DedicatedWorkerGlobalScope as Scope;
140 | 
141 | type FileReports = (load::File, Vec);
142 | 
143 | enum Error {
144 |     Report(Vec),
145 |     Hifijson(String),
146 |     Jaq(jaq_core::Error),
147 | }
148 | 
149 | #[wasm_bindgen]
150 | pub fn run(filter: &str, input: &str, settings: &JsValue, scope: &Scope) {
151 |     let _ = console_log::init_with_level(log::Level::Debug);
152 |     log::trace!("Starting run in Rust ...");
153 | 
154 |     let settings = Settings::try_from(settings).unwrap();
155 |     log::trace!("{settings:?}");
156 | 
157 |     let indent = if settings.tab {
158 |         "\t".to_string()
159 |     } else {
160 |         " ".repeat(settings.indent)
161 |     };
162 | 
163 |     let pp_opts = PpOpts {
164 |         compact: settings.compact,
165 |         indent,
166 |     };
167 | 
168 |     let post_value = |y| {
169 |         let s = FormatterFn(|f: &mut Formatter| match &y {
170 |             Val::Str(s) if settings.raw_output => span(f, "string", escape(s)),
171 |             y => fmt_val(f, &pp_opts, 0, y),
172 |         });
173 |         scope.post_message(&s.to_string().into()).unwrap();
174 |     };
175 |     match process(filter, input, &settings, post_value) {
176 |         Ok(()) => (),
177 |         Err(Error::Report(file_reports)) => {
178 |             for (file, reports) in file_reports {
179 |                 let idx = codesnake::LineIndex::new(&file.code);
180 |                 for e in reports {
181 |                     let error = format!("⚠️ Error: {}", e.message);
182 |                     scope.post_message(&error.into()).unwrap();
183 | 
184 |                     let block = e.into_block(&idx);
185 |                     let block = format!("{}\n{}{}", block.prologue(), block, block.epilogue());
186 |                     scope.post_message(&block.into()).unwrap();
187 |                 }
188 |             }
189 |         }
190 |         Err(Error::Hifijson(e)) => {
191 |             scope
192 |                 .post_message(&format!("⚠️ Parse error: {e}").into())
193 |                 .unwrap();
194 |         }
195 |         Err(Error::Jaq(e)) => {
196 |             scope
197 |                 .post_message(&format!("⚠️ Error: {e}").into())
198 |                 .unwrap();
199 |         }
200 |     }
201 | 
202 |     // signal that we are done
203 |     scope.post_message(&JsValue::NULL).unwrap();
204 | }
205 | 
206 | fn read_str<'a>(
207 |     settings: &Settings,
208 |     input: &'a str,
209 | ) -> Box> + 'a> {
210 |     if settings.raw_input {
211 |         Box::new(raw_input(settings.slurp, input).map(|s| Ok(Val::from(s.to_owned()))))
212 |     } else {
213 |         let vals = json_slice(input.as_bytes());
214 |         Box::new(collect_if(settings.slurp, vals))
215 |     }
216 | }
217 | 
218 | fn raw_input(slurp: bool, input: &str) -> impl Iterator {
219 |     if slurp {
220 |         Box::new(core::iter::once(input))
221 |     } else {
222 |         Box::new(input.lines()) as Box>
223 |     }
224 | }
225 | 
226 | fn json_slice(slice: &[u8]) -> impl Iterator> + '_ {
227 |     let mut lexer = hifijson::SliceLexer::new(slice);
228 |     core::iter::from_fn(move || {
229 |         use hifijson::token::Lex;
230 |         Some(Val::parse(lexer.ws_token()?, &mut lexer).map_err(|e| e.to_string()))
231 |     })
232 | }
233 | 
234 | fn collect_if<'a, T: 'a + FromIterator, E: 'a>(
235 |     slurp: bool,
236 |     iter: impl Iterator> + 'a,
237 | ) -> Box> + 'a> {
238 |     if slurp {
239 |         Box::new(core::iter::once(iter.collect()))
240 |     } else {
241 |         Box::new(iter)
242 |     }
243 | }
244 | 
245 | fn process(filter: &str, input: &str, settings: &Settings, f: impl Fn(Val)) -> Result<(), Error> {
246 |     let (_vals, filter) = parse(filter, &[]).map_err(Error::Report)?;
247 | 
248 |     let inputs = read_str(settings, input);
249 | 
250 |     let inputs = Box::new(inputs) as Box>>;
251 |     let null = Box::new(core::iter::once(Ok(Val::Null))) as Box>;
252 | 
253 |     let inputs = RcIter::new(inputs);
254 |     let null = RcIter::new(null);
255 | 
256 |     for x in if settings.null_input { &null } else { &inputs } {
257 |         let x = x.map_err(Error::Hifijson)?;
258 |         for y in filter.run((Ctx::new([], &inputs), x)) {
259 |             f(y.map_err(Error::Jaq)?);
260 |         }
261 |     }
262 |     Ok(())
263 | }
264 | 
265 | fn parse(code: &str, vars: &[String]) -> Result<(Vec, Filter), Vec> {
266 |     use compile::Compiler;
267 |     use jaq_core::load::{import, Arena, File, Loader};
268 | 
269 |     let vars: Vec<_> = vars.iter().map(|v| format!("${v}")).collect();
270 |     let arena = Arena::default();
271 |     let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs()));
272 |     let modules = loader
273 |         .load(&arena, File { path: (), code })
274 |         .map_err(load_errors)?;
275 | 
276 |     let vals = Vec::new();
277 |     import(&modules, |_path| Err("file loading not supported".into())).map_err(load_errors)?;
278 | 
279 |     let compiler = Compiler::default()
280 |         .with_funs(jaq_std::funs().chain(jaq_json::funs()))
281 |         .with_global_vars(vars.iter().map(|v| &**v));
282 |     let filter = compiler.compile(modules).map_err(compile_errors)?;
283 |     Ok((vals, filter))
284 | }
285 | 
286 | fn load_errors(errs: load::Errors<&str, ()>) -> Vec {
287 |     use load::Error;
288 | 
289 |     let errs = errs.into_iter().map(|(file, err)| {
290 |         let code = file.code;
291 |         let err = match err {
292 |             Error::Io(errs) => errs.into_iter().map(|e| report_io(code, e)).collect(),
293 |             Error::Lex(errs) => errs.into_iter().map(|e| report_lex(code, e)).collect(),
294 |             Error::Parse(errs) => errs.into_iter().map(|e| report_parse(code, e)).collect(),
295 |         };
296 |         (file.map_code(|s| s.into()), err)
297 |     });
298 |     errs.collect()
299 | }
300 | 
301 | fn compile_errors(errs: compile::Errors<&str, ()>) -> Vec {
302 |     let errs = errs.into_iter().map(|(file, errs)| {
303 |         let code = file.code;
304 |         let errs = errs.into_iter().map(|e| report_compile(code, e)).collect();
305 |         (file.map_code(|s| s.into()), errs)
306 |     });
307 |     errs.collect()
308 | }
309 | 
310 | type StringColors = Vec<(String, Option)>;
311 | 
312 | #[derive(Debug)]
313 | struct Report {
314 |     message: String,
315 |     labels: Vec<(core::ops::Range, StringColors, Color)>,
316 | }
317 | 
318 | #[derive(Clone, Debug)]
319 | enum Color {
320 |     Yellow,
321 |     Red,
322 | }
323 | 
324 | impl Color {
325 |     fn apply(&self, d: impl Display) -> String {
326 |         let mut color = format!("{self:?}");
327 |         color.make_ascii_lowercase();
328 |         format!("{d}",)
329 |     }
330 | }
331 | 
332 | fn report_io(code: &str, (path, error): (&str, String)) -> Report {
333 |     let path_range = load::span(code, path);
334 |     Report {
335 |         message: format!("could not load file {}: {}", path, error),
336 |         labels: [(path_range, [(error, None)].into(), Color::Red)].into(),
337 |     }
338 | }
339 | 
340 | fn report_lex(code: &str, (expected, found): load::lex::Error<&str>) -> Report {
341 |     use load::span;
342 |     // truncate found string to its first character
343 |     let found = &found[..found.char_indices().nth(1).map_or(found.len(), |(i, _)| i)];
344 | 
345 |     let found_range = span(code, found);
346 |     let found = match found {
347 |         "" => [("unexpected end of input".to_string(), None)].into(),
348 |         c => [("unexpected character ", None), (c, Some(Color::Red))]
349 |             .map(|(s, c)| (s.into(), c))
350 |             .into(),
351 |     };
352 |     let label = (found_range, found, Color::Red);
353 | 
354 |     let labels = match expected {
355 |         load::lex::Expect::Delim(open) => {
356 |             let text = [("unclosed delimiter ", None), (open, Some(Color::Yellow))]
357 |                 .map(|(s, c)| (s.into(), c));
358 |             Vec::from([(span(code, open), text.into(), Color::Yellow), label])
359 |         }
360 |         _ => Vec::from([label]),
361 |     };
362 | 
363 |     Report {
364 |         message: format!("expected {}", expected.as_str()),
365 |         labels,
366 |     }
367 | }
368 | 
369 | fn report_parse(code: &str, (expected, found): load::parse::Error<&str>) -> Report {
370 |     let found_range = load::span(code, found);
371 | 
372 |     let found = if found.is_empty() {
373 |         "unexpected end of input"
374 |     } else {
375 |         "unexpected token"
376 |     };
377 |     let found = [(found.to_string(), None)].into();
378 | 
379 |     Report {
380 |         message: format!("expected {}", expected.as_str()),
381 |         labels: Vec::from([(found_range, found, Color::Red)]),
382 |     }
383 | }
384 | 
385 | fn report_compile(code: &str, (found, undefined): compile::Error<&str>) -> Report {
386 |     use compile::Undefined::Filter;
387 |     let found_range = load::span(code, found);
388 |     let wnoa = |exp, got| format!("wrong number of arguments (expected {exp}, found {got})");
389 |     let message = match (found, undefined) {
390 |         ("reduce", Filter(arity)) => wnoa("2", arity),
391 |         ("foreach", Filter(arity)) => wnoa("2 or 3", arity),
392 |         (_, undefined) => format!("undefined {}", undefined.as_str()),
393 |     };
394 |     let found = [(message.clone(), None)].into();
395 | 
396 |     Report {
397 |         message,
398 |         labels: Vec::from([(found_range, found, Color::Red)]),
399 |     }
400 | }
401 | 
402 | type CodeBlock = codesnake::Block, String>;
403 | 
404 | impl Report {
405 |     fn into_block(self, idx: &codesnake::LineIndex) -> CodeBlock {
406 |         use codesnake::{Block, CodeWidth, Label};
407 |         let color_maybe = |(text, color): (_, Option)| match color {
408 |             None => text,
409 |             Some(color) => color.apply(text).to_string(),
410 |         };
411 |         let labels = self.labels.into_iter().map(|(range, text, color)| {
412 |             let text = text.into_iter().map(color_maybe).collect::>();
413 |             Label::new(range)
414 |                 .with_text(text.join(""))
415 |                 .with_style(move |s| color.apply(s).to_string())
416 |         });
417 |         Block::new(idx, labels).unwrap().map_code(|c| {
418 |             let c = c.replace('\t', "    ");
419 |             let w = unicode_width::UnicodeWidthStr::width(&*c);
420 |             CodeWidth::new(c, core::cmp::max(w, 1))
421 |         })
422 |     }
423 | }
424 | 


--------------------------------------------------------------------------------
/jaq-play/src/main.js:
--------------------------------------------------------------------------------
 1 | // currently active jaq thread
 2 | let worker = initWorker();
 3 | 
 4 | const param_ids = Object.entries({'q': 'filter', 'j': 'input'});
 5 | 
 6 | function getParams() {
 7 |     const urlParams = new URLSearchParams(window.location.search);
 8 |     for (const [param, id] of param_ids) {
 9 |         const value = urlParams.get(param);
10 |         if (value !== null) {
11 |             document.getElementById(id).value = value;
12 |         }
13 |     }
14 | }
15 | 
16 | function setParams() {
17 |     const url = new URL(window.location)
18 |     for (const [param, id] of param_ids) {
19 |         url.searchParams.set(param, document.getElementById(id).value);
20 |     }
21 |     history.pushState(null, '', url);
22 | }
23 | 
24 | function startWorker() {
25 |     setParams();
26 |     showRunButton(false);
27 | 
28 |     // remove previous output
29 |     document.getElementById('output').replaceChildren();
30 | 
31 |     //console.log("Starting run in JS ...");
32 |     const filter = document.getElementById('filter').value;
33 |     const input = document.getElementById('input').value;
34 |     const settings = getSettings();
35 |     worker.postMessage({filter, input, settings});
36 | }
37 | 
38 | function getSettings() {
39 |     const cbxs = document.querySelectorAll(".settings input[type=checkbox]");
40 |     const nums = document.querySelectorAll(".settings input[type=number]");
41 |     var acc = {};
42 |     cbxs.forEach(node => acc[node.id] = node.checked);
43 |     nums.forEach(node => acc[node.id] = node.value);
44 |     return acc
45 | }
46 | 
47 | function initWorker() {
48 |     let worker = new Worker("./src/worker.js", { type: "module" });
49 |     worker.onmessage = event => receiveFromWorker(event.data);
50 |     return worker
51 | }
52 | 
53 | function receiveFromWorker(data) {
54 |     if (data == null) {
55 |         showRunButton(true);
56 |         return;
57 |     }
58 | 
59 |     let div = document.createElement("div");
60 |     div.innerHTML = data;
61 |     document.getElementById("output").appendChild(div);
62 | }
63 | 
64 | function showRunButton(show) {
65 |     const display = b => b ? "block" : "none";
66 |     document.getElementById("run" ).style.display = display(show);
67 |     document.getElementById("stop").style.display = display(!show);
68 | }
69 | 
70 | function stopWorker() {
71 |     console.log("Stopping worker ...");
72 |     showRunButton(true)
73 |     worker.terminate();
74 |     worker = initWorker();
75 | }
76 | 
77 | document.getElementById("run").onclick = async () => startWorker();
78 | document.getElementById("stop").onclick = async () => stopWorker();
79 | 
80 | document.addEventListener('keydown', event => {
81 |     // CTRL + Enter
82 |     if (event.ctrlKey && event.key == 'Enter') { startWorker() }
83 | });
84 | 
85 | getParams();
86 | 


--------------------------------------------------------------------------------
/jaq-play/src/style.css:
--------------------------------------------------------------------------------
  1 | body {
  2 |   font-family: "Roboto Condensed",sans-serif;
  3 |   display: flex;
  4 |   flex-flow: column;
  5 |   height: 100vh;
  6 |   margin: 0;
  7 | }
  8 | 
  9 | header {
 10 |   align-items: center;
 11 |   display: flex;
 12 |   padding: 0 1rem;
 13 | }
 14 | 
 15 | .top {
 16 |   height: 54px;
 17 |   background: #101112;
 18 | }
 19 | 
 20 | .about {
 21 |   flex-grow: 1;
 22 |   text-align: right;
 23 | }
 24 | 
 25 | .IO header {
 26 |   background: #b7bcc0;
 27 | }
 28 | 
 29 | .filter header {
 30 |   background: #70b0e0;
 31 | }
 32 | 
 33 | .filter {
 34 |   flex: 1;
 35 | }
 36 | 
 37 | .IO {
 38 |   display: flex;
 39 |   justify-content: space-between;
 40 |   flex: 3;
 41 |   /* see https://stackoverflow.com/a/48396712 */
 42 |   min-height: 0;
 43 | }
 44 | 
 45 | section {
 46 |   display: flex;
 47 |   flex-direction: column;
 48 | }
 49 | 
 50 | .IO > section {
 51 |   flex: 1;
 52 |   min-width: 0;
 53 | }
 54 | 
 55 | .logo {
 56 |   height: 100%;
 57 | }
 58 | 
 59 | button {
 60 |   border: none;
 61 |   color: #b7bcc0;
 62 |   background: #282b2d;
 63 |   cursor: pointer;
 64 |   transition: background-color .2s ease-out;
 65 |   padding: .375rem .75rem;
 66 |   font-size: .875rem;
 67 |   border-radius: .3rem;
 68 |   margin-left: 8px;
 69 | }
 70 | 
 71 | button:hover { background: #3a3e41; }
 72 | 
 73 | #stop { display: none; }
 74 | 
 75 | h1 {
 76 |   font-size: 1rem;
 77 |   font-weight: 700;
 78 |   margin: 0.5em 0;
 79 | }
 80 | 
 81 | textarea, #output {
 82 |   font-family: "Roboto Mono",monospace;
 83 |   font-size: 1.2rem;
 84 |   height: 100%; 
 85 |   max-height: 100%;
 86 |   margin: 0;
 87 | }
 88 | 
 89 | textarea {
 90 |   width: 100%;
 91 |   resize: none;
 92 |   box-sizing: border-box;
 93 | }
 94 | 
 95 | /* Inspiration: https://zserge.com/posts/js-editor/ */
 96 | #output {
 97 |   overflow: auto;
 98 |   padding: 2px;
 99 | }
100 | 
101 | .string  { color: green; }
102 | .number  { color: darkorange; }
103 | .boolean { color: blue; }
104 | .null    { color: magenta; }
105 | .key     { color: red; }
106 | 
107 | .red     { color: #dc322f; }
108 | .yellow  { color: #b58900; }
109 | 
110 | 
111 | .settings { display: inline-block; }
112 | 
113 | .settings > div {
114 |   display: none;
115 |   gap: 10px;
116 |   position: absolute;
117 |   background: rgba(16,17,18,.85);
118 |   color: #f0f1f2;
119 |   padding: 1rem;
120 |   border-radius: .125rem;
121 |   backdrop-filter: blur(3px);
122 | }
123 | 
124 | .settings:hover > div { display: flex; }
125 | 
126 | .settings hr { color: rgba(126,135,142,.25); }
127 | 
128 | .settings fieldset {
129 |   border: 0;
130 |   padding: 0;
131 |   margin: 0;
132 | }
133 | 
134 | ul {
135 |   list-style-type: none;
136 |   padding: 0;
137 |   margin: 2px;
138 | }
139 | 


--------------------------------------------------------------------------------
/jaq-play/src/worker.js:
--------------------------------------------------------------------------------
 1 | import init, {run} from "../pkg/jaq_play.js";
 2 | 
 3 | init();
 4 | 
 5 | // Set callback to handle messages passed to the worker.
 6 | self.onmessage = async event => {
 7 |     const { filter, input, settings } = event.data;
 8 |     await run(filter, input, settings, self);
 9 | };
10 | 


--------------------------------------------------------------------------------
/jaq-std/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "jaq-std"
 3 | version = "2.1.1"
 4 | authors = ["Michael Färber "]
 5 | edition = "2021"
 6 | license = "MIT"
 7 | readme = "../README.md"
 8 | description = "Standard library for jaq"
 9 | repository = "https://github.com/01mf02/jaq"
10 | keywords = ["json", "query", "jq"]
11 | rust-version = "1.65"
12 | 
13 | [features]
14 | default = ["std", "format", "log", "math", "regex", "time"]
15 | regex = ["regex-lite"]
16 | std = []
17 | format = ["aho-corasick", "base64", "urlencoding"]
18 | math = ["libm"]
19 | time = ["chrono"]
20 | 
21 | [dependencies]
22 | jaq-core = { version = "2.1.0", path = "../jaq-core" }
23 | 
24 | hifijson = { version = "0.2.0", optional = true }
25 | chrono = { version = "0.4.38", default-features = false, features = ["alloc", "clock"], optional = true }
26 | regex-lite = { version = "0.1", optional = true }
27 | log = { version = "0.4.17", optional = true }
28 | libm = { version = "0.2.7", optional = true }
29 | aho-corasick = { version = "1.0", optional = true }
30 | base64 = { version = "0.22", optional = true }
31 | urlencoding = { version = "2.1.3", optional = true }
32 | 
33 | [dev-dependencies]
34 | jaq-json = { path = "../jaq-json" }
35 | serde_json = "1.0"
36 | 


--------------------------------------------------------------------------------
/jaq-std/src/defs.jq:
--------------------------------------------------------------------------------
  1 | def empty: {}[] as $x | .;
  2 | def null:  [][0];
  3 | 
  4 | def debug(msgs): ((msgs | debug) as $x | empty), .;
  5 | def error(msgs): ((msgs | error) as $x | empty), .;
  6 | 
  7 | def halt_error: halt_error(5);
  8 | 
  9 | # Booleans
 10 | def true:  0 == 0;
 11 | def false: 0 != 0;
 12 | def not: if . then false else true end;
 13 | 
 14 | # Not defined in jq!
 15 | def isboolean: . == true or . == false;
 16 | def isnumber:  . > true and . < "";
 17 | def isstring:  . >= ""  and . < [];
 18 | def isarray:   . >= []  and . < {};
 19 | def isobject:  . >= {};
 20 | 
 21 | # Numbers
 22 | def nan:      0 / 0;
 23 | def infinite: 1 / 0;
 24 | def isnan:      . < nan and nan < .;
 25 | def isinfinite: . == infinite or  . == -infinite;
 26 | def isfinite:   isnumber and (isinfinite | not);
 27 | def isnormal:   isnumber and ((. == 0 or isnan or isinfinite) | not);
 28 | 
 29 | # Math
 30 | def abs: if . < 0 then - . end;
 31 | def logb:
 32 |     if . == 0.0 then -infinite
 33 |   elif isinfinite then infinite
 34 |   elif isnan then .
 35 |   else ilogb | . + 0.0 end;
 36 | def significand:
 37 |     if isinfinite or isnan then .
 38 |   elif . == 0.0 then 0.0
 39 |   else scalbln(.; ilogb | -1 * .) end;
 40 | def pow10:            pow(10.0; .);
 41 | def drem($l; r):      remainder($l; r) | if . == 0 then copysign(.; $l) end;
 42 | def nexttoward(x; y): nextafter(x; y);
 43 | def scalb(x; e):      x * pow(2.0; e);
 44 | def gamma: tgamma;
 45 | 
 46 | # Type
 47 | def type:
 48 |     if . == null then "null"
 49 |   elif isboolean then "boolean"
 50 |   elif . < "" then "number"
 51 |   elif . < [] then "string"
 52 |   elif . < {} then "array"
 53 |   else             "object" end;
 54 | 
 55 | # Selection
 56 | def select(f): if f then . else empty end;
 57 | def values:    select(. != null);
 58 | def nulls:     select(. == null);
 59 | def booleans:  select(isboolean);
 60 | def numbers:   select(isnumber);
 61 | def finites:   select(isfinite);
 62 | def normals:   select(isnormal);
 63 | def strings:   select(isstring);
 64 | def arrays:    select(isarray);
 65 | def objects:   select(isobject);
 66 | def iterables: select(. >= []);
 67 | def scalars:   select(. <  []);
 68 | 
 69 | # Conversion
 70 | def tostring: "\(.)";
 71 | 
 72 | # Generators
 73 | def range(from; to): range(from; to; 1);
 74 | def range(to): range(0; to);
 75 | def repeat(f): def rec: f, rec; rec;
 76 | def recurse(f): def rec: ., (f | rec); rec;
 77 | def recurse: recurse(.[]?);
 78 | def recurse(f; cond): recurse(f | select(cond));
 79 | def while(cond; update): def rec: if cond then ., (update | rec) else empty end; rec;
 80 | def until(cond; update): def rec: if cond then . else update | rec end; rec;
 81 | 
 82 | # Iterators
 83 | def map(f): [.[] | f];
 84 | def map_values(f): .[] |= f;
 85 | def add(f): reduce f as $x (null; . + $x);
 86 | def add: add(.[]);
 87 | def join(x): .[:-1][] += x | add;
 88 | def min_by(f): reduce min_by_or_empty(f) as $x (null; $x);
 89 | def max_by(f): reduce max_by_or_empty(f) as $x (null; $x);
 90 | def min: min_by(.);
 91 | def max: max_by(.);
 92 | def unique_by(f): [group_by(f)[] | .[0]];
 93 | def unique: unique_by(.);
 94 | 
 95 | def getpath($path): reduce $path[] as $p (.; .[$p]);
 96 | def del(f): f |= empty;
 97 | 
 98 | # Arrays
 99 | def first:  .[ 0];
100 | def last:   .[-1];
101 | def nth(n): .[ n];
102 | 
103 | def skip($n; g): foreach g as $x ($n; . - 1; if . < 0 then $x else empty end);
104 | def nth(n; g): last(limit(n + 1; g));
105 | 
106 | # Predicates
107 | def isempty(g): first((g | false), true);
108 | def all(g; cond): isempty(g | cond and empty);
109 | def any(g; cond): isempty(g | cond  or empty) | not;
110 | def all(cond): all(.[]; cond);
111 | def any(cond): any(.[]; cond);
112 | def all: all(.[]; .);
113 | def any: any(.[]; .);
114 | 
115 | # Walking
116 | def walk(f): def rec: (.[]? |= rec) | f; rec;
117 | 
118 | def flatten: [recurse(arrays[]) | select(isarray | not)];
119 | def flatten($d): if $d > 0 then map(if isarray then flatten($d-1) else [.] end) | add end;
120 | 
121 | # Regular expressions
122 | def capture_of_match: map(select(.name) | { (.name): .string} ) | add + {};
123 | 
124 | def    test(re; flags): matches(re; flags) | any;
125 | def    scan(re; flags): matches(re; flags)[] | .[0].string;
126 | def   match(re; flags): matches(re; flags)[] | .[0] + { captures: .[1:] };
127 | def capture(re; flags): matches(re; flags)[] | capture_of_match;
128 | 
129 | def split($sep):
130 |   if isstring and ($sep | isstring) then . / $sep
131 |   else error("split input and separator must be strings") end;
132 | def split (re; flags): split_(re; flags + "g");
133 | def splits(re; flags): split(re; flags)[];
134 | 
135 | def sub(re; f; flags):
136 |   def handle: if isarray then capture_of_match | f end;
137 |   reduce split_matches(re; flags)[] as $x (""; . + ($x | handle));
138 | 
139 | def gsub(re; f; flags): sub(re; f; "g" + flags);
140 | 
141 | def    test(re):    test(re; "");
142 | def    scan(re):    scan(re; "");
143 | def   match(re):   match(re; "");
144 | def capture(re): capture(re; "");
145 | def  splits(re):  splits(re; "");
146 | def  sub(re; f): sub(re; f;  "");
147 | def gsub(re; f): sub(re; f; "g");
148 | 
149 | # I/O
150 | def input: first(inputs);
151 | 
152 | # Date
153 | def   todate:   todateiso8601;
154 | def fromdate: fromdateiso8601;
155 | 
156 | # Formatting
157 | def fmt_row(n; s): if . >= "" then s elif . == null then n else "\(.)" end;
158 | def @csv: .[] |= fmt_row(""; "\"\(escape_csv)\"") | join("," );
159 | def @tsv: .[] |= fmt_row("";      escape_tsv    ) | join("\t");
160 | def @sh: [if isarray then .[] end | fmt_row("null"; "'\(escape_sh)'")] | join(" ");
161 | def @text: "\(.)";
162 | def @html   : tostring | escape_html;
163 | def @uri    : tostring | encode_uri;
164 | def @base64 : tostring | encode_base64;
165 | def @base64d: tostring | decode_base64;
166 | 


--------------------------------------------------------------------------------
/jaq-std/src/math.rs:
--------------------------------------------------------------------------------
  1 | macro_rules! math {
  2 |     // Build a 0-ary filter from a 1-ary math function.
  3 |     ($f: ident, $domain: expr, $codomain: expr) => {
  4 |         #[allow(clippy::redundant_closure_call)]
  5 |         (stringify!($f), v(0), |_, cv| {
  6 |             bome((|| Ok($codomain(libm::$f($domain(&cv.1)?))))())
  7 |         })
  8 |     };
  9 |     // Build a 2-ary filter that ignores '.' from a 2-ary math function.
 10 |     ($f: ident, $domain1: expr, $domain2: expr, $codomain: expr) => {
 11 |         (stringify!($f), v(2), |_, mut cv| {
 12 |             bome((|| {
 13 |                 let y = cv.0.pop_var();
 14 |                 let x = cv.0.pop_var();
 15 |                 Ok($codomain(libm::$f($domain1(&x)?, $domain2(&y)?)))
 16 |             })())
 17 |         })
 18 |     };
 19 |     // Build a 3-ary filter that ignores '.' from a 3-ary math function.
 20 |     ($f: ident, $domain1: expr, $domain2: expr, $domain3: expr, $codomain: expr) => {
 21 |         (stringify!($f), v(3), |_, mut cv| {
 22 |             bome((|| {
 23 |                 let z = cv.0.pop_var();
 24 |                 let y = cv.0.pop_var();
 25 |                 let x = cv.0.pop_var();
 26 |                 Ok($codomain(libm::$f(
 27 |                     $domain1(&x)?,
 28 |                     $domain2(&y)?,
 29 |                     $domain3(&z)?,
 30 |                 )))
 31 |             })())
 32 |         })
 33 |     };
 34 | }
 35 | 
 36 | pub(crate) use math;
 37 | 
 38 | /// Build a filter from float to float
 39 | macro_rules! f_f {
 40 |     ($f: ident) => {
 41 |         crate::math::math!($f, V::as_f64, V::from)
 42 |     };
 43 | }
 44 | 
 45 | /// Build a filter from float to int
 46 | macro_rules! f_i {
 47 |     ($f: ident) => {
 48 |         crate::math::math!($f, V::as_f64, |x| V::from(x as isize))
 49 |     };
 50 | }
 51 | 
 52 | /// Build a filter from float to (float, int)
 53 | macro_rules! f_fi {
 54 |     ($f: ident) => {
 55 |         crate::math::math!($f, V::as_f64, |(x, y)| [V::from(x), V::from(y as isize)]
 56 |             .into_iter()
 57 |             .collect())
 58 |     };
 59 | }
 60 | 
 61 | /// Build a filter from float to (float, float)
 62 | macro_rules! f_ff {
 63 |     ($f: ident) => {
 64 |         crate::math::math!($f, V::as_f64, |(x, y)| [V::from(x), V::from(y)]
 65 |             .into_iter()
 66 |             .collect())
 67 |     };
 68 | }
 69 | 
 70 | /// Build a filter from (float, float) to float
 71 | macro_rules! ff_f {
 72 |     ($f: ident) => {
 73 |         crate::math::math!($f, V::as_f64, V::as_f64, V::from)
 74 |     };
 75 | }
 76 | 
 77 | /// Build a filter from (int, float) to float
 78 | macro_rules! if_f {
 79 |     ($f: ident) => {
 80 |         crate::math::math!($f, V::try_as_i32, V::as_f64, V::from)
 81 |     };
 82 | }
 83 | 
 84 | /// Build a filter from (float, int) to float
 85 | macro_rules! fi_f {
 86 |     ($f: ident) => {
 87 |         crate::math::math!($f, V::as_f64, V::try_as_i32, V::from)
 88 |     };
 89 | }
 90 | 
 91 | /// Build a filter from (float, float, float) to float
 92 | macro_rules! fff_f {
 93 |     ($f: ident) => {
 94 |         crate::math::math!($f, V::as_f64, V::as_f64, V::as_f64, V::from)
 95 |     };
 96 | }
 97 | 
 98 | pub(crate) use f_f;
 99 | pub(crate) use f_ff;
100 | pub(crate) use f_fi;
101 | pub(crate) use f_i;
102 | pub(crate) use ff_f;
103 | pub(crate) use fff_f;
104 | pub(crate) use fi_f;
105 | pub(crate) use if_f;
106 | 


--------------------------------------------------------------------------------
/jaq-std/src/regex.rs:
--------------------------------------------------------------------------------
  1 | //! Helpers to interface with the `regex` crate.
  2 | 
  3 | use alloc::string::{String, ToString};
  4 | use alloc::vec::Vec;
  5 | use regex_lite::{self as regex, Error, Regex, RegexBuilder};
  6 | 
  7 | #[derive(Copy, Clone, Default)]
  8 | pub struct Flags {
  9 |     // global search
 10 |     g: bool,
 11 |     // ignore empty matches
 12 |     n: bool,
 13 |     // case-insensitive
 14 |     i: bool,
 15 |     // multi-line mode: ^ and $ match begin/end of line
 16 |     m: bool,
 17 |     // single-line mode: allow . to match \n
 18 |     s: bool,
 19 |     // greedy
 20 |     l: bool,
 21 |     // extended mode: ignore whitespace and allow line comments (starting with `#`)
 22 |     x: bool,
 23 | }
 24 | 
 25 | impl Flags {
 26 |     pub fn new(flags: &str) -> Result {
 27 |         let mut out = Self::default();
 28 |         for flag in flags.chars() {
 29 |             match flag {
 30 |                 'g' => out.g = true,
 31 |                 'n' => out.n = true,
 32 |                 'i' => out.i = true,
 33 |                 'm' => out.m = true,
 34 |                 's' => out.s = true,
 35 |                 'l' => out.l = true,
 36 |                 'x' => out.x = true,
 37 |                 'p' => {
 38 |                     out.m = true;
 39 |                     out.s = true;
 40 |                 }
 41 |                 c => return Err(c),
 42 |             }
 43 |         }
 44 |         Ok(out)
 45 |     }
 46 | 
 47 |     pub fn ignore_empty(self) -> bool {
 48 |         self.n
 49 |     }
 50 | 
 51 |     pub fn global(self) -> bool {
 52 |         self.g
 53 |     }
 54 | 
 55 |     fn impact(self, builder: &mut RegexBuilder) -> &mut RegexBuilder {
 56 |         builder
 57 |             .case_insensitive(self.i)
 58 |             .multi_line(self.m)
 59 |             .dot_matches_new_line(self.s)
 60 |             .swap_greed(self.l)
 61 |             .ignore_whitespace(self.x)
 62 |     }
 63 | 
 64 |     pub fn regex(self, re: &str) -> Result {
 65 |         let mut builder = RegexBuilder::new(re);
 66 |         self.impact(&mut builder).build()
 67 |     }
 68 | }
 69 | 
 70 | type CharIndices<'a> =
 71 |     core::iter::Chain, core::iter::Once<(usize, char)>>;
 72 | 
 73 | /// Mapping between byte and character indices.
 74 | pub struct ByteChar<'a>(core::iter::Peekable>>);
 75 | 
 76 | impl<'a> ByteChar<'a> {
 77 |     pub fn new(s: &'a str) -> Self {
 78 |         let last = core::iter::once((s.len(), '\0'));
 79 |         Self(s.char_indices().chain(last).enumerate().peekable())
 80 |     }
 81 | 
 82 |     /// Convert byte offset to UTF-8 character offset.
 83 |     ///
 84 |     /// This needs to be called with monotonically increasing values of `byte_offset`.
 85 |     fn char_of_byte(&mut self, byte_offset: usize) -> Option {
 86 |         loop {
 87 |             let (char_i, (byte_i, _char)) = self.0.peek()?;
 88 |             if byte_offset == *byte_i {
 89 |                 return Some(*char_i);
 90 |             } else {
 91 |                 self.0.next();
 92 |             }
 93 |         }
 94 |     }
 95 | }
 96 | 
 97 | pub struct Match {
 98 |     pub offset: usize,
 99 |     pub length: usize,
100 |     pub string: S,
101 |     pub name: Option,
102 | }
103 | 
104 | impl<'a> Match<&'a str> {
105 |     pub fn new(bc: &mut ByteChar, m: regex::Match<'a>, name: Option<&'a str>) -> Self {
106 |         Self {
107 |             offset: bc.char_of_byte(m.start()).unwrap(),
108 |             length: m.as_str().chars().count(),
109 |             string: m.as_str(),
110 |             name,
111 |         }
112 |     }
113 | 
114 |     pub fn fields + From + 'a>(&self) -> impl Iterator + '_ {
115 |         [
116 |             ("offset", (self.offset as isize).into()),
117 |             ("length", (self.length as isize).into()),
118 |             ("string", self.string.to_string().into()),
119 |         ]
120 |         .into_iter()
121 |         .chain(self.name.iter().map(|n| ("name", (*n).to_string().into())))
122 |         .map(|(k, v)| (k.to_string().into(), v))
123 |     }
124 | }
125 | 
126 | pub enum Part {
127 |     Matches(Vec>),
128 |     Mismatch(S),
129 | }
130 | 
131 | /// Apply a regular expression to the given input value.
132 | ///
133 | /// `sm` indicates whether to
134 | /// 1. output strings that do *not* match the regex, and
135 | /// 2. output the matches.
136 | pub fn regex<'a>(s: &'a str, re: &'a Regex, flags: Flags, sm: (bool, bool)) -> Vec> {
137 |     // mismatches & matches
138 |     let (mi, ma) = sm;
139 | 
140 |     let mut last_byte = 0;
141 |     let mut bc = ByteChar::new(s);
142 |     let mut out = Vec::new();
143 | 
144 |     for c in re.captures_iter(s) {
145 |         let whole = c.get(0).unwrap();
146 |         if flags.ignore_empty() && whole.as_str().is_empty() {
147 |             continue;
148 |         }
149 |         let match_names = c.iter().zip(re.capture_names());
150 |         let matches = match_names.filter_map(|(m, n)| Some(Match::new(&mut bc, m?, n)));
151 |         if mi {
152 |             out.push(Part::Mismatch(&s[last_byte..whole.start()]));
153 |             last_byte = whole.end();
154 |         }
155 |         if ma {
156 |             out.push(Part::Matches(matches.collect()));
157 |         }
158 |         if !flags.global() {
159 |             break;
160 |         }
161 |     }
162 |     if mi {
163 |         out.push(Part::Mismatch(&s[last_byte..]));
164 |     }
165 |     out
166 | }
167 | 


--------------------------------------------------------------------------------
/jaq-std/src/time.rs:
--------------------------------------------------------------------------------
  1 | use crate::{Error, ValR, ValT, ValTx};
  2 | use alloc::string::{String, ToString};
  3 | use chrono::{DateTime, Datelike, FixedOffset, NaiveDateTime, TimeZone, Timelike, Utc};
  4 | 
  5 | /// Convert a UNIX epoch timestamp with optional fractions.
  6 | fn epoch_to_datetime(v: &V) -> Result, Error> {
  7 |     let fail = || Error::str(format_args!("cannot parse {v} as epoch timestamp"));
  8 |     let val = match v.as_isize() {
  9 |         Some(i) => i as i64 * 1000000,
 10 |         None => (v.as_f64()? * 1000000.0) as i64,
 11 |     };
 12 |     DateTime::from_timestamp_micros(val).ok_or_else(fail)
 13 | }
 14 | 
 15 | /// Convert a date-time pair to a UNIX epoch timestamp.
 16 | fn datetime_to_epoch(dt: DateTime, frac: bool) -> ValR {
 17 |     if frac {
 18 |         Ok((dt.timestamp_micros() as f64 / 1e6).into())
 19 |     } else {
 20 |         let seconds = dt.timestamp();
 21 |         isize::try_from(seconds)
 22 |             .map(V::from)
 23 |             .or_else(|_| V::from_num(&seconds.to_string()))
 24 |     }
 25 | }
 26 | 
 27 | /// Parse a "broken down time" array.
 28 | fn array_to_datetime(v: &[V]) -> Option> {
 29 |     let [year, month, day, hour, min, sec]: &[V; 6] = v.get(..6)?.try_into().ok()?;
 30 |     let sec = sec.as_f64().ok()?;
 31 |     let u32 = |v: &V| -> Option { v.as_isize()?.try_into().ok() };
 32 |     Utc.with_ymd_and_hms(
 33 |         year.as_isize()?.try_into().ok()?,
 34 |         u32(month)? + 1,
 35 |         u32(day)?,
 36 |         u32(hour)?,
 37 |         u32(min)?,
 38 |         // the `as i8` cast saturates, returning a number in the range [-128, 128]
 39 |         (sec.floor() as i8).try_into().ok()?,
 40 |     )
 41 |     .single()?
 42 |     .with_nanosecond((sec.fract() * 1e9) as u32)
 43 | }
 44 | 
 45 | /// Convert a DateTime to a "broken down time" array
 46 | fn datetime_to_array(dt: DateTime) -> [V; 8] {
 47 |     [
 48 |         V::from(dt.year() as isize),
 49 |         V::from(dt.month0() as isize),
 50 |         V::from(dt.day() as isize),
 51 |         V::from(dt.hour() as isize),
 52 |         V::from(dt.minute() as isize),
 53 |         if dt.nanosecond() > 0 {
 54 |             V::from(dt.second() as f64 + dt.timestamp_subsec_micros() as f64 / 1e6)
 55 |         } else {
 56 |             V::from(dt.second() as isize)
 57 |         },
 58 |         V::from(dt.weekday().num_days_from_sunday() as isize),
 59 |         V::from(dt.ordinal0() as isize),
 60 |     ]
 61 | }
 62 | 
 63 | /// Parse an ISO 8601 timestamp string to a number holding the equivalent UNIX timestamp
 64 | /// (seconds elapsed since 1970/01/01).
 65 | ///
 66 | /// Actually, this parses RFC 3339; see
 67 | ///  for differences.
 68 | /// jq also only parses a very restricted subset of ISO 8601.
 69 | pub fn from_iso8601(s: &str) -> ValR {
 70 |     let dt = DateTime::parse_from_rfc3339(s)
 71 |         .map_err(|e| Error::str(format_args!("cannot parse {s} as ISO-8601 timestamp: {e}")))?;
 72 |     datetime_to_epoch(dt, s.contains('.'))
 73 | }
 74 | 
 75 | /// Format a number as an ISO 8601 timestamp string.
 76 | pub fn to_iso8601(v: &V) -> Result> {
 77 |     let fail = || Error::str(format_args!("cannot format {v} as ISO-8601 timestamp"));
 78 |     if let Some(i) = v.as_isize() {
 79 |         let dt = DateTime::from_timestamp(i as i64, 0).ok_or_else(fail)?;
 80 |         Ok(dt.format("%Y-%m-%dT%H:%M:%SZ").to_string())
 81 |     } else {
 82 |         let f = v.as_f64()?;
 83 |         let dt = DateTime::from_timestamp_micros((f * 1e6) as i64).ok_or_else(fail)?;
 84 |         Ok(dt.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string())
 85 |     }
 86 | }
 87 | 
 88 | /// Format a date (either number or array) in a given timezone.
 89 | pub fn strftime(v: &V, fmt: &str, tz: impl TimeZone) -> ValR {
 90 |     let fail = || Error::str(format_args!("cannot convert {v} to time"));
 91 |     let dt = match v.clone().into_vec() {
 92 |         Ok(v) => array_to_datetime(&v).ok_or_else(fail),
 93 |         Err(_) => epoch_to_datetime(v),
 94 |     }?;
 95 |     let dt = dt.with_timezone(&tz).fixed_offset();
 96 |     Ok(dt.format(fmt).to_string().into())
 97 | }
 98 | 
 99 | /// Convert an epoch timestamp to a "broken down time" array.
100 | pub fn gmtime(v: &V, tz: impl TimeZone) -> ValR {
101 |     let dt = epoch_to_datetime(v)?;
102 |     let dt = dt.with_timezone(&tz).fixed_offset();
103 |     datetime_to_array(dt).into_iter().map(Ok).collect()
104 | }
105 | 
106 | /// Parse a string into a "broken down time" array.
107 | pub fn strptime(s: &str, fmt: &str) -> ValR {
108 |     let dt = NaiveDateTime::parse_from_str(s, fmt)
109 |         .map_err(|e| Error::str(format_args!("cannot parse {s} using {fmt}: {e}")))?;
110 |     let dt = dt.and_utc().fixed_offset();
111 |     datetime_to_array(dt).into_iter().map(Ok).collect()
112 | }
113 | 
114 | /// Parse an array into a UNIX epoch timestamp.
115 | pub fn mktime(v: &V) -> ValR {
116 |     let fail = || Error::str(format_args!("cannot convert {v} to time"));
117 |     let dt = array_to_datetime(&v.clone().into_vec()?).ok_or_else(fail)?;
118 |     datetime_to_epoch(dt, dt.timestamp_subsec_micros() > 0)
119 | }
120 | 


--------------------------------------------------------------------------------
/jaq-std/tests/common/mod.rs:
--------------------------------------------------------------------------------
 1 | use jaq_json::{Error, Val, ValR};
 2 | use serde_json::Value;
 3 | 
 4 | fn yields(x: Val, code: &str, ys: impl Iterator) {
 5 |     use jaq_core::load::{Arena, File, Loader};
 6 |     eprintln!("{}", code.replace('\n', " "));
 7 | 
 8 |     let arena = Arena::default();
 9 |     let loader = Loader::new(jaq_std::defs());
10 |     let modules = loader.load(&arena, File { path: (), code }).unwrap();
11 |     let filter = jaq_core::Compiler::default()
12 |         .with_funs(jaq_std::funs())
13 |         .compile(modules)
14 |         .unwrap();
15 |     filter.yields(x, ys)
16 | }
17 | 
18 | pub fn fail(x: Value, f: &str, err: Error) {
19 |     yields(x.into(), f, core::iter::once(Err(err)))
20 | }
21 | 
22 | pub fn give(x: Value, f: &str, y: Value) {
23 |     yields(x.into(), f, core::iter::once(Ok(y.into())))
24 | }
25 | 
26 | pub fn gives(x: Value, f: &str, ys: [Value; N]) {
27 |     yields(x.into(), f, ys.into_iter().map(|y| Ok(y.into())))
28 | }
29 | 
30 | #[macro_export]
31 | macro_rules! yields {
32 |     ($func_name:ident, $filter:expr, $output: expr) => {
33 |         #[test]
34 |         fn $func_name() {
35 |             give(json!(null), $filter, json!($output))
36 |         }
37 |     };
38 | }
39 | 


--------------------------------------------------------------------------------
/jaq-std/tests/defs.rs:
--------------------------------------------------------------------------------
  1 | //! Tests for filters in the standard library, sorted by name.
  2 | 
  3 | pub mod common;
  4 | 
  5 | use common::{give, gives};
  6 | use serde_json::json;
  7 | 
  8 | #[test]
  9 | fn add() {
 10 |     give(json!({"a": 1, "b": 2}), "add", json!(3));
 11 |     give(json!([[0, 1], [2, 3]]), "add", json!([0, 1, 2, 3]));
 12 | }
 13 | 
 14 | #[test]
 15 | fn all() {
 16 |     give(json!({"a": false, "b": true}), "all", json!(false));
 17 |     give(json!({"a": 1, "b": 2}), "all", json!(true));
 18 | 
 19 |     let f = "def positive(f): all(f; . > 0); positive(.[])";
 20 |     give(json!([1, 2]), f, json!(true));
 21 | }
 22 | 
 23 | #[test]
 24 | fn any() {
 25 |     give(json!({"a": false, "b": true}), "any", json!(true));
 26 | }
 27 | 
 28 | // aliases for fromdateiso8601 and todateiso8601
 29 | yields!(fromdate, r#""1970-01-02T00:00:00Z" | fromdate"#, 86400);
 30 | yields!(todate, r#"86400 | todate"#, "1970-01-02T00:00:00Z");
 31 | yields!(tofromdate, "946684800 | todate | fromdate", 946684800);
 32 | 
 33 | yields!(
 34 |     drem_nan,
 35 |     "[drem(nan, 1; nan, 1)] | (.[0:-1] | all(isnan)) and .[-1] == 0.0",
 36 |     true
 37 | );
 38 | yields!(
 39 |     drem_range,
 40 |     "[drem(3.5, -4; 6, 1.75, 2)]",
 41 |     [-2.5, 0.0, -0.5, 2.0, -0.5, -0.0]
 42 | );
 43 | 
 44 | #[test]
 45 | fn flatten() {
 46 |     let a0 = || json!([1, [{"a": 2}, [3]]]);
 47 |     let a1 = || json!([1, {"a": 2}, [3]]);
 48 |     let a2 = || json!([1, {"a": 2}, 3]);
 49 |     give(a0(), "flatten", json!(a2()));
 50 |     let f = "[flatten(0, 1, 2, 3)]";
 51 |     give(a0(), f, json!([a0(), a1(), a2(), a2()]));
 52 | }
 53 | 
 54 | yields!(
 55 |     flatten_deep,
 56 |     "[[[0], 1], 2, [3, [4]]] | flatten",
 57 |     [0, 1, 2, 3, 4]
 58 | );
 59 | 
 60 | // here, we diverge from jq, which returns just 1
 61 | yields!(flatten_obj, "{a: 1} | flatten", json!([{"a": 1}]));
 62 | // jq gives an error here
 63 | yields!(flatten_num, "0 | flatten", [0]);
 64 | 
 65 | yields!(isfinite_true, "all((0, 1, nan); isfinite)", true);
 66 | yields!(
 67 |     isfinite_false,
 68 |     "any((infinite, -infinite, []); isfinite)",
 69 |     false
 70 | );
 71 | 
 72 | yields!(isnormal_true, "1 | isnormal", true);
 73 | yields!(
 74 |     isnormal_false,
 75 |     "any(0, nan, infinite, -infinite, []; isnormal)",
 76 |     false
 77 | );
 78 | 
 79 | yields!(join_empty, r#"[] | join(" ")"#, json!(null));
 80 | yields!(
 81 |     join_strs,
 82 |     r#"["Hello", "world"] | join(" ")"#,
 83 |     "Hello world"
 84 | );
 85 | // 2 + 1 + 3 + 1 + 4 + 1 + 5
 86 | yields!(join_nums, r#"[2, 3, 4, 5] | join(1)"#, 17);
 87 | 
 88 | yields!(map, "[1, 2] | map(.+1)", [2, 3]);
 89 | 
 90 | // this diverges from jq, which returns [null]
 91 | yields!(last_empty, "[last({}[])]", json!([]));
 92 | yields!(last_some, "last(1, 2, 3)", 3);
 93 | 
 94 | yields!(logb_inf, "infinite | logb | . == infinite", true);
 95 | yields!(logb_nan, "nan | logb | isnan", true);
 96 | yields!(logb_neg_inf, "-infinite | logb | . == infinite", true);
 97 | yields!(
 98 |     logb_range,
 99 |     "[-2.2, -2, -1, 1, 2, 2.2] | map(logb)",
100 |     [1.0, 1.0, 0.0, 0.0, 1.0, 1.0]
101 | );
102 | yields!(logb_zero, "0 | logb | . == -infinite", true);
103 | 
104 | // here we diverge from jq, which returns ["a", "b", "A", "B"]
105 | yields!(
106 |     match_many,
107 |     r#""ABab" | [match("a", "b"; "", "i") | .string]"#,
108 |     ["a", "A", "b", "B"]
109 | );
110 | 
111 | #[test]
112 | fn min_max() {
113 |     give(json!([1, 4, 2]), "min", json!(1));
114 |     give(json!([1, 4, 2]), "max", json!(4));
115 |     // TODO: find examples where `min_by(f)` yields output different from `min`
116 |     // (and move it then to jaq-core/tests/tests.rs)
117 |     give(
118 |         json!([{"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 4}}}, {"a": {"b": {"c": 2}}}]),
119 |         "min_by(.a.b.c)",
120 |         json!({"a": {"b": {"c": 1}}}),
121 |     );
122 |     give(
123 |         json!([{"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 4}}}, {"a": {"b": {"c": 2}}}]),
124 |         "max_by(.a.b.c)",
125 |         json!({"a": {"b": {"c": 4}}}),
126 |     );
127 | }
128 | 
129 | yields!(min_empty, "[] | min_by(.)", json!(null));
130 | // when output is equal, min_by selects the left element and max_by the right one
131 | yields!(
132 |     min_max_eq,
133 |     "[{a: 1, b: 3}, {a: 1, b: 2}] | [(min_by(.a), max_by(.a)) | .b]",
134 |     [3, 2]
135 | );
136 | // multiple-output functions can be used to differentiate elements
137 | yields!(
138 |     max_mult,
139 |     "[{a: 1, b: 3}, {a: 1, b: 2}] | max_by(.a, .b) | .b",
140 |     3
141 | );
142 | 
143 | #[test]
144 | fn nth() {
145 |     let fib = "[0,1] | recurse([.[1], add]) | .[0]";
146 |     give(json!(10), &format!("nth(.; {})", fib), json!(55));
147 | 
148 |     let fib = "[0,1] | recurse([.[1], add])[0]";
149 |     give(json!(10), &format!("nth(.; {})", fib), json!(55));
150 | }
151 | 
152 | yields!(range_many, "[range(-1, 1; 0, 2)]", json!([-1, -1, 0, 1, 1]));
153 | 
154 | #[test]
155 | fn range_reverse() {
156 |     give(json!(null), "[range(1, 2)]", json!([0, 0, 1]));
157 | 
158 |     give(json!(3), "[range(.)] | reverse", json!([2, 1, 0]));
159 | }
160 | 
161 | yields!(
162 |     recurse_update,
163 |     "[0, [1, 2], 3] | recurse |= (.+1)? // .",
164 |     json!([1, [2, 3], 4])
165 | );
166 | 
167 | // the following tests show that sums are evaluated lazily
168 | // (otherwise this would not terminate)
169 | yields!(limit_inf_suml, "[limit(3; recurse(.+1) + 0)]", [0, 1, 2]);
170 | yields!(limit_inf_sumr, "[limit(3; 0 + recurse(.+1))]", [0, 1, 2]);
171 | 
172 | yields!(limit_inf_path, "[limit(2; [1] | .[repeat(0)])]", [1, 1]);
173 | 
174 | #[test]
175 | fn recurse() {
176 |     let x = json!({"a":0,"b":[1]});
177 |     gives(x.clone(), "recurse", [x, json!(0), json!([1]), json!(1)]);
178 | 
179 |     let y = [json!(1), json!(2), json!(3)];
180 |     gives(json!(1), "recurse(.+1; . < 4)", y);
181 | 
182 |     let y = [json!(2), json!(4), json!(16)];
183 |     gives(json!(2), "recurse(. * .; . < 20)", y);
184 | 
185 |     let x = json!([[[0], 1], 2, [3, [4]]]);
186 | 
187 |     let y = json!([[[1], 2], 3, [4, [5]]]);
188 |     give(x.clone(), "(.. | scalars) |= .+1", y);
189 | 
190 |     let f = ".. |= if . < [] then .+1 else . + [42] end";
191 |     let y = json!([[[1, 43], 2, 43], 3, [4, [5, 43], 43], 43]);
192 |     // jq gives: `[[[1, 42], 2, 42], 3, [4, [5, 42], 42], 42]`
193 |     give(x.clone(), f, y);
194 | 
195 |     let f = ".. |= if . < [] then .+1 else [42] + . end";
196 |     let y = json!([43, [43, [43, 1], 2], 3, [43, 4, [43, 5]]]);
197 |     // jq fails here with: "Cannot index number with number"
198 |     give(x.clone(), f, y);
199 | }
200 | 
201 | yields!(
202 |     recurse3,
203 |     "[1 | recurse(if . < 3 then .+1 else empty end)]",
204 |     [1, 2, 3]
205 | );
206 | yields!(
207 |     reduce_recurse,
208 |     "reduce recurse(if . == 1000 then empty else .+1 end) as $x (0; . + $x)",
209 |     500500
210 | );
211 | 
212 | const RECURSE_FLATTEN: &str = "def flatten($d):
213 |   [ { d: $d, x: . } |
214 |     recurse(if .d >= 0 and ([] <= .x and .x < {}) then { d: .d - 1, x: .x[] } else empty end) |
215 |     if .d < 0 or (.x < [] or {} <= .x) then .x else empty end
216 |   ];";
217 | 
218 | yields!(
219 |     recurse_flatten,
220 |     &(RECURSE_FLATTEN.to_owned() + "[[[1], 2], 3] | flatten(1)"),
221 |     json!([[1], 2, 3])
222 | );
223 | 
224 | #[test]
225 | fn repeat() {
226 |     let y = json!([0, 1, 0, 1]);
227 |     give(json!([0, 1]), "[limit(4; repeat(.[]))]", y);
228 | }
229 | 
230 | // the implementation of scalb in jq (or the libm.a library) doesn't
231 | // allow for float exponents; jaq tolerates float exponents in scalb
232 | // and rejects them in scalbln
233 | yields!(
234 |     scalb_eqv_pow2,
235 |     "[-2.2, -1.1, -0.01, 0, 0.01, 1.1, 2.2] | [scalb(1.0; .[])] == [pow(2.0; .[])]",
236 |     true
237 | );
238 | yields!(
239 |     scalb_nan,
240 |     "[scalb(nan, 1; nan, 1)] | (.[0:-1] | all(isnan)) and .[-1] == 2.0",
241 |     true
242 | );
243 | yields!(
244 |     scalb_range,
245 |     "[scalb(-2.5, 0, 2.5; 2, 2.5, 3) * 1000 | round]",
246 |     [-10000, -14142, -20000, 0, 0, 0, 10000, 14142, 20000]
247 | );
248 | 
249 | // here we diverge from jq, which returns ["a", "b", "a", "A", "b", "B"]
250 | yields!(
251 |     scan,
252 |     r#""abAB" | [scan("a", "b"; "g", "gi")]"#,
253 |     // TODO: is this order really desired?
254 |     json!(["a", "a", "A", "b", "b", "B"])
255 | );
256 | 
257 | #[test]
258 | fn select() {
259 |     give(json!([1, 2]), ".[] | select(.>1)", json!(2));
260 |     give(json!([0, 1, 2]), "map(select(.<1, 1<.))", json!([0, 2]));
261 | 
262 |     let v = json!([null, false, true, 1, 1.0, "", "a", [], [0], {}, {"a": 1}]);
263 |     let iterables = json!([[], [0], {}, {"a": 1}]);
264 |     let scalars = json!([null, false, true, 1, 1.0, "", "a"]);
265 |     let values = json!([false, true, 1, 1.0, "", "a", [], [0], {}, {"a": 1}]);
266 |     give(v.clone(), ".[] | nulls", json!(null));
267 |     give(v.clone(), "[.[] | booleans]", json!([false, true]));
268 |     give(v.clone(), "[.[] | numbers]", json!([1, 1.0]));
269 |     give(v.clone(), "[.[] | strings]", json!(["", "a"]));
270 |     give(v.clone(), "[.[] | arrays]", json!([[], [0]]));
271 |     give(v.clone(), "[.[] | objects]", json!([{}, {"a": 1}]));
272 |     give(v.clone(), "[.[] | iterables]", iterables);
273 |     give(v.clone(), "[.[] | scalars]", scalars);
274 |     give(v.clone(), "[.[] | values]", values);
275 | }
276 | 
277 | yields!(
278 |     significand_inf,
279 |     "infinite | significand | . == infinite",
280 |     true
281 | );
282 | yields!(significand_nan, "nan | significand | isnan", true);
283 | yields!(
284 |     significand_neg_inf,
285 |     "-infinite | significand | . == -infinite",
286 |     true
287 | );
288 | yields!(
289 |     significand_range,
290 |     "[-123.456, -2.2, -2, -1, 0, 0.00001, 1, 2, 2.2, 123.456] | map(significand)",
291 |     [-1.929, -1.1, -1.0, -1.0, 0.0, 1.31072, 1.0, 1.0, 1.1, 1.929]
292 | );
293 | 
294 | yields!(tostring_str, r#""\n" | tostring"#, "\n");
295 | yields!(tostring_arr_str, r#"["\n"] | tostring"#, "[\"\\n\"]");
296 | 
297 | #[test]
298 | fn typ() {
299 |     give(json!({"a": 1, "b": 2}), "type", json!("object"));
300 |     give(json!([0, 1]), "type", json!("array"));
301 |     give(json!("Hello"), "type", json!("string"));
302 |     give(json!(1), "type", json!("number"));
303 |     give(json!(1.0), "type", json!("number"));
304 |     give(json!(true), "type", json!("boolean"));
305 |     give(json!(null), "type", json!("null"));
306 | }
307 | 
308 | yields!(sub, r#""XYxyXYxy" | sub("x";"Q")"#, "XYQyXYxy");
309 | yields!(gsub, r#""XYxyXYxy" | gsub("x";"Q")"#, "XYQyXYQy");
310 | yields!(isub, r#""XYxyXYxy" | sub("x";"Q";"i")"#, "QYxyXYxy");
311 | yields!(gisub, r#""XYxyXYxy" | gsub("x";"Q";"i")"#, "QYQyQYQy");
312 | // swap adjacent occurrences of upper- and lower-case characters
313 | yields!(
314 |     gsub_swap,
315 |     r#""XYxyXYxy" | gsub("(?[A-Z])(?[a-z])"; .lower + .upper)"#,
316 |     "XxYyXxYy"
317 | );
318 | // this diverges from jq, which yields ["XxYy", "!XxYy", "Xx!Yy", "!Xx!Yy"]
319 | yields!(
320 |     gsub_many,
321 |     r#""XxYy" | [gsub("(?[A-Z])"; .upper, "!" + .upper)]"#,
322 |     ["XxYy", "Xx!Yy", "!XxYy", "!Xx!Yy"]
323 | );
324 | 
325 | yields!(
326 |     format_text,
327 |     r#"[0, 0 == 0, {}.a, "hello", {}, [] | @text]"#,
328 |     ["0", "true", "null", "hello", "{}", "[]"]
329 | );
330 | yields!(
331 |     format_csv,
332 |     r#"[0, 0 == 0, {}.a, "hello \"quotes\" and, commas"] | @csv"#,
333 |     r#"0,true,,"hello ""quotes"" and, commas""#
334 | );
335 | yields!(
336 |     format_tsv,
337 |     r#"[0, 0 == 0, {}.a, "hello \"quotes\" and \n\r\t\\ escapes"] | @tsv"#,
338 |     "0\ttrue\t\thello \"quotes\" and \\n\\r\\t\\\\ escapes"
339 | );
340 | 
341 | yields!(
342 |     format_sh,
343 |     r#"[0, 0 == 0, {}.a, "O'Hara!", ["Here", "there"] | @sh]"#,
344 |     ["0", "true", "null", r#"'O'\''Hara!'"#, r#"'Here' 'there'"#,]
345 | );
346 | yields!(
347 |     format_sh_rejects_objects,
348 |     r#"{a: "b"} | try @sh catch -1"#,
349 |     -1
350 | );
351 | yields!(
352 |     format_sh_rejects_nested_arrays,
353 |     r#"["fine, but", []] | try @sh catch -1"#,
354 |     -1
355 | );
356 | 


--------------------------------------------------------------------------------
/jaq-std/tests/funs.rs:
--------------------------------------------------------------------------------
  1 | //! Tests for named core filters, sorted by name.
  2 | 
  3 | pub mod common;
  4 | 
  5 | use common::{give, gives};
  6 | use serde_json::json;
  7 | 
  8 | yields!(repeat, "def r(f): f, r(f); [limit(3; r(1, 2))]", [1, 2, 1]);
  9 | 
 10 | yields!(lazy_array, "def f: 1, [f]; limit(1; f)", 1);
 11 | 
 12 | yields!(
 13 |     lazy_foreach,
 14 |     "def f: f; limit(1; foreach (1, f) as $x (0; .))",
 15 |     0
 16 | );
 17 | 
 18 | yields!(nested_rec, "def f: def g: 0, g; g; def h: h; first(f)", 0);
 19 | 
 20 | yields!(
 21 |     rec_two_var_args,
 22 |     "def f($a; $b): [$a, $b], f($a+1; $b+1); [limit(3; f(0; 1))]",
 23 |     [[0, 1], [1, 2], [2, 3]]
 24 | );
 25 | 
 26 | yields!(
 27 |     rec_update,
 28 |     "def upto($x): .[$x], (if $x > 0 then upto($x-1) else {}[] as $x | . end); [1, 2, 3, 4] | upto(1) |= .+1",
 29 |     [2, 3, 3, 4]
 30 | );
 31 | 
 32 | #[test]
 33 | fn ascii() {
 34 |     give(json!("aAaAäの"), "ascii_upcase", json!("AAAAäの"));
 35 |     give(json!("aAaAäの"), "ascii_downcase", json!("aaaaäの"));
 36 | }
 37 | 
 38 | yields!(
 39 |     fromdate,
 40 |     r#""1970-01-02T00:00:00Z" | fromdateiso8601"#,
 41 |     86400
 42 | );
 43 | yields!(
 44 |     fromdate_mu,
 45 |     r#""1970-01-02T00:00:00.123456Z" | fromdateiso8601"#,
 46 |     86400.123456
 47 | );
 48 | yields!(todate, r#"86400 | todateiso8601"#, "1970-01-02T00:00:00Z");
 49 | yields!(
 50 |     todate_mu,
 51 |     "86400.123456 | todateiso8601",
 52 |     "1970-01-02T00:00:00.123456Z"
 53 | );
 54 | yields!(
 55 |     strftime,
 56 |     r#"86400 | strftime("%F %T")"#,
 57 |     "1970-01-02 00:00:00"
 58 | );
 59 | yields!(
 60 |     strftime_arr,
 61 |     r#"[ 1970, 0, 2, 0, 0, 0, 5, 1 ] | strftime("%F %T")"#,
 62 |     "1970-01-02 00:00:00"
 63 | );
 64 | yields!(
 65 |     strftime_mu,
 66 |     r#"86400.123456 | strftime("%F %T.%6f")"#,
 67 |     "1970-01-02 00:00:00.123456"
 68 | );
 69 | yields!(gmtime, r"86400 | gmtime", [1970, 0, 2, 0, 0, 0, 5, 1]);
 70 | yields!(
 71 |     gmtime_mu,
 72 |     r"86400.123456 | gmtime",
 73 |     json!([1970, 0, 2, 0, 0, 0.123456, 5, 1])
 74 | );
 75 | yields!(
 76 |     gmtime_mktime_mu,
 77 |     r"86400.123456 | gmtime | mktime",
 78 |     86400.123456
 79 | );
 80 | yields!(
 81 |     strptime,
 82 |     r#""1970-01-02T00:00:00Z" | strptime("%Y-%m-%dT%H:%M:%SZ")"#,
 83 |     [1970, 0, 2, 0, 0, 0, 5, 1]
 84 | );
 85 | yields!(mktime, "[ 1970, 0, 2, 0, 0, 0, 5, 1 ] | mktime", 86400);
 86 | 
 87 | #[test]
 88 | fn fromtodate() {
 89 |     let fromto = "fromdateiso8601 | todateiso8601";
 90 |     let iso = "2000-01-01T00:00:00Z";
 91 |     give(json!(iso), fromto, json!(iso));
 92 |     let iso_mu = "2000-01-01T00:00:00.123456Z";
 93 |     give(json!(iso_mu), fromto, json!(iso_mu));
 94 | }
 95 | 
 96 | #[test]
 97 | fn explode_implode() {
 98 |     give(json!("❤ の"), "explode", json!([10084, 32, 12398]));
 99 |     give(json!("y̆"), "explode", json!([121, 774]));
100 | 
101 |     give(json!("❤ の"), "explode | implode", json!("❤ の"));
102 |     give(json!("y̆"), "explode | implode", json!("y̆"));
103 | 
104 |     give(json!([1114112]), "try implode catch -1", json!(-1));
105 | }
106 | 
107 | yields!(first_empty, "[first({}[])]", json!([]));
108 | yields!(first_some, "first(1, 2, 3)", 1);
109 | 
110 | yields!(
111 |     encode_base64,
112 |     r#""hello cruel world" | encode_base64"#,
113 |     "aGVsbG8gY3J1ZWwgd29ybGQ="
114 | );
115 | yields!(
116 |     encode_decode_base64,
117 |     r#""hello cruel world" | encode_base64 | decode_base64"#,
118 |     "hello cruel world"
119 | );
120 | 
121 | yields!(
122 |     escape_html,
123 |     r#""
sneaky
" | escape_html"#,
124 |     "<p style='visibility: hidden'>sneaky</p>"
125 | );
126 | yields!(
127 |     encode_uri,
128 |     r#""abc123 ?#+&[]" | encode_uri"#,
129 |     "abc123%20%3F%23%2B%26%5B%5D"
130 | );
131 | 
132 | #[test]
133 | fn group_by() {
134 |     gives(json!([]), "group_by(.)", [json!([])]);
135 |     gives(
136 |         json!([{"key":1, "value": "foo"},{"key":2, "value":"bar"},{"key":1,"value":"baz"}]),
137 |         "group_by(.key)",
138 |         [json!([[{"key":1,"value":"foo"}, {"key":1,"value":"baz"}],[{"key":2,"value":"bar"}]])],
139 |     );
140 | }
141 | 
142 | yields!(utf8bytelength_foo1, r#""foo" | utf8bytelength"#, 3);
143 | yields!(utf8bytelength_foo2, r#""ƒoo" | utf8bytelength"#, 4);
144 | yields!(utf8bytelength_namaste, r#""नमस्ते" | utf8bytelength"#, 18);
145 | 
146 | #[test]
147 | fn limit() {
148 |     // a big WTF: jq outputs "1" here! that looks like another bug ...
149 |     gives(json!(null), "limit(0; 1,2)", []);
150 |     give(json!(null), "[limit(1, 0, 3; 0, 1)]", json!([0, 0, 1]));
151 | 
152 |     // here, jaq diverges from jq, which returns `[0, 1]`
153 |     give(json!(null), "[limit(-1; 0, 1)]", json!([]));
154 | }
155 | 
156 | yields!(limit_overflow, "[limit(0; def f: f | .; f)]", json!([]));
157 | 
158 | yields!(
159 |     math_0_argument_scalar_filters,
160 |     "[-2.2, -1.1, 0, 1.1, 2.2 | sin as $s | cos as $c | $s * $s + $c * $c]",
161 |     [1.0, 1.0, 1.0, 1.0, 1.0]
162 | );
163 | 
164 | yields!(
165 |     math_0_argument_vector_filters,
166 |     "[3, 3.25, 3.5 | modf]",
167 |     [[0.0, 3.0], [0.25, 3.0], [0.5, 3.0]]
168 | );
169 | 
170 | yields!(
171 |     math_2_argument_filters,
172 |     "[pow(0.25, 4, 9; 1, 0.5, 2)]",
173 |     [0.25, 0.5, 0.0625, 4.0, 2.0, 16.0, 9.0, 3.0, 81.0]
174 | );
175 | 
176 | yields!(
177 |     math_3_argument_filters,
178 |     "[fma(2, 1; 3, 4; 4, 5)]",
179 |     [10.0, 11.0, 12.0, 13.0, 7.0, 8.0, 8.0, 9.0]
180 | );
181 | 
182 | yields!(range_pp, "[range(0; 6;  2)]", [0, 2, 4]);
183 | yields!(range_pn, "[range(0; 6; -2)]", json!([]));
184 | yields!(range_np, "[range(0; -6; 2)]", json!([]));
185 | yields!(range_nn, "[range(0; -6; -2)]", [0, -2, -4]);
186 | yields!(range_zz, "[range(0; 0; 0)]", json!([]));
187 | yields!(range_fp, "[range(0.0; 2; 0.5)]", [0.0, 0.5, 1.0, 1.5]);
188 | yields!(range_ip, "[limit(3; range(0; 1/0; 1))]", [0, 1, 2]);
189 | yields!(range_in, "[limit(3; range(0; -1/0; -1))]", [0, -1, -2]);
190 | // here, we diverge from jq, which just returns the empty list
191 | yields!(range_pz, "[limit(3; range(0; 6; 0))]", json!([0, 0, 0]));
192 | yields!(range_nz, "[limit(3; range(0; -6; 0))]", json!([0, 0, 0]));
193 | 
194 | #[test]
195 | fn regex() {
196 |     let date = r#"(\\d{4})-(\\d{2})-(\\d{2})"#;
197 |     let s = "2012-03-14, 2013-01-01 and 2014-07-05";
198 |     let f = |f, re, flags| format!("{f}(\"{re}\"; \"{flags}\")");
199 | 
200 |     let out = json!(["", ", ", " and ", ""]);
201 |     give(json!(s), &f("split_", date, "g"), out);
202 | 
203 |     let c = |o: usize, s: &str| {
204 |         json!({
205 |           "offset": o,
206 |           "length": s.chars().count(),
207 |           "string": s
208 |         })
209 |     };
210 |     let d1 = json!([c(00, "2012-03-14"), c(00, "2012"), c(05, "03"), c(08, "14")]);
211 |     let d2 = json!([c(12, "2013-01-01"), c(12, "2013"), c(17, "01"), c(20, "01")]);
212 |     let d3 = json!([c(27, "2014-07-05"), c(27, "2014"), c(32, "07"), c(35, "05")]);
213 | 
214 |     give(json!(s), &f("matches", date, "g"), json!([d1, d2, d3]));
215 | 
216 |     give(json!(""), &f("matches", "", ""), json!([[c(0, "")]]));
217 |     give(json!(""), &f("matches", "^$", ""), json!([[c(0, "")]]));
218 |     give(
219 |         json!("  "),
220 |         &f("matches", "", "g"),
221 |         json!([[c(0, "")], [c(1, "")], [c(2, "")]]),
222 |     );
223 |     give(json!("  "), &f("matches", "", "gn"), json!([]));
224 | 
225 |     let out = json!(["", d1, ", ", d2, " and ", d3, ""]);
226 |     give(json!(s), &f("split_matches", date, "g"), out);
227 | 
228 |     let out = json!(["", d1, ", 2013-01-01 and 2014-07-05"]);
229 |     give(json!(s), &f("split_matches", date, ""), out);
230 | }
231 | 
232 | yields!(round_int, "[0, 1][1 | round]", 1);
233 | 
234 | yields!(round_pi, " 1   | round", 1);
235 | yields!(round_pf, " 1.0 | round", 1);
236 | yields!(round_ni, "-1   | round", -1);
237 | yields!(round_nf, "-1.0 | round", -1);
238 | 
239 | yields!(round_mid, "-1.5 | round", -2);
240 | yields!(floor_mid, "-1.5 | floor", -2);
241 | yields!(ceili_mid, "-1.5 | ceil ", -1);
242 | 
243 | yields!(round_floor, "-1.4 | round", -1);
244 | yields!(floor_floor, "-1.4 | floor", -2);
245 | yields!(ceili_floor, "-1.4 | ceil ", -1);
246 | 
247 | #[test]
248 | fn startswith() {
249 |     give(json!("foobar"), r#"startswith("")"#, json!(true));
250 |     give(json!("foobar"), r#"startswith("bar")"#, json!(false));
251 |     give(json!("foobar"), r#"startswith("foo")"#, json!(true));
252 |     give(json!(""), r#"startswith("foo")"#, json!(false));
253 | }
254 | 
255 | #[test]
256 | fn endswith() {
257 |     give(json!("foobar"), r#"endswith("")"#, json!(true));
258 |     give(json!("foobar"), r#"endswith("foo")"#, json!(false));
259 |     give(json!("foobar"), r#"endswith("bar")"#, json!(true));
260 |     give(json!(""), r#"endswith("foo")"#, json!(false));
261 | }
262 | 
263 | #[test]
264 | fn ltrimstr() {
265 |     give(json!("foobar"), r#"ltrimstr("")"#, json!("foobar"));
266 |     give(json!("foobar"), r#"ltrimstr("foo")"#, json!("bar"));
267 |     give(json!("foobar"), r#"ltrimstr("bar")"#, json!("foobar"));
268 |     give(json!("اَلْعَرَبِيَّةُ"), r#"ltrimstr("ا")"#, json!("َلْعَرَبِيَّةُ"));
269 | }
270 | 
271 | #[test]
272 | fn rtrimstr() {
273 |     give(json!("foobar"), r#"rtrimstr("")"#, json!("foobar"));
274 |     give(json!("foobar"), r#"rtrimstr("bar")"#, json!("foo"));
275 |     give(json!("foobar"), r#"rtrimstr("foo")"#, json!("foobar"));
276 |     give(json!("اَلْعَرَبِيَّةُ"), r#"rtrimstr("ا")"#, json!("اَلْعَرَبِيَّةُ"));
277 | }
278 | 
279 | #[test]
280 | fn trim() {
281 |     give(json!(""), "trim", json!(""));
282 |     give(json!(" "), "trim", json!(""));
283 |     give(json!("  "), "trim", json!(""));
284 |     give(json!("foo"), "trim", json!("foo"));
285 |     give(json!(" foo  "), "trim", json!("foo"));
286 |     give(json!("  foo  "), "trim", json!("foo"));
287 |     give(json!("foo  "), "trim", json!("foo"));
288 |     give(json!(" اَلْعَرَبِيَّةُ "), "trim", json!("اَلْعَرَبِيَّةُ"));
289 | }
290 | 
291 | #[test]
292 | fn ltrim() {
293 |     give(json!(""), "ltrim", json!(""));
294 |     give(json!(" "), "ltrim", json!(""));
295 |     give(json!("  "), "ltrim", json!(""));
296 |     give(json!("foo"), "ltrim", json!("foo"));
297 |     give(json!(" foo  "), "ltrim", json!("foo  "));
298 |     give(json!("  foo  "), "ltrim", json!("foo  "));
299 |     give(json!("foo  "), "ltrim", json!("foo  "));
300 |     give(json!(" اَلْعَرَبِيَّةُ "), "ltrim", json!("اَلْعَرَبِيَّةُ "));
301 | }
302 | 
303 | #[test]
304 | fn rtrim() {
305 |     give(json!(""), "rtrim", json!(""));
306 |     give(json!(" "), "rtrim", json!(""));
307 |     give(json!("  "), "rtrim", json!(""));
308 |     give(json!("foo"), "rtrim", json!("foo"));
309 |     give(json!("  foo "), "rtrim", json!("  foo"));
310 |     give(json!("  foo  "), "rtrim", json!("  foo"));
311 |     give(json!("  foo"), "rtrim", json!("  foo"));
312 |     give(json!(" اَلْعَرَبِيَّةُ "), "rtrim", json!(" اَلْعَرَبِيَّةُ"));
313 | }
314 | 


--------------------------------------------------------------------------------
/jaq/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "jaq"
 3 | version = "2.2.0"
 4 | authors = ["Michael Färber "]
 5 | edition = "2021"
 6 | license = "MIT"
 7 | readme = "../README.md"
 8 | description = "Just another JSON query tool"
 9 | repository = "https://github.com/01mf02/jaq"
10 | keywords = ["json", "query", "jq"]
11 | categories = ["command-line-utilities", "compilers", "parser-implementations"]
12 | rust-version = "1.65"
13 | 
14 | [features]
15 | default = ["mimalloc"]
16 | 
17 | [dependencies]
18 | jaq-core = { version = "2.1.1", path = "../jaq-core" }
19 | jaq-std  = { version = "2.1.0", path = "../jaq-std" }
20 | jaq-json = { version = "1.1.1", path = "../jaq-json" }
21 | 
22 | codesnake = { version = "0.2" }
23 | env_logger = { version = "0.10.0", default-features = false }
24 | hifijson = "0.2.0"
25 | is-terminal = "0.4.13"
26 | log = { version = "0.4.17" }
27 | memmap2 = "0.9"
28 | mimalloc = { version = "0.1.29", default-features = false, optional = true }
29 | tempfile = "3.3.0"
30 | unicode-width = "0.1.13"
31 | yansi = "1.0.1"
32 | 


--------------------------------------------------------------------------------
/jaq/src/cli.rs:
--------------------------------------------------------------------------------
  1 | //! Command-line argument parsing
  2 | use core::fmt;
  3 | use std::env::ArgsOs;
  4 | use std::ffi::OsString;
  5 | use std::path::PathBuf;
  6 | 
  7 | #[derive(Debug, Default)]
  8 | pub struct Cli {
  9 |     // Input options
 10 |     pub null_input: bool,
 11 |     /// When the option `--slurp` is used additionally,
 12 |     /// then the whole input is read into a single string.
 13 |     pub raw_input: bool,
 14 |     /// When input is read from files,
 15 |     /// jaq yields an array for each file, whereas
 16 |     /// jq produces only a single array.
 17 |     pub slurp: bool,
 18 | 
 19 |     // Output options
 20 |     pub compact_output: bool,
 21 |     pub raw_output: bool,
 22 |     /// This flag enables `--raw-output`.
 23 |     pub join_output: bool,
 24 |     pub in_place: bool,
 25 |     pub sort_keys: bool,
 26 |     pub color_output: bool,
 27 |     pub monochrome_output: bool,
 28 |     pub tab: bool,
 29 |     pub indent: usize,
 30 | 
 31 |     // Compilation options
 32 |     pub from_file: bool,
 33 |     /// If this option is given multiple times, all given directories are searched.
 34 |     pub library_path: Vec,
 35 | 
 36 |     // Key-value options
 37 |     pub arg: Vec<(String, String)>,
 38 |     pub argjson: Vec<(String, String)>,
 39 |     pub slurpfile: Vec<(String, OsString)>,
 40 |     pub rawfile: Vec<(String, OsString)>,
 41 | 
 42 |     // Positional arguments
 43 |     /// If this argument is not given, it is assumed to be `.`, the identity filter.
 44 |     pub filter: Option,
 45 |     pub files: Vec,
 46 |     pub args: Vec,
 47 |     //pub jsonargs: Vec,
 48 |     pub run_tests: Option>,
 49 |     /// If there is some last output value `v`,
 50 |     /// then the exit status code is
 51 |     /// 1 if `v < true` (that is, if `v` is `false` or `null`) and
 52 |     /// 0 otherwise.
 53 |     /// If there is no output value, then the exit status code is 4.
 54 |     ///
 55 |     /// If any error occurs, then this option has no effect.
 56 |     pub exit_status: bool,
 57 |     pub version: bool,
 58 |     pub help: bool,
 59 | }
 60 | 
 61 | #[derive(Debug)]
 62 | pub enum Filter {
 63 |     Inline(String),
 64 |     FromFile(PathBuf),
 65 | }
 66 | 
 67 | impl Cli {
 68 |     fn positional(&mut self, mode: &Mode, arg: OsString) -> Result<(), Error> {
 69 |         if self.filter.is_none() {
 70 |             self.filter = Some(if self.from_file {
 71 |                 Filter::FromFile(arg.into())
 72 |             } else {
 73 |                 Filter::Inline(arg.into_string()?)
 74 |             })
 75 |         } else {
 76 |             match mode {
 77 |                 Mode::Files => self.files.push(arg.into()),
 78 |                 Mode::Args => self.args.push(arg.into_string()?),
 79 |                 //Mode::JsonArgs => self.jsonargs.push(arg.into_string()?),
 80 |             }
 81 |         }
 82 |         Ok(())
 83 |     }
 84 | 
 85 |     fn long(&mut self, mode: &mut Mode, arg: &str, args: &mut ArgsOs) -> Result<(), Error> {
 86 |         let int = |s: OsString| s.into_string().ok()?.parse().ok();
 87 |         match arg {
 88 |             // handle all arguments after "--"
 89 |             "" => args.try_for_each(|arg| self.positional(mode, arg))?,
 90 | 
 91 |             "null-input" => self.short('n', args)?,
 92 |             "raw-input" => self.short('R', args)?,
 93 |             "slurp" => self.short('s', args)?,
 94 | 
 95 |             "compact-output" => self.short('c', args)?,
 96 |             "raw-output" => self.short('r', args)?,
 97 |             "join-output" => self.short('j', args)?,
 98 |             "in-place" => self.short('i', args)?,
 99 |             "sort-keys" => self.short('S', args)?,
100 |             "color-output" => self.short('C', args)?,
101 |             "monochrome-output" => self.short('M', args)?,
102 |             "tab" => self.tab = true,
103 |             "indent" => self.indent = args.next().and_then(int).ok_or(Error::Int("--indent"))?,
104 |             "from-file" => self.short('f', args)?,
105 |             "library-path" => self.short('L', args)?,
106 |             "arg" => {
107 |                 let (name, value) = parse_key_val("--arg", args)?;
108 |                 self.arg.push((name, value.into_string()?));
109 |             }
110 |             "argjson" => {
111 |                 let (name, value) = parse_key_val("--argjson", args)?;
112 |                 self.argjson.push((name, value.into_string()?));
113 |             }
114 |             "slurpfile" => self.slurpfile.push(parse_key_val("--slurpfile", args)?),
115 |             "rawfile" => self.rawfile.push(parse_key_val("--rawfile", args)?),
116 | 
117 |             "args" => *mode = Mode::Args,
118 |             //"jsonargs" => *mode = Mode::JsonArgs,
119 |             "run-tests" => self.run_tests = Some(args.map(PathBuf::from).collect()),
120 |             "exit-status" => self.short('e', args)?,
121 |             "version" => self.short('V', args)?,
122 |             "help" => self.short('h', args)?,
123 | 
124 |             arg => Err(Error::Flag(format!("--{arg}")))?,
125 |         }
126 |         Ok(())
127 |     }
128 | 
129 |     fn short(&mut self, arg: char, args: &mut ArgsOs) -> Result<(), Error> {
130 |         match arg {
131 |             'n' => self.null_input = true,
132 |             'R' => self.raw_input = true,
133 |             's' => self.slurp = true,
134 | 
135 |             'c' => self.compact_output = true,
136 |             'r' => self.raw_output = true,
137 |             'j' => self.join_output = true,
138 |             'i' => self.in_place = true,
139 |             'S' => self.sort_keys = true,
140 |             'C' => self.color_output = true,
141 |             'M' => self.monochrome_output = true,
142 | 
143 |             'f' => self.from_file = true,
144 |             'L' => self
145 |                 .library_path
146 |                 .push(args.next().ok_or(Error::Path("-L"))?.into()),
147 |             'e' => self.exit_status = true,
148 |             'V' => self.version = true,
149 |             'h' => self.help = true,
150 |             arg => Err(Error::Flag(format!("-{arg}")))?,
151 |         }
152 |         Ok(())
153 |     }
154 | 
155 |     pub fn parse() -> Result {
156 |         let mut cli = Self {
157 |             indent: 2,
158 |             ..Self::default()
159 |         };
160 |         let mut mode = Mode::Files;
161 |         let mut args = std::env::args_os();
162 |         args.next();
163 |         while let Some(arg) = args.next() {
164 |             match arg.to_str() {
165 |                 // we've got a valid UTF-8 argument here
166 |                 Some(s) => match s.strip_prefix("--") {
167 |                     Some(rest) => cli.long(&mut mode, rest, &mut args)?,
168 |                     None => match s.strip_prefix("-") {
169 |                         Some(rest) => rest.chars().try_for_each(|c| cli.short(c, &mut args))?,
170 |                         None => cli.positional(&mode, arg)?,
171 |                     },
172 |                 },
173 |                 // we've got invalid UTF-8, so it is no valid flag
174 |                 // note that we do not check here whether arg starts with `-`,
175 |                 // because this seems to be quite difficult to do in a portable way
176 |                 None => cli.positional(&mode, arg)?,
177 |             }
178 |         }
179 |         Ok(cli)
180 |     }
181 | 
182 |     pub fn color_if(&self, f: impl Fn() -> bool) -> bool {
183 |         if self.monochrome_output {
184 |             false
185 |         } else if self.color_output {
186 |             true
187 |         } else {
188 |             f()
189 |         }
190 |     }
191 | }
192 | 
193 | #[derive(Debug)]
194 | pub enum Error {
195 |     Flag(String),
196 |     Utf8(OsString),
197 |     KeyValue(&'static str),
198 |     Int(&'static str),
199 |     Path(&'static str),
200 | }
201 | 
202 | impl fmt::Display for Error {
203 |     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
204 |         match self {
205 |             Self::Flag(s) => write!(f, "unknown flag: {s}"),
206 |             Self::Utf8(s) => write!(f, "invalid UTF-8: {s:?}"),
207 |             Self::KeyValue(o) => write!(f, "{o} expects a key and a value"),
208 |             Self::Int(o) => write!(f, "{o} expects an integer"),
209 |             Self::Path(o) => write!(f, "{o} expects a path"),
210 |         }
211 |     }
212 | }
213 | 
214 | /// Conversion of errors from [`OsString::into_string`].
215 | impl From for Error {
216 |     fn from(e: OsString) -> Self {
217 |         Self::Utf8(e)
218 |     }
219 | }
220 | 
221 | fn parse_key_val(arg: &'static str, args: &mut ArgsOs) -> Result<(String, OsString), Error> {
222 |     let err = || Error::KeyValue(arg);
223 |     let key = args.next().ok_or_else(err)?.into_string()?;
224 |     let val = args.next().ok_or_else(err)?;
225 |     Ok((key, val))
226 | }
227 | 
228 | /// Interpretation of positional arguments.
229 | enum Mode {
230 |     Args,
231 |     //JsonArgs,
232 |     Files,
233 | }
234 | 


--------------------------------------------------------------------------------
/jaq/src/help.txt:
--------------------------------------------------------------------------------
 1 | Just Another Query Tool
 2 | 
 3 | Usage: jaq [OPTION]... [FILTER] [ARG]...
 4 | 
 5 | Arguments:
 6 |   [FILTER]  Filter to execute
 7 |   [ARG]...  Positional arguments, by default used as input files
 8 | 
 9 | Input options:
10 |   -n, --null-input          Use null as single input value
11 |   -R, --raw-input           Read lines of the input as sequence of strings
12 |   -s, --slurp               Read (slurp) all input values into one array
13 | 
14 | Output options:
15 |   -c, --compact-output      Print JSON compactly, omitting whitespace
16 |   -r, --raw-output          Write strings without escaping them with quotes
17 |   -j, --join-output         Do not print a newline after each value
18 |   -i, --in-place            Overwrite input file with its output
19 |   -S, --sort-keys           Print objects sorted by their keys
20 |   -C, --color-output        Always color output
21 |   -M, --monochrome-output   Do not color output
22 |       --tab                 Use tabs for indentation rather than spaces
23 |       --indent           Use N spaces for indentation [default: 2]
24 | 
25 | Compilation options:
26 |   -f, --from-file           Read filter from a file given by filter argument
27 |   -L, --library-path   Search for modules and data in given directory
28 | 
29 | Variable options:
30 |       --arg           Set variable `$A` to string `V`
31 |       --argjson       Set variable `$A` to JSON value `V`
32 |       --slurpfile     Set variable `$A` to array containing the JSON values in file `F`
33 |       --rawfile       Set variable `$A` to string containing the contents of file `F`
34 |       --args                Collect remaining positional arguments into `$ARGS.positional`
35 | 
36 | Remaining options:
37 |       --run-tests     Run tests from a file
38 |   -e, --exit-status         Use the last output value as exit status code
39 |   -V, --version             Print version
40 |   -h, --help                Print help
41 | 


--------------------------------------------------------------------------------
/jaq/tests/a.jq:
--------------------------------------------------------------------------------
1 | include "b" {search: "."};
2 | import "c" as c {search: "mods"};
3 | import "data" as $data {search: "."};
4 | def a: b + c::c;
5 | def d: 2;
6 | def d: 3;
7 | def data: $data;
8 | 


--------------------------------------------------------------------------------
/jaq/tests/b.jq:
--------------------------------------------------------------------------------
1 | def b: "b";
2 | 


--------------------------------------------------------------------------------
/jaq/tests/data.json:
--------------------------------------------------------------------------------
1 | 1
2 | 2
3 | 


--------------------------------------------------------------------------------
/jaq/tests/golden.rs:
--------------------------------------------------------------------------------
  1 | use std::{env, io, process, str};
  2 | 
  3 | fn golden_test(args: &[&str], input: &str, out_ex: &str) -> io::Result<()> {
  4 |     let mut child = process::Command::new(env!("CARGO_BIN_EXE_jaq"))
  5 |         .args(args)
  6 |         .stdin(process::Stdio::piped())
  7 |         .stdout(process::Stdio::piped())
  8 |         .spawn()?;
  9 | 
 10 |     use io::Write;
 11 |     child.stdin.take().unwrap().write_all(input.as_bytes())?;
 12 |     let output = child.wait_with_output()?;
 13 |     assert!(output.status.success());
 14 | 
 15 |     let out_act = str::from_utf8(&output.stdout).expect("invalid UTF-8 in output");
 16 |     // remove '\r' from output for compatibility with Windows
 17 |     let out_act = out_act.replace('\r', "");
 18 |     if out_ex.trim() != out_act.trim() {
 19 |         println!("Expected output:\n{}\n---", out_ex);
 20 |         println!("Actual output:\n{}\n---", out_act);
 21 |         process::exit(2);
 22 |     }
 23 |     Ok(())
 24 | }
 25 | 
 26 | macro_rules! test {
 27 |     ($name:ident, $args:expr, $input:expr, $output:expr) => {
 28 |         #[test]
 29 |         fn $name() -> io::Result<()> {
 30 |             golden_test($args, $input, $output)
 31 |         }
 32 |     };
 33 | }
 34 | 
 35 | test!(no_args, &[], "[0, 1]", "[\n  0,\n  1\n]");
 36 | test!(one, &["1"], "0", "1");
 37 | test!(sparse, &["."], "[2,3]", "[\n  2,\n  3\n]");
 38 | 
 39 | test!(
 40 |     arg,
 41 |     &["--arg", "x", "y", "--arg", "a", "b", "$x + $a"],
 42 |     "0",
 43 |     "\"yb\""
 44 | );
 45 | 
 46 | test!(
 47 |     argjson,
 48 |     &[
 49 |         "--argjson",
 50 |         "a",
 51 |         "[1,2,3]",
 52 |         "--argjson",
 53 |         "b",
 54 |         r#""abc""#,
 55 |         "$a,$b"
 56 |     ],
 57 |     "0",
 58 |     r#"[
 59 |   1,
 60 |   2,
 61 |   3
 62 | ]
 63 | "abc""#
 64 | );
 65 | 
 66 | test!(
 67 |     args,
 68 |     &["-c", "--args", "--arg", "x", "y", "$ARGS", "a", "--", "--test", "--"],
 69 |     "0",
 70 |     r#"{"positional":["a","--test","--"],"named":{"x":"y"}}"#
 71 | );
 72 | 
 73 | test!(
 74 |     join_output,
 75 |     &["-j", "."],
 76 |     r#"[] "foo" "bar" 1 2 {}"#,
 77 |     "[]foobar12{}"
 78 | );
 79 | 
 80 | test!(
 81 |     raw_output,
 82 |     &["-r", "."],
 83 |     r#""foo" "bar" ["baz"]"#,
 84 |     r#"foo
 85 | bar
 86 | [
 87 |   "baz"
 88 | ]"#
 89 | );
 90 | 
 91 | test!(
 92 |     compact,
 93 |     &["-c", "."],
 94 |     r#"[2,3]
 95 | {"a":1,   "b":["c"   ]}"#,
 96 |     r#"[2,3]
 97 | {"a":1,"b":["c"]}"#
 98 | );
 99 | 
100 | test!(
101 |     sort_keys,
102 |     &["-Sc", "."],
103 |     r#"{"b": 1, "a": 2}"#,
104 |     r#"{"a":2,"b":1}"#
105 | );
106 | 
107 | test!(
108 |     inputs,
109 |     &["-c", r#"{".": .}, {input: input}"#],
110 |     "0\n1\n2\n3",
111 |     r#"{".":0}
112 | {"input":1}
113 | {".":2}
114 | {"input":3}"#
115 | );
116 | 
117 | test!(
118 |     null_input,
119 |     &["-nc", r#"{".": .}, {inputs: [inputs]}"#],
120 |     "0\n1\n2\n3",
121 |     r#"{".":null}
122 | {"inputs":[0,1,2,3]}"#
123 | );
124 | 
125 | const ONE23: &str = "One\nTwo\nThree\n";
126 | 
127 | test!(raw_input_slurp, &["-Rs"], ONE23, r#""One\nTwo\nThree\n""#);
128 | 
129 | test!(
130 |     raw_input,
131 |     &["-R"],
132 |     ONE23,
133 |     r#""One"
134 | "Two"
135 | "Three""#
136 | );
137 | 
138 | test!(
139 |     fmt_str,
140 |     &[],
141 |     r#""\u0000\u200b\r\t\n asdf""#,
142 |     r#""\u0000​\r\t\n asdf""#
143 | );
144 | 
145 | test!(
146 |     fmt_obj_key,
147 |     &["-c"],
148 |     r#"{"विश\u094dव": 1}"#,
149 |     r#"{"विश्व":1}"#
150 | );
151 | 
152 | test!(
153 |     mods,
154 |     &["-c", "-L", "tests", r#"include "a"; [a, data, d]"#],
155 |     "0",
156 |     r#"["bcddd",[1,2],3]"#
157 | );
158 | 


--------------------------------------------------------------------------------
/jaq/tests/mods/c.jq:
--------------------------------------------------------------------------------
1 | import "d" as d1 {search: "."};
2 | import "d" as d2 {search: "../mods"};
3 | import "d" as d3 {search: ["foo", "."]};
4 | def c: "c" + d1::d + d2::d + d3::d;
5 | 


--------------------------------------------------------------------------------
/jaq/tests/mods/d.jq:
--------------------------------------------------------------------------------
1 | include "b" {search: ".."};
2 | def d: "d";
3 | 


--------------------------------------------------------------------------------