├── .github ├── CODEOWNERS ├── labels.yaml ├── renovate.json5 ├── scripts │ └── package.sh └── workflows │ ├── ci.yml │ ├── labels.yml │ ├── publish-packages.yml │ ├── release-plz.yml │ └── renovate.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── benches └── benchmark.rs ├── cliff.toml ├── release-plz.toml ├── src ├── bin │ └── dts │ │ ├── args.rs │ │ ├── assets │ │ └── example.json │ │ ├── highlighting.rs │ │ ├── main.rs │ │ ├── output.rs │ │ ├── paging.rs │ │ └── utils.rs ├── de.rs ├── encoding.rs ├── error.rs ├── filter │ ├── jaq.rs │ ├── jq.rs │ └── mod.rs ├── key.rs ├── lib.rs ├── parsers │ ├── error.rs │ ├── flat_key.rs │ ├── grammars │ │ ├── flat_key.pest │ │ └── gron.pest │ ├── gron.rs │ └── mod.rs ├── ser.rs ├── sink.rs ├── source.rs └── value.rs └── tests ├── fixtures ├── example.compact.hcl ├── example.filtered.json ├── example.hcl ├── example.js ├── example.js.ungron.json ├── example.json ├── example.merged.json ├── example.multi-doc.yaml ├── example.toml ├── example.yaml ├── filter.jq ├── friends.csv ├── math.hcl ├── math.json ├── math.simplified.json └── users.csv └── integration_tests.rs /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @martinohmann 2 | -------------------------------------------------------------------------------- /.github/labels.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Renovate 3 | - name: renovate/container 4 | color: "ffc300" 5 | - name: renovate/github-action 6 | color: "ffc300" 7 | - name: renovate/github-release 8 | color: "ffc300" 9 | # Semantic Type 10 | - name: type/digest 11 | color: "ffeC19" 12 | - name: type/patch 13 | color: "ffec19" 14 | - name: type/minor 15 | color: "ff9800" 16 | - name: type/major 17 | color: "f6412d" 18 | # Uncategorized 19 | - name: bug 20 | color: "ee0701" 21 | - name: do-not-merge 22 | color: "ee0701" 23 | - name: hold 24 | color: "ee0701" 25 | - name: docs 26 | color: "f4d1b7" 27 | - name: enhancement 28 | color: "84b6eb" 29 | - name: broken-links 30 | color: "7b55d7" 31 | - name: question 32 | color: "cc317c" 33 | - name: community 34 | color: "0e8a16" 35 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | ":automergeBranch", 5 | ":automergeDigest", 6 | ":semanticCommitTypeAll(chore)", 7 | "github>martinohmann/.github//renovate/default.json5" 8 | ], 9 | "schedule": ["on the first day of the month"], 10 | "packageRules": [ 11 | { 12 | "matchUpdateTypes": ["major"], 13 | "labels": ["type/major"] 14 | }, 15 | { 16 | "matchUpdateTypes": ["minor"], 17 | "labels": ["type/minor"] 18 | }, 19 | { 20 | "matchUpdateTypes": ["patch"], 21 | "labels": ["type/patch"] 22 | }, 23 | { 24 | "matchDatasources": ["docker"], 25 | "addLabels": ["renovate/container"] 26 | }, 27 | { 28 | "matchDatasources": ["github-releases", "github-tags"], 29 | "addLabels": ["renovate/github-release"] 30 | }, 31 | { 32 | "matchManagers": ["github-actions"], 33 | "addLabels": ["renovate/github-action"] 34 | }, 35 | { 36 | "description": ["jaq group"], 37 | "groupName": "jaq", 38 | "matchPackageNames": ["/jaq.*/"], 39 | "matchManagers": ["cargo"], 40 | "group": { 41 | "commitMessageTopic": "{{{groupName}}} group" 42 | }, 43 | "separateMinorPatch": true 44 | }, 45 | { 46 | "description": ["clap group"], 47 | "groupName": "clap", 48 | "matchPackageNames": ["/clap.*/"], 49 | "matchManagers": ["cargo"], 50 | "group": { 51 | "commitMessageTopic": "{{{groupName}}} group" 52 | }, 53 | "separateMinorPatch": true 54 | }, 55 | { 56 | "description": ["pest group"], 57 | "groupName": "pest", 58 | "matchPackageNames": ["/pest.*/"], 59 | "matchManagers": ["cargo"], 60 | "group": { 61 | "commitMessageTopic": "{{{groupName}}} group" 62 | }, 63 | "separateMinorPatch": true 64 | }, 65 | ] 66 | } 67 | -------------------------------------------------------------------------------- /.github/scripts/package.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Packages up releases as tar archives. 4 | 5 | set -euo pipefail 6 | 7 | GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" 8 | 9 | strip_binary() { 10 | local target="$1" 11 | local bin_path="$2" 12 | local stripped_bin_path="$3" 13 | 14 | case "$target" in 15 | arm-unknown-linux-*) 16 | strip="arm-linux-gnueabihf-strip" ;; 17 | aarch64-unknown-linux-gnu) 18 | strip="aarch64-linux-gnu-strip" ;; 19 | *) 20 | strip="strip" ;; 21 | esac 22 | 23 | echo "stripping binary $bin_path -> $stripped_bin_path" 24 | 25 | "$strip" -o "$stripped_bin_path" "$bin_path" 26 | } 27 | 28 | create_package() { 29 | local archive_dir="$1" 30 | local bin_path="$2" 31 | 32 | echo "copying package files to $archive_dir" 33 | 34 | cp "$bin_path" "$archive_dir" 35 | cp "README.md" "LICENSE" "CHANGELOG.md" "$archive_dir" 36 | } 37 | 38 | create_archive() { 39 | local target="$1" 40 | local package_dir="$2" 41 | local package_basename="$3" 42 | local archive_name="$4" 43 | 44 | case "$target" in 45 | *-darwin) 46 | sha512sum="gsha512sum" ;; 47 | *) 48 | sha512sum="sha512sum" ;; 49 | esac 50 | 51 | pushd "$package_dir" >/dev/null || exit 1 52 | echo "creating archive ${package_dir}/${archive_name}" 53 | tar czf "$archive_name" "$package_basename"/* 54 | 55 | echo "creating checksum file for archive ${package_dir}/${archive_name}.sha512" 56 | "$sha512sum" "$archive_name" > "${archive_name}.sha512" 57 | popd >/dev/null || exit 1 58 | } 59 | 60 | package() { 61 | local target="$1" 62 | local version="$2" 63 | 64 | bin_name=dts 65 | bin_path="target/${target}/release/${bin_name}" 66 | 67 | if ! [ -f "$bin_path" ]; then 68 | echo "release binary missing, build via:" 69 | echo 70 | echo " cargo build --release --locked --target $target" 71 | exit 1 72 | fi 73 | 74 | artifacts_dir=release-artifacts 75 | stripped_bin_path="${artifacts_dir}/${bin_name}" 76 | 77 | rm -rf "$artifacts_dir" 78 | mkdir -p "$artifacts_dir" 79 | 80 | strip_binary "$target" "$bin_path" "$stripped_bin_path" 81 | 82 | package_basename="${bin_name}-v${version}-${target}" 83 | archive_name="${package_basename}.tar.gz" 84 | package_dir="${artifacts_dir}/package" 85 | archive_dir="${package_dir}/${package_basename}/" 86 | archive_path="${package_dir}/${archive_name}" 87 | 88 | mkdir -p "$archive_dir" 89 | 90 | create_package "$archive_dir" "$stripped_bin_path" 91 | create_archive "$target" "$package_dir" "$package_basename" \ 92 | "$archive_name" 93 | 94 | rm -rf "$archive_dir" 95 | 96 | # shellcheck disable=SC2129 97 | echo "package_dir=${package_dir}" >> "$GITHUB_OUTPUT" 98 | echo "archive_name=${archive_name}" >> "$GITHUB_OUTPUT" 99 | echo "archive_path=${archive_path}" >> "$GITHUB_OUTPUT" 100 | } 101 | 102 | if [ $# -lt 2 ]; then 103 | echo "usage: $0 " 104 | exit 1 105 | fi 106 | 107 | package "$@" 108 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: ci 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - main 9 | tags: 10 | - '*' 11 | 12 | jobs: 13 | test: 14 | env: 15 | # Emit backtraces on panics. 16 | RUST_BACKTRACE: 1 17 | name: test ${{ matrix.target }} (${{ matrix.os }}) 18 | runs-on: ${{ matrix.os }} 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | include: 23 | - target: aarch64-apple-darwin 24 | os: macos-latest 25 | - target: aarch64-unknown-linux-gnu 26 | os: ubuntu-latest 27 | use-cross: true 28 | - target: arm-unknown-linux-gnueabihf 29 | os: ubuntu-latest 30 | use-cross: true 31 | - target: arm-unknown-linux-musleabihf 32 | os: ubuntu-latest 33 | use-cross: true 34 | - target: x86_64-apple-darwin 35 | os: macos-latest 36 | - target: x86_64-unknown-linux-gnu 37 | os: ubuntu-latest 38 | - target: x86_64-unknown-linux-musl 39 | os: ubuntu-latest 40 | use-cross: true 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 44 | 45 | - name: Install Rust 46 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 47 | with: 48 | toolchain: stable 49 | target: ${{ matrix.target }} 50 | profile: minimal 51 | override: true 52 | 53 | - name: Load cache 54 | uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 55 | with: 56 | path: | 57 | ~/.cargo/bin/ 58 | ~/.cargo/registry/index/ 59 | ~/.cargo/registry/cache/ 60 | ~/.cargo/git/db/ 61 | target/ 62 | key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} 63 | 64 | - name: Build 65 | uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 66 | with: 67 | use-cross: ${{ matrix.use-cross }} 68 | command: build 69 | args: --target=${{ matrix.target }} 70 | 71 | - name: Run tests 72 | if: ${{ !matrix.use-cross }} 73 | uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 74 | with: 75 | use-cross: ${{ matrix.use-cross }} 76 | command: test 77 | args: --target=${{ matrix.target }} 78 | 79 | rustfmt: 80 | name: rustfmt 81 | runs-on: ubuntu-latest 82 | steps: 83 | - name: Checkout repository 84 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 85 | 86 | - name: Install Rust 87 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 88 | with: 89 | toolchain: stable 90 | override: true 91 | profile: minimal 92 | components: rustfmt 93 | 94 | - name: Check formatting 95 | uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 96 | with: 97 | command: fmt 98 | args: --all -- --check 99 | 100 | docs: 101 | name: docs 102 | runs-on: ubuntu-latest 103 | steps: 104 | - name: Checkout repository 105 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 106 | 107 | - name: Install Rust 108 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 109 | with: 110 | toolchain: stable 111 | profile: minimal 112 | override: true 113 | 114 | - name: Check documentation 115 | uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 116 | env: 117 | RUSTDOCFLAGS: -D warnings 118 | with: 119 | command: doc 120 | args: --no-deps --document-private-items 121 | -------------------------------------------------------------------------------- /.github/workflows/labels.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Sync labels 3 | 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: ["main"] 8 | paths: [".github/labels.yaml"] 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | labels: 14 | name: Sync Labels 15 | permissions: 16 | contents: read 17 | issues: write 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | 23 | - name: Sync Labels 24 | uses: EndBug/label-sync@52074158190acb45f3077f9099fea818aa43f97a # v2.3.3 25 | with: 26 | config-file: .github/labels.yaml 27 | token: "${{ secrets.GITHUB_TOKEN }}" 28 | delete-other-labels: true 29 | -------------------------------------------------------------------------------- /.github/workflows/publish-packages.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: publish-packages 3 | on: 4 | workflow_dispatch: 5 | push: 6 | tags: 7 | - '**[0-9]+.[0-9]+.[0-9]+*' 8 | 9 | jobs: 10 | publish-packages: 11 | name: Package ${{ matrix.target }} (${{ matrix.os }}) 12 | runs-on: ${{ matrix.os }} 13 | env: 14 | # Emit backtraces on panics. 15 | RUST_BACKTRACE: 1 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | include: 20 | - target: aarch64-apple-darwin 21 | os: macos-latest 22 | - target: aarch64-unknown-linux-gnu 23 | os: ubuntu-latest 24 | use-cross: true 25 | - target: arm-unknown-linux-gnueabihf 26 | os: ubuntu-latest 27 | use-cross: true 28 | - target: arm-unknown-linux-musleabihf 29 | os: ubuntu-latest 30 | use-cross: true 31 | - target: x86_64-apple-darwin 32 | os: macos-latest 33 | - target: x86_64-unknown-linux-gnu 34 | os: ubuntu-latest 35 | - target: x86_64-unknown-linux-musl 36 | os: ubuntu-latest 37 | use-cross: true 38 | steps: 39 | - name: Checkout source code 40 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 41 | 42 | - name: Install prerequisites 43 | shell: bash 44 | run: | 45 | case ${{ matrix.target }} in 46 | arm-unknown-linux-*) 47 | sudo apt-get -y update 48 | sudo apt-get -y install gcc-arm-linux-gnueabihf ;; 49 | aarch64-unknown-linux-gnu) 50 | sudo apt-get -y update 51 | sudo apt-get -y install gcc-aarch64-linux-gnu ;; 52 | *-darwin) 53 | brew install coreutils ;; 54 | esac 55 | 56 | - name: Install Rust toolchain 57 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 58 | with: 59 | toolchain: stable 60 | target: ${{ matrix.target }} 61 | override: true 62 | profile: minimal 63 | 64 | - name: Package version 65 | id: package-version 66 | shell: bash 67 | run: | 68 | version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n1)" 69 | 70 | echo "version=${version}" >> $GITHUB_OUTPUT 71 | 72 | - name: Show version information (Rust, cargo, GCC) 73 | shell: bash 74 | run: | 75 | gcc --version || true 76 | rustup -V 77 | rustup toolchain list 78 | rustup default 79 | cargo -V 80 | rustc -V 81 | echo 'Package version: ${{ steps.package-version.outputs.version }}' 82 | 83 | - name: Build 84 | uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 85 | with: 86 | use-cross: ${{ matrix.use-cross }} 87 | command: build 88 | args: --locked --release --target=${{ matrix.target }} 89 | 90 | - name: Package 91 | id: package 92 | run: | 93 | .github/scripts/package.sh ${{ matrix.target }} \ 94 | ${{ steps.package-version.outputs.version }} 95 | 96 | - name: Upload artifacts 97 | uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 98 | with: 99 | name: ${{ steps.package.outputs.archive_name }} 100 | path: ${{ steps.package.outputs.archive_path }} 101 | 102 | - name: Publish release archives 103 | uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2 104 | with: 105 | generate_release_notes: false 106 | tag_name: "v${{ steps.package-version.outputs.version }}" 107 | files: ${{ steps.package.outputs.package_dir }}/* 108 | -------------------------------------------------------------------------------- /.github/workflows/release-plz.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: release-plz 3 | 4 | permissions: 5 | pull-requests: write 6 | contents: write 7 | 8 | on: 9 | push: 10 | branches: 11 | - main 12 | 13 | jobs: 14 | release-plz: 15 | name: Release-plz 16 | runs-on: ubuntu-latest 17 | if: ${{ github.repository_owner == 'martinohmann' }} 18 | steps: 19 | - name: Checkout repository 20 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 21 | with: 22 | fetch-depth: 0 23 | token: ${{ secrets.RELEASE_PLZ_TOKEN }} 24 | 25 | - name: Install Rust toolchain 26 | uses: dtolnay/rust-toolchain@stable 27 | 28 | - name: Run release-plz 29 | uses: MarcoIeni/release-plz-action@dde7b63054529c440305a924e5849c68318bcc9a # v0.5 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.RELEASE_PLZ_TOKEN }} 32 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 33 | -------------------------------------------------------------------------------- /.github/workflows/renovate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Renovate" 3 | 4 | on: 5 | workflow_dispatch: 6 | inputs: 7 | dryRun: 8 | description: Dry Run 9 | default: "false" 10 | required: false 11 | logLevel: 12 | description: Log Level 13 | default: debug 14 | required: false 15 | version: 16 | description: Renovate version 17 | default: latest 18 | required: false 19 | schedule: 20 | - cron: "0 18 * * *" 21 | push: 22 | branches: ["main"] 23 | paths: 24 | - .github/renovate.json5 25 | - .github/renovate/**.json5 26 | 27 | concurrency: 28 | group: ${{ github.workflow }}-${{ github.event.number || github.ref }} 29 | cancel-in-progress: true 30 | 31 | env: 32 | WORKFLOW_DRY_RUN: false 33 | WORKFLOW_LOG_LEVEL: debug 34 | WORKFLOW_VERSION: latest 35 | RENOVATE_PLATFORM: github 36 | RENOVATE_PLATFORM_COMMIT: true 37 | RENOVATE_ONBOARDING_CONFIG_FILE_NAME: .github/renovate.json5 38 | RENOVATE_AUTODISCOVER: true 39 | RENOVATE_AUTODISCOVER_FILTER: "${{ github.repository }}" 40 | 41 | permissions: 42 | contents: read 43 | pull-requests: write 44 | 45 | jobs: 46 | renovate: 47 | name: Renovate 48 | runs-on: ubuntu-latest 49 | steps: 50 | - name: Checkout 51 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 52 | 53 | - name: Override default config from dispatch variables 54 | shell: bash 55 | run: | 56 | echo "RENOVATE_DRY_RUN=${{ github.event.inputs.dryRun || env.WORKFLOW_DRY_RUN }}" >> "${GITHUB_ENV}" 57 | echo "LOG_LEVEL=${{ github.event.inputs.logLevel || env.WORKFLOW_LOG_LEVEL }}" >> "${GITHUB_ENV}" 58 | 59 | - name: Renovate 60 | uses: renovatebot/github-action@8058cfe11252651a837a58e2e3370fbc0e72c658 # v42.0.4 61 | with: 62 | configurationFile: "${{ env.RENOVATE_ONBOARDING_CONFIG_FILE_NAME }}" 63 | token: "${{ secrets.GITHUB_TOKEN }}" 64 | renovate-version: "${{ github.event.inputs.version || env.WORKFLOW_VERSION }}" 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /release-artifacts 2 | /testdata 3 | /target 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dts" 3 | version = "0.6.10" 4 | authors = ["Martin Ohmann "] 5 | license = "MIT" 6 | description = "A tool to deserialize, transform and serialize data between different encodings" 7 | repository = "https://github.com/martinohmann/dts" 8 | documentation = "https://docs.rs/dts/" 9 | keywords = ["hcl", "json", "yaml", "jq"] 10 | categories = ["encoding"] 11 | readme = "README.md" 12 | edition = "2024" 13 | rust-version = "1.85" 14 | exclude = [ 15 | ".github/", 16 | ] 17 | 18 | [features] 19 | default = ["color", "jaq"] 20 | color = ["bat", "bat/paging", "clap/color"] 21 | jaq = ["jaq-core", "jaq-interpret", "jaq-parse", "jaq-std"] 22 | 23 | [dependencies] 24 | anyhow = "1.0.86" 25 | crossbeam-utils = "0.8.16" 26 | csv = "1.2.2" 27 | glob = "0.3.1" 28 | clap_complete = "4.5.1" 29 | grep-cli = "0.1.8" 30 | hcl-rs = { version = "0.18.0", features = ["perf"] } 31 | jaq-core = { version = "1.4.0", optional = true } 32 | jaq-interpret = { version = "1.5.0", optional = true } 33 | jaq-parse = { version = "1.0.2", optional = true } 34 | jaq-std = { version = "1.4.0", optional = true } 35 | json5 = "0.4.1" 36 | once_cell = "1.19.0" 37 | pathdiff = "0.2.1" 38 | pest = "2.7.7" 39 | pest_derive = "2.7.7" 40 | rayon = "1.7.0" 41 | regex = "1.7.3" 42 | serde-xml-rs = "0.8.0" 43 | serde_qs = "0.15.0" 44 | serde_yaml = "0.9.34" 45 | shell-words = "1.1.0" 46 | termcolor = "1.4.1" 47 | thiserror = "1.0.59" 48 | toml = "0.8.12" 49 | unescape = "0.1.0" 50 | ureq = "3.0.0" 51 | url = "2.5.4" 52 | 53 | [dependencies.bat] 54 | optional = true 55 | default-features = false 56 | features = ["regex-onig"] 57 | version = "0.25.0" 58 | 59 | [dependencies.clap] 60 | default-features = false 61 | features = ["std", "derive", "env", "help", "suggestions"] 62 | version = "4.5.13" 63 | 64 | [dependencies.serde] 65 | features = ["derive"] 66 | version = "1.0.203" 67 | 68 | [dependencies.serde_json] 69 | features = ["preserve_order"] 70 | version = "1.0.137" 71 | 72 | [dev-dependencies] 73 | assert_cmd = "2.0.14" 74 | criterion = "0.6" 75 | pretty_assertions = "1.4.0" 76 | predicates = "3.0.3" 77 | temp-env = "0.3.1" 78 | 79 | [[bench]] 80 | name = "benchmark" 81 | harness = false 82 | 83 | [[test]] 84 | name = "integration" 85 | path = "tests/integration_tests.rs" 86 | 87 | [profile.release] 88 | codegen-units = 1 89 | lto = true 90 | opt-level = "s" 91 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 martinohmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dts 2 | 3 | [![Build Status](https://github.com/martinohmann/dts/workflows/ci/badge.svg)](https://github.com/martinohmann/dts/actions?query=workflow%3Aci) 4 | ![MIT License](https://img.shields.io/github/license/martinohmann/dts?color=blue) 5 | 6 | A simple tool to _**deserialize**_ data from an input encoding, _**transform**_ 7 | it and _**serialize**_ it back into an output encoding. 8 | 9 | Uses [`jq`](https://stedolan.github.io/jq/) for data transformation and 10 | requires rust >= 1.56.0. 11 | 12 | ## Installation 13 | 14 | Check out the [releases page](https://github.com/martinohmann/dts/releases) 15 | for prebuilt versions of `dts`. 16 | 17 | Statically-linked binaries are also available: look for archives with `musl` in 18 | the filename. 19 | 20 | ### From crates.io 21 | 22 | ```sh 23 | cargo install dts 24 | ``` 25 | 26 | ### From source 27 | 28 | Clone the repository and run: 29 | 30 | ```sh 31 | cargo install --locked --path . 32 | ``` 33 | 34 | ## Usage 35 | 36 | ```sh 37 | dts [...] [-j ] [-O ...] 38 | ``` 39 | 40 | For a full list of available flags consult the help: 41 | 42 | ```sh 43 | dts --help 44 | ``` 45 | 46 | ## Examples 47 | 48 | Convert YAML to TOML: 49 | 50 | ```sh 51 | dts input.yaml -o toml 52 | ``` 53 | 54 | Load all YAML files from sub directories and merge them into one: 55 | 56 | ```sh 57 | dts . --glob '**/*.yaml' output.yaml 58 | ``` 59 | 60 | Transform the input data using a [`jq`](https://stedolan.github.io/jq/) expression: 61 | 62 | ```sh 63 | dts tests/fixtures/example.json -j '.users | map(select(.age < 30))' 64 | ``` 65 | 66 | Use `jq` filter expression from a file: 67 | 68 | ```sh 69 | dts tests/fixtures/example.json -j @my-filter.jq 70 | ``` 71 | 72 | Read data from stdin: 73 | 74 | ```sh 75 | echo '{"foo": {"bar": "baz"}}' | dts -i json -o yaml 76 | ``` 77 | 78 | ## Output colors and themes 79 | 80 | `dts` supports output coloring and syntax highlighting. The coloring behaviour 81 | can be controlled via the `--color` flag or `DTS_COLOR` environment variable. 82 | 83 | If the default theme used for syntax highlighting does not suit you, you can 84 | change it via the `--theme` flag or `DTS_THEME` environment variable. 85 | 86 | Available themes can be listed via: 87 | 88 | ```sh 89 | dts --list-themes 90 | ``` 91 | 92 | **Hint**: The `color` feature can be disabled at compile time if you don't want 93 | to have colors at all. See the [feature flags](#feature-flags) section below. 94 | 95 | ## Supported Encodings 96 | 97 | Right now `dts` supports the following encodings: 98 | 99 | - JSON 100 | - YAML 101 | - TOML 102 | - JSON5 _(deserialize only)_ 103 | - CSV 104 | - QueryString 105 | - XML 106 | - Text 107 | - Gron 108 | - HCL _(deserialize, serialize only supports HCL attributes)_ 109 | 110 | ## Feature flags 111 | 112 | To build `dts` without its default features enabled, run: 113 | 114 | ```sh 115 | cargo build --no-default-features --release 116 | ``` 117 | 118 | The following feature flags are available: 119 | 120 | * `color`: Enables support for colored output. This feature is enabled by 121 | default. 122 | 123 | If you just want to disable colors by default with the option to enable them 124 | conditionally, you can also set the [`NO_COLOR`](https://no-color.org/) 125 | environment variable or set `DTS_COLOR=never`. 126 | 127 | * `jaq`: Use [`jaq-core`](https://docs.rs/jaq-core/latest/jaq_core/) to 128 | process transformation filters instead of shelling out to `jq`. This feature 129 | is enabled by default. 130 | 131 | ## License 132 | 133 | The source code of dts is released under the MIT License. See the bundled 134 | LICENSE file for details. 135 | -------------------------------------------------------------------------------- /benches/benchmark.rs: -------------------------------------------------------------------------------- 1 | use assert_cmd::Command; 2 | use criterion::{Criterion, criterion_group, criterion_main}; 3 | use dts::key::*; 4 | use serde_json::json; 5 | 6 | fn benchmark_transform(c: &mut Criterion) { 7 | c.bench_function("expand_keys", |b| { 8 | b.iter(|| { 9 | let value = json!({ 10 | "foo": [], 11 | "foo.bar[0]": "baz", 12 | "foo.bar[1]": "qux", 13 | "bar": {}, 14 | "bar.qux": [], 15 | "bar.qux[0]": null, 16 | "bar.qux[10]": "qux", 17 | "qux": {}, 18 | "qux.one": [], 19 | "qux.one[0]": {}, 20 | "qux.one[0].two": 3, 21 | "qux.one[1]": {}, 22 | "qux.one[3].four": [], 23 | "qux.one[3].four[0]": "five", 24 | "qux.one[30].four[1]": 6, 25 | "bam": [], 26 | "bam[0]": "a", 27 | "bam[1]": "b", 28 | "bam[5]": [], 29 | "bam[5][0]": "c", 30 | "bam[5][1]": "d", 31 | "bam[5][6]": "e", 32 | "adsf[\"foo\\\"-bar\"].baz[\"very\"][10].deep": 42, 33 | "adsf[\"foo\\\"-bar\"].buz[\"adsf\"].foo": null, 34 | }); 35 | 36 | expand_keys(value) 37 | }) 38 | }); 39 | 40 | c.bench_function("flatten_keys", |b| { 41 | b.iter(|| { 42 | let value = json!({ 43 | "foo": { 44 | "bar": ["baz", "qux"] 45 | }, 46 | "bar": { 47 | "qux": [null, "qux"] 48 | }, 49 | "qux": { 50 | "one": [ 51 | {"two": 3}, 52 | {"four": ["five", 6]} 53 | ] 54 | }, 55 | "bam": ["a", "b", ["c", "d", "e"]] 56 | }); 57 | 58 | flatten_keys(value, "data") 59 | }) 60 | }); 61 | 62 | c.bench_function("dts", |b| { 63 | b.iter(|| { 64 | Command::cargo_bin("dts") 65 | .unwrap() 66 | .arg("tests/fixtures") 67 | .args(&["--glob", "*", "-C", "-j", ".[]"]) 68 | .assert() 69 | .success() 70 | }) 71 | }); 72 | } 73 | 74 | criterion_group!(benches, benchmark_transform); 75 | criterion_main!(benches); 76 | -------------------------------------------------------------------------------- /cliff.toml: -------------------------------------------------------------------------------- 1 | # git-cliff ~ default configuration file 2 | # https://git-cliff.org/docs/configuration 3 | # 4 | # Lines starting with "#" are comments. 5 | # Configuration options are organized into tables and keys. 6 | # See documentation for more information on available options. 7 | 8 | [changelog] 9 | # changelog header 10 | header = """ 11 | # Changelog\n 12 | All notable changes to this project will be documented in this file.\n 13 | """ 14 | # template for the changelog body 15 | # https://keats.github.io/tera/docs/#introduction 16 | body = """ 17 | {% if version %}\ 18 | {% if previous.version %}\ 19 | ## [{{ version | trim_start_matches(pat="v") }}](/compare/{{ previous.version }}..{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }} 20 | {% else %}\ 21 | ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} 22 | {% endif %}\ 23 | {% else %}\ 24 | ## [unreleased] 25 | {% endif %}\ 26 | 27 | {% macro commit(commit) -%} 28 | - {% if commit.scope %}**{{ commit.scope }}**: {% endif %}{% if commit.breaking %}[**breaking**] {% endif %}\ 29 | {{ commit.message | upper_first }} - ([{{ commit.id | truncate(length=7, end="") }}](/commit/{{ commit.id }}))\ 30 | {% endmacro -%} 31 | 32 | {% for group, commits in commits | group_by(attribute="group") %} 33 | ### {{ group | striptags | trim | upper_first }} 34 | {% for commit in commits 35 | | filter(attribute="scope") 36 | | sort(attribute="scope") %} 37 | {{ self::commit(commit=commit) }} 38 | {%- endfor -%} 39 | {% raw %}\n{% endraw %}\ 40 | {%- for commit in commits %} 41 | {%- if not commit.scope -%} 42 | {{ self::commit(commit=commit) }} 43 | {% endif -%} 44 | {% endfor -%} 45 | {% endfor %}\n 46 | """ 47 | # remove the leading and trailing whitespace from the template 48 | trim = true 49 | # changelog footer 50 | footer = """ 51 | 52 | """ 53 | # postprocessors 54 | postprocessors = [ 55 | { pattern = '', replace = "https://github.com/martinohmann/dts" }, 56 | ] 57 | [git] 58 | # parse the commits based on https://www.conventionalcommits.org 59 | conventional_commits = true 60 | # filter out the commits that are not conventional 61 | filter_unconventional = true 62 | # process each line of a commit as an individual commit 63 | split_commits = false 64 | # regex for preprocessing the commit messages 65 | commit_preprocessors = [ 66 | # { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, # replace issue numbers 67 | ] 68 | # regex for parsing and grouping commits 69 | commit_parsers = [ 70 | { message = "^feat", group = "Features" }, 71 | { message = "^fix", group = "Bug Fixes" }, 72 | { message = "^doc", group = "Documentation" }, 73 | { message = "^perf", group = "Performance" }, 74 | { message = "^refactor", group = "Refactor" }, 75 | { message = "^test", group = "Testing" }, 76 | { message = "^chore\\(deps\\)", group = "Dependencies" }, 77 | { message = "^chore|ci", group = "Miscellaneous Tasks" }, 78 | { body = ".*security", group = "Security" }, 79 | { message = "^revert", group = "Revert" }, 80 | ] 81 | # protect breaking changes from being skipped due to matching a skipping commit_parser 82 | protect_breaking_commits = false 83 | # filter out the commits that are not matched by commit parsers 84 | filter_commits = false 85 | # regex for matching git tags 86 | tag_pattern = "v[0-9].*" 87 | 88 | # regex for skipping tags 89 | skip_tags = "v0.1.0-beta.1" 90 | # regex for ignoring tags 91 | ignore_tags = "" 92 | # sort the tags topologically 93 | topo_order = false 94 | # sort the commits inside sections by oldest/newest order 95 | sort_commits = "oldest" 96 | # limit the number of commits included in the changelog. 97 | # limit_commits = 42 98 | -------------------------------------------------------------------------------- /release-plz.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | # path of the git-cliff configuration 3 | changelog_config = "cliff.toml" 4 | 5 | # enable changelog updates 6 | changelog_update = true 7 | 8 | # update dependencies with `cargo update` 9 | dependencies_update = true 10 | 11 | # create tags for the releases 12 | git_tag_enable = true 13 | 14 | # enable GitHub releases 15 | git_release_enable = true 16 | 17 | # disallow updating repositories with uncommitted changes 18 | allow_dirty = false 19 | 20 | # disallow packaging with uncommitted changes 21 | publish_allow_dirty = false 22 | 23 | # disable running `cargo-semver-checks` 24 | semver_check = false 25 | -------------------------------------------------------------------------------- /src/bin/dts/args.rs: -------------------------------------------------------------------------------- 1 | //! Command line arguments for dts. 2 | 3 | #[cfg(feature = "color")] 4 | use crate::output::ColorChoice; 5 | use crate::paging::PagingChoice; 6 | use anyhow::{Result, anyhow}; 7 | use clap::{Args, Parser, ValueHint}; 8 | use clap_complete::Shell; 9 | use dts::{Encoding, Sink, Source, de::DeserializeOptions, ser::SerializeOptions}; 10 | use regex::Regex; 11 | use unescape::unescape; 12 | 13 | /// Simple tool to transcode between different encodings. 14 | /// 15 | /// The tool first deserializes data from the input into an internal representation which resembles 16 | /// JSON. As an optional step certain transformations can be applied before serializing back into 17 | /// the output encoding. 18 | /// 19 | /// Refer to the documentation of the input, transform and output options below. 20 | #[derive(Parser, Debug)] 21 | #[command( 22 | name = "dts", 23 | version, 24 | after_help = "Hint: `dts -h` only provides a usage summary. Run `dts --help` for the full details to each flag." 25 | )] 26 | pub struct Options { 27 | /// Input sources. 28 | /// 29 | /// If multiple files are provided, the decoded data is read into an array. Input files many 30 | /// also be remote URLs. Data may also be provided on stdin. If stdin is used in combination 31 | /// with one or more input files, the data from stdin will be read into the first element of 32 | /// the resulting array. 33 | #[arg(name = "SOURCE", value_hint = ValueHint::AnyPath)] 34 | pub sources: Vec, 35 | 36 | /// Output sink. Can be specified multiple times. Defaults to stdout if omitted. 37 | /// 38 | /// It is possible to provide multiple output files if the data resembles an array. Each output 39 | /// file will receive an array element. The last output file collects the remaining elements if 40 | /// there are more elements than files. 41 | /// 42 | /// Passing '-' as filename or providing no output files will write the data to stdout instead. 43 | #[arg(short = 'O', long = "sink", value_name = "SINK", value_hint = ValueHint::FilePath)] 44 | pub sinks: Vec, 45 | 46 | /// Options for deserializing the input. 47 | #[clap(flatten)] 48 | pub input: InputOptions, 49 | 50 | /// Options for data transformations performed after deserializing from the input encoding but 51 | /// before serializing back into the output encoding. 52 | #[clap(flatten)] 53 | pub transform: TransformOptions, 54 | 55 | /// Options for serializing the output. 56 | #[clap(flatten)] 57 | pub output: OutputOptions, 58 | 59 | /// If provided, outputs the completion file for the given shell. 60 | #[arg(value_enum, long, value_name = "SHELL", group = "generate-completion")] 61 | pub generate_completion: Option, 62 | 63 | /// List available color themes and exit. 64 | #[cfg(feature = "color")] 65 | #[arg(long, conflicts_with = "generate-completion")] 66 | pub list_themes: bool, 67 | } 68 | 69 | /// Options that configure the behaviour of input deserialization. 70 | #[derive(Args, Debug)] 71 | pub struct InputOptions { 72 | /// Set the input encoding. 73 | /// 74 | /// If absent, dts will attempt to detect the encoding from the input file extension (if 75 | /// present) or from the first line of input. 76 | #[arg(value_enum, short = 'i', long, help_heading = "Input Options")] 77 | pub input_encoding: Option, 78 | 79 | /// Indicate that CSV input does not include a header row. 80 | /// 81 | /// If this flag is absent, the first line of CSV input is treated as headers and will be 82 | /// discarded. 83 | #[arg(long, help_heading = "Input Options")] 84 | pub csv_without_headers: bool, 85 | 86 | /// Use CSV headers as keys for the row columns. 87 | /// 88 | /// When reading CSV, this flag will deserialize the input into an array of maps with each 89 | /// field keyed by the corresponding header value. Otherwise, the input is deserialized into an 90 | /// array of arrays. 91 | #[arg(short = 'H', long, help_heading = "Input Options")] 92 | pub csv_headers_as_keys: bool, 93 | 94 | /// Custom delimiter for CSV input. 95 | #[arg(short = 'd', long, value_parser = parse_csv_delimiter, help_heading = "Input Options")] 96 | pub csv_input_delimiter: Option, 97 | 98 | /// Regex pattern to split text input at. 99 | #[arg(short = 's', long, help_heading = "Input Options")] 100 | pub text_split_pattern: Option, 101 | 102 | /// Glob pattern for directories. 103 | /// 104 | /// Required if any of the input paths is a directory. Ignored otherwise. 105 | #[arg(long, help_heading = "Input Options")] 106 | pub glob: Option, 107 | 108 | /// Read input into a map keyed by file path of the origin file. 109 | /// 110 | /// If multiple input files or at least one directory is provided, this reads the result into 111 | /// a map keyed by file path instead of an array. If only one input file is provided, this 112 | /// option is ignored. 113 | #[arg(short = 'P', long, help_heading = "Input Options")] 114 | pub file_paths: bool, 115 | 116 | /// Continue on errors that occur while reading or deserializing input data. 117 | /// 118 | /// If the flag is provided, `dts` will continue to read and deserialize the remaining input 119 | /// sources. For example, this is useful if you want to deserialize files using a glob pattern 120 | /// and one of the files is malformed. In this case a warning is logged to stderr and the 121 | /// source is skipped. This flag is ignored if input is read only from a single source that is 122 | /// not a directory. 123 | #[arg(short = 'C', long, help_heading = "Input Options")] 124 | pub continue_on_error: bool, 125 | 126 | /// Simplify input if the encoding supports it. 127 | /// 128 | /// Some encodings like HCL support partial expression evaluation, where an expression like 129 | /// `1 + 2` can be evaluated to `3`. This flag controls if input simplifications like this 130 | /// should be performed or not. 131 | #[arg(long, help_heading = "Input Options")] 132 | pub simplify: bool, 133 | } 134 | 135 | impl From<&InputOptions> for DeserializeOptions { 136 | fn from(opts: &InputOptions) -> Self { 137 | Self { 138 | csv_headers_as_keys: opts.csv_headers_as_keys, 139 | csv_without_headers: opts.csv_without_headers, 140 | csv_delimiter: opts.csv_input_delimiter, 141 | text_split_pattern: opts.text_split_pattern.clone(), 142 | simplify: opts.simplify, 143 | } 144 | } 145 | } 146 | 147 | /// Options that configure the behaviour of data transformation. 148 | #[cfg(feature = "jaq")] 149 | #[derive(Args, Debug)] 150 | pub struct TransformOptions { 151 | /// A jq expression for transforming the input data. 152 | /// 153 | /// If the expression starts with an `@` it is treated as a local file path and the expression 154 | /// is read from there instead. 155 | /// 156 | /// See for supported operators, filters and 157 | /// functions. 158 | #[arg( 159 | short = 'j', 160 | long = "jq", 161 | value_name = "EXPRESSION", 162 | help_heading = "Transform Options" 163 | )] 164 | pub jq_expression: Option, 165 | } 166 | 167 | /// Options that configure the behaviour of data transformation. 168 | #[cfg(not(feature = "jaq"))] 169 | #[derive(Args, Debug)] 170 | pub struct TransformOptions { 171 | /// A jq expression for transforming the input data. 172 | /// 173 | /// The usage of this flag requires the `jq` executable to be present in the `PATH`. You may 174 | /// also point `dts` to a different `jq` executable by setting the `DTS_JQ` environment 175 | /// variable. 176 | /// 177 | /// If the expression starts with an `@` it is treated as a local file path and the expression 178 | /// is read from there instead. 179 | /// 180 | /// See for supported operators, filters and 181 | /// functions. 182 | #[arg( 183 | short = 'j', 184 | long = "jq", 185 | value_name = "EXPRESSION", 186 | help_heading = "Transform Options" 187 | )] 188 | pub jq_expression: Option, 189 | } 190 | 191 | /// Options that configure the behaviour of output serialization. 192 | #[derive(Args, Debug)] 193 | pub struct OutputOptions { 194 | /// Set the output encoding. 195 | /// 196 | /// If absent, the encoding will be detected from the output file extension. 197 | /// 198 | /// If the encoding is not explicitly set and it cannot be inferred from the output file 199 | /// extension (or the output is stdout), the fallback is to encode output as JSON. 200 | #[arg(value_enum, short = 'o', long, help_heading = "Output Options")] 201 | pub output_encoding: Option, 202 | 203 | /// Controls when to use colors. 204 | /// 205 | /// The default setting is `auto`, which means dts will try to guess when to use colors. For 206 | /// example, if dts is printing to a terminal, it will use colors. If it is redirected to a 207 | /// file or a pipe, it will suppress color output. Output is also not colored if the TERM 208 | /// environment variable isn't set or the terminal is `dumb`. 209 | /// 210 | /// Use color `always` to enforce coloring. 211 | #[cfg(feature = "color")] 212 | #[arg( 213 | value_enum, 214 | long, 215 | value_name = "WHEN", 216 | default_value = "auto", 217 | env = "DTS_COLOR", 218 | help_heading = "Output Options" 219 | )] 220 | pub color: ColorChoice, 221 | 222 | /// Controls the color theme to use. 223 | /// 224 | /// See --list-themes for available color themes. 225 | #[cfg(feature = "color")] 226 | #[arg(long, env = "DTS_THEME", help_heading = "Output Options")] 227 | pub theme: Option, 228 | 229 | /// Controls when to page output. 230 | /// 231 | /// The default setting is `auto`. dts will try to guess when to page output when `auto` is 232 | /// enabled. For example, if the output does fit onto the screen it may not be paged depending 233 | /// on the pager in use. 234 | /// 235 | /// Use `always` to enforce paging even if the output fits onto the screen. 236 | #[arg( 237 | value_enum, 238 | long, 239 | value_name = "WHEN", 240 | default_value = "auto", 241 | env = "DTS_PAGING", 242 | help_heading = "Output Options" 243 | )] 244 | pub paging: PagingChoice, 245 | 246 | /// Controls the output pager to use. 247 | /// 248 | /// By default the pager configured via the `PAGER` environment variable will be used. The 249 | /// fallback is `less`. 250 | #[arg(long, env = "DTS_PAGER", help_heading = "Output Options")] 251 | pub pager: Option, 252 | 253 | /// Emit output data in a compact format. 254 | /// 255 | /// This will disable pretty printing for encodings that support it. 256 | #[arg(short = 'c', long, help_heading = "Output Options")] 257 | pub compact: bool, 258 | 259 | /// Add a trailing newline to the output. 260 | #[arg(short = 'n', long, help_heading = "Output Options")] 261 | pub newline: bool, 262 | 263 | /// Use object keys of the first item as CSV headers. 264 | /// 265 | /// When the input is an array of objects and the output encoding is CSV, the field names of 266 | /// the first object will be used as CSV headers. Field values of all following objects will be 267 | /// matched to the right CSV column based on their key. Missing fields produce empty columns 268 | /// while excess fields are ignored. 269 | #[arg(short = 'K', long, help_heading = "Output Options")] 270 | pub keys_as_csv_headers: bool, 271 | 272 | /// Custom delimiter for CSV output. 273 | #[arg(short = 'D', long, value_parser = parse_csv_delimiter, help_heading = "Output Options")] 274 | pub csv_output_delimiter: Option, 275 | 276 | /// Custom separator to join text output with. 277 | #[arg(short = 'J', long, value_parser = parse_unescaped, help_heading = "Output Options")] 278 | pub text_join_separator: Option, 279 | 280 | /// Treat output arrays as multiple YAML documents. 281 | /// 282 | /// If the output is an array and the output format is YAML, treat the array members as 283 | /// multiple YAML documents that get written to the same file. 284 | #[arg(long, help_heading = "Output Options")] 285 | pub multi_doc_yaml: bool, 286 | 287 | /// Overwrite output files if they exist. 288 | #[arg(long)] 289 | pub overwrite: bool, 290 | } 291 | 292 | impl From<&OutputOptions> for SerializeOptions { 293 | fn from(opts: &OutputOptions) -> Self { 294 | Self { 295 | compact: opts.compact, 296 | newline: opts.newline, 297 | keys_as_csv_headers: opts.keys_as_csv_headers, 298 | csv_delimiter: opts.csv_output_delimiter, 299 | text_join_separator: opts.text_join_separator.clone(), 300 | multi_doc_yaml: opts.multi_doc_yaml, 301 | } 302 | } 303 | } 304 | 305 | fn parse_csv_delimiter(s: &str) -> Result { 306 | let unescaped = parse_unescaped(s)?; 307 | let bytes = unescaped.as_bytes(); 308 | 309 | if bytes.len() == 1 { 310 | Ok(bytes[0]) 311 | } else { 312 | Err(anyhow!("expected single byte delimiter")) 313 | } 314 | } 315 | 316 | fn parse_unescaped(s: &str) -> Result { 317 | unescape(s).ok_or_else(|| anyhow!("string contains invalid escape sequences: `{}`", s)) 318 | } 319 | -------------------------------------------------------------------------------- /src/bin/dts/assets/example.json: -------------------------------------------------------------------------------- 1 | {"foo": "bar", "baz": [1, 2], "qux": null} 2 | -------------------------------------------------------------------------------- /src/bin/dts/highlighting.rs: -------------------------------------------------------------------------------- 1 | //! Utilities to syntax highlight output. 2 | 3 | use crate::{ 4 | output::ColorChoice, 5 | paging::{PagingChoice, PagingConfig}, 6 | utils::resolve_cmd, 7 | }; 8 | use bat::{Input, PagingMode, assets::HighlightingAssets, config::Config, controller::Controller}; 9 | use dts::Encoding; 10 | use std::io::{self, Write}; 11 | use std::path::Path; 12 | use termcolor::{ColorSpec, StandardStream, WriteColor}; 13 | 14 | /// ColoredStdoutWriter writes data to stdout and may or may not colorize it. 15 | pub struct ColoredStdoutWriter<'a> { 16 | highlighter: SyntaxHighlighter<'a>, 17 | encoding: Encoding, 18 | theme: Option<&'a str>, 19 | buf: Vec, 20 | } 21 | 22 | impl<'a> ColoredStdoutWriter<'a> { 23 | /// Creates a new `ColoredStdoutWriter` which colorizes output based on the provided `Encoding` 24 | /// and `theme`, and prints it using the provided `SyntaxHighlighter`. 25 | pub fn new( 26 | highlighter: SyntaxHighlighter<'a>, 27 | encoding: Encoding, 28 | theme: Option<&'a str>, 29 | ) -> Self { 30 | ColoredStdoutWriter { 31 | highlighter, 32 | encoding, 33 | theme, 34 | buf: Vec::with_capacity(256), 35 | } 36 | } 37 | } 38 | 39 | impl io::Write for ColoredStdoutWriter<'_> { 40 | fn write(&mut self, buf: &[u8]) -> io::Result { 41 | self.buf.write(buf) 42 | } 43 | 44 | fn flush(&mut self) -> io::Result<()> { 45 | let buf = std::mem::take(&mut self.buf); 46 | 47 | if buf.is_empty() { 48 | return Ok(()); 49 | } 50 | 51 | self.highlighter.print(self.encoding, &buf, self.theme) 52 | } 53 | } 54 | 55 | impl Drop for ColoredStdoutWriter<'_> { 56 | fn drop(&mut self) { 57 | let _ = self.flush(); 58 | } 59 | } 60 | 61 | impl From for PagingMode { 62 | fn from(choice: PagingChoice) -> Self { 63 | match choice { 64 | PagingChoice::Always => PagingMode::Always, 65 | PagingChoice::Auto => PagingMode::QuitIfOneScreen, 66 | PagingChoice::Never => PagingMode::Never, 67 | } 68 | } 69 | } 70 | 71 | /// Can syntax highlight a buffer and then print the result to stdout. 72 | pub struct SyntaxHighlighter<'a> { 73 | assets: HighlightingAssets, 74 | paging_config: PagingConfig<'a>, 75 | } 76 | 77 | impl<'a> SyntaxHighlighter<'a> { 78 | /// Creates a new `SyntaxHighlighter`. 79 | pub fn new(paging_config: PagingConfig<'a>) -> Self { 80 | SyntaxHighlighter { 81 | assets: HighlightingAssets::from_binary(), 82 | paging_config, 83 | } 84 | } 85 | 86 | /// Returns an iterator over all supported color themes. 87 | fn themes(&self) -> impl Iterator { 88 | self.assets.themes() 89 | } 90 | 91 | /// Returns the color theme that should be used to color the output. Checks if the requested 92 | /// theme is available, otherwise fall back to `base16` as default. 93 | fn pick_theme(&self, theme: Option<&str>) -> String { 94 | theme 95 | .and_then(|requested| { 96 | let requested = requested.to_lowercase(); 97 | self.themes() 98 | .find(|known| known.to_lowercase() == requested) 99 | .map(|theme| theme.to_owned()) 100 | }) 101 | .unwrap_or_else(|| String::from("base16")) 102 | } 103 | 104 | /// Returns a suitable output pager. 105 | fn pager(&self) -> String { 106 | // Since we are using `bat` to do the syntax highlighting for us we have to ensure that the 107 | // pager is not `bat` itself. In this case we'll just fall back to using the default pager. 108 | let pager = self.paging_config.pager(); 109 | 110 | if let Some((pager_bin, _)) = resolve_cmd(&pager) { 111 | if !pager_bin.ends_with("bat") && !pager_bin.ends_with("bat.exe") { 112 | return pager; 113 | } 114 | } 115 | 116 | self.paging_config.default_pager() 117 | } 118 | 119 | /// Highlights `buf` using the given `Encoding` hint and `theme` and prints the result to 120 | /// stdout. 121 | pub fn print(&self, encoding: Encoding, buf: &[u8], theme: Option<&str>) -> io::Result<()> { 122 | let pager = self.pager(); 123 | let paging_choice = self.paging_config.paging_choice(); 124 | let theme = self.pick_theme(theme); 125 | 126 | let config = Config { 127 | colored_output: true, 128 | true_color: true, 129 | pager: Some(&pager), 130 | paging_mode: paging_choice.into(), 131 | theme, 132 | ..Default::default() 133 | }; 134 | 135 | let pseudo_filename = Path::new("out").with_extension(encoding.as_str()); 136 | let inputs = vec![Input::from_bytes(buf).name(pseudo_filename).into()]; 137 | 138 | let ctrl = Controller::new(&config, &self.assets); 139 | ctrl.run(inputs, None).map_err(io::Error::other)?; 140 | Ok(()) 141 | } 142 | } 143 | 144 | /// Prints available themes to stdout. 145 | pub fn print_themes(color_choice: ColorChoice) -> io::Result<()> { 146 | let example = include_bytes!("assets/example.json"); 147 | let highlighter = SyntaxHighlighter::new(PagingConfig::default()); 148 | 149 | if color_choice.should_colorize() { 150 | let max_len = highlighter.themes().map(str::len).max().unwrap_or(0); 151 | 152 | let mut stdout = StandardStream::stdout(color_choice.into()); 153 | 154 | for theme in highlighter.themes() { 155 | stdout.set_color(ColorSpec::new().set_bold(true))?; 156 | write!(&mut stdout, "{:1$}", theme, max_len + 2)?; 157 | stdout.reset()?; 158 | 159 | highlighter.print(Encoding::Json, example, Some(theme))?; 160 | } 161 | } else { 162 | for theme in highlighter.themes() { 163 | println!("{}", theme); 164 | } 165 | } 166 | 167 | Ok(()) 168 | } 169 | -------------------------------------------------------------------------------- /src/bin/dts/main.rs: -------------------------------------------------------------------------------- 1 | mod args; 2 | #[cfg(feature = "color")] 3 | mod highlighting; 4 | mod output; 5 | mod paging; 6 | mod utils; 7 | 8 | #[cfg(feature = "color")] 9 | use crate::highlighting::{ColoredStdoutWriter, SyntaxHighlighter, print_themes}; 10 | use crate::{ 11 | args::{InputOptions, Options, OutputOptions, TransformOptions}, 12 | output::StdoutWriter, 13 | paging::PagingConfig, 14 | }; 15 | use anyhow::{Context, Result, anyhow}; 16 | use clap::{Command, CommandFactory, Parser}; 17 | use clap_complete::{Shell, generate}; 18 | use dts::{Encoding, Error, Sink, Source, de::Deserializer, filter::Filter, ser::Serializer}; 19 | use rayon::prelude::*; 20 | use serde_json::Value; 21 | use std::fs::{self, File}; 22 | use std::io::{self, BufWriter, IsTerminal}; 23 | 24 | fn deserialize(source: &Source, opts: &InputOptions) -> Result { 25 | let reader = source 26 | .to_reader() 27 | .with_context(|| format!("failed to create reader for source `{}`", source))?; 28 | 29 | let encoding = opts 30 | .input_encoding 31 | .or_else(|| reader.encoding()) 32 | .context("unable to detect input encoding, please provide it explicitly via -i")?; 33 | 34 | let mut de = Deserializer::with_options(reader, opts.into()); 35 | 36 | de.deserialize(encoding) 37 | .with_context(|| format!("failed to deserialize `{}` from `{}`", encoding, source)) 38 | } 39 | 40 | fn deserialize_many(sources: &[Source], opts: &InputOptions) -> Result { 41 | let results = if opts.continue_on_error { 42 | sources 43 | .par_iter() 44 | .filter_map(|src| match deserialize(src, opts) { 45 | Ok(val) => Some((src, val)), 46 | Err(_) => { 47 | eprintln!("Warning: Source `{}` skipped due to errors", src); 48 | None 49 | } 50 | }) 51 | .collect::>() 52 | } else { 53 | sources 54 | .par_iter() 55 | .map(|src| deserialize(src, opts).map(|val| (src, val))) 56 | .collect::>>()? 57 | }; 58 | 59 | if opts.file_paths { 60 | Ok(Value::Object( 61 | results 62 | .into_iter() 63 | .map(|res| (res.0.to_string(), res.1)) 64 | .collect(), 65 | )) 66 | } else { 67 | Ok(Value::Array(results.into_iter().map(|res| res.1).collect())) 68 | } 69 | } 70 | 71 | fn transform(value: Value, opts: &TransformOptions) -> Result { 72 | match &opts.jq_expression { 73 | Some(expr) => { 74 | let expr = match expr.strip_prefix('@') { 75 | Some(path) => fs::read_to_string(path)?, 76 | None => expr.to_owned(), 77 | }; 78 | 79 | let filter = Filter::new(&expr)?; 80 | 81 | filter.apply(value).context("failed to transform value") 82 | } 83 | None => Ok(value), 84 | } 85 | } 86 | 87 | fn serialize(sink: &Sink, value: Value, opts: &OutputOptions) -> Result<()> { 88 | let encoding = opts 89 | .output_encoding 90 | .or_else(|| sink.encoding()) 91 | .unwrap_or(Encoding::Json); 92 | 93 | let paging_config = PagingConfig::new(opts.paging, opts.pager.as_deref()); 94 | 95 | let writer: Box = match sink { 96 | #[cfg(feature = "color")] 97 | Sink::Stdout => { 98 | if opts.color.should_colorize() { 99 | let highlighter = SyntaxHighlighter::new(paging_config); 100 | let theme = opts.theme.as_deref(); 101 | Box::new(ColoredStdoutWriter::new(highlighter, encoding, theme)) 102 | } else { 103 | Box::new(StdoutWriter::new(paging_config)) 104 | } 105 | } 106 | #[cfg(not(feature = "color"))] 107 | Sink::Stdout => Box::new(StdoutWriter::new(paging_config)), 108 | Sink::Path(path) => Box::new( 109 | File::create(path) 110 | .with_context(|| format!("failed to create writer for sink `{}`", sink))?, 111 | ), 112 | }; 113 | 114 | let mut ser = Serializer::with_options(BufWriter::new(writer), opts.into()); 115 | 116 | match ser.serialize(encoding, value) { 117 | Ok(()) => Ok(()), 118 | Err(Error::Io(err)) if err.kind() == io::ErrorKind::BrokenPipe => Ok(()), 119 | Err(err) => Err(err), 120 | } 121 | .with_context(|| format!("failed to serialize `{}` to `{}`", encoding, sink)) 122 | } 123 | 124 | fn serialize_many(sinks: &[Sink], value: Value, opts: &OutputOptions) -> Result<()> { 125 | let values = match value { 126 | Value::Array(mut values) => { 127 | if sinks.len() < values.len() { 128 | // There are more values than files. The last file takes an array of the left 129 | // over values. 130 | let rest = values.split_off(sinks.len() - 1); 131 | values.push(Value::Array(rest)); 132 | } 133 | 134 | values 135 | } 136 | _ => { 137 | return Err(anyhow!( 138 | "when using multiple output files, the data must be an array" 139 | )); 140 | } 141 | }; 142 | 143 | if sinks.len() > values.len() { 144 | eprintln!( 145 | "Warning: skipping {} output files due to lack of data", 146 | sinks.len() - values.len() 147 | ); 148 | } 149 | 150 | sinks 151 | .iter() 152 | .zip(values) 153 | .try_for_each(|(file, value)| serialize(file, value, opts)) 154 | } 155 | 156 | fn print_completions(cmd: &mut Command, shell: Shell) { 157 | generate(shell, cmd, cmd.get_name().to_string(), &mut io::stdout()); 158 | } 159 | 160 | fn main() -> Result<()> { 161 | let opts = Options::parse(); 162 | 163 | if let Some(shell) = opts.generate_completion { 164 | let mut cmd = Options::command(); 165 | print_completions(&mut cmd, shell); 166 | std::process::exit(0); 167 | } 168 | 169 | #[cfg(feature = "color")] 170 | if opts.list_themes { 171 | print_themes(opts.output.color)?; 172 | std::process::exit(0); 173 | } 174 | 175 | let mut sources = Vec::with_capacity(opts.sources.len()); 176 | 177 | // If sources contains directories, force deserialization into a collection (array or object 178 | // with sources as keys depending on the input options) even if all directory globs only 179 | // produce a zero or one sources. This will ensure that deserializing the files that resulted 180 | // from directory globs always produces a consistent structure of the data. 181 | let dir_sources = opts.sources.iter().any(|s| s.is_dir()); 182 | 183 | for source in opts.sources { 184 | match source.as_path() { 185 | Some(path) => { 186 | if path.is_dir() { 187 | let pattern = opts 188 | .input 189 | .glob 190 | .as_ref() 191 | .context("--glob is required if sources contain directories")?; 192 | 193 | let mut matches = source.glob_files(pattern)?; 194 | 195 | sources.append(&mut matches); 196 | } else { 197 | sources.push(path.into()); 198 | } 199 | } 200 | None => sources.push(source), 201 | } 202 | } 203 | 204 | if sources.is_empty() && !io::stdin().is_terminal() { 205 | // Input is piped on stdin. 206 | sources.push(Source::Stdin); 207 | } 208 | 209 | let sinks = opts.sinks; 210 | 211 | // Validate sinks to prevent accidentally overwriting existing files. 212 | for sink in &sinks { 213 | if let Sink::Path(path) = sink { 214 | if !path.exists() { 215 | continue; 216 | } 217 | 218 | if !path.is_file() { 219 | return Err(anyhow!( 220 | "output file `{}` exists but is not a file", 221 | path.display() 222 | )); 223 | } else if !opts.output.overwrite { 224 | return Err(anyhow!( 225 | "output file `{}` exists, pass --overwrite to overwrite it", 226 | path.display() 227 | )); 228 | } 229 | } 230 | } 231 | 232 | let value = match (sources.len(), dir_sources) { 233 | (0, false) => return Err(anyhow!("input file or data on stdin expected")), 234 | (1, false) => deserialize(&sources[0], &opts.input)?, 235 | (_, _) => deserialize_many(&sources, &opts.input)?, 236 | }; 237 | 238 | let value = transform(value, &opts.transform)?; 239 | 240 | if sinks.len() <= 1 { 241 | serialize(sinks.first().unwrap_or(&Sink::Stdout), value, &opts.output) 242 | } else { 243 | serialize_many(&sinks, value, &opts.output) 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/bin/dts/output.rs: -------------------------------------------------------------------------------- 1 | //! Contains an `io::Write` implementation that is capable to pipe output through a pager and 2 | //! utilities to decide if output should be colored or not. 3 | 4 | use crate::{ 5 | paging::{PagingChoice, PagingConfig}, 6 | utils::resolve_cmd, 7 | }; 8 | use clap::ValueEnum; 9 | use std::io::{self, IsTerminal, Stdout}; 10 | use std::process::{Child, Command, Stdio}; 11 | 12 | /// ColorChoice represents the color preference of a user. 13 | #[derive(ValueEnum, Debug, PartialEq, Eq, Clone, Copy, Default)] 14 | pub enum ColorChoice { 15 | /// Always color output even if stdout is a file. 16 | Always, 17 | /// Automatically detect if output should be colored. Coloring is disabled if stdout is not 18 | /// interactive or a dumb term, or the user explicitly disabled colors via `NO_COLOR` 19 | /// environment variable. 20 | Auto, 21 | /// Never color output. 22 | #[default] 23 | Never, 24 | } 25 | 26 | impl From for termcolor::ColorChoice { 27 | fn from(choice: ColorChoice) -> Self { 28 | match choice { 29 | ColorChoice::Always => termcolor::ColorChoice::Always, 30 | ColorChoice::Auto => termcolor::ColorChoice::Auto, 31 | ColorChoice::Never => termcolor::ColorChoice::Never, 32 | } 33 | } 34 | } 35 | 36 | impl ColorChoice { 37 | /// Returns true if the `ColorChoice` indicates that coloring is enabled. 38 | pub fn should_colorize(&self) -> bool { 39 | match *self { 40 | ColorChoice::Always => true, 41 | ColorChoice::Never => false, 42 | ColorChoice::Auto => self.env_allows_color() && io::stdout().is_terminal(), 43 | } 44 | } 45 | 46 | #[cfg(not(windows))] 47 | fn env_allows_color(&self) -> bool { 48 | match std::env::var_os("TERM") { 49 | None => return false, 50 | Some(k) => { 51 | if k == "dumb" { 52 | return false; 53 | } 54 | } 55 | } 56 | 57 | std::env::var_os("NO_COLOR").is_none() 58 | } 59 | 60 | #[cfg(windows)] 61 | fn env_allows_color(&self) -> bool { 62 | // On Windows, if TERM isn't set, then we shouldn't automatically 63 | // assume that colors aren't allowed. This is unlike Unix environments 64 | // where TERM is more rigorously set. 65 | if let Some(k) = std::env::var_os("TERM") { 66 | if k == "dumb" { 67 | return false; 68 | } 69 | } 70 | 71 | std::env::var_os("NO_COLOR").is_none() 72 | } 73 | } 74 | 75 | /// StdoutWriter either writes data directly to stdout or passes it through a pager first. 76 | #[derive(Debug)] 77 | pub enum StdoutWriter { 78 | Pager(Child), 79 | Stdout(Stdout), 80 | } 81 | 82 | impl StdoutWriter { 83 | /// Creates a new `StdoutWriter` which may page output based on the `PagingConfig`. 84 | pub fn new(config: PagingConfig<'_>) -> Self { 85 | match config.paging_choice() { 86 | PagingChoice::Always | PagingChoice::Auto => StdoutWriter::pager(config), 87 | PagingChoice::Never => StdoutWriter::stdout(), 88 | } 89 | } 90 | 91 | /// Tries to launch the pager. Falls back to `io::Stdout` in case of errors. 92 | fn pager(config: PagingConfig<'_>) -> Self { 93 | match resolve_cmd(config.pager()) { 94 | Some((pager_bin, args)) => { 95 | let mut cmd = Command::new(&pager_bin); 96 | 97 | if pager_bin.ends_with("less") || pager_bin.ends_with("less.exe") { 98 | if args.is_empty() { 99 | if let PagingChoice::Auto = config.paging_choice() { 100 | cmd.arg("--quit-if-one-screen"); 101 | } 102 | 103 | cmd.arg("--no-init"); 104 | } else { 105 | cmd.args(args); 106 | } 107 | 108 | cmd.env("LESSCHARSET", "UTF-8"); 109 | } else { 110 | cmd.args(args); 111 | } 112 | 113 | cmd.stdin(Stdio::piped()) 114 | .spawn() 115 | .map(StdoutWriter::Pager) 116 | .unwrap_or_else(|_| StdoutWriter::stdout()) 117 | } 118 | None => StdoutWriter::stdout(), 119 | } 120 | } 121 | 122 | fn stdout() -> Self { 123 | StdoutWriter::Stdout(io::stdout()) 124 | } 125 | } 126 | 127 | impl io::Write for StdoutWriter { 128 | fn write(&mut self, buf: &[u8]) -> io::Result { 129 | match self { 130 | StdoutWriter::Pager(Child { stdin, .. }) => stdin.as_mut().unwrap().write(buf), 131 | StdoutWriter::Stdout(stdout) => stdout.write(buf), 132 | } 133 | } 134 | 135 | fn flush(&mut self) -> io::Result<()> { 136 | match self { 137 | StdoutWriter::Pager(Child { stdin, .. }) => stdin.as_mut().unwrap().flush(), 138 | StdoutWriter::Stdout(stdout) => stdout.flush(), 139 | } 140 | } 141 | } 142 | 143 | impl Drop for StdoutWriter { 144 | fn drop(&mut self) { 145 | if let StdoutWriter::Pager(cmd) = self { 146 | let _ = cmd.wait(); 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/bin/dts/paging.rs: -------------------------------------------------------------------------------- 1 | //! Utilities to facilitate output paging. 2 | 3 | use clap::ValueEnum; 4 | 5 | /// PagingChoice represents the paging preference of a user. 6 | #[derive(ValueEnum, Debug, PartialEq, Eq, Clone, Copy, Default)] 7 | pub enum PagingChoice { 8 | /// Always page output even if you would fit on the screen. 9 | Always, 10 | /// Automatically decide when to page output. This will page output only if it does not fit on 11 | /// the screen. 12 | Auto, 13 | /// Never page output. 14 | #[default] 15 | Never, 16 | } 17 | 18 | /// PagingConfig holds configuration related to output paging. 19 | #[derive(Default, Clone)] 20 | pub struct PagingConfig<'a> { 21 | choice: PagingChoice, 22 | pager: Option<&'a str>, 23 | } 24 | 25 | impl<'a> PagingConfig<'a> { 26 | /// Creates a new `PagingConfig` with given `PagingChoice` and an optional pager. 27 | pub fn new(choice: PagingChoice, pager: Option<&'a str>) -> Self { 28 | PagingConfig { choice, pager } 29 | } 30 | 31 | /// Returns the default pager command. 32 | pub fn default_pager(&self) -> String { 33 | String::from("less") 34 | } 35 | 36 | /// Returns a suitable output pager. This will either use a) the explicitly configured pager b) 37 | /// the contents of the `PAGER` environment variable or c) the default pager which is `less`. 38 | pub fn pager(&self) -> String { 39 | match self.pager { 40 | Some(cmd) => cmd.to_owned(), 41 | None => std::env::var("PAGER").unwrap_or_else(|_| self.default_pager()), 42 | } 43 | } 44 | 45 | /// Returns the configured `PagingChoice`. 46 | pub fn paging_choice(&self) -> PagingChoice { 47 | self.choice 48 | } 49 | } 50 | 51 | #[cfg(test)] 52 | mod test { 53 | use super::*; 54 | 55 | #[test] 56 | fn pager() { 57 | let config = PagingConfig::new(PagingChoice::Auto, Some("my-pager")); 58 | assert_eq!(&config.pager(), "my-pager"); 59 | 60 | let config = PagingConfig::new(PagingChoice::Auto, None); 61 | assert_eq!(&config.pager(), &config.default_pager()); 62 | 63 | let config = PagingConfig::new(PagingChoice::Auto, None); 64 | temp_env::with_var("PAGER", Some("more"), || { 65 | assert_eq!(&config.pager(), "more") 66 | }); 67 | 68 | temp_env::with_var("PAGER", None::<&str>, || { 69 | assert_eq!(&config.pager(), &config.default_pager()) 70 | }); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/bin/dts/utils.rs: -------------------------------------------------------------------------------- 1 | //! Misc utilities. 2 | 3 | use std::path::PathBuf; 4 | 5 | /// Resolves a shell command to a binary and its args. 6 | /// 7 | /// The purpose of doing this instead of passing the path to the program directly to Command::new 8 | /// is that Command::new will hand relative paths to CreateProcess on Windows, which will 9 | /// implicitly search the current working directory for the executable. This could be undesirable 10 | /// for security reasons. 11 | /// 12 | /// Returns `None` if `s` is not a valid shell command (e.g. mismatching quotes). On windows it 13 | /// also returns `None` if the binary cannot be found in `PATH`. 14 | pub fn resolve_cmd>(s: S) -> Option<(PathBuf, Vec)> { 15 | shell_words::split(s.as_ref()).ok().and_then(|parts| { 16 | parts.split_first().and_then(|(cmd, args)| { 17 | grep_cli::resolve_binary(cmd) 18 | .ok() 19 | .map(|bin_path| (bin_path, args.to_vec())) 20 | }) 21 | }) 22 | } 23 | -------------------------------------------------------------------------------- /src/de.rs: -------------------------------------------------------------------------------- 1 | //! This module provides a `Deserializer` which supports deserializing input data with various 2 | //! encodings into a `Value`. 3 | 4 | use crate::{Encoding, Result, key::expand_keys, parsers::gron}; 5 | use hcl::eval::Evaluate; 6 | use regex::Regex; 7 | use serde::Deserialize; 8 | use serde_json::{Map, Value}; 9 | 10 | /// Options for the `Deserializer`. The options are context specific and may only be honored when 11 | /// deserializing from a certain `Encoding`. 12 | #[derive(Debug, Default, Clone)] 13 | pub struct DeserializeOptions { 14 | /// Indicates that an input CSV does not include a header line. If `false`, the first line is 15 | /// discarded. 16 | pub csv_without_headers: bool, 17 | /// Indicates that the header fields of an input CSV should be used as keys for each row's 18 | /// columns. This means that the deserialized row data will be of type object. Otherwise row 19 | /// data will be of type array. 20 | pub csv_headers_as_keys: bool, 21 | /// Optional custom delimiter for CSV input. 22 | pub csv_delimiter: Option, 23 | /// Optional regex pattern to split text input at. 24 | pub text_split_pattern: Option, 25 | /// Simplify input if the encoding supports it. 26 | pub simplify: bool, 27 | } 28 | 29 | impl DeserializeOptions { 30 | /// Creates new `DeserializeOptions`. 31 | pub fn new() -> Self { 32 | Self::default() 33 | } 34 | } 35 | 36 | /// A `DeserializerBuilder` can be used to build a `Deserializer` with certain 37 | /// `DeserializeOptions`. 38 | /// 39 | /// ## Example 40 | /// 41 | /// ``` 42 | /// use dts::{de::DeserializerBuilder, Encoding}; 43 | /// 44 | /// let buf = r#"["foo"]"#.as_bytes(); 45 | /// 46 | /// let deserializer = DeserializerBuilder::new() 47 | /// .csv_delimiter(b'\t') 48 | /// .build(buf); 49 | /// ``` 50 | #[derive(Debug, Default, Clone)] 51 | pub struct DeserializerBuilder { 52 | opts: DeserializeOptions, 53 | } 54 | 55 | impl DeserializerBuilder { 56 | /// Creates a new `DeserializerBuilder`. 57 | pub fn new() -> Self { 58 | Self::default() 59 | } 60 | 61 | /// Indicates that an input CSV does not include a header line. If `false`, the first line is 62 | /// discarded. 63 | pub fn csv_without_headers(&mut self, yes: bool) -> &mut Self { 64 | self.opts.csv_without_headers = yes; 65 | self 66 | } 67 | 68 | /// Indicates that the header fields of an input CSV should be used as keys for each row's 69 | /// columns. This means that the deserialized row data will be of type object. Otherwise row 70 | /// data will be of type array. 71 | pub fn csv_headers_as_keys(&mut self, yes: bool) -> &mut Self { 72 | self.opts.csv_headers_as_keys = yes; 73 | self 74 | } 75 | 76 | /// Sets a custom CSV delimiter. 77 | pub fn csv_delimiter(&mut self, delim: u8) -> &mut Self { 78 | self.opts.csv_delimiter = Some(delim); 79 | self 80 | } 81 | 82 | /// Sets regex pattern to split text at. 83 | pub fn text_split_pattern(&mut self, pattern: Regex) -> &mut Self { 84 | self.opts.text_split_pattern = Some(pattern); 85 | self 86 | } 87 | 88 | /// Simplify input if the encoding supports it. 89 | pub fn simplifiy(&mut self, yes: bool) -> &mut Self { 90 | self.opts.simplify = yes; 91 | self 92 | } 93 | 94 | /// Builds the `Deserializer` for the given reader. 95 | pub fn build(&self, reader: R) -> Deserializer 96 | where 97 | R: std::io::Read, 98 | { 99 | Deserializer::with_options(reader, self.opts.clone()) 100 | } 101 | } 102 | 103 | /// A `Deserializer` can deserialize input data from a reader into a `Value`. 104 | pub struct Deserializer { 105 | reader: R, 106 | opts: DeserializeOptions, 107 | } 108 | 109 | impl Deserializer 110 | where 111 | R: std::io::Read, 112 | { 113 | /// Creates a new `Deserializer` for reader with default options. 114 | pub fn new(reader: R) -> Self { 115 | Self::with_options(reader, Default::default()) 116 | } 117 | 118 | /// Creates a new `Deserializer` for reader with options. 119 | pub fn with_options(reader: R, opts: DeserializeOptions) -> Self { 120 | Self { reader, opts } 121 | } 122 | 123 | /// Reads input data from the given reader and deserializes it in a `Value`. 124 | /// 125 | /// ## Example 126 | /// 127 | /// ``` 128 | /// use dts::{de::DeserializerBuilder, Encoding}; 129 | /// use serde_json::json; 130 | /// # use std::error::Error; 131 | /// # 132 | /// # fn main() -> Result<(), Box> { 133 | /// let buf = r#"["foo"]"#.as_bytes(); 134 | /// 135 | /// let mut de = DeserializerBuilder::new().build(buf); 136 | /// let value = de.deserialize(Encoding::Json)?; 137 | /// 138 | /// assert_eq!(value, json!(["foo"])); 139 | /// # Ok(()) 140 | /// # } 141 | /// ``` 142 | pub fn deserialize(&mut self, encoding: Encoding) -> Result { 143 | match encoding { 144 | Encoding::Yaml => self.deserialize_yaml(), 145 | Encoding::Json => self.deserialize_json(), 146 | Encoding::Toml => self.deserialize_toml(), 147 | Encoding::Json5 => self.deserialize_json5(), 148 | Encoding::Csv => self.deserialize_csv(), 149 | Encoding::QueryString => self.deserialize_query_string(), 150 | Encoding::Xml => self.deserialize_xml(), 151 | Encoding::Text => self.deserialize_text(), 152 | Encoding::Gron => self.deserialize_gron(), 153 | Encoding::Hcl => self.deserialize_hcl(), 154 | } 155 | } 156 | 157 | fn deserialize_yaml(&mut self) -> Result { 158 | let mut values = serde_yaml::Deserializer::from_reader(&mut self.reader) 159 | .map(Value::deserialize) 160 | .collect::, _>>()?; 161 | 162 | // If this was not multi-document YAML, just take the first document's value without 163 | // wrapping it into an array. 164 | if values.len() == 1 { 165 | Ok(values.swap_remove(0)) 166 | } else { 167 | Ok(Value::Array(values)) 168 | } 169 | } 170 | 171 | fn deserialize_json(&mut self) -> Result { 172 | Ok(serde_json::from_reader(&mut self.reader)?) 173 | } 174 | 175 | fn deserialize_toml(&mut self) -> Result { 176 | let mut s = String::new(); 177 | self.reader.read_to_string(&mut s)?; 178 | Ok(toml::from_str(&s)?) 179 | } 180 | 181 | fn deserialize_json5(&mut self) -> Result { 182 | let mut s = String::new(); 183 | self.reader.read_to_string(&mut s)?; 184 | Ok(json5::from_str(&s)?) 185 | } 186 | 187 | fn deserialize_csv(&mut self) -> Result { 188 | let keep_first_line = self.opts.csv_without_headers || self.opts.csv_headers_as_keys; 189 | 190 | let mut csv_reader = csv::ReaderBuilder::new() 191 | .trim(csv::Trim::All) 192 | .has_headers(!keep_first_line) 193 | .delimiter(self.opts.csv_delimiter.unwrap_or(b',')) 194 | .from_reader(&mut self.reader); 195 | 196 | let mut iter = csv_reader.deserialize(); 197 | 198 | let value = if self.opts.csv_headers_as_keys { 199 | match iter.next() { 200 | Some(headers) => { 201 | let headers: Vec = headers?; 202 | 203 | Value::Array( 204 | iter.map(|record| { 205 | Ok(headers.iter().cloned().zip(record?.into_iter()).collect()) 206 | }) 207 | .collect::>()?, 208 | ) 209 | } 210 | None => Value::Array(Vec::new()), 211 | } 212 | } else { 213 | Value::Array( 214 | iter.map(|v| Ok(serde_json::to_value(v?)?)) 215 | .collect::>()?, 216 | ) 217 | }; 218 | 219 | Ok(value) 220 | } 221 | 222 | fn deserialize_query_string(&mut self) -> Result { 223 | let mut s = String::new(); 224 | self.reader.read_to_string(&mut s)?; 225 | Ok(Value::Object(serde_qs::from_str(&s)?)) 226 | } 227 | 228 | fn deserialize_xml(&mut self) -> Result { 229 | Ok(serde_xml_rs::from_reader(&mut self.reader)?) 230 | } 231 | 232 | fn deserialize_text(&mut self) -> Result { 233 | let mut s = String::new(); 234 | self.reader.read_to_string(&mut s)?; 235 | 236 | let pattern = match &self.opts.text_split_pattern { 237 | Some(pattern) => pattern.clone(), 238 | None => Regex::new("\n").unwrap(), 239 | }; 240 | 241 | Ok(Value::Array( 242 | pattern 243 | .split(&s) 244 | .map(serde_json::to_value) 245 | .collect::>()?, 246 | )) 247 | } 248 | 249 | fn deserialize_gron(&mut self) -> Result { 250 | let mut s = String::new(); 251 | self.reader.read_to_string(&mut s)?; 252 | 253 | let map = gron::parse(&s)? 254 | .iter() 255 | .map(|statement| { 256 | Ok(( 257 | statement.path().to_owned(), 258 | serde_json::from_str(statement.value())?, 259 | )) 260 | }) 261 | .collect::>>()?; 262 | 263 | Ok(expand_keys(Value::Object(map))) 264 | } 265 | 266 | fn deserialize_hcl(&mut self) -> Result { 267 | let value = if self.opts.simplify { 268 | let mut body: hcl::Body = hcl::from_reader(&mut self.reader)?; 269 | let ctx = hcl::eval::Context::new(); 270 | let _ = body.evaluate_in_place(&ctx); 271 | hcl::from_body(body)? 272 | } else { 273 | hcl::from_reader(&mut self.reader)? 274 | }; 275 | 276 | Ok(value) 277 | } 278 | } 279 | 280 | #[cfg(test)] 281 | mod test { 282 | use super::*; 283 | use pretty_assertions::assert_eq; 284 | use serde_json::json; 285 | 286 | #[track_caller] 287 | fn assert_builder_deserializes_to( 288 | builder: &mut DeserializerBuilder, 289 | encoding: Encoding, 290 | input: &str, 291 | expected: Value, 292 | ) { 293 | let mut de = builder.build(input.as_bytes()); 294 | let value = de.deserialize(encoding).unwrap(); 295 | assert_eq!(value, expected); 296 | } 297 | 298 | #[track_caller] 299 | fn assert_deserializes_to(encoding: Encoding, input: &str, expected: Value) { 300 | assert_builder_deserializes_to(&mut DeserializerBuilder::new(), encoding, input, expected); 301 | } 302 | 303 | #[test] 304 | fn test_deserialize_yaml() { 305 | assert_deserializes_to(Encoding::Yaml, "---\nfoo: bar", json!({"foo": "bar"})); 306 | assert_deserializes_to( 307 | Encoding::Yaml, 308 | "---\nfoo: bar\n---\nbaz: qux", 309 | json!([{"foo": "bar"}, {"baz": "qux"}]), 310 | ); 311 | } 312 | 313 | #[test] 314 | fn test_deserialize_csv() { 315 | assert_deserializes_to( 316 | Encoding::Csv, 317 | "header1,header2\ncol1,col2", 318 | json!([["col1", "col2"]]), 319 | ); 320 | assert_builder_deserializes_to( 321 | &mut DeserializerBuilder::new().csv_without_headers(true), 322 | Encoding::Csv, 323 | "row1col1,row1col2\nrow2col1,row2col2", 324 | json!([["row1col1", "row1col2"], ["row2col1", "row2col2"]]), 325 | ); 326 | assert_builder_deserializes_to( 327 | &mut DeserializerBuilder::new().csv_headers_as_keys(true), 328 | Encoding::Csv, 329 | "header1,header2\nrow1col1,row1col2\nrow2col1,row2col2", 330 | json!([{"header1":"row1col1", "header2":"row1col2"}, {"header1":"row2col1", "header2":"row2col2"}]), 331 | ); 332 | assert_builder_deserializes_to( 333 | &mut DeserializerBuilder::new().csv_delimiter(b'|'), 334 | Encoding::Csv, 335 | "header1|header2\ncol1|col2", 336 | json!([["col1", "col2"]]), 337 | ); 338 | } 339 | 340 | #[test] 341 | fn test_deserialize_text() { 342 | assert_deserializes_to( 343 | Encoding::Text, 344 | "one\ntwo\nthree\n", 345 | json!(["one", "two", "three", ""]), 346 | ); 347 | assert_deserializes_to(Encoding::Text, "", json!([""])); 348 | } 349 | } 350 | -------------------------------------------------------------------------------- /src/encoding.rs: -------------------------------------------------------------------------------- 1 | //! Supported encodings for serialization and deserialization. 2 | 3 | use clap::ValueEnum; 4 | use once_cell::sync::Lazy; 5 | use regex::Regex; 6 | use std::fmt; 7 | use std::path::Path; 8 | 9 | /// Encodings supported by this crate. 10 | /// 11 | /// Not all of the supported encodings are supported to serialize and deserialize into. Some, like 12 | /// hjson only allow deserialization of encoded data but are not able to serialize back into the 13 | /// original representation. 14 | #[non_exhaustive] 15 | #[derive(ValueEnum, Debug, PartialEq, Eq, Clone, Copy)] 16 | pub enum Encoding { 17 | /// JavaScript Object Notation 18 | Json, 19 | /// Yet Another Markup Language 20 | #[clap(alias = "yml")] 21 | Yaml, 22 | /// TOML configuration format 23 | Toml, 24 | /// ES5 JSON 25 | Json5, 26 | /// Comma separated values 27 | Csv, 28 | /// URL query string 29 | #[clap(alias = "qs")] 30 | QueryString, 31 | /// Extensible Markup Language 32 | Xml, 33 | /// Plaintext document 34 | #[clap(alias = "txt")] 35 | Text, 36 | /// Gron 37 | Gron, 38 | /// HCL 39 | Hcl, 40 | } 41 | 42 | // Patterns to detect a source encoding by looking at the first line of input. The patterns are 43 | // lazily constructed upon first usage as they are only needed if there is no other encoding hint 44 | // (e.g. encoding inferred from file extension or explicitly provided on the command line). 45 | // 46 | // These patterns are very basic and will only detect some of the more common first lines. Thus 47 | // they may not match valid pattern for a given encoding on purpose due to ambiguities. For example 48 | // the first line `["foo"]` may be a JSON array or a TOML table header. Make sure to avoid matching 49 | // anything that is ambiguous. 50 | static FIRST_LINES: Lazy> = Lazy::new(|| { 51 | vec![ 52 | // XML or HTML start. 53 | ( 54 | Encoding::Xml, 55 | Regex::new( 56 | r#"^(?x: 57 | <\?xml\s 58 | | \s*<(?:[\w-]+):Envelope\s+ 59 | | \s*(?i: [|]* { 67 | // 68 | // Expression for matching quoted strings is very basic. 69 | ( 70 | Encoding::Hcl, 71 | Regex::new( 72 | r#"^(?xi: 73 | [a-z_][a-z0-9_-]*\s+ 74 | (?:(?:[a-z_][a-z0-9_-]*|"[^"]*")\s+)*\{ 75 | )"#, 76 | ) 77 | .unwrap(), 78 | ), 79 | // YAML document start or document separator. 80 | (Encoding::Yaml, Regex::new(r"^(?:%YAML.*|---\s*)$").unwrap()), 81 | // TOML array of tables or table. 82 | ( 83 | Encoding::Toml, 84 | Regex::new( 85 | r#"^(?xi: 86 | # array of tables 87 | \[\[\s*[a-z0-9_-]+(?:\s*\.\s*(?:[a-z0-9_-]+|"[^"]*"))*\s*\]\]\s* 88 | # table 89 | | \[\s*[a-z0-9_-]+(?:\s*\.\s*(?:[a-z0-9_-]+|"[^"]*"))*\s*\]\s* 90 | )$"#, 91 | ) 92 | .unwrap(), 93 | ), 94 | // JSON object start or array start. 95 | ( 96 | Encoding::Json, 97 | Regex::new(r#"^(?:\{\s*(?:"|$)|\[\s*$)"#).unwrap(), 98 | ), 99 | ] 100 | }); 101 | 102 | impl Encoding { 103 | /// Creates an `Encoding` from a path by looking at the file extension. 104 | /// 105 | /// Returns `None` if the extension is absent or if the extension does not match any of the 106 | /// supported encodings. 107 | pub fn from_path

(path: P) -> Option 108 | where 109 | P: AsRef, 110 | { 111 | let ext = path.as_ref().extension()?.to_str()?; 112 | 113 | match ext { 114 | "json" => Some(Encoding::Json), 115 | "yaml" | "yml" => Some(Encoding::Yaml), 116 | "toml" => Some(Encoding::Toml), 117 | "json5" => Some(Encoding::Json5), 118 | "csv" => Some(Encoding::Csv), 119 | "xml" => Some(Encoding::Xml), 120 | "txt" | "text" => Some(Encoding::Text), 121 | "hcl" | "tf" => Some(Encoding::Hcl), 122 | _ => None, 123 | } 124 | } 125 | 126 | /// Tries to detect the `Encoding` by looking at the first line of the input. 127 | /// 128 | /// Returns `None` if the encoding cannot be detected from the first line. 129 | pub fn from_first_line(line: &str) -> Option { 130 | if line.is_empty() { 131 | // Fast path. 132 | return None; 133 | } 134 | 135 | for (encoding, regex) in FIRST_LINES.iter() { 136 | if regex.is_match(line) { 137 | return Some(*encoding); 138 | } 139 | } 140 | 141 | None 142 | } 143 | 144 | /// Returns the name of the `Encoding`. 145 | pub fn as_str(&self) -> &'static str { 146 | match self { 147 | Encoding::Json => "json", 148 | Encoding::Yaml => "yaml", 149 | Encoding::Toml => "toml", 150 | Encoding::Json5 => "json5", 151 | Encoding::Csv => "csv", 152 | Encoding::QueryString => "query-string", 153 | Encoding::Xml => "xml", 154 | Encoding::Text => "text", 155 | Encoding::Gron => "gron", 156 | Encoding::Hcl => "hcl", 157 | } 158 | } 159 | } 160 | 161 | impl fmt::Display for Encoding { 162 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 163 | fmt::Display::fmt(self.as_str(), f) 164 | } 165 | } 166 | 167 | #[cfg(test)] 168 | mod tests { 169 | use super::*; 170 | use pretty_assertions::assert_eq; 171 | 172 | #[test] 173 | fn test_encoding_from_path() { 174 | assert_eq!(Encoding::from_path("foo.yaml"), Some(Encoding::Yaml)); 175 | assert_eq!(Encoding::from_path("foo.yml"), Some(Encoding::Yaml)); 176 | assert_eq!(Encoding::from_path("foo.json"), Some(Encoding::Json)); 177 | assert_eq!(Encoding::from_path("foo.json5"), Some(Encoding::Json5)); 178 | assert_eq!(Encoding::from_path("foo.toml"), Some(Encoding::Toml)); 179 | assert_eq!(Encoding::from_path("foo.bak"), None); 180 | assert_eq!(Encoding::from_path("foo"), None); 181 | } 182 | 183 | #[test] 184 | fn test_encoding_from_first_line() { 185 | // no match 186 | assert_eq!(Encoding::from_first_line(""), None); 187 | assert_eq!(Encoding::from_first_line(r#"["foo"]"#), None); 188 | 189 | // match 190 | assert_eq!( 191 | Encoding::from_first_line(r#"resource "aws_s3_bucket" "my-bucket" {"#), 192 | Some(Encoding::Hcl) 193 | ); 194 | assert_eq!(Encoding::from_first_line("{ "), Some(Encoding::Json)); 195 | assert_eq!(Encoding::from_first_line("[ "), Some(Encoding::Json)); 196 | assert_eq!( 197 | Encoding::from_first_line(r#"{"foo": 1 }"#), 198 | Some(Encoding::Json) 199 | ); 200 | assert_eq!( 201 | Encoding::from_first_line(r#"[foo .bar."baz".qux]"#), 202 | Some(Encoding::Toml) 203 | ); 204 | assert_eq!( 205 | Encoding::from_first_line(r#"[[foo .bar."baz".qux]] "#), 206 | Some(Encoding::Toml) 207 | ); 208 | assert_eq!(Encoding::from_first_line("%YAML 1.2"), Some(Encoding::Yaml)); 209 | assert_eq!( 210 | Encoding::from_first_line(""), 211 | Some(Encoding::Xml) 212 | ); 213 | assert_eq!( 214 | Encoding::from_first_line(r#""#), 215 | Some(Encoding::Xml) 216 | ); 217 | assert_eq!( 218 | Encoding::from_first_line( 219 | r#""# 220 | ), 221 | Some(Encoding::Xml) 222 | ); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | //! Defines the `Error` and `Result` types used by this crate. 2 | 3 | use crate::{Encoding, parsers::ParseError}; 4 | use std::error::Error as StdError; 5 | use std::fmt::Display; 6 | use std::io; 7 | use thiserror::Error; 8 | 9 | /// A type alias for `Result`. 10 | pub type Result = std::result::Result; 11 | 12 | /// The error returned by all fallible operations within this crate. 13 | #[non_exhaustive] 14 | #[derive(Error, Debug)] 15 | pub enum Error { 16 | /// Represents a generic error message. 17 | #[error("{0}")] 18 | Message(String), 19 | 20 | /// Represents errors of operations that are not supported by a certain encoding. 21 | #[error("operation is not supported for encoding `{0}`")] 22 | UnsupportedEncoding(Encoding), 23 | 24 | /// Error emitted by parsers from this crate. 25 | #[error(transparent)] 26 | ParseError(#[from] ParseError), 27 | 28 | /// Represents generic IO errors. 29 | #[error(transparent)] 30 | Io(#[from] io::Error), 31 | 32 | /// Represents an invalid glob pattern. 33 | #[error("invalid glob pattern `{pattern}`")] 34 | GlobPatternError { 35 | /// The pattern that caused the error. 36 | pattern: String, 37 | /// The underlying error. 38 | source: glob::PatternError, 39 | }, 40 | 41 | /// Represents an error fetching a remote data source. 42 | #[error("unable to fetch remote data source")] 43 | RequestError(Box), 44 | 45 | /// Represents errors emitted by serializers and deserializers. 46 | #[error(transparent)] 47 | Serde(Box), 48 | } 49 | 50 | impl Error { 51 | pub(crate) fn new(msg: T) -> Error 52 | where 53 | T: Display, 54 | { 55 | Error::Message(msg.to_string()) 56 | } 57 | 58 | pub(crate) fn serde(err: E) -> Error 59 | where 60 | E: Into>, 61 | { 62 | Error::Serde(err.into()) 63 | } 64 | 65 | pub(crate) fn io(err: E) -> Error 66 | where 67 | E: Into, 68 | { 69 | Error::Io(err.into()) 70 | } 71 | 72 | pub(crate) fn glob_pattern(pattern: T, source: glob::PatternError) -> Error 73 | where 74 | T: Display, 75 | { 76 | Error::GlobPatternError { 77 | pattern: pattern.to_string(), 78 | source, 79 | } 80 | } 81 | } 82 | 83 | impl From for Error { 84 | fn from(err: serde_json::Error) -> Self { 85 | if err.is_io() { 86 | Error::io(err) 87 | } else { 88 | Error::serde(err) 89 | } 90 | } 91 | } 92 | 93 | impl From for Error { 94 | fn from(err: serde_yaml::Error) -> Self { 95 | if let Some(source) = err.source() { 96 | if let Some(io_err) = source.downcast_ref::() { 97 | return Error::io(io_err.kind()); 98 | } 99 | } 100 | 101 | Error::serde(err) 102 | } 103 | } 104 | 105 | impl From for Error { 106 | fn from(err: json5::Error) -> Self { 107 | Error::serde(err) 108 | } 109 | } 110 | 111 | impl From for Error { 112 | fn from(err: toml::ser::Error) -> Self { 113 | Error::serde(err) 114 | } 115 | } 116 | 117 | impl From for Error { 118 | fn from(err: toml::de::Error) -> Self { 119 | Error::serde(err) 120 | } 121 | } 122 | 123 | impl From for Error { 124 | fn from(err: csv::Error) -> Self { 125 | if err.is_io_error() { 126 | match err.into_kind() { 127 | csv::ErrorKind::Io(io_err) => Error::io(io_err), 128 | _ => unreachable!(), 129 | } 130 | } else { 131 | Error::serde(err) 132 | } 133 | } 134 | } 135 | 136 | impl From for Error { 137 | fn from(err: serde_qs::Error) -> Self { 138 | match err { 139 | serde_qs::Error::Io(io_err) => Error::io(io_err), 140 | other => Error::serde(other), 141 | } 142 | } 143 | } 144 | 145 | impl From for Error { 146 | fn from(err: serde_xml_rs::Error) -> Self { 147 | Error::serde(err) 148 | } 149 | } 150 | 151 | impl From for Error { 152 | fn from(err: hcl::Error) -> Self { 153 | match err { 154 | hcl::Error::Io(io_err) => Error::io(io_err), 155 | other => Error::serde(other), 156 | } 157 | } 158 | } 159 | 160 | impl From for Error { 161 | fn from(err: ureq::Error) -> Self { 162 | Error::RequestError(Box::new(err)) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/filter/jaq.rs: -------------------------------------------------------------------------------- 1 | //! A wrapper for `jaq`. 2 | 3 | use crate::{Error, Result}; 4 | use jaq_interpret::{Ctx, FilterT, ParseCtx, RcIter, Val}; 5 | use serde_json::Value; 6 | use std::fmt; 7 | 8 | #[derive(Debug)] 9 | struct ParseError { 10 | expr: String, 11 | errs: Vec, 12 | } 13 | 14 | impl fmt::Display for ParseError { 15 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 16 | write!(f, "invalid filter expression `{}`: ", self.expr)?; 17 | 18 | for (i, err) in self.errs.iter().enumerate() { 19 | if i > 0 { 20 | write!(f, "; {}", err)?; 21 | } else { 22 | write!(f, "{}", err)?; 23 | } 24 | } 25 | 26 | Ok(()) 27 | } 28 | } 29 | 30 | impl std::error::Error for ParseError {} 31 | 32 | pub(crate) struct Filter { 33 | filter: jaq_interpret::Filter, 34 | } 35 | 36 | impl Filter { 37 | pub(crate) fn new(expr: &str) -> Result { 38 | let mut defs = ParseCtx::new(Vec::new()); 39 | defs.insert_natives(jaq_core::core()); 40 | defs.insert_defs(jaq_std::std()); 41 | 42 | let (main, errs) = jaq_parse::parse(expr, jaq_parse::main()); 43 | 44 | if errs.is_empty() { 45 | Ok(Filter { 46 | filter: defs.compile(main.unwrap()), 47 | }) 48 | } else { 49 | Err(Error::new(ParseError { 50 | expr: expr.to_owned(), 51 | errs, 52 | })) 53 | } 54 | } 55 | 56 | pub(crate) fn apply(&self, value: Value) -> Result { 57 | let empty: Vec> = Vec::new(); 58 | let iter = RcIter::new(empty.into_iter()); 59 | let mut values = self 60 | .filter 61 | .run((Ctx::new(Vec::new(), &iter), Val::from(value))) 62 | .map(|out| Ok(Value::from(out.map_err(Error::new)?))) 63 | .collect::, Error>>()?; 64 | 65 | if values.len() == 1 { 66 | Ok(values.remove(0)) 67 | } else { 68 | Ok(Value::Array(values)) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/filter/jq.rs: -------------------------------------------------------------------------------- 1 | //! A wrapper for `jq`. 2 | 3 | use crate::{Error, Result}; 4 | use serde_json::Value; 5 | use std::io::{self, BufRead, Write}; 6 | use std::path::{Path, PathBuf}; 7 | use std::process::{Child, Command, Stdio}; 8 | use std::thread; 9 | 10 | #[derive(Debug, Clone)] 11 | pub(crate) struct Filter { 12 | expr: String, 13 | executable: PathBuf, 14 | } 15 | 16 | impl Filter { 17 | pub(crate) fn new(expr: &str) -> Result { 18 | let exe = std::env::var("DTS_JQ") 19 | .ok() 20 | .unwrap_or_else(|| String::from("jq")); 21 | Filter::with_executable(expr, exe) 22 | } 23 | 24 | pub(crate) fn apply(&self, value: Value) -> Result { 25 | let mut cmd = self.spawn_cmd()?; 26 | let mut stdin = cmd.stdin.take().unwrap(); 27 | 28 | let buf = serde_json::to_vec(&value)?; 29 | 30 | thread::spawn(move || stdin.write_all(&buf)); 31 | 32 | let output = cmd.wait_with_output()?; 33 | 34 | if output.status.success() { 35 | process_output(&output.stdout) 36 | } else { 37 | Err(Error::new(String::from_utf8_lossy(&output.stderr))) 38 | } 39 | } 40 | 41 | fn with_executable

(expr: &str, executable: P) -> Result 42 | where 43 | P: AsRef, 44 | { 45 | let executable = executable.as_ref(); 46 | 47 | let output = Command::new(executable) 48 | .arg("--version") 49 | .output() 50 | .map_err(|err| { 51 | if let io::ErrorKind::NotFound = err.kind() { 52 | Error::new(format!("executable `{}` not found", executable.display())) 53 | } else { 54 | Error::Io(err) 55 | } 56 | })?; 57 | 58 | let executable = executable.to_path_buf(); 59 | let version = String::from_utf8_lossy(&output.stdout); 60 | 61 | if version.starts_with("jq-") { 62 | Ok(Filter { 63 | expr: expr.to_owned(), 64 | executable, 65 | }) 66 | } else { 67 | Err(Error::new(format!( 68 | "executable `{}` exists but does appear to be `jq`", 69 | executable.display() 70 | ))) 71 | } 72 | } 73 | 74 | fn spawn_cmd(&self) -> io::Result { 75 | Command::new(&self.executable) 76 | .arg("--compact-output") 77 | .arg("--monochrome-output") 78 | .arg(&self.expr) 79 | .stdin(Stdio::piped()) 80 | .stdout(Stdio::piped()) 81 | .stderr(Stdio::piped()) 82 | .spawn() 83 | } 84 | } 85 | 86 | fn process_output(buf: &[u8]) -> Result { 87 | let mut values = buf 88 | .lines() 89 | .map(|line| serde_json::from_str(&line.unwrap())) 90 | .collect::, _>>()?; 91 | 92 | if values.len() == 1 { 93 | Ok(values.remove(0)) 94 | } else { 95 | Ok(Value::Array(values)) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/filter/mod.rs: -------------------------------------------------------------------------------- 1 | //! Provides functionality to filter a `serde_json::Value` based on a filter expression. 2 | 3 | use crate::Result; 4 | use serde_json::Value; 5 | 6 | #[cfg(feature = "jaq")] 7 | mod jaq; 8 | #[cfg(not(feature = "jaq"))] 9 | mod jq; 10 | 11 | #[cfg(feature = "jaq")] 12 | use jaq::Filter as FilterImpl; 13 | #[cfg(not(feature = "jaq"))] 14 | use jq::Filter as FilterImpl; 15 | 16 | /// A jq-like filter for transforming a `Value` into a different `Value` based on the contents of 17 | /// a filter expression. 18 | /// 19 | /// This can be used to transform a `Value` using a `jq` expression. 20 | /// 21 | /// ## Example 22 | /// 23 | /// ``` 24 | /// use dts::filter::Filter; 25 | /// use serde_json::json; 26 | /// # use std::error::Error; 27 | /// # 28 | /// # fn main() -> Result<(), Box> { 29 | /// let value = json!([5, 4, 10]); 30 | /// 31 | /// let filter = Filter::new("map(select(. > 5))")?; 32 | /// let result = filter.apply(value)?; 33 | /// 34 | /// assert_eq!(result, json!([10])); 35 | /// # Ok(()) 36 | /// # } 37 | /// ``` 38 | pub struct Filter { 39 | inner: FilterImpl, 40 | } 41 | 42 | impl Filter { 43 | /// Constructs the filter from the `&str` expression. 44 | /// 45 | /// Depending on the underlying implementation this may return an error if parsing the 46 | /// expression fails. 47 | pub fn new(expr: &str) -> Result { 48 | let inner = FilterImpl::new(expr)?; 49 | Ok(Filter { inner }) 50 | } 51 | 52 | /// Applies the filter to a `Value` and returns the result. 53 | pub fn apply(&self, value: Value) -> Result { 54 | self.inner.apply(value) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/key.rs: -------------------------------------------------------------------------------- 1 | //! Object key transformation utilities. 2 | 3 | use crate::{ 4 | parsers::flat_key::{self, KeyPart, KeyParts, StringKeyParts}, 5 | value::ValueExt, 6 | }; 7 | use rayon::prelude::*; 8 | use serde_json::{Map, Value}; 9 | use std::collections::BTreeMap; 10 | use std::iter; 11 | 12 | /// Flattens value to an object with flat keys. 13 | /// 14 | /// ## Examples 15 | /// 16 | /// Nested map with array: 17 | /// 18 | /// ``` 19 | /// # use pretty_assertions::assert_eq; 20 | /// use dts::key::flatten_keys; 21 | /// use serde_json::json; 22 | /// 23 | /// let value = json!({"foo": {"bar": ["baz", "qux"]}}); 24 | /// 25 | /// let value = flatten_keys(value, "data"); 26 | /// 27 | /// assert_eq!( 28 | /// value, 29 | /// json!({ 30 | /// "data": {}, 31 | /// "data.foo": {}, 32 | /// "data.foo.bar": [], 33 | /// "data.foo.bar[0]": "baz", 34 | /// "data.foo.bar[1]": "qux" 35 | /// }) 36 | /// ); 37 | /// ``` 38 | /// 39 | /// Array value with prefix "array": 40 | /// 41 | /// ``` 42 | /// # use pretty_assertions::assert_eq; 43 | /// use dts::key::flatten_keys; 44 | /// use serde_json::json; 45 | /// 46 | /// let value = json!(["foo", "bar", "baz"]); 47 | /// 48 | /// let value = flatten_keys(value, "array"); 49 | /// 50 | /// assert_eq!( 51 | /// value, 52 | /// json!({ 53 | /// "array": [], 54 | /// "array[0]": "foo", 55 | /// "array[1]": "bar", 56 | /// "array[2]": "baz" 57 | /// }) 58 | /// ); 59 | /// ``` 60 | /// 61 | /// Single primitive value: 62 | /// 63 | /// ``` 64 | /// # use pretty_assertions::assert_eq; 65 | /// use dts::key::flatten_keys; 66 | /// use serde_json::json; 67 | /// 68 | /// let value = json!("foo"); 69 | /// 70 | /// assert_eq!(flatten_keys(value, "data"), json!({"data": "foo"})); 71 | /// ``` 72 | pub fn flatten_keys

(value: Value, prefix: P) -> Value 73 | where 74 | P: AsRef, 75 | { 76 | let mut flattener = KeyFlattener::new(prefix.as_ref()); 77 | Value::Object(Map::from_iter(flattener.flatten(value))) 78 | } 79 | 80 | /// Recursively expands flat keys to nested objects. 81 | /// 82 | /// ``` 83 | /// # use pretty_assertions::assert_eq; 84 | /// use dts::key::expand_keys; 85 | /// use serde_json::json; 86 | /// 87 | /// let value = json!([{"foo.bar": 1, "foo[\"bar-baz\"]": 2}]); 88 | /// let expected = json!([{"foo": {"bar": 1, "bar-baz": 2}}]); 89 | /// 90 | /// assert_eq!(expand_keys(value), expected); 91 | /// ``` 92 | pub fn expand_keys(value: Value) -> Value { 93 | match value { 94 | Value::Object(object) => object 95 | .into_iter() 96 | .collect::>() 97 | .into_par_iter() 98 | .map(|(key, value)| match flat_key::parse(&key).ok() { 99 | Some(mut parts) => { 100 | parts.reverse(); 101 | expand_key_parts(&mut parts, value) 102 | } 103 | None => Value::Object(Map::from_iter(iter::once((key, value)))), 104 | }) 105 | .reduce( 106 | || Value::Null, 107 | |mut a, mut b| { 108 | a.deep_merge(&mut b); 109 | a 110 | }, 111 | ), 112 | Value::Array(array) => Value::Array(array.into_iter().map(expand_keys).collect()), 113 | value => value, 114 | } 115 | } 116 | 117 | fn expand_key_parts(parts: &mut KeyParts, value: Value) -> Value { 118 | match parts.pop() { 119 | Some(key) => match key { 120 | KeyPart::Ident(ident) => { 121 | let mut object = Map::with_capacity(1); 122 | object.insert(ident, expand_key_parts(parts, value)); 123 | Value::Object(object) 124 | } 125 | KeyPart::Index(index) => { 126 | let mut array = vec![Value::Null; index + 1]; 127 | array[index] = expand_key_parts(parts, value); 128 | Value::Array(array) 129 | } 130 | }, 131 | None => value, 132 | } 133 | } 134 | 135 | struct KeyFlattener<'a> { 136 | prefix: &'a str, 137 | stack: StringKeyParts, 138 | } 139 | 140 | impl<'a> KeyFlattener<'a> { 141 | fn new(prefix: &'a str) -> Self { 142 | Self { 143 | prefix, 144 | stack: StringKeyParts::new(), 145 | } 146 | } 147 | 148 | fn flatten(&mut self, value: Value) -> BTreeMap { 149 | let mut map = BTreeMap::new(); 150 | self.stack.push_ident(self.prefix); 151 | self.flatten_value(&mut map, value); 152 | self.stack.pop(); 153 | map 154 | } 155 | 156 | fn flatten_value(&mut self, map: &mut BTreeMap, value: Value) { 157 | match value { 158 | Value::Array(array) => { 159 | map.insert(self.key(), Value::Array(Vec::new())); 160 | for (index, value) in array.into_iter().enumerate() { 161 | self.stack.push_index(index); 162 | self.flatten_value(map, value); 163 | self.stack.pop(); 164 | } 165 | } 166 | Value::Object(object) => { 167 | map.insert(self.key(), Value::Object(Map::new())); 168 | for (key, value) in object.into_iter() { 169 | self.stack.push_ident(&key); 170 | self.flatten_value(map, value); 171 | self.stack.pop(); 172 | } 173 | } 174 | value => { 175 | map.insert(self.key(), value); 176 | } 177 | } 178 | } 179 | 180 | fn key(&self) -> String { 181 | self.stack.to_string() 182 | } 183 | } 184 | 185 | #[cfg(test)] 186 | mod tests { 187 | use super::*; 188 | use pretty_assertions::assert_eq; 189 | use serde_json::json; 190 | 191 | #[test] 192 | fn test_expand_keys() { 193 | let value = json!({ 194 | "data": {}, 195 | "data.foo": {}, 196 | "data.foo.bar": [], 197 | "data.foo.bar[0]": "baz", 198 | "data.foo.bar[1]": "qux" 199 | }); 200 | 201 | assert_eq!( 202 | expand_keys(value), 203 | json!({"data": {"foo": {"bar": ["baz", "qux"]}}}) 204 | ); 205 | } 206 | 207 | #[test] 208 | fn test_flatten_keys() { 209 | let value = json!({"foo": {"bar": ["baz", "qux"]}}); 210 | 211 | assert_eq!( 212 | flatten_keys(value, "data"), 213 | json!({ 214 | "data": {}, 215 | "data.foo": {}, 216 | "data.foo.bar": [], 217 | "data.foo.bar[0]": "baz", 218 | "data.foo.bar[1]": "qux" 219 | }) 220 | ); 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | #![warn(missing_docs)] 3 | 4 | use std::fs::canonicalize; 5 | use std::path::{Path, PathBuf}; 6 | 7 | pub use encoding::*; 8 | pub use error::*; 9 | pub use sink::Sink; 10 | pub use source::{Source, SourceReader}; 11 | 12 | pub mod de; 13 | mod encoding; 14 | mod error; 15 | pub mod filter; 16 | pub mod key; 17 | mod parsers; 18 | pub mod ser; 19 | mod sink; 20 | mod source; 21 | mod value; 22 | 23 | trait PathExt { 24 | fn relative_to

(&self, path: P) -> Option 25 | where 26 | P: AsRef; 27 | 28 | fn relative_to_cwd(&self) -> Option { 29 | std::env::current_dir() 30 | .ok() 31 | .and_then(|base| self.relative_to(base)) 32 | } 33 | 34 | fn glob_files(&self, pattern: &str) -> Result>; 35 | } 36 | 37 | impl PathExt for T 38 | where 39 | T: AsRef, 40 | { 41 | fn relative_to

(&self, base: P) -> Option 42 | where 43 | P: AsRef, 44 | { 45 | let (path, base) = (canonicalize(self).ok()?, canonicalize(base).ok()?); 46 | pathdiff::diff_paths(path, base) 47 | } 48 | 49 | fn glob_files(&self, pattern: &str) -> Result> { 50 | let full_pattern = self.as_ref().join(pattern); 51 | 52 | glob::glob(&full_pattern.to_string_lossy()) 53 | .map_err(|err| Error::glob_pattern(full_pattern.display(), err))? 54 | .filter_map(|result| match result { 55 | Ok(path) => path.is_file().then_some(Ok(path)), 56 | Err(err) => Some(Err(err.into_error().into())), 57 | }) 58 | .collect() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/parsers/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Display}; 2 | use thiserror::Error; 3 | 4 | /// Error emitted by all parsers in this module. 5 | #[derive(Error, Debug)] 6 | #[error("failed to parse {kind}:\n{msg}")] 7 | pub struct ParseError { 8 | kind: ParseErrorKind, 9 | msg: String, 10 | } 11 | 12 | impl ParseError { 13 | pub(crate) fn new(kind: ParseErrorKind, msg: T) -> ParseError 14 | where 15 | T: Display, 16 | { 17 | ParseError { 18 | kind, 19 | msg: msg.to_string(), 20 | } 21 | } 22 | } 23 | 24 | /// The kind of `ParseError`. 25 | #[derive(Debug)] 26 | pub enum ParseErrorKind { 27 | /// Error parsing flat keys. 28 | FlatKey, 29 | /// Error parsing gron. 30 | Gron, 31 | } 32 | 33 | impl Display for ParseErrorKind { 34 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 35 | match self { 36 | ParseErrorKind::FlatKey => write!(f, "flat key"), 37 | ParseErrorKind::Gron => write!(f, "gron"), 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/parsers/flat_key.rs: -------------------------------------------------------------------------------- 1 | use super::{ParseError, ParseErrorKind}; 2 | use crate::Result; 3 | use pest::Parser as ParseTrait; 4 | use pest_derive::Parser; 5 | use std::fmt; 6 | 7 | #[derive(Parser)] 8 | #[grammar = "parsers/grammars/flat_key.pest"] 9 | struct FlatKeyParser; 10 | 11 | /// Parses `KeyParts` from a `&str`. 12 | pub fn parse(key: &str) -> Result { 13 | let parts = FlatKeyParser::parse(Rule::Parts, key) 14 | .map_err(|e| ParseError::new(ParseErrorKind::FlatKey, e))? 15 | .filter_map(|pair| match pair.as_rule() { 16 | Rule::Key => Some(KeyPart::Ident(pair.as_str().to_owned())), 17 | Rule::StringDq => Some(KeyPart::Ident(pair.as_str().replace("\\\"", "\""))), 18 | Rule::StringSq => Some(KeyPart::Ident(pair.as_str().replace("\\'", "'"))), 19 | Rule::Index => Some(KeyPart::Index(pair.as_str().parse::().unwrap())), 20 | Rule::EOI => None, 21 | _ => unreachable!(), 22 | }) 23 | .collect(); 24 | 25 | Ok(parts) 26 | } 27 | 28 | #[derive(Debug, PartialEq, Eq)] 29 | pub enum KeyPart { 30 | Index(usize), 31 | Ident(String), 32 | } 33 | 34 | impl fmt::Display for KeyPart { 35 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 36 | match self { 37 | KeyPart::Index(index) => write!(f, "[{}]", index), 38 | KeyPart::Ident(key) => { 39 | let no_quote = key.chars().all(|c| c == '_' || c.is_ascii_alphanumeric()); 40 | 41 | if no_quote { 42 | write!(f, "{}", key) 43 | } else { 44 | write!(f, "[\"{}\"]", key.replace('\"', "\\\"")) 45 | } 46 | } 47 | } 48 | } 49 | } 50 | 51 | #[derive(Debug, Default, PartialEq, Eq)] 52 | pub struct KeyParts { 53 | inner: Vec, 54 | } 55 | 56 | impl KeyParts { 57 | pub fn pop(&mut self) -> Option { 58 | self.inner.pop() 59 | } 60 | 61 | pub fn reverse(&mut self) { 62 | self.inner.reverse() 63 | } 64 | } 65 | 66 | impl fmt::Display for KeyParts { 67 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 68 | let parts = StringKeyParts::from(self); 69 | fmt::Display::fmt(&parts, f) 70 | } 71 | } 72 | 73 | impl FromIterator for KeyParts { 74 | fn from_iter(iter: T) -> Self 75 | where 76 | T: IntoIterator, 77 | { 78 | Self { 79 | inner: iter.into_iter().collect(), 80 | } 81 | } 82 | } 83 | 84 | impl IntoIterator for KeyParts { 85 | type Item = KeyPart; 86 | type IntoIter = std::vec::IntoIter; 87 | 88 | fn into_iter(self) -> Self::IntoIter { 89 | self.inner.into_iter() 90 | } 91 | } 92 | 93 | #[derive(Debug, Default, PartialEq, Eq)] 94 | pub struct StringKeyParts { 95 | inner: Vec, 96 | } 97 | 98 | impl StringKeyParts { 99 | pub fn new() -> Self { 100 | Self::default() 101 | } 102 | 103 | pub fn pop(&mut self) -> Option { 104 | self.inner.pop() 105 | } 106 | 107 | pub fn push(&mut self, part: KeyPart) { 108 | self.inner.push(part.to_string()) 109 | } 110 | 111 | pub fn push_index(&mut self, index: usize) { 112 | self.push(KeyPart::Index(index)) 113 | } 114 | 115 | pub fn push_ident(&mut self, ident: &str) { 116 | self.push(KeyPart::Ident(ident.to_string())) 117 | } 118 | } 119 | 120 | impl From<&KeyParts> for StringKeyParts { 121 | fn from(parts: &KeyParts) -> Self { 122 | Self { 123 | inner: parts.inner.iter().map(ToString::to_string).collect(), 124 | } 125 | } 126 | } 127 | 128 | impl fmt::Display for StringKeyParts { 129 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 130 | for (i, key) in self.inner.iter().enumerate() { 131 | if i > 0 && !key.starts_with('[') { 132 | write!(f, ".{}", key)?; 133 | } else { 134 | write!(f, "{}", key)?; 135 | } 136 | } 137 | 138 | Ok(()) 139 | } 140 | } 141 | 142 | #[cfg(test)] 143 | mod test { 144 | use super::*; 145 | use pretty_assertions::assert_eq; 146 | 147 | #[test] 148 | fn test_parse() { 149 | assert!(parse("foo.[").is_err()); 150 | assert_eq!( 151 | parse("foo").unwrap(), 152 | KeyParts::from_iter(vec![KeyPart::Ident("foo".into())]) 153 | ); 154 | assert_eq!( 155 | parse("foo.bar[5].baz").unwrap(), 156 | KeyParts::from_iter(vec![ 157 | KeyPart::Ident("foo".into()), 158 | KeyPart::Ident("bar".into()), 159 | KeyPart::Index(5), 160 | KeyPart::Ident("baz".into()) 161 | ]) 162 | ); 163 | assert_eq!( 164 | parse("foo.bar_baz[0]").unwrap(), 165 | KeyParts::from_iter(vec![ 166 | KeyPart::Ident("foo".into()), 167 | KeyPart::Ident("bar_baz".into()), 168 | KeyPart::Index(0), 169 | ]) 170 | ); 171 | } 172 | 173 | #[test] 174 | fn test_roundtrip() { 175 | let s = "foo[\"京\\\"\tasdf\"][0]"; 176 | 177 | let parsed = parse(s).unwrap(); 178 | 179 | let expected = KeyParts::from_iter(vec![ 180 | KeyPart::Ident("foo".into()), 181 | KeyPart::Ident("京\"\tasdf".into()), 182 | KeyPart::Index(0), 183 | ]); 184 | 185 | assert_eq!(parsed, expected); 186 | assert_eq!(&parsed.to_string(), s); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/parsers/grammars/flat_key.pest: -------------------------------------------------------------------------------- 1 | WHITESPACE = _{ " " | "\t" | "\r\n" | "\n"} 2 | 3 | StringLit = _{ StringLitDq | StringLitSq } 4 | StringLitDq = _{ "\"" ~ StringDq ~ "\"" } 5 | StringLitSq = _{ "'" ~ StringSq ~ "'" } 6 | StringDq = { (("\\" ~ ("\"" | "\\")) | (!"\"" ~ ANY))* } 7 | StringSq = { (("\\" ~ ("'" | "\\")) | (!"'" ~ ANY))* } 8 | 9 | ElementAccess = _{ ("." ~ Key) | KeyBrackets } 10 | Index = ${ ASCII_DIGIT+ } 11 | Key = ${ (ASCII_ALPHANUMERIC | "_")* } 12 | KeyBrackets = _{ "[" ~ (StringLit | Index) ~ "]" } 13 | 14 | Path = _{ Key ~ ElementAccess* } 15 | Parts = _{ SOI ~ Path ~ EOI } 16 | -------------------------------------------------------------------------------- /src/parsers/grammars/gron.pest: -------------------------------------------------------------------------------- 1 | WHITESPACE = _{ " " | "\t" | "\r\n" | "\n"} 2 | 3 | StringLit = _{ StringLitDq | StringLitSq } 4 | StringLitDq = _{ "\"" ~ StringDq ~ "\"" } 5 | StringLitSq = _{ "'" ~ StringSq ~ "'" } 6 | StringDq = { (("\\" ~ ("\"" | "\\")) | (!"\"" ~ ANY))* } 7 | StringSq = { (("\\" ~ ("'" | "\\")) | (!"'" ~ ANY))* } 8 | 9 | ElementAccess = _{ ("." ~ Key) | KeyBrackets } 10 | Index = _{ ASCII_DIGIT+ } 11 | Key = _{ (ASCII_ALPHANUMERIC | "_")* } 12 | KeyBrackets = _{ "[" ~ (StringLit | Index) ~ "]" } 13 | 14 | SubPath = _{ Key ~ ElementAccess* } 15 | Path = @{ SubPath } 16 | 17 | Value = @{ (Null | Boolean | Number | String | Array | Object) } 18 | 19 | Boolean = @{ "true" | "false" } 20 | Null = @{ "null" } 21 | String = @{ StringLit } 22 | Number = @{ "-"? ~ ("0" | ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*) ~ ("." ~ ASCII_DIGIT+)? ~ (^"e" ~ ("+" | "-")? ~ ASCII_DIGIT+)? } 23 | Array = @{ "[" ~ (Value ~ ("," ~ Value)*)* ~ "]" } 24 | Object = @{ "{" ~ (ObjectInner ~ ("," ~ ObjectInner)*)* ~ "}" } 25 | ObjectKey = { WHITESPACE* ~ StringDq ~ WHITESPACE* } 26 | ObjectInner = { ObjectKey ~ ":" ~ Value } 27 | 28 | Statement = { Path ~ "=" ~ Value ~ ";"? } 29 | Statements = _{ SOI ~ Statement* ~ EOI } 30 | -------------------------------------------------------------------------------- /src/parsers/gron.rs: -------------------------------------------------------------------------------- 1 | use super::{ParseError, ParseErrorKind}; 2 | use crate::Result; 3 | use pest::Parser as ParseTrait; 4 | use pest_derive::Parser; 5 | use std::slice::Iter; 6 | 7 | #[derive(Parser)] 8 | #[grammar = "parsers/grammars/gron.pest"] 9 | struct GronParser; 10 | 11 | /// Parses `Statements` from a `&str`. 12 | pub fn parse(s: &str) -> Result, ParseError> { 13 | let statements = GronParser::parse(Rule::Statements, s) 14 | .map_err(|e| ParseError::new(ParseErrorKind::Gron, e))? 15 | .filter_map(|pair| match pair.as_rule() { 16 | Rule::Statement => { 17 | let mut inner = pair.into_inner(); 18 | // Guaranteed by the grammar that these will exist so unchecked unwrap here is 19 | // safe. 20 | let path = inner.next().unwrap().as_str(); 21 | let value = inner.next().unwrap().as_str(); 22 | 23 | Some(Statement::new(path, value)) 24 | } 25 | Rule::EOI => None, 26 | _ => unreachable!(), 27 | }) 28 | .collect(); 29 | 30 | Ok(statements) 31 | } 32 | 33 | #[derive(Debug, PartialEq, Eq)] 34 | pub struct Statement<'a> { 35 | path: &'a str, 36 | value: &'a str, 37 | } 38 | 39 | impl<'a> Statement<'a> { 40 | pub fn new(path: &'a str, value: &'a str) -> Self { 41 | Self { path, value } 42 | } 43 | 44 | pub fn path(&self) -> &'a str { 45 | self.path 46 | } 47 | 48 | pub fn value(&self) -> &'a str { 49 | self.value 50 | } 51 | } 52 | 53 | #[derive(Debug, PartialEq, Eq)] 54 | pub struct Statements<'a> { 55 | inner: Vec>, 56 | } 57 | 58 | impl<'a> Statements<'a> { 59 | pub fn iter(&self) -> Iter<'a, Statement> { 60 | self.inner.iter() 61 | } 62 | } 63 | 64 | impl<'a> FromIterator> for Statements<'a> { 65 | fn from_iter(iter: T) -> Self 66 | where 67 | T: IntoIterator>, 68 | { 69 | Self { 70 | inner: iter.into_iter().collect(), 71 | } 72 | } 73 | } 74 | 75 | impl<'a> IntoIterator for Statements<'a> { 76 | type Item = Statement<'a>; 77 | type IntoIter = std::vec::IntoIter; 78 | 79 | fn into_iter(self) -> Self::IntoIter { 80 | self.inner.into_iter() 81 | } 82 | } 83 | 84 | #[cfg(test)] 85 | mod test { 86 | use super::*; 87 | use pretty_assertions::assert_eq; 88 | 89 | #[test] 90 | fn test_parse() { 91 | assert_eq!( 92 | parse("foo.bar = \"baz\";").unwrap(), 93 | Statements::from_iter(vec![Statement::new("foo.bar", "\"baz\"")]) 94 | ); 95 | assert_eq!( 96 | parse("foo.bar[5].baz = []").unwrap(), 97 | Statements::from_iter(vec![Statement::new("foo.bar[5].baz", "[]")]) 98 | ); 99 | assert_eq!( 100 | parse("foo = \"bar\"; baz = 1").unwrap(), 101 | Statements::from_iter(vec![ 102 | Statement::new("foo", "\"bar\""), 103 | Statement::new("baz", "1") 104 | ]) 105 | ); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/parsers/mod.rs: -------------------------------------------------------------------------------- 1 | mod error; 2 | pub mod flat_key; 3 | pub mod gron; 4 | 5 | pub use error::*; 6 | -------------------------------------------------------------------------------- /src/ser.rs: -------------------------------------------------------------------------------- 1 | //! This module provides a `Serializer` which supports serializing values into various output 2 | //! encodings. 3 | 4 | use crate::{Encoding, Error, Result, key::flatten_keys, value::ValueExt}; 5 | use serde_json::Value; 6 | use std::fmt::Write; 7 | 8 | /// Options for the `Serializer`. The options are context specific and may only be honored when 9 | /// serializing into a certain `Encoding`. 10 | #[derive(Debug, Default, Clone, PartialEq, Eq)] 11 | pub struct SerializeOptions { 12 | /// Emit output data in a compact format. This will disable pretty printing for encodings that 13 | /// support it. 14 | pub compact: bool, 15 | /// Append a trailing newline to the serialized data. 16 | pub newline: bool, 17 | /// When the input is an array of objects and the output encoding is CSV, the field names of 18 | /// the first object will be used as CSV headers. Field values of all following objects will 19 | /// be matched to the right CSV column based on their key. Missing fields produce empty columns 20 | /// while excess fields are ignored. 21 | pub keys_as_csv_headers: bool, 22 | /// Optional custom delimiter for CSV output. 23 | pub csv_delimiter: Option, 24 | /// Optional seprator to join text output with. 25 | pub text_join_separator: Option, 26 | /// Treat output arrays as multiple YAML documents. 27 | pub multi_doc_yaml: bool, 28 | } 29 | 30 | impl SerializeOptions { 31 | /// Creates new `SerializeOptions`. 32 | pub fn new() -> Self { 33 | Self::default() 34 | } 35 | } 36 | 37 | /// A `SerializerBuilder` can be used to build a `Serializer` with certain 38 | /// `SerializeOptions`. 39 | /// 40 | /// ## Example 41 | /// 42 | /// ``` 43 | /// use dts::{ser::SerializerBuilder, Encoding}; 44 | /// 45 | /// let writer = std::io::stdout(); 46 | /// let mut serializer = SerializerBuilder::new() 47 | /// .newline(true) 48 | /// .build(writer); 49 | /// ``` 50 | #[derive(Debug, Default, Clone)] 51 | pub struct SerializerBuilder { 52 | opts: SerializeOptions, 53 | } 54 | 55 | impl SerializerBuilder { 56 | /// Creates a new `SerializerBuilder`. 57 | pub fn new() -> Self { 58 | Self::default() 59 | } 60 | 61 | /// Emit output data in a compact format. This will disable pretty printing for encodings that 62 | /// support it. 63 | pub fn compact(&mut self, yes: bool) -> &mut Self { 64 | self.opts.compact = yes; 65 | self 66 | } 67 | 68 | /// Append a trailing newline to the serialized data. 69 | pub fn newline(&mut self, yes: bool) -> &mut Self { 70 | self.opts.newline = yes; 71 | self 72 | } 73 | 74 | /// When the input is an array of objects and the output encoding is CSV, the field names of 75 | /// the first object will be used as CSV headers. Field values of all following objects will 76 | /// be matched to the right CSV column based on their key. Missing fields produce empty columns 77 | /// while excess fields are ignored. 78 | pub fn keys_as_csv_headers(&mut self, yes: bool) -> &mut Self { 79 | self.opts.keys_as_csv_headers = yes; 80 | self 81 | } 82 | 83 | /// Sets a custom CSV delimiter. 84 | pub fn csv_delimiter(&mut self, delim: u8) -> &mut Self { 85 | self.opts.csv_delimiter = Some(delim); 86 | self 87 | } 88 | 89 | /// Sets a custom separator to join text output with. 90 | pub fn text_join_separator(&mut self, sep: S) -> &mut Self 91 | where 92 | S: AsRef, 93 | { 94 | self.opts.text_join_separator = Some(sep.as_ref().to_owned()); 95 | self 96 | } 97 | 98 | /// Treat output arrays as multiple YAML documents. 99 | pub fn multi_doc_yaml(&mut self, yes: bool) -> &mut Self { 100 | self.opts.multi_doc_yaml = yes; 101 | self 102 | } 103 | 104 | /// Builds the `Serializer` for the given writer. 105 | pub fn build(&self, writer: W) -> Serializer 106 | where 107 | W: std::io::Write, 108 | { 109 | Serializer::with_options(writer, self.opts.clone()) 110 | } 111 | } 112 | 113 | /// A `Serializer` can serialize a `Value` into an encoded byte stream. 114 | pub struct Serializer { 115 | writer: W, 116 | opts: SerializeOptions, 117 | } 118 | 119 | impl Serializer 120 | where 121 | W: std::io::Write, 122 | { 123 | /// Creates a new `Serializer` for writer with default options. 124 | pub fn new(writer: W) -> Self { 125 | Self::with_options(writer, Default::default()) 126 | } 127 | 128 | /// Creates a new `Serializer` for writer with options. 129 | pub fn with_options(writer: W, opts: SerializeOptions) -> Self { 130 | Self { writer, opts } 131 | } 132 | 133 | /// Serializes the given `Value` and writes the encoded data to the writer. 134 | /// 135 | /// ## Example 136 | /// 137 | /// ``` 138 | /// use dts::{ser::SerializerBuilder, Encoding}; 139 | /// use serde_json::json; 140 | /// # use std::error::Error; 141 | /// # 142 | /// # fn main() -> Result<(), Box> { 143 | /// let mut buf = Vec::new(); 144 | /// let mut ser = SerializerBuilder::new().compact(true).build(&mut buf); 145 | /// ser.serialize(Encoding::Json, json!(["foo"]))?; 146 | /// 147 | /// assert_eq!(&buf, r#"["foo"]"#.as_bytes()); 148 | /// # Ok(()) 149 | /// # } 150 | /// ``` 151 | pub fn serialize(&mut self, encoding: Encoding, value: Value) -> Result<()> { 152 | match encoding { 153 | Encoding::Yaml => self.serialize_yaml(value)?, 154 | Encoding::Json => self.serialize_json(value)?, 155 | Encoding::Toml => self.serialize_toml(value)?, 156 | Encoding::Csv => self.serialize_csv(value)?, 157 | Encoding::QueryString => self.serialize_query_string(value)?, 158 | Encoding::Xml => self.serialize_xml(value)?, 159 | Encoding::Text => self.serialize_text(value)?, 160 | Encoding::Gron => self.serialize_gron(value)?, 161 | Encoding::Hcl => self.serialize_hcl(value)?, 162 | encoding => return Err(Error::UnsupportedEncoding(encoding)), 163 | }; 164 | 165 | if self.opts.newline { 166 | self.writer.write_all(b"\n")?; 167 | } 168 | 169 | Ok(()) 170 | } 171 | 172 | fn serialize_yaml(&mut self, value: Value) -> Result<()> { 173 | match value { 174 | Value::Array(array) if self.opts.multi_doc_yaml => array 175 | .into_iter() 176 | .try_for_each(|document| self.serialize_yaml_document(&document)), 177 | value => self.serialize_yaml_document(&value), 178 | } 179 | } 180 | 181 | fn serialize_yaml_document(&mut self, value: &Value) -> Result<()> { 182 | self.writer.write_all(b"---\n")?; 183 | serde_yaml::to_writer(&mut self.writer, value)?; 184 | Ok(()) 185 | } 186 | 187 | fn serialize_json(&mut self, value: Value) -> Result<()> { 188 | if self.opts.compact { 189 | serde_json::to_writer(&mut self.writer, &value)? 190 | } else { 191 | serde_json::to_writer_pretty(&mut self.writer, &value)? 192 | } 193 | 194 | Ok(()) 195 | } 196 | 197 | fn serialize_toml(&mut self, value: Value) -> Result<()> { 198 | let value = toml::Value::try_from(value)?; 199 | 200 | let s = if self.opts.compact { 201 | toml::ser::to_string(&value)? 202 | } else { 203 | toml::ser::to_string_pretty(&value)? 204 | }; 205 | 206 | Ok(self.writer.write_all(s.as_bytes())?) 207 | } 208 | 209 | fn serialize_csv(&mut self, value: Value) -> Result<()> { 210 | // Because individual row items may produce errors during serialization because they are of 211 | // unexpected type, write into a buffer first and only flush out to the writer only if 212 | // serialization of all rows succeeded. This avoids writing out partial data. 213 | let mut buf = Vec::new(); 214 | { 215 | let mut csv_writer = csv::WriterBuilder::new() 216 | .delimiter(self.opts.csv_delimiter.unwrap_or(b',')) 217 | .from_writer(&mut buf); 218 | 219 | let mut headers: Option> = None; 220 | let empty_value = Value::String("".into()); 221 | 222 | for row in value.into_array().into_iter() { 223 | let row_data = if !self.opts.keys_as_csv_headers { 224 | row.into_array() 225 | .into_iter() 226 | .map(Value::into_string) 227 | .collect::>() 228 | } else { 229 | let row = row.into_object("csv"); 230 | 231 | // The first row dictates the header fields. 232 | if headers.is_none() { 233 | let header_data = row.keys().cloned().collect(); 234 | csv_writer.serialize(&header_data)?; 235 | headers = Some(header_data); 236 | } 237 | 238 | headers 239 | .as_ref() 240 | .unwrap() 241 | .iter() 242 | .map(|header| row.get(header).unwrap_or(&empty_value)) 243 | .cloned() 244 | .map(Value::into_string) 245 | .collect::>() 246 | }; 247 | 248 | csv_writer.serialize(row_data)?; 249 | } 250 | } 251 | 252 | Ok(self.writer.write_all(&buf)?) 253 | } 254 | 255 | fn serialize_query_string(&mut self, value: Value) -> Result<()> { 256 | Ok(serde_qs::to_writer(&value, &mut self.writer)?) 257 | } 258 | 259 | fn serialize_xml(&mut self, value: Value) -> Result<()> { 260 | Ok(serde_xml_rs::to_writer(&mut self.writer, &value)?) 261 | } 262 | 263 | fn serialize_text(&mut self, value: Value) -> Result<()> { 264 | let sep = self 265 | .opts 266 | .text_join_separator 267 | .clone() 268 | .unwrap_or_else(|| String::from('\n')); 269 | 270 | let text = value 271 | .into_array() 272 | .into_iter() 273 | .map(Value::into_string) 274 | .collect::>() 275 | .join(&sep); 276 | 277 | Ok(self.writer.write_all(text.as_bytes())?) 278 | } 279 | 280 | fn serialize_gron(&mut self, value: Value) -> Result<()> { 281 | let output = flatten_keys(value, "json") 282 | .as_object() 283 | .unwrap() 284 | .into_iter() 285 | .fold(String::new(), |mut output, (k, v)| { 286 | let _ = writeln!(output, "{k} = {v};"); 287 | output 288 | }); 289 | 290 | Ok(self.writer.write_all(output.as_bytes())?) 291 | } 292 | 293 | fn serialize_hcl(&mut self, value: Value) -> Result<()> { 294 | if self.opts.compact { 295 | let fmt = hcl::format::Formatter::builder() 296 | .compact(self.opts.compact) 297 | .build(&mut self.writer); 298 | let mut ser = hcl::ser::Serializer::with_formatter(fmt); 299 | ser.serialize(&value)?; 300 | } else { 301 | hcl::to_writer(&mut self.writer, &value)?; 302 | } 303 | 304 | Ok(()) 305 | } 306 | } 307 | 308 | #[cfg(test)] 309 | mod test { 310 | use super::*; 311 | use pretty_assertions::assert_eq; 312 | use serde_json::json; 313 | use std::str; 314 | 315 | #[track_caller] 316 | fn assert_serializes_to(encoding: Encoding, value: Value, expected: &str) { 317 | assert_builder_serializes_to(&mut SerializerBuilder::new(), encoding, value, expected) 318 | } 319 | 320 | #[track_caller] 321 | fn assert_builder_serializes_to( 322 | builder: &mut SerializerBuilder, 323 | encoding: Encoding, 324 | value: Value, 325 | expected: &str, 326 | ) { 327 | let mut buf = Vec::new(); 328 | let mut ser = builder.build(&mut buf); 329 | 330 | ser.serialize(encoding, value).unwrap(); 331 | assert_eq!(str::from_utf8(&buf).unwrap(), expected); 332 | } 333 | 334 | #[test] 335 | fn test_serialize_json() { 336 | assert_builder_serializes_to( 337 | &mut SerializerBuilder::new().compact(true), 338 | Encoding::Json, 339 | json!(["one", "two"]), 340 | "[\"one\",\"two\"]", 341 | ); 342 | assert_serializes_to( 343 | Encoding::Json, 344 | json!(["one", "two"]), 345 | "[\n \"one\",\n \"two\"\n]", 346 | ); 347 | } 348 | 349 | #[test] 350 | fn test_serialize_csv() { 351 | assert_serializes_to( 352 | Encoding::Csv, 353 | json!([["one", "two"], ["three", "four"]]), 354 | "one,two\nthree,four\n", 355 | ); 356 | assert_builder_serializes_to( 357 | &mut SerializerBuilder::new().keys_as_csv_headers(true), 358 | Encoding::Csv, 359 | json!([ 360 | {"one": "val1", "two": "val2"}, 361 | {"one": "val3", "three": "val4"}, 362 | {"two": "val5"} 363 | ]), 364 | "one,two\nval1,val2\nval3,\n,val5\n", 365 | ); 366 | assert_builder_serializes_to( 367 | &mut SerializerBuilder::new().keys_as_csv_headers(true), 368 | Encoding::Csv, 369 | json!({"one": "val1", "two": "val2"}), 370 | "one,two\nval1,val2\n", 371 | ); 372 | assert_serializes_to(Encoding::Csv, json!("non-array"), "non-array\n"); 373 | assert_serializes_to( 374 | Encoding::Csv, 375 | json!([{"non-array": "row"}]), 376 | "\"{\"\"non-array\"\":\"\"row\"\"}\"\n", 377 | ); 378 | assert_builder_serializes_to( 379 | &mut SerializerBuilder::new().keys_as_csv_headers(true), 380 | Encoding::Csv, 381 | json!([["non-object-row"]]), 382 | "csv\n\"[\"\"non-object-row\"\"]\"\n", 383 | ); 384 | } 385 | 386 | #[test] 387 | fn test_serialize_text() { 388 | assert_serializes_to(Encoding::Text, json!(["one", "two"]), "one\ntwo"); 389 | assert_serializes_to( 390 | Encoding::Text, 391 | json!([{"foo": "bar"}, "baz"]), 392 | "{\"foo\":\"bar\"}\nbaz", 393 | ); 394 | assert_serializes_to(Encoding::Text, json!({"foo": "bar"}), "{\"foo\":\"bar\"}"); 395 | } 396 | 397 | #[test] 398 | fn test_serialize_hcl() { 399 | assert_serializes_to(Encoding::Hcl, json!([{"foo": "bar"}]), "foo = \"bar\"\n"); 400 | assert_serializes_to( 401 | Encoding::Hcl, 402 | json!({"foo": "bar", "bar": 2}), 403 | "foo = \"bar\"\nbar = 2\n", 404 | ); 405 | } 406 | } 407 | -------------------------------------------------------------------------------- /src/sink.rs: -------------------------------------------------------------------------------- 1 | use crate::{Encoding, PathExt, Result}; 2 | use std::fmt; 3 | use std::path::{Path, PathBuf}; 4 | use std::str::FromStr; 5 | 6 | /// A target to write serialized output to. 7 | #[derive(Debug, Clone, PartialEq, Eq)] 8 | pub enum Sink { 9 | /// Stdout sink. 10 | Stdout, 11 | /// Local path sink. 12 | Path(PathBuf), 13 | } 14 | 15 | impl Sink { 16 | /// Tries to detect the encoding of the sink. Returns `None` if the encoding cannot be 17 | /// detected. 18 | pub fn encoding(&self) -> Option { 19 | match self { 20 | Self::Stdout => None, 21 | Self::Path(path) => Encoding::from_path(path), 22 | } 23 | } 24 | } 25 | 26 | impl From<&str> for Sink { 27 | fn from(s: &str) -> Self { 28 | if s == "-" { 29 | Self::Stdout 30 | } else { 31 | Self::Path(PathBuf::from(s)) 32 | } 33 | } 34 | } 35 | 36 | impl From<&Path> for Sink { 37 | fn from(path: &Path) -> Self { 38 | Self::Path(path.to_path_buf()) 39 | } 40 | } 41 | 42 | impl FromStr for Sink { 43 | type Err = std::convert::Infallible; 44 | 45 | fn from_str(s: &str) -> Result { 46 | Ok(From::from(s)) 47 | } 48 | } 49 | 50 | impl fmt::Display for Sink { 51 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 52 | match self { 53 | Self::Stdout => write!(f, ""), 54 | Self::Path(path) => path 55 | .relative_to_cwd() 56 | .unwrap_or_else(|| path.clone()) 57 | .display() 58 | .fmt(f), 59 | } 60 | } 61 | } 62 | 63 | #[cfg(test)] 64 | mod test { 65 | use super::*; 66 | use pretty_assertions::assert_eq; 67 | 68 | #[test] 69 | fn test_from_str() { 70 | assert_eq!(Sink::from_str("-"), Ok(Sink::Stdout)); 71 | assert_eq!( 72 | Sink::from_str("foo.json"), 73 | Ok(Sink::Path(PathBuf::from("foo.json"))) 74 | ); 75 | } 76 | 77 | #[test] 78 | fn test_encoding() { 79 | assert_eq!(Sink::from("-").encoding(), None); 80 | assert_eq!(Sink::from("foo").encoding(), None); 81 | assert_eq!(Sink::from("foo.json").encoding(), Some(Encoding::Json)); 82 | } 83 | 84 | #[test] 85 | fn test_to_string() { 86 | assert_eq!(&Sink::Stdout.to_string(), ""); 87 | assert_eq!(&Sink::from("Cargo.toml").to_string(), "Cargo.toml"); 88 | assert_eq!( 89 | &Sink::from(std::fs::canonicalize("src/lib.rs").unwrap().as_path()).to_string(), 90 | "src/lib.rs" 91 | ); 92 | assert_eq!( 93 | &Sink::from("/non-existent/path").to_string(), 94 | "/non-existent/path" 95 | ); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/source.rs: -------------------------------------------------------------------------------- 1 | use crate::{Encoding, Error, PathExt, Result}; 2 | use std::fmt; 3 | use std::fs; 4 | use std::io::{self, BufRead, BufReader, Cursor, Read}; 5 | use std::path::{Path, PathBuf}; 6 | use std::str::FromStr; 7 | use url::Url; 8 | 9 | /// A source for data that needs to be deserialized. 10 | #[derive(Debug, Clone, PartialEq, Eq)] 11 | pub enum Source { 12 | /// Stdin source. 13 | Stdin, 14 | /// Local file or directory source. 15 | Path(PathBuf), 16 | /// Remote URL source. 17 | Url(Url), 18 | } 19 | 20 | impl Source { 21 | /// Returns `Some` if the source is a local path, `None` otherwise. 22 | pub fn as_path(&self) -> Option<&Path> { 23 | match self { 24 | Self::Path(path) => Some(path), 25 | _ => None, 26 | } 27 | } 28 | 29 | /// Returns `true` if the `Source` is a local path and the path exists on disk and is pointing 30 | /// at a directory. 31 | pub fn is_dir(&self) -> bool { 32 | self.as_path().map(|path| path.is_dir()).unwrap_or(false) 33 | } 34 | 35 | /// Tries to detect the encoding of the source. Returns `None` if the encoding cannot be 36 | /// detected. 37 | pub fn encoding(&self) -> Option { 38 | match self { 39 | Self::Stdin => None, 40 | Self::Path(path) => Encoding::from_path(path), 41 | Self::Url(url) => Encoding::from_path(url.as_str()), 42 | } 43 | } 44 | 45 | /// If source is a local path, this returns sources for all files matching the glob pattern. 46 | /// 47 | /// ## Errors 48 | /// 49 | /// Returns an error if the sink is not of variant `Sink::Path`, the pattern is invalid or if 50 | /// there is a `io::Error` while reading the file system. 51 | pub fn glob_files(&self, pattern: &str) -> Result> { 52 | match self.as_path() { 53 | Some(path) => Ok(path 54 | .glob_files(pattern)? 55 | .iter() 56 | .map(|path| Self::from(path.as_path())) 57 | .collect()), 58 | None => Err(Error::new("not a path source")), 59 | } 60 | } 61 | 62 | /// Returns a `SourceReader` to read from the source. 63 | /// 64 | /// ## Errors 65 | /// 66 | /// May return an error if the source is `Source::Path` and the file cannot be opened of if 67 | /// source is `Source::Url` and there is an error requesting the remote url. 68 | pub fn to_reader(&self) -> Result { 69 | let reader: Box = match self { 70 | Self::Stdin => Box::new(io::stdin()), 71 | Self::Path(path) => Box::new(fs::File::open(path)?), 72 | Self::Url(url) => Box::new(ureq::get(url.as_ref()).call()?.into_body().into_reader()), 73 | }; 74 | 75 | SourceReader::new(reader, self.encoding()) 76 | } 77 | } 78 | 79 | impl From<&str> for Source { 80 | fn from(s: &str) -> Self { 81 | if s == "-" { 82 | Self::Stdin 83 | } else { 84 | if let Ok(url) = Url::parse(s) { 85 | if url.scheme() != "file" { 86 | return Self::Url(url); 87 | } 88 | } 89 | 90 | Self::Path(PathBuf::from(s)) 91 | } 92 | } 93 | } 94 | 95 | impl From<&Path> for Source { 96 | fn from(path: &Path) -> Self { 97 | Self::Path(path.to_path_buf()) 98 | } 99 | } 100 | 101 | impl FromStr for Source { 102 | type Err = std::convert::Infallible; 103 | 104 | fn from_str(s: &str) -> Result { 105 | Ok(From::from(s)) 106 | } 107 | } 108 | 109 | impl fmt::Display for Source { 110 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 111 | match self { 112 | Self::Stdin => write!(f, ""), 113 | Self::Url(url) => url.fmt(f), 114 | Self::Path(path) => path 115 | .relative_to_cwd() 116 | .unwrap_or_else(|| path.clone()) 117 | .display() 118 | .fmt(f), 119 | } 120 | } 121 | } 122 | 123 | /// A type that can read from a `Source`. It is able to detect the `Source`'s encoding by looking 124 | /// at the first line of the input. 125 | pub struct SourceReader { 126 | first_line: Cursor>, 127 | remainder: BufReader>, 128 | encoding: Option, 129 | } 130 | 131 | impl SourceReader { 132 | /// Creates a new `SourceReader` for an `io::Read` implementation and an optional encoding 133 | /// hint. 134 | /// 135 | /// Reads the first line from `reader` upon creation. 136 | /// 137 | /// ## Errors 138 | /// 139 | /// Returns an error if reading the first line from the reader fails. 140 | pub fn new(reader: Box, encoding: Option) -> Result { 141 | let mut remainder = BufReader::new(reader); 142 | let mut buf = Vec::new(); 143 | 144 | remainder.read_until(b'\n', &mut buf)?; 145 | 146 | let first_line = Cursor::new(buf); 147 | 148 | Ok(SourceReader { 149 | first_line, 150 | remainder, 151 | encoding, 152 | }) 153 | } 154 | 155 | /// Tries to detect the encoding of the source. If the source provides an encoding hint it is 156 | /// returned as is. Otherwise the `SourceReader` attempts to detect the encoding based on the 157 | /// contents of the first line of the input data. 158 | /// 159 | /// Returns `None` if the encoding cannot be detected. 160 | pub fn encoding(&self) -> Option { 161 | self.encoding.or_else(|| { 162 | std::str::from_utf8(self.first_line.get_ref()) 163 | .ok() 164 | .and_then(Encoding::from_first_line) 165 | }) 166 | } 167 | } 168 | 169 | impl Read for SourceReader { 170 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 171 | if self.first_line.position() < self.first_line.get_ref().len() as u64 { 172 | self.first_line.read(buf) 173 | } else { 174 | self.remainder.read(buf) 175 | } 176 | } 177 | } 178 | 179 | #[cfg(test)] 180 | mod test { 181 | use super::*; 182 | use pretty_assertions::assert_eq; 183 | 184 | #[test] 185 | fn test_from_str() { 186 | assert_eq!(Source::from_str("-"), Ok(Source::Stdin)); 187 | assert_eq!( 188 | Source::from_str("foo.json"), 189 | Ok(Source::Path(PathBuf::from("foo.json"))) 190 | ); 191 | assert_eq!( 192 | Source::from_str("http://localhost/foo.json"), 193 | Ok(Source::Url( 194 | Url::from_str("http://localhost/foo.json").unwrap() 195 | )) 196 | ); 197 | } 198 | 199 | #[test] 200 | fn test_encoding() { 201 | assert_eq!(Source::from("-").encoding(), None); 202 | assert_eq!(Source::from("foo").encoding(), None); 203 | assert_eq!(Source::from("foo.json").encoding(), Some(Encoding::Json)); 204 | assert_eq!( 205 | Source::from("http://localhost/bar.yaml").encoding(), 206 | Some(Encoding::Yaml) 207 | ); 208 | } 209 | 210 | #[test] 211 | fn test_to_string() { 212 | assert_eq!(&Source::Stdin.to_string(), ""); 213 | assert_eq!(&Source::from("Cargo.toml").to_string(), "Cargo.toml"); 214 | assert_eq!( 215 | &Source::from(std::fs::canonicalize("src/lib.rs").unwrap().as_path()).to_string(), 216 | "src/lib.rs" 217 | ); 218 | assert_eq!( 219 | &Source::from("/non-existent/path").to_string(), 220 | "/non-existent/path" 221 | ); 222 | assert_eq!( 223 | &Source::from("http://localhost/bar.yaml").to_string(), 224 | "http://localhost/bar.yaml", 225 | ); 226 | } 227 | 228 | #[test] 229 | fn test_glob_files() { 230 | assert!( 231 | Source::from("src/") 232 | .glob_files("*.rs") 233 | .unwrap() 234 | .contains(&Source::from("src/lib.rs")) 235 | ); 236 | assert!(Source::from("-").glob_files("*.json").is_err()); 237 | assert!( 238 | Source::from("http://localhost/") 239 | .glob_files("*.json") 240 | .is_err(), 241 | ); 242 | assert!(matches!( 243 | Source::from("src/").glob_files("***"), 244 | Err(Error::GlobPatternError { .. }) 245 | )); 246 | } 247 | 248 | #[test] 249 | fn test_source_reader() { 250 | let input = Cursor::new("---\nfoo: bar\n"); 251 | let mut reader = SourceReader::new(Box::new(input), None).unwrap(); 252 | 253 | assert_eq!(reader.encoding(), Some(Encoding::Yaml)); 254 | 255 | let mut buf = String::new(); 256 | reader.read_to_string(&mut buf).unwrap(); 257 | 258 | assert_eq!(&buf, "---\nfoo: bar\n"); 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /src/value.rs: -------------------------------------------------------------------------------- 1 | //! Extension methods for `serde_json::Value`. 2 | 3 | use serde_json::{Map, Value}; 4 | use std::fmt; 5 | use std::iter; 6 | 7 | /// A trait to add extension methods to `serde_json::Value`. 8 | pub trait ValueExt { 9 | /// Converts value into an array. If the value is of variant `Value::Array`, the wrapped value 10 | /// will be returned. Otherwise the result is a `Vec` which contains the `Value`. 11 | fn into_array(self) -> Vec; 12 | 13 | /// Converts value into an object. If the value is of variant `Value::Object`, the wrapped value 14 | /// will be returned. Otherwise the result is a `Map` which contains a single entry with the 15 | /// provided key. 16 | fn into_object(self, key: K) -> Map 17 | where 18 | K: fmt::Display; 19 | 20 | /// Converts the value to its string representation but ensures that the resulting string is 21 | /// not quoted. 22 | fn into_string(self) -> String; 23 | 24 | /// Deep merges `other` into `self`, replacing all values in `other` that were merged into 25 | /// `self` with `Value::Null`. 26 | fn deep_merge(&mut self, other: &mut Value); 27 | } 28 | 29 | impl ValueExt for Value { 30 | fn into_array(self) -> Vec { 31 | match self { 32 | Value::Array(array) => array, 33 | value => vec![value], 34 | } 35 | } 36 | 37 | fn into_object(self, key: K) -> Map 38 | where 39 | K: fmt::Display, 40 | { 41 | match self { 42 | Value::Object(object) => object, 43 | value => Map::from_iter(iter::once((key.to_string(), value))), 44 | } 45 | } 46 | 47 | fn into_string(self) -> String { 48 | match self { 49 | Value::String(s) => s, 50 | value => value.to_string(), 51 | } 52 | } 53 | 54 | fn deep_merge(&mut self, other: &mut Value) { 55 | match (self, other) { 56 | (Value::Object(lhs), Value::Object(rhs)) => { 57 | rhs.iter_mut().for_each(|(key, value)| { 58 | lhs.entry(key.to_string()) 59 | .and_modify(|lhs| lhs.deep_merge(value)) 60 | .or_insert_with(|| value.take()); 61 | }); 62 | } 63 | (Value::Array(lhs), Value::Array(rhs)) => { 64 | lhs.resize(lhs.len().max(rhs.len()), Value::Null); 65 | 66 | rhs.iter_mut() 67 | .enumerate() 68 | .for_each(|(i, rhs)| lhs[i].deep_merge(rhs)); 69 | } 70 | (_, Value::Null) => (), 71 | (lhs, rhs) => *lhs = rhs.take(), 72 | } 73 | } 74 | } 75 | 76 | #[cfg(test)] 77 | mod tests { 78 | use super::*; 79 | use pretty_assertions::assert_eq; 80 | use serde_json::json; 81 | 82 | #[test] 83 | fn test_into_array() { 84 | assert_eq!(json!("foo").into_array(), vec![json!("foo")]); 85 | assert_eq!(json!(["foo"]).into_array(), vec![json!("foo")]); 86 | assert_eq!( 87 | json!({"foo": "bar"}).into_array(), 88 | vec![json!({"foo": "bar"})] 89 | ); 90 | } 91 | 92 | #[test] 93 | fn test_into_object() { 94 | assert_eq!( 95 | json!("foo").into_object("the-key"), 96 | Map::from_iter(iter::once(("the-key".into(), json!("foo")))) 97 | ); 98 | assert_eq!( 99 | json!(["foo", "bar"]).into_object("the-key"), 100 | Map::from_iter(iter::once(("the-key".into(), json!(["foo", "bar"])))) 101 | ); 102 | assert_eq!( 103 | json!({"foo": "bar"}).into_object("the-key"), 104 | Map::from_iter(iter::once(("foo".into(), json!("bar")))) 105 | ); 106 | } 107 | 108 | #[test] 109 | fn test_into_string() { 110 | assert_eq!( 111 | json!({"foo": "bar"}).into_string(), 112 | String::from(r#"{"foo":"bar"}"#) 113 | ); 114 | assert_eq!( 115 | json!(["foo", "bar"]).into_string(), 116 | String::from(r#"["foo","bar"]"#) 117 | ); 118 | assert_eq!(json!("foo").into_string(), String::from("foo")); 119 | assert_eq!(json!(true).into_string(), String::from("true")); 120 | assert_eq!(json!(1).into_string(), String::from("1")); 121 | assert_eq!(Value::Null.into_string(), String::from("null")); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /tests/fixtures/example.compact.hcl: -------------------------------------------------------------------------------- 1 | users = [{ "id" = "6189643191d9e7fe060114f0", "isActive" = true, "balance" = "$2,509.69", "picture" = "http://placehold.it/32x32", "age" = 34, "eyeColor" = "blue", "name" = "Rena Franklin", "gender" = "female", "company" = "VITRICOMP", "email" = "renafranklin@vitricomp.com", "phone" = "+1 (851) 579-2333", "address" = "548 Manhattan Court, Somerset, North Dakota, 9737", "about" = "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n", "registered" = "2019-04-22T05:49:21 -02:00", "tags" = ["enim", "officia", "do", "ullamco", "voluptate"], "friends" = [{ "id" = "61896431a6546aad2d840a88", "name" = "Florine Acevedo" }, { "id" = "618964dc6fabe8b6bc673abb", "name" = "Roxanne Woods" }, { "id" = "618964dcee90f14682191e15", "name" = "Hines Jackson" }], "latitude" = 31.180897, "longitude" = -106.807272 }, { "id" = "61896431b3d9c0729b0d35e6", "isActive" = true, "balance" = "$1,533.07", "picture" = "http://placehold.it/32x32", "age" = 27, "eyeColor" = "brown", "name" = "Dena Stone", "gender" = "female", "company" = "ZAJ", "email" = "denastone@zaj.com", "phone" = "+1 (822) 541-3135", "address" = "416 Oxford Street, Balm, Indiana, 5900", "about" = "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n", "registered" = "2019-03-05T05:26:01 -01:00", "tags" = ["aute", "elit", "mollit", "sit"], "friends" = [{ "id" = "6189643146646d84cd519d62", "name" = "Dawn Joyce" }], "latitude" = 65.453177, "longitude" = 102.591909 }, { "id" = "618964315cabd70b6541dcae", "isActive" = false, "balance" = "$3,970.08", "picture" = "http://placehold.it/32x32", "age" = 37, "eyeColor" = "blue", "name" = "Stacy Horton", "gender" = "female", "company" = "AVIT", "email" = "stacyhorton@avit.com", "phone" = "+1 (922) 549-2426", "address" = "780 Nautilus Avenue, Delwood, Pennsylvania, 8513", "about" = "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n", "registered" = "2014-09-05T08:05:33 -02:00", "tags" = ["non", "deserunt", "eiusmod", "sit", "aliquip", "reprehenderit", "voluptate"], "friends" = [{ "id" = "618964317d23d69f9acebddf", "name" = "Natalie Rosario" }, { "id" = "61896431c417b533ef7ba58c", "name" = "Colon Kinney" }], "latitude" = 89.138415, "longitude" = -139.356254 }] 2 | -------------------------------------------------------------------------------- /tests/fixtures/example.filtered.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "61896431b3d9c0729b0d35e6", 4 | "isActive": true, 5 | "balance": "$1,533.07", 6 | "picture": "http://placehold.it/32x32", 7 | "age": 27, 8 | "eyeColor": "brown", 9 | "name": "Dena Stone", 10 | "gender": "female", 11 | "company": "ZAJ", 12 | "email": "denastone@zaj.com", 13 | "phone": "+1 (822) 541-3135", 14 | "address": "416 Oxford Street, Balm, Indiana, 5900", 15 | "about": "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n", 16 | "registered": "2019-03-05T05:26:01 -01:00", 17 | "tags": [ 18 | "aute", 19 | "elit", 20 | "mollit", 21 | "sit" 22 | ], 23 | "friends": [ 24 | { 25 | "id": "6189643146646d84cd519d62", 26 | "name": "Dawn Joyce" 27 | } 28 | ], 29 | "latitude": 65.453177, 30 | "longitude": 102.591909 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /tests/fixtures/example.hcl: -------------------------------------------------------------------------------- 1 | users = [ 2 | { 3 | "id" = "6189643191d9e7fe060114f0" 4 | "isActive" = true 5 | "balance" = "$2,509.69" 6 | "picture" = "http://placehold.it/32x32" 7 | "age" = 34 8 | "eyeColor" = "blue" 9 | "name" = "Rena Franklin" 10 | "gender" = "female" 11 | "company" = "VITRICOMP" 12 | "email" = "renafranklin@vitricomp.com" 13 | "phone" = "+1 (851) 579-2333" 14 | "address" = "548 Manhattan Court, Somerset, North Dakota, 9737" 15 | "about" = "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n" 16 | "registered" = "2019-04-22T05:49:21 -02:00" 17 | "tags" = [ 18 | "enim", 19 | "officia", 20 | "do", 21 | "ullamco", 22 | "voluptate" 23 | ] 24 | "friends" = [ 25 | { 26 | "id" = "61896431a6546aad2d840a88" 27 | "name" = "Florine Acevedo" 28 | }, 29 | { 30 | "id" = "618964dc6fabe8b6bc673abb" 31 | "name" = "Roxanne Woods" 32 | }, 33 | { 34 | "id" = "618964dcee90f14682191e15" 35 | "name" = "Hines Jackson" 36 | } 37 | ] 38 | "latitude" = 31.180897 39 | "longitude" = -106.807272 40 | }, 41 | { 42 | "id" = "61896431b3d9c0729b0d35e6" 43 | "isActive" = true 44 | "balance" = "$1,533.07" 45 | "picture" = "http://placehold.it/32x32" 46 | "age" = 27 47 | "eyeColor" = "brown" 48 | "name" = "Dena Stone" 49 | "gender" = "female" 50 | "company" = "ZAJ" 51 | "email" = "denastone@zaj.com" 52 | "phone" = "+1 (822) 541-3135" 53 | "address" = "416 Oxford Street, Balm, Indiana, 5900" 54 | "about" = "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n" 55 | "registered" = "2019-03-05T05:26:01 -01:00" 56 | "tags" = [ 57 | "aute", 58 | "elit", 59 | "mollit", 60 | "sit" 61 | ] 62 | "friends" = [ 63 | { 64 | "id" = "6189643146646d84cd519d62" 65 | "name" = "Dawn Joyce" 66 | } 67 | ] 68 | "latitude" = 65.453177 69 | "longitude" = 102.591909 70 | }, 71 | { 72 | "id" = "618964315cabd70b6541dcae" 73 | "isActive" = false 74 | "balance" = "$3,970.08" 75 | "picture" = "http://placehold.it/32x32" 76 | "age" = 37 77 | "eyeColor" = "blue" 78 | "name" = "Stacy Horton" 79 | "gender" = "female" 80 | "company" = "AVIT" 81 | "email" = "stacyhorton@avit.com" 82 | "phone" = "+1 (922) 549-2426" 83 | "address" = "780 Nautilus Avenue, Delwood, Pennsylvania, 8513" 84 | "about" = "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n" 85 | "registered" = "2014-09-05T08:05:33 -02:00" 86 | "tags" = [ 87 | "non", 88 | "deserunt", 89 | "eiusmod", 90 | "sit", 91 | "aliquip", 92 | "reprehenderit", 93 | "voluptate" 94 | ] 95 | "friends" = [ 96 | { 97 | "id" = "618964317d23d69f9acebddf" 98 | "name" = "Natalie Rosario" 99 | }, 100 | { 101 | "id" = "61896431c417b533ef7ba58c" 102 | "name" = "Colon Kinney" 103 | } 104 | ] 105 | "latitude" = 89.138415 106 | "longitude" = -139.356254 107 | } 108 | ] 109 | -------------------------------------------------------------------------------- /tests/fixtures/example.js: -------------------------------------------------------------------------------- 1 | json = {}; 2 | json.users = []; 3 | json.users[0] = {}; 4 | json.users[0].about = "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n"; 5 | json.users[0].address = "548 Manhattan Court, Somerset, North Dakota, 9737"; 6 | json.users[0].age = 34; 7 | json.users[0].balance = "$2,509.69"; 8 | json.users[0].company = "VITRICOMP"; 9 | json.users[0].email = "renafranklin@vitricomp.com"; 10 | json.users[0].eyeColor = "blue"; 11 | json.users[0].friends = []; 12 | json.users[0].friends[0] = {}; 13 | json.users[0].friends[0].id = "61896431a6546aad2d840a88"; 14 | json.users[0].friends[0].name = "Florine Acevedo"; 15 | json.users[0].friends[1] = {}; 16 | json.users[0].friends[1].id = "618964dc6fabe8b6bc673abb"; 17 | json.users[0].friends[1].name = "Roxanne Woods"; 18 | json.users[0].friends[2] = {}; 19 | json.users[0].friends[2].id = "618964dcee90f14682191e15"; 20 | json.users[0].friends[2].name = "Hines Jackson"; 21 | json.users[0].gender = "female"; 22 | json.users[0].id = "6189643191d9e7fe060114f0"; 23 | json.users[0].isActive = true; 24 | json.users[0].latitude = 31.180897; 25 | json.users[0].longitude = -106.807272; 26 | json.users[0].name = "Rena Franklin"; 27 | json.users[0].phone = "+1 (851) 579-2333"; 28 | json.users[0].picture = "http://placehold.it/32x32"; 29 | json.users[0].registered = "2019-04-22T05:49:21 -02:00"; 30 | json.users[0].tags = []; 31 | json.users[0].tags[0] = "enim"; 32 | json.users[0].tags[1] = "officia"; 33 | json.users[0].tags[2] = "do"; 34 | json.users[0].tags[3] = "ullamco"; 35 | json.users[0].tags[4] = "voluptate"; 36 | json.users[1] = {}; 37 | json.users[1].about = "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n"; 38 | json.users[1].address = "416 Oxford Street, Balm, Indiana, 5900"; 39 | json.users[1].age = 27; 40 | json.users[1].balance = "$1,533.07"; 41 | json.users[1].company = "ZAJ"; 42 | json.users[1].email = "denastone@zaj.com"; 43 | json.users[1].eyeColor = "brown"; 44 | json.users[1].friends = []; 45 | json.users[1].friends[0] = {}; 46 | json.users[1].friends[0].id = "6189643146646d84cd519d62"; 47 | json.users[1].friends[0].name = "Dawn Joyce"; 48 | json.users[1].gender = "female"; 49 | json.users[1].id = "61896431b3d9c0729b0d35e6"; 50 | json.users[1].isActive = true; 51 | json.users[1].latitude = 65.453177; 52 | json.users[1].longitude = 102.591909; 53 | json.users[1].name = "Dena Stone"; 54 | json.users[1].phone = "+1 (822) 541-3135"; 55 | json.users[1].picture = "http://placehold.it/32x32"; 56 | json.users[1].registered = "2019-03-05T05:26:01 -01:00"; 57 | json.users[1].tags = []; 58 | json.users[1].tags[0] = "aute"; 59 | json.users[1].tags[1] = "elit"; 60 | json.users[1].tags[2] = "mollit"; 61 | json.users[1].tags[3] = "sit"; 62 | json.users[2] = {}; 63 | json.users[2].about = "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n"; 64 | json.users[2].address = "780 Nautilus Avenue, Delwood, Pennsylvania, 8513"; 65 | json.users[2].age = 37; 66 | json.users[2].balance = "$3,970.08"; 67 | json.users[2].company = "AVIT"; 68 | json.users[2].email = "stacyhorton@avit.com"; 69 | json.users[2].eyeColor = "blue"; 70 | json.users[2].friends = []; 71 | json.users[2].friends[0] = {}; 72 | json.users[2].friends[0].id = "618964317d23d69f9acebddf"; 73 | json.users[2].friends[0].name = "Natalie Rosario"; 74 | json.users[2].friends[1] = {}; 75 | json.users[2].friends[1].id = "61896431c417b533ef7ba58c"; 76 | json.users[2].friends[1].name = "Colon Kinney"; 77 | json.users[2].gender = "female"; 78 | json.users[2].id = "618964315cabd70b6541dcae"; 79 | json.users[2].isActive = false; 80 | json.users[2].latitude = 89.138415; 81 | json.users[2].longitude = -139.356254; 82 | json.users[2].name = "Stacy Horton"; 83 | json.users[2].phone = "+1 (922) 549-2426"; 84 | json.users[2].picture = "http://placehold.it/32x32"; 85 | json.users[2].registered = "2014-09-05T08:05:33 -02:00"; 86 | json.users[2].tags = []; 87 | json.users[2].tags[0] = "non"; 88 | json.users[2].tags[1] = "deserunt"; 89 | json.users[2].tags[2] = "eiusmod"; 90 | json.users[2].tags[3] = "sit"; 91 | json.users[2].tags[4] = "aliquip"; 92 | json.users[2].tags[5] = "reprehenderit"; 93 | json.users[2].tags[6] = "voluptate"; 94 | -------------------------------------------------------------------------------- /tests/fixtures/example.js.ungron.json: -------------------------------------------------------------------------------- 1 | { 2 | "users": [ 3 | { 4 | "about": "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n", 5 | "address": "548 Manhattan Court, Somerset, North Dakota, 9737", 6 | "age": 34, 7 | "balance": "$2,509.69", 8 | "company": "VITRICOMP", 9 | "email": "renafranklin@vitricomp.com", 10 | "eyeColor": "blue", 11 | "friends": [ 12 | { 13 | "id": "61896431a6546aad2d840a88", 14 | "name": "Florine Acevedo" 15 | }, 16 | { 17 | "id": "618964dc6fabe8b6bc673abb", 18 | "name": "Roxanne Woods" 19 | }, 20 | { 21 | "id": "618964dcee90f14682191e15", 22 | "name": "Hines Jackson" 23 | } 24 | ], 25 | "gender": "female", 26 | "id": "6189643191d9e7fe060114f0", 27 | "isActive": true, 28 | "latitude": 31.180897, 29 | "longitude": -106.807272, 30 | "name": "Rena Franklin", 31 | "phone": "+1 (851) 579-2333", 32 | "picture": "http://placehold.it/32x32", 33 | "registered": "2019-04-22T05:49:21 -02:00", 34 | "tags": [ 35 | "enim", 36 | "officia", 37 | "do", 38 | "ullamco", 39 | "voluptate" 40 | ] 41 | }, 42 | { 43 | "about": "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n", 44 | "address": "416 Oxford Street, Balm, Indiana, 5900", 45 | "age": 27, 46 | "balance": "$1,533.07", 47 | "company": "ZAJ", 48 | "email": "denastone@zaj.com", 49 | "eyeColor": "brown", 50 | "friends": [ 51 | { 52 | "id": "6189643146646d84cd519d62", 53 | "name": "Dawn Joyce" 54 | } 55 | ], 56 | "gender": "female", 57 | "id": "61896431b3d9c0729b0d35e6", 58 | "isActive": true, 59 | "latitude": 65.453177, 60 | "longitude": 102.591909, 61 | "name": "Dena Stone", 62 | "phone": "+1 (822) 541-3135", 63 | "picture": "http://placehold.it/32x32", 64 | "registered": "2019-03-05T05:26:01 -01:00", 65 | "tags": [ 66 | "aute", 67 | "elit", 68 | "mollit", 69 | "sit" 70 | ] 71 | }, 72 | { 73 | "about": "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n", 74 | "address": "780 Nautilus Avenue, Delwood, Pennsylvania, 8513", 75 | "age": 37, 76 | "balance": "$3,970.08", 77 | "company": "AVIT", 78 | "email": "stacyhorton@avit.com", 79 | "eyeColor": "blue", 80 | "friends": [ 81 | { 82 | "id": "618964317d23d69f9acebddf", 83 | "name": "Natalie Rosario" 84 | }, 85 | { 86 | "id": "61896431c417b533ef7ba58c", 87 | "name": "Colon Kinney" 88 | } 89 | ], 90 | "gender": "female", 91 | "id": "618964315cabd70b6541dcae", 92 | "isActive": false, 93 | "latitude": 89.138415, 94 | "longitude": -139.356254, 95 | "name": "Stacy Horton", 96 | "phone": "+1 (922) 549-2426", 97 | "picture": "http://placehold.it/32x32", 98 | "registered": "2014-09-05T08:05:33 -02:00", 99 | "tags": [ 100 | "non", 101 | "deserunt", 102 | "eiusmod", 103 | "sit", 104 | "aliquip", 105 | "reprehenderit", 106 | "voluptate" 107 | ] 108 | } 109 | ] 110 | } 111 | -------------------------------------------------------------------------------- /tests/fixtures/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "users": [ 3 | { 4 | "id": "6189643191d9e7fe060114f0", 5 | "isActive": true, 6 | "balance": "$2,509.69", 7 | "picture": "http://placehold.it/32x32", 8 | "age": 34, 9 | "eyeColor": "blue", 10 | "name": "Rena Franklin", 11 | "gender": "female", 12 | "company": "VITRICOMP", 13 | "email": "renafranklin@vitricomp.com", 14 | "phone": "+1 (851) 579-2333", 15 | "address": "548 Manhattan Court, Somerset, North Dakota, 9737", 16 | "about": "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n", 17 | "registered": "2019-04-22T05:49:21 -02:00", 18 | "tags": [ 19 | "enim", 20 | "officia", 21 | "do", 22 | "ullamco", 23 | "voluptate" 24 | ], 25 | "friends": [ 26 | { 27 | "id": "61896431a6546aad2d840a88", 28 | "name": "Florine Acevedo" 29 | }, 30 | { 31 | "id": "618964dc6fabe8b6bc673abb", 32 | "name": "Roxanne Woods" 33 | }, 34 | { 35 | "id": "618964dcee90f14682191e15", 36 | "name": "Hines Jackson" 37 | } 38 | ], 39 | "latitude": 31.180897, 40 | "longitude": -106.807272 41 | }, 42 | { 43 | "id": "61896431b3d9c0729b0d35e6", 44 | "isActive": true, 45 | "balance": "$1,533.07", 46 | "picture": "http://placehold.it/32x32", 47 | "age": 27, 48 | "eyeColor": "brown", 49 | "name": "Dena Stone", 50 | "gender": "female", 51 | "company": "ZAJ", 52 | "email": "denastone@zaj.com", 53 | "phone": "+1 (822) 541-3135", 54 | "address": "416 Oxford Street, Balm, Indiana, 5900", 55 | "about": "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n", 56 | "registered": "2019-03-05T05:26:01 -01:00", 57 | "tags": [ 58 | "aute", 59 | "elit", 60 | "mollit", 61 | "sit" 62 | ], 63 | "friends": [ 64 | { 65 | "id": "6189643146646d84cd519d62", 66 | "name": "Dawn Joyce" 67 | } 68 | ], 69 | "latitude": 65.453177, 70 | "longitude": 102.591909 71 | }, 72 | { 73 | "id": "618964315cabd70b6541dcae", 74 | "isActive": false, 75 | "balance": "$3,970.08", 76 | "picture": "http://placehold.it/32x32", 77 | "age": 37, 78 | "eyeColor": "blue", 79 | "name": "Stacy Horton", 80 | "gender": "female", 81 | "company": "AVIT", 82 | "email": "stacyhorton@avit.com", 83 | "phone": "+1 (922) 549-2426", 84 | "address": "780 Nautilus Avenue, Delwood, Pennsylvania, 8513", 85 | "about": "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n", 86 | "registered": "2014-09-05T08:05:33 -02:00", 87 | "tags": [ 88 | "non", 89 | "deserunt", 90 | "eiusmod", 91 | "sit", 92 | "aliquip", 93 | "reprehenderit", 94 | "voluptate" 95 | ], 96 | "friends": [ 97 | { 98 | "id": "618964317d23d69f9acebddf", 99 | "name": "Natalie Rosario" 100 | }, 101 | { 102 | "id": "61896431c417b533ef7ba58c", 103 | "name": "Colon Kinney" 104 | } 105 | ], 106 | "latitude": 89.138415, 107 | "longitude": -139.356254 108 | } 109 | ] 110 | } 111 | -------------------------------------------------------------------------------- /tests/fixtures/example.merged.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "618964315cabd70b6541dcae", 3 | "isActive": false, 4 | "balance": "$3,970.08", 5 | "picture": "http://placehold.it/32x32", 6 | "age": 37, 7 | "eyeColor": "blue", 8 | "name": "Stacy Horton", 9 | "gender": "female", 10 | "company": "AVIT", 11 | "email": "stacyhorton@avit.com", 12 | "phone": "+1 (922) 549-2426", 13 | "address": "780 Nautilus Avenue, Delwood, Pennsylvania, 8513", 14 | "about": "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n", 15 | "registered": "2014-09-05T08:05:33 -02:00", 16 | "tags": [ 17 | "non", 18 | "deserunt", 19 | "eiusmod", 20 | "sit", 21 | "aliquip", 22 | "reprehenderit", 23 | "voluptate" 24 | ], 25 | "friends": [ 26 | { 27 | "id": "618964317d23d69f9acebddf", 28 | "name": "Natalie Rosario" 29 | }, 30 | { 31 | "id": "61896431c417b533ef7ba58c", 32 | "name": "Colon Kinney" 33 | } 34 | ], 35 | "latitude": 89.138415, 36 | "longitude": -139.356254 37 | } 38 | -------------------------------------------------------------------------------- /tests/fixtures/example.multi-doc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | id: 6189643191d9e7fe060114f0 3 | isActive: true 4 | balance: $2,509.69 5 | picture: http://placehold.it/32x32 6 | age: 34 7 | eyeColor: blue 8 | name: Rena Franklin 9 | gender: female 10 | company: VITRICOMP 11 | email: renafranklin@vitricomp.com 12 | phone: +1 (851) 579-2333 13 | address: 548 Manhattan Court, Somerset, North Dakota, 9737 14 | about: "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n" 15 | registered: 2019-04-22T05:49:21 -02:00 16 | tags: 17 | - enim 18 | - officia 19 | - do 20 | - ullamco 21 | - voluptate 22 | friends: 23 | - id: 61896431a6546aad2d840a88 24 | name: Florine Acevedo 25 | - id: 618964dc6fabe8b6bc673abb 26 | name: Roxanne Woods 27 | - id: 618964dcee90f14682191e15 28 | name: Hines Jackson 29 | latitude: 31.180897 30 | longitude: -106.807272 31 | --- 32 | id: 61896431b3d9c0729b0d35e6 33 | isActive: true 34 | balance: $1,533.07 35 | picture: http://placehold.it/32x32 36 | age: 27 37 | eyeColor: brown 38 | name: Dena Stone 39 | gender: female 40 | company: ZAJ 41 | email: denastone@zaj.com 42 | phone: +1 (822) 541-3135 43 | address: 416 Oxford Street, Balm, Indiana, 5900 44 | about: "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n" 45 | registered: 2019-03-05T05:26:01 -01:00 46 | tags: 47 | - aute 48 | - elit 49 | - mollit 50 | - sit 51 | friends: 52 | - id: 6189643146646d84cd519d62 53 | name: Dawn Joyce 54 | latitude: 65.453177 55 | longitude: 102.591909 56 | --- 57 | id: 618964315cabd70b6541dcae 58 | isActive: false 59 | balance: $3,970.08 60 | picture: http://placehold.it/32x32 61 | age: 37 62 | eyeColor: blue 63 | name: Stacy Horton 64 | gender: female 65 | company: AVIT 66 | email: stacyhorton@avit.com 67 | phone: +1 (922) 549-2426 68 | address: 780 Nautilus Avenue, Delwood, Pennsylvania, 8513 69 | about: "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n" 70 | registered: 2014-09-05T08:05:33 -02:00 71 | tags: 72 | - non 73 | - deserunt 74 | - eiusmod 75 | - sit 76 | - aliquip 77 | - reprehenderit 78 | - voluptate 79 | friends: 80 | - id: 618964317d23d69f9acebddf 81 | name: Natalie Rosario 82 | - id: 61896431c417b533ef7ba58c 83 | name: Colon Kinney 84 | latitude: 89.138415 85 | longitude: -139.356254 86 | -------------------------------------------------------------------------------- /tests/fixtures/example.toml: -------------------------------------------------------------------------------- 1 | [[users]] 2 | about = """ 3 | Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r 4 | """ 5 | address = "548 Manhattan Court, Somerset, North Dakota, 9737" 6 | age = 34 7 | balance = "$2,509.69" 8 | company = "VITRICOMP" 9 | email = "renafranklin@vitricomp.com" 10 | eyeColor = "blue" 11 | gender = "female" 12 | id = "6189643191d9e7fe060114f0" 13 | isActive = true 14 | latitude = 31.180897 15 | longitude = -106.807272 16 | name = "Rena Franklin" 17 | phone = "+1 (851) 579-2333" 18 | picture = "http://placehold.it/32x32" 19 | registered = "2019-04-22T05:49:21 -02:00" 20 | tags = ["enim", "officia", "do", "ullamco", "voluptate"] 21 | 22 | [[users.friends]] 23 | id = "61896431a6546aad2d840a88" 24 | name = "Florine Acevedo" 25 | 26 | [[users.friends]] 27 | id = "618964dc6fabe8b6bc673abb" 28 | name = "Roxanne Woods" 29 | 30 | [[users.friends]] 31 | id = "618964dcee90f14682191e15" 32 | name = "Hines Jackson" 33 | 34 | [[users]] 35 | about = """ 36 | Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r 37 | """ 38 | address = "416 Oxford Street, Balm, Indiana, 5900" 39 | age = 27 40 | balance = "$1,533.07" 41 | company = "ZAJ" 42 | email = "denastone@zaj.com" 43 | eyeColor = "brown" 44 | gender = "female" 45 | id = "61896431b3d9c0729b0d35e6" 46 | isActive = true 47 | latitude = 65.453177 48 | longitude = 102.591909 49 | name = "Dena Stone" 50 | phone = "+1 (822) 541-3135" 51 | picture = "http://placehold.it/32x32" 52 | registered = "2019-03-05T05:26:01 -01:00" 53 | tags = ["aute", "elit", "mollit", "sit"] 54 | 55 | [[users.friends]] 56 | id = "6189643146646d84cd519d62" 57 | name = "Dawn Joyce" 58 | 59 | [[users]] 60 | about = """ 61 | Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r 62 | """ 63 | address = "780 Nautilus Avenue, Delwood, Pennsylvania, 8513" 64 | age = 37 65 | balance = "$3,970.08" 66 | company = "AVIT" 67 | email = "stacyhorton@avit.com" 68 | eyeColor = "blue" 69 | gender = "female" 70 | id = "618964315cabd70b6541dcae" 71 | isActive = false 72 | latitude = 89.138415 73 | longitude = -139.356254 74 | name = "Stacy Horton" 75 | phone = "+1 (922) 549-2426" 76 | picture = "http://placehold.it/32x32" 77 | registered = "2014-09-05T08:05:33 -02:00" 78 | tags = ["non", "deserunt", "eiusmod", "sit", "aliquip", "reprehenderit", "voluptate"] 79 | 80 | [[users.friends]] 81 | id = "618964317d23d69f9acebddf" 82 | name = "Natalie Rosario" 83 | 84 | [[users.friends]] 85 | id = "61896431c417b533ef7ba58c" 86 | name = "Colon Kinney" 87 | -------------------------------------------------------------------------------- /tests/fixtures/example.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | users: 3 | - id: 6189643191d9e7fe060114f0 4 | isActive: true 5 | balance: $2,509.69 6 | picture: http://placehold.it/32x32 7 | age: 34 8 | eyeColor: blue 9 | name: Rena Franklin 10 | gender: female 11 | company: VITRICOMP 12 | email: renafranklin@vitricomp.com 13 | phone: +1 (851) 579-2333 14 | address: 548 Manhattan Court, Somerset, North Dakota, 9737 15 | about: "Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo.\r\n" 16 | registered: 2019-04-22T05:49:21 -02:00 17 | tags: 18 | - enim 19 | - officia 20 | - do 21 | - ullamco 22 | - voluptate 23 | friends: 24 | - id: 61896431a6546aad2d840a88 25 | name: Florine Acevedo 26 | - id: 618964dc6fabe8b6bc673abb 27 | name: Roxanne Woods 28 | - id: 618964dcee90f14682191e15 29 | name: Hines Jackson 30 | latitude: 31.180897 31 | longitude: -106.807272 32 | - id: 61896431b3d9c0729b0d35e6 33 | isActive: true 34 | balance: $1,533.07 35 | picture: http://placehold.it/32x32 36 | age: 27 37 | eyeColor: brown 38 | name: Dena Stone 39 | gender: female 40 | company: ZAJ 41 | email: denastone@zaj.com 42 | phone: +1 (822) 541-3135 43 | address: 416 Oxford Street, Balm, Indiana, 5900 44 | about: "Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex.\r\n" 45 | registered: 2019-03-05T05:26:01 -01:00 46 | tags: 47 | - aute 48 | - elit 49 | - mollit 50 | - sit 51 | friends: 52 | - id: 6189643146646d84cd519d62 53 | name: Dawn Joyce 54 | latitude: 65.453177 55 | longitude: 102.591909 56 | - id: 618964315cabd70b6541dcae 57 | isActive: false 58 | balance: $3,970.08 59 | picture: http://placehold.it/32x32 60 | age: 37 61 | eyeColor: blue 62 | name: Stacy Horton 63 | gender: female 64 | company: AVIT 65 | email: stacyhorton@avit.com 66 | phone: +1 (922) 549-2426 67 | address: 780 Nautilus Avenue, Delwood, Pennsylvania, 8513 68 | about: "Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis.\r\n" 69 | registered: 2014-09-05T08:05:33 -02:00 70 | tags: 71 | - non 72 | - deserunt 73 | - eiusmod 74 | - sit 75 | - aliquip 76 | - reprehenderit 77 | - voluptate 78 | friends: 79 | - id: 618964317d23d69f9acebddf 80 | name: Natalie Rosario 81 | - id: 61896431c417b533ef7ba58c 82 | name: Colon Kinney 83 | latitude: 89.138415 84 | longitude: -139.356254 85 | -------------------------------------------------------------------------------- /tests/fixtures/filter.jq: -------------------------------------------------------------------------------- 1 | .users | map(select(.age < 30)) 2 | -------------------------------------------------------------------------------- /tests/fixtures/friends.csv: -------------------------------------------------------------------------------- 1 | id,name 2 | 61896431a6546aad2d840a88,Florine Acevedo 3 | 618964dc6fabe8b6bc673abb,Roxanne Woods 4 | 618964dcee90f14682191e15,Hines Jackson 5 | 6189643146646d84cd519d62,Dawn Joyce 6 | 618964317d23d69f9acebddf,Natalie Rosario 7 | 61896431c417b533ef7ba58c,Colon Kinney 8 | -------------------------------------------------------------------------------- /tests/fixtures/math.hcl: -------------------------------------------------------------------------------- 1 | a = 1 + 2 2 | b = 3 3 | c = a + b 4 | -------------------------------------------------------------------------------- /tests/fixtures/math.json: -------------------------------------------------------------------------------- 1 | { 2 | "a": "${1 + 2}", 3 | "b": 3, 4 | "c": "${a + b}" 5 | } -------------------------------------------------------------------------------- /tests/fixtures/math.simplified.json: -------------------------------------------------------------------------------- 1 | { 2 | "a": 3, 3 | "b": 3, 4 | "c": "${a + b}" 5 | } -------------------------------------------------------------------------------- /tests/fixtures/users.csv: -------------------------------------------------------------------------------- 1 | id,isActive,balance,picture,age,eyeColor,name,gender,company,email,phone,address,about,registered,tags,friends,latitude,longitude 2 | 6189643191d9e7fe060114f0,true,"$2,509.69",http://placehold.it/32x32,34,blue,Rena Franklin,female,VITRICOMP,renafranklin@vitricomp.com,+1 (851) 579-2333,"548 Manhattan Court, Somerset, North Dakota, 9737","Ex laboris magna officia quis adipisicing laboris exercitation. Ut non incididunt velit Lorem. Eu officia ut eiusmod labore excepteur amet velit commodo mollit est proident sit mollit. Qui amet consequat aute velit tempor sint labore aliquip nulla consequat. Irure deserunt sit commodo laboris sit non consectetur excepteur officia commodo. 3 | ",2019-04-22T05:49:21 -02:00,"[""enim"",""officia"",""do"",""ullamco"",""voluptate""]","[{""id"":""61896431a6546aad2d840a88"",""name"":""Florine Acevedo""},{""id"":""618964dc6fabe8b6bc673abb"",""name"":""Roxanne Woods""},{""id"":""618964dcee90f14682191e15"",""name"":""Hines Jackson""}]",31.180897,-106.807272 4 | 61896431b3d9c0729b0d35e6,true,"$1,533.07",http://placehold.it/32x32,27,brown,Dena Stone,female,ZAJ,denastone@zaj.com,+1 (822) 541-3135,"416 Oxford Street, Balm, Indiana, 5900","Aute mollit quis reprehenderit consequat do laborum. Veniam reprehenderit aliqua in velit eu tempor tempor ullamco culpa qui ad. Ad fugiat est excepteur sint do occaecat eiusmod voluptate velit anim laboris. Esse duis Lorem amet in aliqua cillum enim. Et proident ullamco qui non cillum occaecat est do deserunt. Quis pariatur dolore velit eu duis ullamco occaecat exercitation proident labore minim consequat voluptate ex. 5 | ",2019-03-05T05:26:01 -01:00,"[""aute"",""elit"",""mollit"",""sit""]","[{""id"":""6189643146646d84cd519d62"",""name"":""Dawn Joyce""}]",65.453177,102.591909 6 | 618964315cabd70b6541dcae,false,"$3,970.08",http://placehold.it/32x32,37,blue,Stacy Horton,female,AVIT,stacyhorton@avit.com,+1 (922) 549-2426,"780 Nautilus Avenue, Delwood, Pennsylvania, 8513","Commodo tempor nostrud ex adipisicing adipisicing. Anim culpa consequat enim tempor deserunt esse nostrud dolore. Anim sunt incididunt ut occaecat consectetur deserunt laboris excepteur cillum dolor. Nulla in enim in tempor commodo nisi. Cillum nostrud eiusmod ullamco ea deserunt duis anim exercitation quis. 7 | ",2014-09-05T08:05:33 -02:00,"[""non"",""deserunt"",""eiusmod"",""sit"",""aliquip"",""reprehenderit"",""voluptate""]","[{""id"":""618964317d23d69f9acebddf"",""name"":""Natalie Rosario""},{""id"":""61896431c417b533ef7ba58c"",""name"":""Colon Kinney""}]",89.138415,-139.356254 8 | -------------------------------------------------------------------------------- /tests/integration_tests.rs: -------------------------------------------------------------------------------- 1 | use assert_cmd::Command; 2 | use predicates::prelude::*; 3 | use std::fs::read_to_string as read; 4 | 5 | #[test] 6 | fn json_to_yaml() { 7 | Command::cargo_bin("dts") 8 | .unwrap() 9 | .arg("tests/fixtures/example.json") 10 | .args(&["-o", "yaml"]) 11 | .assert() 12 | .success() 13 | .stdout(read("tests/fixtures/example.yaml").unwrap()); 14 | } 15 | 16 | #[test] 17 | fn json_to_yaml_stdin() { 18 | Command::cargo_bin("dts") 19 | .unwrap() 20 | .args(&["-i", "json", "-o", "yaml"]) 21 | .pipe_stdin("tests/fixtures/example.json") 22 | .unwrap() 23 | .assert() 24 | .success() 25 | .stdout(read("tests/fixtures/example.yaml").unwrap()); 26 | } 27 | 28 | #[test] 29 | fn yaml_to_pretty_json() { 30 | Command::cargo_bin("dts") 31 | .unwrap() 32 | .arg("tests/fixtures/example.yaml") 33 | .args(&["-o", "json", "-n"]) 34 | .assert() 35 | .success() 36 | .stdout(read("tests/fixtures/example.json").unwrap()); 37 | } 38 | 39 | #[test] 40 | fn json_to_toml() { 41 | Command::cargo_bin("dts") 42 | .unwrap() 43 | .arg("tests/fixtures/example.json") 44 | .args(&["-o", "toml", "-c"]) 45 | .assert() 46 | .success() 47 | .stdout(read("tests/fixtures/example.toml").unwrap()); 48 | } 49 | 50 | #[test] 51 | fn json_to_csv_filtered_flattened_with_keys() { 52 | Command::cargo_bin("dts") 53 | .unwrap() 54 | .arg("tests/fixtures/example.json") 55 | .args(&["-o", "csv", "-j", ".users[].friends[]", "-K"]) 56 | .assert() 57 | .success() 58 | .stdout(read("tests/fixtures/friends.csv").unwrap()); 59 | } 60 | 61 | #[test] 62 | fn json_to_csv_collections_as_json() { 63 | Command::cargo_bin("dts") 64 | .unwrap() 65 | .arg("tests/fixtures/example.json") 66 | .args(&["-o", "csv", "-j", ".users[]", "-K"]) 67 | .assert() 68 | .success() 69 | .stdout(read("tests/fixtures/users.csv").unwrap()); 70 | } 71 | 72 | #[test] 73 | fn json_to_gron() { 74 | Command::cargo_bin("dts") 75 | .unwrap() 76 | .arg("tests/fixtures/example.json") 77 | .args(&["-o", "gron"]) 78 | .assert() 79 | .success() 80 | .stdout(read("tests/fixtures/example.js").unwrap()); 81 | } 82 | 83 | #[test] 84 | fn json_to_hcl() { 85 | Command::cargo_bin("dts") 86 | .unwrap() 87 | .arg("tests/fixtures/example.json") 88 | .args(&["-o", "hcl"]) 89 | .assert() 90 | .success() 91 | .stdout(read("tests/fixtures/example.hcl").unwrap()); 92 | } 93 | 94 | #[test] 95 | fn json_to_hcl_compact() { 96 | Command::cargo_bin("dts") 97 | .unwrap() 98 | .arg("tests/fixtures/example.json") 99 | .args(&["-o", "hcl", "--compact"]) 100 | .assert() 101 | .success() 102 | .stdout(read("tests/fixtures/example.compact.hcl").unwrap()); 103 | } 104 | 105 | #[test] 106 | fn hcl_to_json() { 107 | Command::cargo_bin("dts") 108 | .unwrap() 109 | .arg("tests/fixtures/math.hcl") 110 | .args(&["-o", "json"]) 111 | .assert() 112 | .success() 113 | .stdout(read("tests/fixtures/math.json").unwrap()); 114 | } 115 | 116 | #[test] 117 | fn hcl_to_json_simplified() { 118 | Command::cargo_bin("dts") 119 | .unwrap() 120 | .arg("tests/fixtures/math.hcl") 121 | .args(&["-o", "json", "--simplify"]) 122 | .assert() 123 | .success() 124 | .stdout(read("tests/fixtures/math.simplified.json").unwrap()); 125 | } 126 | 127 | #[test] 128 | fn gron_to_json() { 129 | Command::cargo_bin("dts") 130 | .unwrap() 131 | .arg("tests/fixtures/example.js") 132 | .args(&["-i", "gron", "-n", "-j", ".json"]) 133 | .assert() 134 | .success() 135 | .stdout(read("tests/fixtures/example.js.ungron.json").unwrap()); 136 | } 137 | 138 | #[test] 139 | fn encoding_required_for_stdin() { 140 | Command::cargo_bin("dts") 141 | .unwrap() 142 | .pipe_stdin("tests/fixtures/example.js") 143 | .unwrap() 144 | .assert() 145 | .failure() 146 | .stderr(predicate::str::contains( 147 | "unable to detect input encoding, please provide it explicitly via -i", 148 | )); 149 | } 150 | 151 | #[test] 152 | fn encoding_inferred_from_first_line() { 153 | Command::cargo_bin("dts") 154 | .unwrap() 155 | .pipe_stdin("tests/fixtures/example.json") 156 | .unwrap() 157 | .assert() 158 | .success(); 159 | } 160 | 161 | #[test] 162 | fn multiple_sinks_require_array() { 163 | Command::cargo_bin("dts") 164 | .unwrap() 165 | .args(&["-i", "json", "-O", "-", "-O", "-"]) 166 | .write_stdin("{}") 167 | .assert() 168 | .failure() 169 | .stderr(predicate::str::contains( 170 | "when using multiple output files, the data must be an array", 171 | )); 172 | } 173 | 174 | #[test] 175 | fn glob_required_for_dirs() { 176 | Command::cargo_bin("dts") 177 | .unwrap() 178 | .arg("tests/") 179 | .assert() 180 | .failure() 181 | .stderr(predicate::str::contains( 182 | "--glob is required if sources contain directories", 183 | )); 184 | } 185 | 186 | #[test] 187 | fn merge_json() { 188 | Command::cargo_bin("dts") 189 | .unwrap() 190 | .arg("tests/fixtures/example.json") 191 | .args(&["-j", "reduce .users[] as $item ({}; . + $item)", "-n"]) 192 | .assert() 193 | .success() 194 | .stdout(read("tests/fixtures/example.merged.json").unwrap()); 195 | } 196 | 197 | #[test] 198 | fn filter_expression_from_file() { 199 | Command::cargo_bin("dts") 200 | .unwrap() 201 | .arg("tests/fixtures/example.json") 202 | .args(&["-j", "@tests/fixtures/filter.jq", "-n"]) 203 | .assert() 204 | .success() 205 | .stdout(read("tests/fixtures/example.filtered.json").unwrap()); 206 | } 207 | 208 | #[test] 209 | fn continue_on_error() { 210 | // Test for the failure first without the --continue-on-error flag to catch potential 211 | // regressions. 212 | Command::cargo_bin("dts") 213 | .unwrap() 214 | .arg("tests/fixtures/example.js") 215 | .arg("tests/fixtures/example.json") 216 | .args(&[ 217 | "-i", 218 | "json", 219 | "-j", 220 | ".[] | reduce .users[] as $item ({}; . + $item)", 221 | "-n", 222 | ]) 223 | .assert() 224 | .failure(); 225 | 226 | Command::cargo_bin("dts") 227 | .unwrap() 228 | .arg("tests/fixtures/example.js") 229 | .arg("tests/fixtures/example.json") 230 | .args(&[ 231 | "-i", 232 | "json", 233 | "-j", 234 | ".[] | reduce .users[] as $item ({}; . + $item)", 235 | "-n", 236 | "--continue-on-error", 237 | ]) 238 | .assert() 239 | .success() 240 | .stdout(read("tests/fixtures/example.merged.json").unwrap()); 241 | } 242 | 243 | #[test] 244 | fn yaml_to_multi_doc_yaml() { 245 | Command::cargo_bin("dts") 246 | .unwrap() 247 | .arg("tests/fixtures/example.yaml") 248 | .args(&["-o", "yaml", "--multi-doc-yaml", "-j", ".users[]"]) 249 | .assert() 250 | .success() 251 | .stdout(read("tests/fixtures/example.multi-doc.yaml").unwrap()); 252 | } 253 | --------------------------------------------------------------------------------