├── .dockerignore ├── .github └── workflows │ ├── build.yml │ └── verify-docker-build.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── NOTICE ├── README.md ├── code-gen-projects ├── README.md ├── input │ ├── bad │ │ ├── enum_type │ │ │ ├── invalid_case_enum_varaint.ion │ │ │ ├── invalid_enum_variant.ion │ │ │ └── mismatched_enum_type.ion │ │ ├── nested_struct │ │ │ ├── mismatched_sequence_type.ion │ │ │ └── mismatched_type.ion │ │ ├── scalar │ │ │ └── mismatched_type.ion │ │ ├── sequence │ │ │ ├── mismatched_sequence_element_type.ion │ │ │ └── mismatched_sequence_type.ion │ │ ├── sequence_with_enum_element │ │ │ └── invalid_value.ion │ │ ├── sequence_with_import │ │ │ ├── mismatched_sequence_element_type.ion │ │ │ └── mismatched_sequence_type.ion │ │ ├── struct_with_enum_fields │ │ │ ├── mismatched_sequence_element_type.ion │ │ │ ├── mismatched_sequence_type.ion │ │ │ ├── mismatched_type.ion │ │ │ └── missing_required_fields.ion │ │ ├── struct_with_fields │ │ │ ├── mismatched_sequence_element_type.ion │ │ │ ├── mismatched_sequence_type.ion │ │ │ ├── mismatched_type.ion │ │ │ └── missing_required_fields.ion │ │ └── struct_with_inline_import │ │ │ └── mismatched_import_type.ion │ └── good │ │ ├── enum_type │ │ ├── valid_value_1.ion │ │ ├── valid_value_2.ion │ │ ├── valid_value_3.ion │ │ └── valid_value_4.ion │ │ ├── nested_struct │ │ ├── empty_values.ion │ │ ├── valid_fields.ion │ │ ├── valid_optional_fields.ion │ │ └── valid_unordered_fields.ion │ │ ├── scalar │ │ ├── empty_value.ion │ │ └── valid_value.ion │ │ ├── sequence │ │ ├── empty_sequence.ion │ │ └── valid_elements.ion │ │ ├── sequence_with_enum_element │ │ └── valid_value.ion │ │ ├── sequence_with_import │ │ ├── empty_sequence.ion │ │ └── valid_elements.ion │ │ ├── struct_with_enum_fields │ │ ├── empty_values.ion │ │ ├── valid_fields.ion │ │ ├── valid_optional_fields.ion │ │ └── valid_unordered_fields.ion │ │ ├── struct_with_fields │ │ ├── empty_values.ion │ │ ├── valid_fields.ion │ │ ├── valid_optional_fields.ion │ │ └── valid_unordered_fields.ion │ │ └── struct_with_inline_import │ │ ├── valid_fields.ion │ │ ├── valid_optional_fields.ion │ │ └── valid_unordered_fields.ion ├── java │ └── code-gen-demo │ │ ├── .gitattributes │ │ ├── .gitignore │ │ ├── build.gradle.kts │ │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle.kts │ │ └── src │ │ └── test │ │ └── java │ │ └── org │ │ └── example │ │ └── CodeGenTest.java ├── rust │ └── code-gen-demo │ │ ├── .gitignore │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src │ │ └── lib.rs └── schema │ ├── enum_type.isl │ ├── nested_struct.isl │ ├── scalar.isl │ ├── sequence.isl │ ├── sequence_with_enum_element.isl │ ├── sequence_with_import.isl │ ├── struct_with_enum_fields.isl │ ├── struct_with_fields.isl │ ├── struct_with_inline_import.isl │ └── utils │ └── fruits.isl ├── images ├── example_inspect_limit_bytes.png ├── example_inspect_output.png └── example_inspect_skip_bytes.png ├── src └── bin │ └── ion │ ├── ansi_codes.rs │ ├── assets │ ├── README.md │ ├── ion.newlines.packdump │ ├── ion.nonewlines.packdump │ └── ion.sublime-syntax │ ├── auto_decompress.rs │ ├── commands │ ├── cat.rs │ ├── command_namespace.rs │ ├── complaint.rs │ ├── from │ │ ├── json.rs │ │ └── mod.rs │ ├── generate │ │ ├── README.md │ │ ├── context.rs │ │ ├── generator.rs │ │ ├── mod.rs │ │ ├── model.rs │ │ ├── result.rs │ │ ├── templates │ │ │ ├── java │ │ │ │ ├── class.templ │ │ │ │ ├── enum.templ │ │ │ │ ├── nested_type.templ │ │ │ │ ├── scalar.templ │ │ │ │ ├── sequence.templ │ │ │ │ └── util_macros.templ │ │ │ ├── mod.rs │ │ │ └── rust │ │ │ │ ├── enum.templ │ │ │ │ ├── import.templ │ │ │ │ ├── nested_type.templ │ │ │ │ ├── result.templ │ │ │ │ ├── scalar.templ │ │ │ │ ├── sequence.templ │ │ │ │ ├── struct.templ │ │ │ │ └── util_macros.templ │ │ └── utils.rs │ ├── hash.rs │ ├── head.rs │ ├── inspect.rs │ ├── jq.rs │ ├── mod.rs │ ├── primitive.rs │ ├── schema │ │ ├── check.rs │ │ ├── mod.rs │ │ └── validate.rs │ ├── stats.rs │ ├── symtab │ │ ├── filter.rs │ │ └── mod.rs │ └── to │ │ ├── json.rs │ │ └── mod.rs │ ├── file_writer.rs │ ├── hex_reader.rs │ ├── input.rs │ ├── input_grouping.rs │ ├── main.rs │ ├── output.rs │ └── transcribe.rs └── tests ├── cli.rs └── code-gen-tests.rs /.dockerignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.idea 3 | 4 | *.md 5 | 6 | Dockerfile 7 | LICENSE 8 | NOTICE 9 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI Build 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | build: 7 | name: Build and Test 8 | runs-on: ${{ matrix.os }} 9 | # We want to run on external PRs, but not on internal ones as push automatically builds 10 | # H/T: https://github.com/Dart-Code/Dart-Code/commit/612732d5879730608baa9622bf7f5e5b7b51ae65 11 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'amazon-ion/ion-cli' 12 | strategy: 13 | matrix: 14 | os: [ ubuntu-latest, macos-latest, windows-latest ] 15 | 16 | steps: 17 | - name: Git Checkout 18 | uses: actions/checkout@v2 19 | with: 20 | submodules: recursive 21 | - name: Rust Toolchain 22 | uses: actions-rs/toolchain@v1 23 | with: 24 | profile: minimal 25 | toolchain: stable 26 | override: true 27 | - name: Cargo Build 28 | uses: actions-rs/cargo@v1 29 | with: 30 | command: build 31 | args: --verbose --workspace 32 | - name: Cargo Test 33 | uses: actions-rs/cargo@v1 34 | with: 35 | command: test 36 | args: --verbose --workspace 37 | - name: Rustfmt Check 38 | uses: actions-rs/cargo@v1 39 | with: 40 | command: fmt 41 | args: --verbose -- --check 42 | -------------------------------------------------------------------------------- /.github/workflows/verify-docker-build.yml: -------------------------------------------------------------------------------- 1 | name: Verify Docker image builds 2 | on: 3 | push: 4 | branches: 5 | - master 6 | 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | push_to_registry: 13 | name: Build docker image 14 | runs-on: ubuntu-20.04 15 | steps: 16 | - 17 | name: Checkout repository 18 | uses: actions/checkout@v2 19 | - 20 | name: Setup Docker buildx 21 | uses: docker/setup-buildx-action@v1 22 | - 23 | name: Build image 24 | run: docker buildx build -t ion-cli:test-build . 25 | 26 | 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /target 3 | *.swp 4 | *.swo 5 | *.swn 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/.gitmodules -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ion-cli" 3 | version = "0.11.0" 4 | authors = ["The Ion Team "] 5 | edition = "2021" 6 | description = "Command line tool for working with the Ion data format." 7 | repository = "https://github.com/amzn/ion-cli" 8 | license = "Apache-2.0" 9 | categories = ["command-line-utilities", "development-tools", "encoding", "parsing"] 10 | keywords = ["format", "parse", "encode"] 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [dependencies] 15 | anyhow = "1.0" 16 | clap = { version = "4.5.8", features = ["cargo", "env", "wrap_help"] } 17 | colored = "2.0.0" 18 | sha2 = "0.9" 19 | sha3 = "0.9" 20 | flate2 = "1.0" 21 | infer = "0.15.0" 22 | # ion-rs version must be pinned because we are using experimental features 23 | # See https://github.com/amazon-ion/ion-cli/issues/155 24 | ion-rs = { version = "1.0.0-rc.11", features = ["experimental", "experimental-ion-hash"] } 25 | tempfile = "3.2.0" 26 | ion-schema = "0.15.0" 27 | lowcharts = "0.5.8" 28 | serde = { version = "1.0.163", features = ["derive"] } 29 | serde_json = { version = "1.0.81", features = ["arbitrary_precision", "preserve_order"] } 30 | base64 = "0.21.1" 31 | tera = { version = "1.18.1" } 32 | convert_case = { version = "0.6.0" } 33 | thiserror = "1.0.50" 34 | zstd = "0.13.0" 35 | termcolor = "1.4.1" 36 | derive_builder = "0.20.0" 37 | itertools = "0.13.0" 38 | jaq-core = "2.1.1" 39 | jaq-std = "2.1.0" 40 | bigdecimal = "0.4.8" 41 | syntect = "5.2.0" 42 | syntect-assets = "0.23.6" 43 | 44 | [target.'cfg(not(target_os = "windows"))'.dependencies] 45 | pager = "0.16.1" 46 | 47 | [dev-dependencies] 48 | rstest = "~0.17.0" 49 | assert_cmd = "~1.0.5" 50 | tempfile = "~3.5.0" 51 | 52 | [features] 53 | default = [] 54 | 55 | [[bin]] 56 | name = "ion" 57 | test = true 58 | bench = false 59 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.80.1-bullseye as builder 2 | WORKDIR /usr/src/ion-cli 3 | COPY . . 4 | RUN cargo install --verbose --path . 5 | 6 | FROM debian:11.1-slim 7 | COPY --from=builder /usr/local/cargo/bin/ion /usr/bin/ion 8 | CMD /usr/bin/ion 9 | VOLUME /data 10 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `ion-cli` 2 | 3 | [![Crate](https://img.shields.io/crates/v/ion-cli.svg)](https://crates.io/crates/ion-cli) 4 | [![License](https://img.shields.io/hexpm/l/plug.svg)](https://github.com/amazon-ion/ion-cli/blob/main/LICENSE) 5 | [![CI Build](https://github.com/amazon-ion/ion-cli/workflows/CI%20Build/badge.svg)](https://github.com/amazon-ion/ion-cli/actions?query=workflow%3A%22CI+Build%22) 6 | 7 | This repository is home to the `ion` command line tool, which provides subcommands 8 | for working with [the Ion data format](https://amzn.github.io/ion-docs/docs/spec.html). 9 | 10 | ## Table of contents 11 | 12 | * [Examples](#examples) 13 | * [Viewing the contents of an Ion file](#viewing-the-contents-of-an-ion-file) 14 | * [Converting between Ion formats](#converting-between-ion-formats) 15 | * [Converting between Ion and other formats with `to` and 16 | `from`](#converting-between-ion-and-other-formats-with-to-and-from) 17 | * [Ion code generation](#ion-code-generation) 18 | * [Analyzing binary Ion file encodings with `inspect`](#analyzing-binary-ion-file-encodings-with-inspect) 19 | * [Installation](#installation) 20 | * [via `brew`](#via-brew) 21 | * [via `cargo`](#via-cargo) 22 | * [Build Instructions](#build-instructions) 23 | * [From source](#from-source) 24 | * [Using Docker](#using-docker) 25 | 26 | ## Examples 27 | 28 | These examples use the `.ion` file extension for text Ion and the `.10n` file 29 | extension for binary Ion. This is simply a convention; the tool does not 30 | evaluate the file extension. 31 | 32 | Unless otherwise noted, these commands can accept any Ion format as input. 33 | 34 | ### Viewing the contents of an Ion file 35 | 36 | The `ion cat` command reads the contents of the specified files (or `STDIN`) sequentially 37 | and writes their content to `STDOUT` in the requested Ion format. 38 | 39 | ```shell 40 | ion cat my_file.ion 41 | ``` 42 | 43 | You can use the `--format`/`-f` flag to specify the desired format. The supported formats are: 44 | 45 | * `pretty` - Generously spaced, human-friendly text Ion. This is the default. 46 | * `text` - Minimally spaced text Ion. 47 | * `lines` - Text Ion that places each value on its own line. 48 | * `binary`- Binary Ion 49 | 50 | ### Converting between Ion formats 51 | 52 | Convert Ion text (or JSON) to Ion binary: 53 | 54 | ```shell 55 | ion cat --format binary my_text_file.ion -o my_binary_file.ion 56 | ``` 57 | 58 | Convert Ion binary to generously-spaced, human-friendly text: 59 | 60 | ```shell 61 | ion cat --format pretty my_binary_file.ion -o my_text_file.ion 62 | ``` 63 | 64 | Convert Ion binary to minimally-spaced, compact text: 65 | 66 | ```shell 67 | ion cat --format text my_binary_file.ion -o my_text_file.ion 68 | ``` 69 | 70 | ### Converting between Ion and other formats with `to` and `from` 71 | 72 | The `to` and `from` commands can convert Ion to and from other formats. 73 | Currently, JSON is supported. 74 | 75 | Convert Ion to JSON: 76 | 77 | ```shell 78 | ion to -X json my_file.10n 79 | ``` 80 | 81 | Convert JSON to Ion: 82 | 83 | ```shell 84 | ion from -X json my_file.json 85 | ``` 86 | 87 | ### Ion Code generation 88 | 89 | Code generation is supported with `generate` subcommand on the CLI. 90 | For more information on how to use code generator, 91 | see [Ion code generator user guide](https://github.com/amazon-ion/ion-cli/tree/main/src/bin/ion/commands/generate/README.md). 92 | 93 | ### Analyzing binary Ion file encodings with `inspect` 94 | 95 | The `inspect` command can display the hex bytes of a binary Ion file alongside 96 | the equivalent text Ion for easier analysis. 97 | 98 | ```shell 99 | # Write some text Ion to a file 100 | echo '{foo: null, bar: true, baz: [1, 2, 3]}' > my_file.ion 101 | 102 | # Convert the text Ion to binary Ion 103 | ion cat --format binary my_file.ion > my_file.10n 104 | 105 | # Show the binary encoding alongside its equivalent text 106 | ion inspect my_file.10n 107 | ``` 108 | 109 | ![example_inspect_output.png](images/example_inspect_output.png) 110 | 111 | ---- 112 | **The `--skip-bytes` flag** 113 | 114 | To skip to a particular offset in the stream, you can use the `--skip-bytes` flag. 115 | 116 | ```shell 117 | ion inspect --skip-bytes 30 my_file.10n 118 | ``` 119 | 120 | ![img.png](images/example_inspect_skip_bytes.png) 121 | 122 | Notice that the text column adds comments indicating where data has been skipped. 123 | Also, if the requested index is nested inside one or more containers, the beginnings 124 | of those containers (along with their lengths and offsets) will still be included 125 | in the output. 126 | 127 | ----- 128 | **The `--limit-bytes` flag** 129 | 130 | You can limit the amount of data that `inspect` displays by using the `--limit-bytes` 131 | flag: 132 | 133 | ```shell 134 | ion inspect --skip-bytes 30 --limit-bytes 2 my_file.10n 135 | ``` 136 | 137 | ![img.png](images/example_inspect_limit_bytes.png) 138 | 139 | ### Schema subcommands 140 | 141 | All the subcommand to load or validate schema are under the `schema` subcommand. 142 | 143 | To load a schema: 144 | 145 | ```bash 146 | ion schema -X load --directory --schema 147 | ``` 148 | 149 | To validate an ion value against a schema type: 150 | 151 | ```bash 152 | ion schema -X validate --directory --schema --input --type 153 | ``` 154 | 155 | For more information on how to use the schema subcommands using CLI, run the following command: 156 | 157 | ```bash 158 | ion schema help 159 | ``` 160 | 161 | ## Installation 162 | 163 | ### via `brew` 164 | 165 | The easiest way to install the `ion-cli` is via [Homebrew](https://brew.sh/). 166 | 167 | Once the `brew` command is available, run: 168 | 169 | ```bash 170 | brew tap amazon-ion/ion-cli 171 | brew install ion-cli 172 | ``` 173 | 174 | To install the (potentially unstable) latest changes from the tip of `main` rather than the latest release, use: 175 | ```bash 176 | brew install ion-cli --HEAD 177 | ``` 178 | 179 | ### via `cargo` 180 | 181 | The `ion-cli` can also be installed by using Rust's package manager, `cargo`. 182 | If you don't already have `cargo`, you can install it by visiting 183 | [rustup.rs](https://rustup.rs/). 184 | 185 | To install `ion-cli`, run the following command: 186 | 187 | ```shell 188 | cargo install ion-cli 189 | ``` 190 | 191 | ## Build instructions 192 | 193 | ### From source 194 | 195 | 1. Clone the repository: 196 | ``` 197 | git clone https://github.com/amzn/ion-cli.git 198 | ``` 199 | 200 | 2. Step into the newly created directory: 201 | ``` 202 | cd ion-cli 203 | ``` 204 | 205 | 3. Install Rust/Cargo [by visiting rustup.rs](https://rustup.rs/). 206 | 207 | 4. Build the `ion` tool: 208 | ``` 209 | cargo install --path . 210 | ``` 211 | This will put a copy of the `ion` executable in `~/.cargo/bin`. 212 | 213 | 5. Confirm that `~/.cargo/bin` is on your `$PATH`. `rustup` will probably take care of this for you. 214 | 215 | 6. Confirm that the executable is available by running: 216 | ``` 217 | ion help 218 | ``` 219 | 220 | ### Using Docker 221 | 222 | 1. Install Docker (see OS specific instructions on the [Docker website](https://docs.docker.com/get-docker/)) 223 | 2. Clone the repository (recursive clone not necessary) 224 | ``` 225 | git clone https://github.com/amzn/ion-cli.git 226 | ``` 227 | 3. Step into the newly created directory 228 | ``` 229 | cd ion-cli 230 | ``` 231 | 4. Build and run the image 232 | ``` 233 | # build the image 234 | docker build -t : . 235 | 236 | 237 | # run the CLI binary inside the Docker image 238 | docker run -it --rm [optional flags...] : ion 239 | 240 | # examples: 241 | 242 | # build docker image with current release version 243 | docker build -t ion-cli:0.1.1 . 244 | 245 | # print the help message 246 | docker run -it --rm ion-cli:0.1.1 ion -V 247 | 248 | # mount current directory to /data volume and cat an ion file 249 | docker run -it --rm -v $PWD:/data ion-cli:0.1.1 ion cat /data/test.ion 250 | 251 | ``` 252 | 253 | ## Security 254 | 255 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 256 | 257 | ## License 258 | 259 | This project is licensed under the Apache-2.0 License. 260 | -------------------------------------------------------------------------------- /code-gen-projects/README.md: -------------------------------------------------------------------------------- 1 | # Code generation projects 2 | 3 | This directory contains 2 projects that are used in tests for code generation and serve as an example of 4 | how to use `ion-cli` code generator under the `generate` subcommand with an existing project. 5 | 6 | ## Table of contents 7 | 8 | * [/input](#input) 9 | * [/schema](#schema) 10 | * [/java](#java) 11 | * [Gradle build process](#gradle-build-process) 12 | * [Tests](#tests) 13 | * [How to run the tests?](#how-to-run-the-tests) 14 | * [/rust](#rust) 15 | * [Cargo build process](#cargo-build-process) 16 | * [Tests](#tests-1) 17 | * [How to run the tests?](#how-to-run-the-tests-1) 18 | 19 | ## /input 20 | 21 | This directory contains some good and bad test Ion files based on corresponding schema in `/schema`. 22 | 23 | ## /schema 24 | 25 | This directory contains all the schema files used in testing code generation with `ion-cli` `generate` subcommand. 26 | 27 | ## /java 28 | 29 | This directory contains a Java project called `code-gen-demo` which is a gradle project which has tests that uses the 30 | generated code based 31 | on schema file provided in `/schema` and test Ion file provided in `/input`. 32 | 33 | ### Gradle build process 34 | 35 | To generate code as part of the build process of this project, a gradle build task is defined inside `build.gradle.kts`. 36 | This task performs following steps: 37 | 38 | - Gets the executable path for `ion-cli` through an environment variable `ION_CLI`. If the environment variable is not 39 | set then it uses the local executable named `ion`. 40 | - Sets the schema directory as `/schema` which will be used by `generate` subcommand to generate code for the schema 41 | files inside it. 42 | - Sets the path to output directory where the code will be generated and sets it as source directory. 43 | - It runs the `ion-cli` `generate` subcommand with the set schema directory and a namespace where the code will be 44 | generated. 45 | 46 | Following is a sample build task you can add in an existing gradle project to generate code for your schemas, 47 | 48 | ``` 49 | val ionSchemaSourceCodeDir = "YOUR_SOURCE_SCHEMA_DIRECTORY" 50 | val generatedIonSchemaModelDir = "${layout.buildDirectory.get()}/generated/java" 51 | sourceSets { 52 | main { 53 | java.srcDir(generatedIonSchemaModelDir) 54 | } 55 | } 56 | 57 | 58 | tasks { 59 | val ionCodegen = create("ionCodegen") { 60 | inputs.files(ionSchemaSourceCodeDir) 61 | outputs.file(generatedIonSchemaModelDir) 62 | 63 | val ionCli = System.getenv("ION_CLI") ?: "ion" 64 | 65 | commandLine(ionCli) 66 | .args( 67 | "-X", "generate", 68 | "-l", "java", 69 | "-n", "NAMESPACE_FOR_GENERATED_CODE", 70 | "-d", ionSchemaSourceCodeDir, 71 | "-o", generatedIonSchemaModelDir, 72 | ) 73 | .workingDir(rootProject.projectDir) 74 | } 75 | 76 | withType { 77 | options.encoding = "UTF-8" 78 | if (JavaVersion.current() != JavaVersion.VERSION_1_8) { 79 | options.release.set(8) 80 | } 81 | dependsOn(ionCodegen) 82 | } 83 | } 84 | ``` 85 | 86 | ### Tests 87 | 88 | The tests for the generated code are defined in `CodeGenTests.java`. It has the following tests: 89 | 90 | - Tests for getter and setters of the generated code 91 | - Roundtrip test for bad input Ion files which should result in Exception while reading. 92 | - Roundtrip test for good input Ion files. Roundtrip has following steps: 93 | - Roundtrip test first read an Ion file into the generated model using `readFrom` API of the model 94 | - Then writes that model using `writeTo` API of the model. 95 | - Compares the written Ion data and original input Ion data. 96 | 97 | ### How to run the tests? 98 | 99 | Here are the steps to follow for running tests: 100 | 101 | 1. Install ion-cli with either `brew install ion-cli` or `cargo install ion-cli`. 102 | 1. If you installed with brew then your executable is there in `ion` and you don't need to set up `ION_CLI` 103 | environment variable. 104 | 2. If you installed with `cargo` then your executable would be in `$HOME/.cargo/bin` and you need to setup the 105 | environment variable `ION_CLI` to point to the executable's path. If you need latest commits from cargo which are 106 | not released yet, then do `cargo install ion-cli --git https://github.com/amazon-ion/ion-cli.git` or 107 | `brew install ion-cli --HEAD`. 108 | 2. All the tests uses an environment variable `ION_INPUT` which has the path to input Ion files. So if you want to 109 | test out this project locally set the environment variable `ION_INPUT` to point to `code-gen-projects/input.`_ 110 | 3. `cd code-gen-projects/java/code-gen-demo` 111 | 4. Finally, to run the tests, just do: 112 | 113 | ```bash 114 | ION_INPUT=../../input ./gradlew test 115 | ``` 116 | 117 | _Note: If you have used `cargo` and have to setup `ION_CLI` then 118 | use `ION_CLI=$HOME/.cargo/bin/ion ION_INPUT=../../input ./gradlew test`._ 119 | 120 | At any point if gradle complains about error to write to the output directory then it might be because there is already 121 | generated code in that directory(i.e. `code-gen-projects/java/code-gen-demo/build/generated/java/*`). So removing that 122 | directory and then trying out (i.e. remove `generated/java` directory) should make it work. 123 | 124 | ## /rust 125 | 126 | This directory contains a Rust project called `code-gen-demo` which is a cargo project which has tests that uses the 127 | generated code based 128 | on schema file provided in `/schema` and test Ion file provided in `/input`. 129 | 130 | ### Cargo build process 131 | 132 | To generate code as part of the build process of this cargo project, a cargo build script is defined in `build.rs`. 133 | This task performs following steps: 134 | 135 | - Gets the executable path for `ion-cli` through an environment variable `ION_CLI`. If the environment variable is not 136 | set then it uses the local executable named `ion`. 137 | - Sets the schema directory as `/schema` which will be used by `generate` subcommand to generate code for the schema 138 | files inside it. 139 | - Sets the path to output directory where the code will be generated (e.g. `OUT_DIR`). 140 | - It runs the `ion-cli` `generate` subcommand with the set schema directory and a namespace where the code will be 141 | generated. 142 | 143 | Following is sample build script you can add in your existing cargo project to generate code using `ion-cli`. 144 | 145 | ```rust 146 | fn main() { 147 | let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); 148 | let out_dir = env::var("OUT_DIR").unwrap(); 149 | 150 | // Invokes the ion-cli executable using environment variable ION_CLI if present, otherwise uses local executable named `ion` 151 | let ion_cli = env::var("ION_CLI").unwrap_or("ion".to_string()); 152 | println!("cargo:warn=Running command: {}", ion_cli); 153 | let mut cmd = std::process::Command::new(ion_cli); 154 | cmd.arg("-X") 155 | .arg("generate") 156 | .arg("-l") 157 | .arg("rust") 158 | .arg("-d") 159 | .arg("YOUR_SOURCE_SCHEMA_DIRECTORY") 160 | .arg("-o") 161 | .arg(&out_dir); 162 | 163 | println!("cargo:warn=Running: {:?}", cmd); 164 | 165 | let output = cmd.output().expect("failed to execute process"); 166 | 167 | println!("status: {}", output.status); 168 | io::stdout().write_all(&output.stdout).unwrap(); 169 | io::stderr().write_all(&output.stderr).unwrap(); 170 | 171 | assert!(output.status.success()); 172 | 173 | println!("cargo:rerun-if-changed=input/"); 174 | println!("cargo:rerun-if-changed=schema/"); 175 | } 176 | ``` 177 | 178 | ### Tests 179 | 180 | The tests for the generated code are defined in `tests` module in `lib.rs`. It has the following tests: 181 | 182 | - Roundtrip test for bad input Ion files which should result in Exception while reading. 183 | - Roundtrip test for good input Ion files. Roundtrip has following steps: 184 | - Roundtrip test first read an Ion file into the generated model using `readFrom` API of the model 185 | - Then writes that model using `writeTo` API of the model. 186 | - Compares the written Ion data and original input Ion data. 187 | 188 | ### How to run the tests? 189 | 190 | Here are the steps to follow for running tests: 191 | 192 | 1. Install ion-cli with either `brew install ion-cli` or `cargo install ion-cli`. 193 | 1. If you installed with brew then your executable is there in `ion` and you need to setup the 194 | environment variable `ION_CLI` to point to the executable's path. 195 | 2. If you installed with `cargo` then your executable would be in `$HOME/.cargo/bin` and you need to setup the 196 | environment variable `ION_CLI` to point to the executable's path. If you need latest commits from cargo which are 197 | not released yet, then do `cargo install ion-cli --git https://github.com/amazon-ion/ion-cli.git`. 198 | 2. `cd code-gen-projects/rust/code-gen-demo` 199 | 3. Finally, to run the tests, just do: 200 | 201 | ```bash 202 | cargo test 203 | ``` 204 | 205 | _Note: If you have used `cargo` and have to setup `ION_CLI` then 206 | use `ION_CLI=$HOME/.cargo/bin/ion cargo test`._ -------------------------------------------------------------------------------- /code-gen-projects/input/bad/enum_type/invalid_case_enum_varaint.ion: -------------------------------------------------------------------------------- 1 | FoobarBaz // expected FooBarBaz, found FoobarBaz -------------------------------------------------------------------------------- /code-gen-projects/input/bad/enum_type/invalid_enum_variant.ion: -------------------------------------------------------------------------------- 1 | hello // expected (foo, bar, baz or FooBarBaz) found hello -------------------------------------------------------------------------------- /code-gen-projects/input/bad/enum_type/mismatched_enum_type.ion: -------------------------------------------------------------------------------- 1 | "foo" // expected a symbol value foo for enum, found string "foo" -------------------------------------------------------------------------------- /code-gen-projects/input/bad/nested_struct/mismatched_sequence_type.ion: -------------------------------------------------------------------------------- 1 | // nested struct with mismatched sequence type 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: { 6 | D: false, 7 | E: (1 2 3) // expected list 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/nested_struct/mismatched_type.ion: -------------------------------------------------------------------------------- 1 | // nested struct with type mismatched fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: { 6 | D: 1e0, // expected type: bool 7 | E: [1, 2, 3] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/scalar/mismatched_type.ion: -------------------------------------------------------------------------------- 1 | 12 // expected string 2 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/sequence/mismatched_sequence_element_type.ion: -------------------------------------------------------------------------------- 1 | [1, 2, 3] // expected list of strings 2 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/sequence/mismatched_sequence_type.ion: -------------------------------------------------------------------------------- 1 | ("foo" "bar" "baz") // expected list 2 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/sequence_with_enum_element/invalid_value.ion: -------------------------------------------------------------------------------- 1 | [foobar] // expected values are either foo , bar or baz, found foobar. -------------------------------------------------------------------------------- /code-gen-projects/input/bad/sequence_with_import/mismatched_sequence_element_type.ion: -------------------------------------------------------------------------------- 1 | [ mango ] // expected apple, banana or strawberry, found mango -------------------------------------------------------------------------------- /code-gen-projects/input/bad/sequence_with_import/mismatched_sequence_type.ion: -------------------------------------------------------------------------------- 1 | (apple banana) // expected list, found sexp -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_enum_fields/mismatched_sequence_element_type.ion: -------------------------------------------------------------------------------- 1 | // struct with mismatched sequence element 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: (1 2 3), // expected sexpression of strings 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_enum_fields/mismatched_sequence_type.ion: -------------------------------------------------------------------------------- 1 | // simple struct with type mismatched sequence type 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: ["foo", "bar", "baz"], // expected sexp 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_enum_fields/mismatched_type.ion: -------------------------------------------------------------------------------- 1 | // simple struct with type mismatched fields 2 | { 3 | A: "hello", 4 | B: false, // expected field type: int 5 | C: ("foo" "bar" "baz"), 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_enum_fields/missing_required_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | // C: ("foo" "bar" "baz"), // since `C` is a required field, this is an invalid struct 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_fields/mismatched_sequence_element_type.ion: -------------------------------------------------------------------------------- 1 | // struct with mismatched sequence element 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: (1 2 3), // expected sexpression of strings 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_fields/mismatched_sequence_type.ion: -------------------------------------------------------------------------------- 1 | // simple struct with type mismatched sequence type 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: ["foo", "bar", "baz"], // expected sexp 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_fields/mismatched_type.ion: -------------------------------------------------------------------------------- 1 | // simple struct with type mismatched fields 2 | { 3 | A: "hello", 4 | B: false, // expected field type: int 5 | C: ("foo" "bar" "baz"), 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_fields/missing_required_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | // C: ("foo" "bar" "baz"), // since `C` is a required field, this is an invalid struct 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/bad/struct_with_inline_import/mismatched_import_type.ion: -------------------------------------------------------------------------------- 1 | // simple struct with type mismatched import field 2 | { 3 | A: "hello", 4 | B: false, // expected field type symbol 5 | } 6 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/enum_type/valid_value_1.ion: -------------------------------------------------------------------------------- 1 | foo -------------------------------------------------------------------------------- /code-gen-projects/input/good/enum_type/valid_value_2.ion: -------------------------------------------------------------------------------- 1 | bar -------------------------------------------------------------------------------- /code-gen-projects/input/good/enum_type/valid_value_3.ion: -------------------------------------------------------------------------------- 1 | baz -------------------------------------------------------------------------------- /code-gen-projects/input/good/enum_type/valid_value_4.ion: -------------------------------------------------------------------------------- 1 | FooBarBaz -------------------------------------------------------------------------------- /code-gen-projects/input/good/nested_struct/empty_values.ion: -------------------------------------------------------------------------------- 1 | // nested struct with empty string, list and zeros 2 | { 3 | C: { 4 | D: false, 5 | E: [], 6 | }, 7 | A: "", 8 | B: 0, 9 | } 10 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/nested_struct/valid_fields.ion: -------------------------------------------------------------------------------- 1 | // nested struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: { 6 | D: false, 7 | E: [1, 2, 3] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/nested_struct/valid_optional_fields.ion: -------------------------------------------------------------------------------- 1 | // nested struct with some optional fields that are not provided 2 | { 3 | A: "hello", 4 | // B: 12, // since `B` is optional field, this is a valid struct 5 | C: { 6 | D: false, 7 | E: [1, 2, 3] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/nested_struct/valid_unordered_fields.ion: -------------------------------------------------------------------------------- 1 | // nested struct with unordered fields 2 | { 3 | B: 12, 4 | A: "hello", 5 | C: { 6 | D: false, 7 | E: [1, 2, 3] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/scalar/empty_value.ion: -------------------------------------------------------------------------------- 1 | // empty string 2 | "" 3 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/scalar/valid_value.ion: -------------------------------------------------------------------------------- 1 | // a scalar value of string type 2 | "Hello World!" 3 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/sequence/empty_sequence.ion: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/sequence/valid_elements.ion: -------------------------------------------------------------------------------- 1 | ["foo", "bar", "baz"] 2 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/sequence_with_enum_element/valid_value.ion: -------------------------------------------------------------------------------- 1 | [foo, bar, baz] -------------------------------------------------------------------------------- /code-gen-projects/input/good/sequence_with_import/empty_sequence.ion: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /code-gen-projects/input/good/sequence_with_import/valid_elements.ion: -------------------------------------------------------------------------------- 1 | [ apple, strawberry ] -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_enum_fields/empty_values.ion: -------------------------------------------------------------------------------- 1 | // struct with empty list, empty string and zeros 2 | { 3 | C: (), 4 | A: "", 5 | B: 0, 6 | D: 0e0, 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_enum_fields/valid_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: ("foo" "bar" "baz"), 6 | D: 10e2, 7 | E: foo 8 | } 9 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_enum_fields/valid_optional_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: ("foo" "bar" "baz"), 6 | // D: 10e2, // since `D` is optional field, this is a valid struct 7 | E: foo, 8 | } 9 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_enum_fields/valid_unordered_fields.ion: -------------------------------------------------------------------------------- 1 | // struct with unordered fields 2 | { 3 | C: ("foo" "bar" "baz"), 4 | A: "hello", 5 | B: 12, 6 | E: foo, 7 | D: 10e2, 8 | } 9 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_fields/empty_values.ion: -------------------------------------------------------------------------------- 1 | // struct with empty list, empty string and zeros 2 | { 3 | C: (), 4 | A: "", 5 | B: 0, 6 | D: 0e0, 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_fields/valid_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: ("foo" "bar" "baz"), 6 | D: 10e2 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_fields/valid_optional_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: 12, 5 | C: ("foo" "bar" "baz"), 6 | // D: 10e2, // since `D` is optional field, this is a valid struct 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_fields/valid_unordered_fields.ion: -------------------------------------------------------------------------------- 1 | // struct with unordered fields 2 | { 3 | C: ("foo" "bar" "baz"), 4 | A: "hello", 5 | B: 12, 6 | D: 10e2, 7 | } 8 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_inline_import/valid_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | B: apple, 5 | } 6 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_inline_import/valid_optional_fields.ion: -------------------------------------------------------------------------------- 1 | // simple struct with all valid fields 2 | { 3 | A: "hello", 4 | // B: apple, // since `B` is an optional field, this is a valid struct 5 | } 6 | -------------------------------------------------------------------------------- /code-gen-projects/input/good/struct_with_inline_import/valid_unordered_fields.ion: -------------------------------------------------------------------------------- 1 | // struct with unordered fields 2 | { 3 | B: banana, 4 | A: "hello", 5 | } 6 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # Linux start script should use lf 5 | /gradlew text eol=lf 6 | 7 | # These are Windows script files and should use crlf 8 | *.bat text eol=crlf 9 | 10 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .gradle 3 | 4 | # Ignore Gradle build output directory 5 | build 6 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * This generated file contains a sample Java library project to get you started. 5 | * For more details on building Java & JVM projects, please refer to https://docs.gradle.org/8.6/userguide/building_java_projects.html in the Gradle documentation. 6 | */ 7 | 8 | plugins { 9 | // Apply the java plugin for API and implementation separation. 10 | java 11 | } 12 | 13 | repositories { 14 | // Use Maven Central for resolving dependencies. 15 | mavenCentral() 16 | } 17 | 18 | dependencies { 19 | implementation("com.amazon.ion:ion-java:1.11.4") 20 | 21 | // Use JUnit Jupiter for testing. 22 | testRuntimeOnly("org.junit.platform:junit-platform-launcher") 23 | testImplementation("org.junit.jupiter:junit-jupiter:5.7.1") 24 | } 25 | 26 | val ionSchemaSourceCodeDir = "../../schema/" 27 | val generatedIonSchemaModelDir = "${layout.buildDirectory.get()}/generated/java" 28 | sourceSets { 29 | main { 30 | java.srcDir(generatedIonSchemaModelDir) 31 | } 32 | } 33 | 34 | 35 | tasks { 36 | val ionCodegen = create("ionCodegen") { 37 | inputs.files(ionSchemaSourceCodeDir) 38 | outputs.file(generatedIonSchemaModelDir) 39 | 40 | val ionCli = System.getenv("ION_CLI") ?: "ion" 41 | 42 | commandLine(ionCli) 43 | .args( 44 | "-X", "generate", 45 | "-l", "java", 46 | "-n", "org.example", 47 | "-A", ionSchemaSourceCodeDir, 48 | "-o", generatedIonSchemaModelDir, 49 | ) 50 | .workingDir(rootProject.projectDir) 51 | } 52 | 53 | withType { 54 | options.encoding = "UTF-8" 55 | // The `release` option is not available for the Java 8 compiler, but if we're building with Java 8 we don't 56 | // need it anyway. 57 | if (JavaVersion.current() != JavaVersion.VERSION_1_8) { 58 | options.release.set(8) 59 | } 60 | 61 | dependsOn(ionCodegen) 62 | } 63 | } 64 | 65 | tasks.named("test") { 66 | // Use JUnit Platform for unit tests. 67 | useJUnitPlatform() 68 | testLogging { 69 | showStandardStreams = true 70 | events("skipped", "failed") 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/code-gen-projects/java/code-gen-demo/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | # This is normally unused 84 | # shellcheck disable=SC2034 85 | APP_BASE_NAME=${0##*/} 86 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 87 | APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit 88 | 89 | # Use the maximum available, or set MAX_FD != -1 to use that value. 90 | MAX_FD=maximum 91 | 92 | warn () { 93 | echo "$*" 94 | } >&2 95 | 96 | die () { 97 | echo 98 | echo "$*" 99 | echo 100 | exit 1 101 | } >&2 102 | 103 | # OS specific support (must be 'true' or 'false'). 104 | cygwin=false 105 | msys=false 106 | darwin=false 107 | nonstop=false 108 | case "$( uname )" in #( 109 | CYGWIN* ) cygwin=true ;; #( 110 | Darwin* ) darwin=true ;; #( 111 | MSYS* | MINGW* ) msys=true ;; #( 112 | NONSTOP* ) nonstop=true ;; 113 | esac 114 | 115 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 116 | 117 | 118 | # Determine the Java command to use to start the JVM. 119 | if [ -n "$JAVA_HOME" ] ; then 120 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 121 | # IBM's JDK on AIX uses strange locations for the executables 122 | JAVACMD=$JAVA_HOME/jre/sh/java 123 | else 124 | JAVACMD=$JAVA_HOME/bin/java 125 | fi 126 | if [ ! -x "$JAVACMD" ] ; then 127 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 128 | 129 | Please set the JAVA_HOME variable in your environment to match the 130 | location of your Java installation." 131 | fi 132 | else 133 | JAVACMD=java 134 | if ! command -v java >/dev/null 2>&1 135 | then 136 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | fi 142 | 143 | # Increase the maximum file descriptors if we can. 144 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 145 | case $MAX_FD in #( 146 | max*) 147 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 148 | # shellcheck disable=SC2039,SC3045 149 | MAX_FD=$( ulimit -H -n ) || 150 | warn "Could not query maximum file descriptor limit" 151 | esac 152 | case $MAX_FD in #( 153 | '' | soft) :;; #( 154 | *) 155 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 156 | # shellcheck disable=SC2039,SC3045 157 | ulimit -n "$MAX_FD" || 158 | warn "Could not set maximum file descriptor limit to $MAX_FD" 159 | esac 160 | fi 161 | 162 | # Collect all arguments for the java command, stacking in reverse order: 163 | # * args from the command line 164 | # * the main class name 165 | # * -classpath 166 | # * -D...appname settings 167 | # * --module-path (only if needed) 168 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 169 | 170 | # For Cygwin or MSYS, switch paths to Windows format before running java 171 | if "$cygwin" || "$msys" ; then 172 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 173 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 174 | 175 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 176 | 177 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 178 | for arg do 179 | if 180 | case $arg in #( 181 | -*) false ;; # don't mess with options #( 182 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 183 | [ -e "$t" ] ;; #( 184 | *) false ;; 185 | esac 186 | then 187 | arg=$( cygpath --path --ignore --mixed "$arg" ) 188 | fi 189 | # Roll the args list around exactly as many times as the number of 190 | # args, so each arg winds up back in the position where it started, but 191 | # possibly modified. 192 | # 193 | # NB: a `for` loop captures its iteration list before it begins, so 194 | # changing the positional parameters here affects neither the number of 195 | # iterations, nor the values presented in `arg`. 196 | shift # remove old arg 197 | set -- "$@" "$arg" # push replacement arg 198 | done 199 | fi 200 | 201 | 202 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 203 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 204 | 205 | # Collect all arguments for the java command: 206 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 207 | # and any embedded shellness will be escaped. 208 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 209 | # treated as '${Hostname}' itself on the command line. 210 | 211 | set -- \ 212 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 213 | -classpath "$CLASSPATH" \ 214 | org.gradle.wrapper.GradleWrapperMain \ 215 | "$@" 216 | 217 | # Stop when "xargs" is not available. 218 | if ! command -v xargs >/dev/null 2>&1 219 | then 220 | die "xargs is not available" 221 | fi 222 | 223 | # Use "xargs" to parse quoted args. 224 | # 225 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 226 | # 227 | # In Bash we could simply go: 228 | # 229 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 230 | # set -- "${ARGS[@]}" "$@" 231 | # 232 | # but POSIX shell has neither arrays nor command substitution, so instead we 233 | # post-process each arg (as a line of input to sed) to backslash-escape any 234 | # character that might be a shell metacharacter, then use eval to reverse 235 | # that process (while maintaining the separation between arguments), and wrap 236 | # the whole thing up as a single "set" statement. 237 | # 238 | # This will of course break if any of these variables contains a newline or 239 | # an unmatched quote. 240 | # 241 | 242 | eval "set -- $( 243 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 244 | xargs -n1 | 245 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 246 | tr '\n' ' ' 247 | )" '"$@"' 248 | 249 | exec "$JAVACMD" "$@" 250 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 1>&2 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 48 | echo. 1>&2 49 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 50 | echo location of your Java installation. 1>&2 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 1>&2 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 62 | echo. 1>&2 63 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 64 | echo location of your Java installation. 1>&2 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /code-gen-projects/java/code-gen-demo/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * For more detailed information on multi-project builds, please refer to https://docs.gradle.org/8.6/userguide/multi_project_builds.html in the Gradle documentation. 6 | */ 7 | 8 | plugins { 9 | // Apply the foojay-resolver plugin to allow automatic download of JDKs 10 | id("org.gradle.toolchains.foojay-resolver-convention") version "0.7.0" 11 | } 12 | 13 | rootProject.name = "code-gen-demo" -------------------------------------------------------------------------------- /code-gen-projects/rust/code-gen-demo/.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /target 3 | *.swp 4 | *.swo 5 | *.swn 6 | 7 | -------------------------------------------------------------------------------- /code-gen-projects/rust/code-gen-demo/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | # Empty workspace to make this separate from `ion-cli` 3 | 4 | [package] 5 | name = "code-gen-demo" 6 | version = "0.1.0" 7 | edition = "2021" 8 | 9 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 10 | 11 | [dev-dependencies] 12 | ion-rs = { version = "1.0.0-rc.2", features = ["experimental"] } 13 | test-generator = "0.3" -------------------------------------------------------------------------------- /code-gen-projects/rust/code-gen-demo/build.rs: -------------------------------------------------------------------------------- 1 | // build.rs 2 | 3 | use std::env; 4 | use std::io; 5 | use std::io::*; 6 | 7 | fn main() { 8 | let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); 9 | let out_dir = env::var("OUT_DIR").unwrap(); 10 | 11 | // Invoke cargo CLI 12 | let ion_cli = env::var("ION_CLI").unwrap_or("ion".to_string()); 13 | println!("cargo:warn=Running command: {}", ion_cli); 14 | let mut cmd = std::process::Command::new(ion_cli); 15 | cmd.arg("-X") 16 | .arg("generate") 17 | .arg("-l") 18 | .arg("rust") 19 | .arg("-A") 20 | .arg(format!("{}/../../schema", crate_dir)) 21 | .arg("-o") 22 | .arg(&out_dir); 23 | 24 | println!("cargo:warn=Running: {:?}", cmd); 25 | 26 | let output = cmd.output().expect("failed to execute process"); 27 | 28 | println!("status: {}", output.status); 29 | io::stdout().write_all(&output.stdout).unwrap(); 30 | io::stderr().write_all(&output.stderr).unwrap(); 31 | 32 | assert!(output.status.success()); 33 | 34 | println!("cargo:rerun-if-changed=input/"); 35 | println!("cargo:rerun-if-changed=schema/"); 36 | } 37 | -------------------------------------------------------------------------------- /code-gen-projects/rust/code-gen-demo/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub fn add(left: usize, right: usize) -> usize { 2 | left + right 3 | } 4 | 5 | #[cfg(test)] 6 | mod tests { 7 | use super::*; 8 | use ion_rs::Element; 9 | use ion_rs::IonType; 10 | use ion_rs::ReaderBuilder; 11 | use ion_rs::TextWriterBuilder; 12 | use std::fs; 13 | use std::path::MAIN_SEPARATOR_STR as PATH_SEPARATOR; 14 | use test_generator::test_resources; 15 | 16 | include!(concat!(env!("OUT_DIR"), "/ion_generated_code.rs")); 17 | 18 | /// Determines if the given file name is in the ROUNDTRIP_TESTS_SKIP_LIST list. This deals with platform 19 | /// path separator differences from '/' separators in the path list. 20 | #[inline] 21 | pub fn skip_list_contains_path(file_name: &str) -> bool { 22 | ROUNDTRIP_TESTS_SKIP_LIST 23 | .iter() 24 | // TODO construct the paths in a not so hacky way 25 | .map(|p| p.replace('/', PATH_SEPARATOR)) 26 | .any(|p| p == file_name) 27 | } 28 | 29 | pub const ROUNDTRIP_TESTS_SKIP_LIST: &[&str] = &[ 30 | "../../input/good/nested_struct/valid_optional_fields.ion", 31 | "../../input/good/struct_with_fields/valid_optional_fields.ion", 32 | "../../input/bad/struct_with_fields/missing_required_fields.ion", 33 | ]; 34 | 35 | #[test] 36 | fn it_works() { 37 | let result = add(2, 2); 38 | assert_eq!(result, 4); 39 | } 40 | 41 | #[test_resources("../../input/good/struct_with_fields/**/*.ion")] 42 | fn roundtrip_good_test_generated_code_structs_with_fields(file_name: &str) -> SerdeResult<()> { 43 | // if file name is under the ROUNDTRIP_TESTS_SKIP_LIST then do nothing. 44 | if skip_list_contains_path(&file_name) { 45 | return Ok(()); 46 | } 47 | let ion_string = fs::read_to_string(file_name).unwrap(); 48 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 49 | let mut buffer = Vec::new(); 50 | let mut text_writer = TextWriterBuilder::default().build(&mut buffer)?; 51 | // read given Ion value using Ion reader 52 | reader.next()?; 53 | let structs_with_fields: StructWithFields = StructWithFields::read_from(&mut reader)?; 54 | // write the generated abstract data type using Ion writer 55 | structs_with_fields.write_to(&mut text_writer)?; 56 | text_writer.flush()?; 57 | // compare given Ion value with round tripped Ion value written using abstract data type's `write_to` API 58 | assert_eq!( 59 | Element::read_one(text_writer.output().as_slice())?, 60 | (Element::read_one(&ion_string)?) 61 | ); 62 | 63 | Ok(()) 64 | } 65 | 66 | #[test_resources("../../input/bad/struct_with_fields/**/*.ion")] 67 | fn roundtrip_bad_test_generated_code_structs_with_fields(file_name: &str) -> SerdeResult<()> { 68 | if skip_list_contains_path(&file_name) { 69 | return Ok(()); 70 | } 71 | let ion_string = fs::read_to_string(file_name).unwrap(); 72 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 73 | // read given Ion value using Ion reader 74 | reader.next()?; 75 | let result = StructWithFields::read_from(&mut reader); 76 | assert!(result.is_err()); 77 | 78 | Ok(()) 79 | } 80 | 81 | #[test_resources("../../input/good/nested_struct/**/*.ion")] 82 | fn roundtrip_good_test_generated_code_nested_structs(file_name: &str) -> SerdeResult<()> { 83 | if skip_list_contains_path(&file_name) { 84 | return Ok(()); 85 | } 86 | let ion_string = fs::read_to_string(file_name).unwrap(); 87 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 88 | let mut buffer = Vec::new(); 89 | let mut text_writer = TextWriterBuilder::default().build(&mut buffer)?; 90 | // read given Ion value using Ion reader 91 | reader.next()?; 92 | let nested_struct: NestedStruct = NestedStruct::read_from(&mut reader)?; 93 | // write the generated abstract data type using Ion writer 94 | nested_struct.write_to(&mut text_writer)?; 95 | text_writer.flush()?; 96 | // compare given Ion value with round tripped Ion value written using abstract data type's `write_to` API 97 | assert_eq!( 98 | Element::read_one(text_writer.output().as_slice())?, 99 | (Element::read_one(&ion_string)?) 100 | ); 101 | 102 | Ok(()) 103 | } 104 | 105 | #[test_resources("../../input/bad/nested_struct/**/*.ion")] 106 | fn roundtrip_bad_test_generated_code_nested_structs(file_name: &str) -> SerdeResult<()> { 107 | let ion_string = fs::read_to_string(file_name).unwrap(); 108 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 109 | // read given Ion value using Ion reader 110 | reader.next()?; 111 | let result = NestedStruct::read_from(&mut reader); 112 | assert!(result.is_err()); 113 | 114 | Ok(()) 115 | } 116 | 117 | #[test_resources("../../input/good/scalar/**/*.ion")] 118 | fn roundtrip_good_test_generated_code_scalar(file_name: &str) -> SerdeResult<()> { 119 | let ion_string = fs::read_to_string(file_name).unwrap(); 120 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 121 | let mut buffer = Vec::new(); 122 | let mut text_writer = TextWriterBuilder::default().build(&mut buffer)?; 123 | // read given Ion value using Ion reader 124 | reader.next()?; 125 | let scalar: Scalar = Scalar::read_from(&mut reader)?; 126 | // write the generated abstract data type using Ion writer 127 | scalar.write_to(&mut text_writer)?; 128 | text_writer.flush()?; 129 | // compare given Ion value with round tripped Ion value written using abstract data type's `write_to` API 130 | assert_eq!( 131 | Element::read_one(text_writer.output().as_slice())?, 132 | (Element::read_one(&ion_string)?) 133 | ); 134 | 135 | Ok(()) 136 | } 137 | 138 | #[test_resources("../../input/bad/scalar/**/*.ion")] 139 | fn roundtrip_bad_test_generated_code_scalar(file_name: &str) -> SerdeResult<()> { 140 | let ion_string = fs::read_to_string(file_name).unwrap(); 141 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 142 | // read given Ion value using Ion reader 143 | reader.next()?; 144 | let result = Scalar::read_from(&mut reader); 145 | assert!(result.is_err()); 146 | 147 | Ok(()) 148 | } 149 | 150 | #[test_resources("../../input/good/sequence/**/*.ion")] 151 | fn roundtrip_good_test_generated_code_sequence(file_name: &str) -> SerdeResult<()> { 152 | let ion_string = fs::read_to_string(file_name).unwrap(); 153 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 154 | let mut buffer = Vec::new(); 155 | let mut text_writer = TextWriterBuilder::default().build(&mut buffer)?; 156 | // read given Ion value using Ion reader 157 | reader.next()?; 158 | let sequence: Sequence = Sequence::read_from(&mut reader)?; 159 | // write the generated abstract data type using Ion writer 160 | sequence.write_to(&mut text_writer)?; 161 | text_writer.flush()?; 162 | // compare given Ion value with round tripped Ion value written using abstract data type's `write_to` API 163 | assert_eq!( 164 | Element::read_one(text_writer.output().as_slice())?, 165 | (Element::read_one(&ion_string)?) 166 | ); 167 | 168 | Ok(()) 169 | } 170 | 171 | #[test_resources("../../input/bad/sequence/**/*.ion")] 172 | fn roundtrip_bad_test_generated_code_sequence(file_name: &str) -> SerdeResult<()> { 173 | let ion_string = fs::read_to_string(file_name).unwrap(); 174 | let mut reader = ReaderBuilder::new().build(ion_string.clone())?; 175 | // read given Ion value using Ion reader 176 | reader.next()?; 177 | let result = Sequence::read_from(&mut reader); 178 | assert!(result.is_err()); 179 | 180 | Ok(()) 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /code-gen-projects/schema/enum_type.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: enum_type, 3 | valid_values: [foo, bar, baz, FooBarBaz] 4 | } -------------------------------------------------------------------------------- /code-gen-projects/schema/nested_struct.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: nested_struct, 3 | type: struct, 4 | fields: { 5 | A: string, 6 | B: int, 7 | C: { 8 | type: struct, 9 | fields: { 10 | D: bool, 11 | E: { type: list, element: int } 12 | } 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /code-gen-projects/schema/scalar.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: scalar, 3 | type: string 4 | } -------------------------------------------------------------------------------- /code-gen-projects/schema/sequence.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: sequence, 3 | type: list, 4 | element: string 5 | } 6 | -------------------------------------------------------------------------------- /code-gen-projects/schema/sequence_with_enum_element.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: sequence_with_enum_element, 3 | type: list, 4 | element: { valid_values: [foo, bar, baz] } 5 | } -------------------------------------------------------------------------------- /code-gen-projects/schema/sequence_with_import.isl: -------------------------------------------------------------------------------- 1 | schema_header::{ 2 | imports: [ 3 | { id: "utils/fruits.isl", type: fruits } 4 | ] 5 | } 6 | 7 | type::{ 8 | name: sequence_with_import, 9 | type: list, 10 | element: fruits 11 | } 12 | 13 | schema_footer::{} -------------------------------------------------------------------------------- /code-gen-projects/schema/struct_with_enum_fields.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: struct_with_enum_fields, 3 | type: struct, 4 | fields: { 5 | A: string, 6 | B: int, 7 | C: { element: string, type: sexp, occurs: required }, 8 | D: float, 9 | E: { valid_values: [foo, bar, baz] } 10 | } 11 | } 12 | 13 | -------------------------------------------------------------------------------- /code-gen-projects/schema/struct_with_fields.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: struct_with_fields, 3 | type: struct, 4 | fields: { 5 | A: string, 6 | B: int, 7 | C: { element: string, type: sexp, occurs: required }, 8 | D: float, 9 | } 10 | } 11 | 12 | -------------------------------------------------------------------------------- /code-gen-projects/schema/struct_with_inline_import.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: struct_with_inline_import, 3 | type: struct, 4 | fields: { 5 | A: string, 6 | B: { id: "utils/fruits.isl", type: fruits } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /code-gen-projects/schema/utils/fruits.isl: -------------------------------------------------------------------------------- 1 | type::{ 2 | name: fruits, 3 | valid_values: [apple, banana, strawberry] 4 | } -------------------------------------------------------------------------------- /images/example_inspect_limit_bytes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/images/example_inspect_limit_bytes.png -------------------------------------------------------------------------------- /images/example_inspect_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/images/example_inspect_output.png -------------------------------------------------------------------------------- /images/example_inspect_skip_bytes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/images/example_inspect_skip_bytes.png -------------------------------------------------------------------------------- /src/bin/ion/ansi_codes.rs: -------------------------------------------------------------------------------- 1 | //! Ansi Codes are a convenient way to add styling to text in a terminal. 2 | //! There are libraries that can accomplish the same thing, but when you want to have a large block 3 | //! of static text, sometimes it's simpler to just use `format!()` and include named substitutions 4 | //! (like `{BOLD}`) to turn styling on and off. 5 | 6 | // TODO: Add more constants as needed. 7 | 8 | pub(crate) const NO_STYLE: &str = "\x1B[0m"; 9 | pub(crate) const BOLD: &str = "\x1B[1m"; 10 | pub(crate) const ITALIC: &str = "\x1B[3m"; 11 | pub(crate) const UNDERLINE: &str = "\x1B[4m"; 12 | 13 | pub(crate) const RED: &str = "\x1B[0;31m"; 14 | pub(crate) const GREEN: &str = "\x1B[0;32m"; 15 | -------------------------------------------------------------------------------- /src/bin/ion/assets/README.md: -------------------------------------------------------------------------------- 1 | The packdump files are generated via the `synpack` command from the `gendata` example in `syntect`. 2 | 3 | Check out `syntect` and then run: 4 | ```shell 5 | cargo run --features=metadata --example gendata -- \ 6 | synpack ion.sublime-syntax ion.newlines.packdump ion.nonewlines.packdump 7 | ``` 8 | 9 | `ion.sublime-syntax` is sourced from `partiql/partiql-rust-cli`. 10 | -------------------------------------------------------------------------------- /src/bin/ion/assets/ion.newlines.packdump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/src/bin/ion/assets/ion.newlines.packdump -------------------------------------------------------------------------------- /src/bin/ion/assets/ion.nonewlines.packdump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-ion/ion-cli/de4bbb1b5b11b76eb25df9583cb0ab81d3254b02/src/bin/ion/assets/ion.nonewlines.packdump -------------------------------------------------------------------------------- /src/bin/ion/assets/ion.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | # See https://www.sublimetext.com/docs/syntax.htm 3 | --- 4 | name: ion 5 | version: "2" 6 | file_extensions: 7 | - ion 8 | scope: source.ion 9 | contexts: 10 | keywords: 11 | - match: "\\b(?i:true|false)\\b" 12 | scope: constant.language.bool.ion 13 | - match: "\\b(?i:null.null|null.bool|null.int|null.float|null.decimal|null.timestamp|null.string|null.symbol|null.blob|null.clob|null.struct|null.list|null.sexp|null)\\b" 14 | scope: constant.language.null.ion 15 | main: 16 | - include: value 17 | value: 18 | - include: whitespace 19 | - include: comment 20 | - include: annotation 21 | - include: string 22 | - include: number 23 | - include: keywords 24 | - include: symbol 25 | - include: clob 26 | - include: blob 27 | - include: struct 28 | - include: list 29 | - include: sexp 30 | sexp: 31 | - match: "\\(" 32 | scope: punctuation.definition.sexp.begin.ion 33 | push: sexp__0 34 | sexp__0: 35 | - match: "\\)" 36 | scope: punctuation.definition.sexp.end.ion 37 | pop: true 38 | - include: comment 39 | - include: value 40 | - match: "[\\!\\#\\%\\&\\*\\+\\-\\.\\/\\;\\<\\=\\>\\?\\@\\^\\`\\|\\~]+" 41 | scope: storage.type.symbol.operator.ion 42 | comment: 43 | - match: "\\/\\/[^\\n]*" 44 | scope: comment.line.ion 45 | - match: "\\/\\*" 46 | scope: comment.block.ion 47 | push: comment__1 48 | comment__1: 49 | - match: "[*]\\/" 50 | scope: comment.block.ion 51 | pop: true 52 | - match: "[^*\\/]+" 53 | scope: comment.block.ion 54 | - match: "[*\\/]+" 55 | scope: comment.block.ion 56 | list: 57 | - match: "\\[" 58 | scope: punctuation.definition.list.begin.ion 59 | push: list__0 60 | list__0: 61 | - match: "\\]" 62 | scope: punctuation.definition.list.end.ion 63 | pop: true 64 | - include: comment 65 | - include: value 66 | - match: "," 67 | scope: punctuation.definition.list.separator.ion 68 | struct: 69 | - match: "\\{" 70 | scope: punctuation.definition.struct.begin.ion 71 | push: struct__0 72 | struct__0: 73 | - match: "\\}" 74 | scope: punctuation.definition.struct.end.ion 75 | pop: true 76 | - include: comment 77 | - include: value 78 | - match: ",|:" 79 | scope: punctuation.definition.struct.separator.ion 80 | blob: 81 | - match: "(\\{\\{)([^\"]*)(\\}\\})" 82 | captures: 83 | 1: punctuation.definition.blob.begin.ion 84 | 2: string.other.blob.ion 85 | 3: punctuation.definition.blob.end.ion 86 | clob: 87 | - match: "(\\{\\{)(\"[^\"]*\")(\\}\\})" 88 | captures: 89 | 1: punctuation.definition.clob.begin.ion 90 | 2: string.other.clob.ion 91 | 3: punctuation.definition.clob.end.ion 92 | symbol: 93 | - match: "(['])((?:(?:\\\\')|(?:[^']))*?)(['])" 94 | scope: storage.type.symbol.quoted.ion 95 | - match: "[\\$_a-zA-Z][\\$_a-zA-Z0-9]*" 96 | scope: storage.type.symbol.identifier.ion 97 | number: 98 | - match: "\\d{4}(?:-\\d{2})?(?:-\\d{2})?T(?:\\d{2}:\\d{2})(?::\\d{2})?(?:\\.\\d+)?(?:Z|[-+]\\d{2}:\\d{2})?" 99 | scope: constant.numeric.timestamp.ion 100 | - match: "\\d{4}-\\d{2}-\\d{2}T?" 101 | scope: constant.numeric.timestamp.ion 102 | - match: "-?0[bB][01](?:_?[01])*" 103 | scope: constant.numeric.integer.binary.ion 104 | - match: "-?0[xX][0-9a-fA-F](?:_?[0-9a-fA-F])*" 105 | scope: constant.numeric.integer.hex.ion 106 | - match: "-?(?:0|[1-9](?:_?\\d)*)(?:\\.(?:\\d(?:_?\\d)*)?)?(?:[eE][+-]?\\d+)" 107 | scope: constant.numeric.float.ion 108 | - match: "(?:[-+]inf)|(?:nan)" 109 | scope: constant.numeric.float.ion 110 | - match: "-?(?:0|[1-9](?:_?\\d)*)(?:(?:(?:\\.(?:\\d(?:_?\\d)*)?)(?:[dD][+-]?\\d+)|\\.(?:\\d(?:_?\\d)*)?)|(?:[dD][+-]?\\d+))" 111 | scope: constant.numeric.decimal.ion 112 | - match: "-?(?:0|[1-9](?:_?\\d)*)" 113 | scope: constant.numeric.integer.ion 114 | string: 115 | - match: "([\"])((?:(?:\\\\\")|(?:[^\"]))*?)([\"])" 116 | captures: 117 | 1: punctuation.definition.string.begin.ion 118 | 2: string.quoted.double.ion 119 | 3: punctuation.definition.string.end.ion 120 | - match: "'{3}" 121 | scope: punctuation.definition.string.begin.ion 122 | push: string__1 123 | string__1: 124 | - match: "'{3}" 125 | scope: punctuation.definition.string.end.ion 126 | pop: true 127 | - match: "(?:\\\\'|[^'])+" 128 | scope: string.quoted.triple.ion 129 | - match: "'" 130 | scope: string.quoted.triple.ion 131 | annotation: 132 | - match: "('(?:[^']|\\\\\\\\|\\\\')*')\\s*(::)" 133 | captures: 134 | 1: variable.language.annotation.ion 135 | 2: punctuation.definition.annotation.ion 136 | - match: "([\\$_a-zA-Z][\\$_a-zA-Z0-9]*)\\s*(::)" 137 | captures: 138 | 1: variable.language.annotation.ion 139 | 2: punctuation.definition.annotation.ion 140 | whitespace: 141 | - match: "\\s+" 142 | scope: text.ion -------------------------------------------------------------------------------- /src/bin/ion/auto_decompress.rs: -------------------------------------------------------------------------------- 1 | use infer::Type; 2 | use std::io; 3 | use std::io::{BufReader, Cursor, Read}; 4 | 5 | use crate::input::CompressionDetected; 6 | use ion_rs::IonResult; 7 | 8 | /// Auto-detects a compressed byte stream and wraps the original reader 9 | /// into a reader that transparently decompresses. 10 | pub type AutoDecompressingReader = BufReader>; 11 | 12 | pub fn decompress( 13 | mut reader: R, 14 | header_len: usize, 15 | ) -> IonResult<(CompressionDetected, AutoDecompressingReader)> 16 | where 17 | R: Read + 'static, 18 | { 19 | // read header 20 | let mut header_bytes = vec![0; header_len]; 21 | let nread = read_reliably(&mut reader, &mut header_bytes)?; 22 | header_bytes.truncate(nread); 23 | 24 | let detected_type = infer::get(&header_bytes); 25 | let header = Cursor::new(header_bytes); 26 | let stream = header.chain(reader); 27 | 28 | // detect compression type and wrap reader in a decompressor 29 | match detected_type.as_ref().map(Type::extension) { 30 | Some("gz") => { 31 | // "rewind" to let the decompressor read magic bytes again 32 | let zreader = Box::new(flate2::read::MultiGzDecoder::new(stream)); 33 | Ok((CompressionDetected::Gzip, BufReader::new(zreader))) 34 | } 35 | Some("zst") => { 36 | let zreader = Box::new(zstd::stream::read::Decoder::new(stream)?); 37 | Ok((CompressionDetected::Zstd, BufReader::new(zreader))) 38 | } 39 | _ => Ok((CompressionDetected::None, BufReader::new(Box::new(stream)))), 40 | } 41 | } 42 | 43 | /// Similar to [`Read::read()`], but loops in case of fragmented reads. 44 | pub fn read_reliably(reader: &mut R, buf: &mut [u8]) -> io::Result { 45 | let mut nread = 0; 46 | while nread < buf.len() { 47 | match reader.read(&mut buf[nread..]) { 48 | Ok(0) => break, 49 | Ok(n) => nread += n, 50 | Err(e) => return Err(e), 51 | } 52 | } 53 | Ok(nread) 54 | } 55 | -------------------------------------------------------------------------------- /src/bin/ion/commands/cat.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::{ArgMatches, Command}; 3 | use ion_rs::*; 4 | 5 | use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; 6 | use crate::transcribe::write_all_as; 7 | 8 | pub struct CatCommand; 9 | 10 | impl IonCliCommand for CatCommand { 11 | fn name(&self) -> &'static str { 12 | "cat" 13 | } 14 | 15 | fn about(&self) -> &'static str { 16 | "Prints all Ion input files to the specified output in the requested format." 17 | } 18 | 19 | fn is_stable(&self) -> bool { 20 | true 21 | } 22 | 23 | fn is_porcelain(&self) -> bool { 24 | false 25 | } 26 | 27 | fn configure_args(&self, command: Command) -> Command { 28 | command 29 | .alias("dump") 30 | .with_input() 31 | .with_output() 32 | .with_format() 33 | .with_color() 34 | .with_ion_version() 35 | } 36 | 37 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 38 | CommandIo::new(args)?.for_each_input(|output, input| { 39 | let mut reader = Reader::new(AnyEncoding, input.into_source())?; 40 | let encoding = *output.encoding(); 41 | let format = *output.format(); 42 | write_all_as(&mut reader, output, encoding, format)?; 43 | Ok(()) 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/bin/ion/commands/command_namespace.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::{IonCliCommand, WithIonCliArgument, UNSTABLE_FLAG}; 2 | use clap::{ArgMatches, Command as ClapCommand}; 3 | use std::process; 4 | 5 | /// A trait that handles the implementation of [IonCliCommand] for command namespaces. 6 | pub trait IonCliNamespace { 7 | fn name(&self) -> &'static str; 8 | fn about(&self) -> &'static str; 9 | fn subcommands(&self) -> Vec>; 10 | } 11 | 12 | impl IonCliCommand for T { 13 | // Namespaces can't be used on their own, so we'll pretend that they are all stable and 14 | // let the leaf commands handle stability. 15 | fn is_stable(&self) -> bool { 16 | true 17 | } 18 | 19 | // Namespaces can't be used on their own, so we'll pretend that they are all plumbing and 20 | // let the leaf commands handle plumbing vs porcelain. 21 | fn is_porcelain(&self) -> bool { 22 | false 23 | } 24 | 25 | fn name(&self) -> &'static str { 26 | IonCliNamespace::name(self) 27 | } 28 | 29 | fn about(&self) -> &'static str { 30 | IonCliNamespace::about(self) 31 | } 32 | 33 | fn configure_args(&self, command: ClapCommand) -> ClapCommand { 34 | // Create a `ClapCommand` representing each of this command's subcommands. 35 | let clap_subcommands: Vec<_> = self 36 | .subcommands() 37 | .iter() 38 | .map(|s| s.clap_command()) 39 | .collect(); 40 | 41 | let mut command = command 42 | .subcommand_required(true) 43 | .subcommands(clap_subcommands); 44 | 45 | // If there are subcommands, add them to the configuration and set 'subcommand_required'. 46 | let has_unstable_subcommand = self.subcommands().iter().any(|sc| !sc.is_stable()); 47 | if has_unstable_subcommand { 48 | command = command.show_unstable_flag(); 49 | } 50 | command 51 | } 52 | 53 | fn run(&self, command_path: &mut Vec, args: &ArgMatches) -> anyhow::Result<()> { 54 | // Safe to unwrap because if this is a namespace are subcommands, then clap has already 55 | // ensured that a known subcommand is present in args. 56 | let (subcommand_name, subcommand_args) = args.subcommand().unwrap(); 57 | let subcommands = self.subcommands(); 58 | let subcommand: &dyn IonCliCommand = subcommands 59 | .iter() 60 | .find(|sc| sc.name() == subcommand_name) 61 | .unwrap() 62 | .as_ref(); 63 | 64 | match (subcommand.is_stable(), args.get_flag(UNSTABLE_FLAG)) { 65 | // Warn if using an unnecessary `-X` 66 | (true, true) => eprintln!( 67 | "'{}' is stable and does not require opt-in", 68 | subcommand_name 69 | ), 70 | // Error if missing a required `-X` 71 | (false, false) => { 72 | eprintln!( 73 | "'{}' is unstable and requires explicit opt-in", 74 | subcommand_name 75 | ); 76 | process::exit(1) 77 | } 78 | _ => {} 79 | } 80 | 81 | command_path.push(subcommand_name.to_owned()); 82 | subcommand.run(command_path, subcommand_args) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/bin/ion/commands/complaint.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::IonCliCommand; 2 | use clap::{ArgMatches, Command}; 3 | 4 | pub struct SucksCommand; 5 | 6 | impl IonCliCommand for SucksCommand { 7 | fn is_stable(&self) -> bool { 8 | true 9 | } 10 | 11 | fn name(&self) -> &'static str { 12 | "sucks" 13 | } 14 | 15 | fn about(&self) -> &'static str { 16 | "" 17 | } 18 | 19 | fn configure_args(&self, command: Command) -> Command { 20 | command.hide(true) 21 | } 22 | 23 | fn run(&self, _command_path: &mut Vec, _args: &ArgMatches) -> anyhow::Result<()> { 24 | println!( 25 | " 26 | We're very sorry to hear that! 27 | 28 | Rather than complaining into the void, why not file an issue? 29 | https://github.com/amazon-ion/ion-docs/issues/new/choose 30 | " 31 | ); 32 | Ok(()) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/bin/ion/commands/from/json.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::{ArgMatches, Command}; 3 | 4 | use crate::commands::cat::CatCommand; 5 | use crate::commands::{IonCliCommand, WithIonCliArgument}; 6 | 7 | pub struct FromJsonCommand; 8 | 9 | impl IonCliCommand for FromJsonCommand { 10 | fn name(&self) -> &'static str { 11 | "json" 12 | } 13 | 14 | fn about(&self) -> &'static str { 15 | "Converts data from JSON to Ion." 16 | } 17 | 18 | fn is_stable(&self) -> bool { 19 | false // TODO: Should this be true? 20 | } 21 | 22 | fn is_porcelain(&self) -> bool { 23 | false 24 | } 25 | 26 | fn configure_args(&self, command: Command) -> Command { 27 | // Args must be identical to CatCommand so that we can safely delegate 28 | command 29 | .with_input() 30 | .with_output() 31 | .with_format() 32 | .with_ion_version() 33 | } 34 | 35 | fn run(&self, command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 36 | // Because JSON data is valid Ion, the `cat` command may be reused for converting JSON. 37 | // TODO ideally, this would perform some smarter "up-conversion". 38 | CatCommand.run(command_path, args) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/bin/ion/commands/from/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::command_namespace::IonCliNamespace; 2 | use crate::commands::IonCliCommand; 3 | 4 | use crate::commands::from::json::FromJsonCommand; 5 | 6 | pub mod json; 7 | 8 | pub struct FromNamespace; 9 | 10 | impl IonCliNamespace for FromNamespace { 11 | fn name(&self) -> &'static str { 12 | "from" 13 | } 14 | 15 | fn about(&self) -> &'static str { 16 | "'from' is a namespace for commands that convert other data formats to Ion." 17 | } 18 | 19 | fn subcommands(&self) -> Vec> { 20 | vec![Box::new(FromJsonCommand)] 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/context.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::generate::model::DataModelNode; 2 | use serde::Serialize; 3 | 4 | /// Represents a context that will be used for code generation 5 | pub struct CodeGenContext { 6 | // Represents the nested types for the current abstract data type 7 | pub(crate) nested_types: Vec, 8 | } 9 | 10 | impl CodeGenContext { 11 | pub fn new() -> Self { 12 | Self { 13 | nested_types: vec![], 14 | } 15 | } 16 | } 17 | 18 | /// Represents a sequenced type which could either be a list or s-expression. 19 | /// This is used by `AbstractDataType` to represent sequence type for `Sequence` variant. 20 | #[derive(Debug, Clone, PartialEq, Serialize)] 21 | #[allow(dead_code)] 22 | pub enum SequenceType { 23 | List, 24 | SExp, 25 | } 26 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/mod.rs: -------------------------------------------------------------------------------- 1 | mod context; 2 | mod generator; 3 | mod result; 4 | mod templates; 5 | mod utils; 6 | 7 | mod model; 8 | 9 | use crate::commands::generate::generator::CodeGenerator; 10 | use crate::commands::generate::model::NamespaceNode; 11 | use crate::commands::generate::utils::{JavaLanguage, RustLanguage}; 12 | use crate::commands::IonCliCommand; 13 | use anyhow::{bail, Result}; 14 | use clap::{Arg, ArgAction, ArgMatches, Command, ValueHint}; 15 | use colored::Colorize; 16 | use ion_schema::authority::{DocumentAuthority, FileSystemDocumentAuthority}; 17 | use ion_schema::system::SchemaSystem; 18 | use std::fs; 19 | use std::path::{Path, PathBuf}; 20 | 21 | pub struct GenerateCommand; 22 | 23 | impl IonCliCommand for GenerateCommand { 24 | fn name(&self) -> &'static str { 25 | "generate" 26 | } 27 | 28 | fn about(&self) -> &'static str { 29 | "Generates code using given schema file." 30 | } 31 | 32 | fn is_stable(&self) -> bool { 33 | false 34 | } 35 | 36 | fn is_porcelain(&self) -> bool { 37 | false 38 | } 39 | 40 | fn configure_args(&self, command: Command) -> Command { 41 | command 42 | .arg( 43 | Arg::new("output") 44 | .long("output") 45 | .short('o') 46 | .help("Output directory [default: current directory]"), 47 | ) 48 | // `--namespace` is required when Java language is specified for code generation 49 | .arg( 50 | Arg::new("namespace") 51 | .long("namespace") 52 | .short('n') 53 | .required_if_eq("language", "java") 54 | .help("Provide namespace for generated Java code (e.g. `org.example`)"), 55 | ) 56 | .arg( 57 | Arg::new("language") 58 | .long("language") 59 | .short('l') 60 | .required(true) 61 | .value_parser(["java", "rust"]) 62 | .help("Programming language for the generated code"), 63 | ) 64 | .arg( 65 | Arg::new("authority") 66 | .long("authority") 67 | .short('A') 68 | .required(true) 69 | .action(ArgAction::Append) 70 | .value_name("directory") 71 | .value_hint(ValueHint::DirPath) 72 | .help("The root(s) of the file system authority(s)"), 73 | ) 74 | } 75 | 76 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 77 | // Extract programming language for code generation 78 | let language: &str = args.get_one::("language").unwrap().as_str(); 79 | 80 | // Extract namespace for code generation 81 | let namespace = args.get_one::("namespace"); 82 | 83 | // Extract output path information where the generated code will be saved 84 | // Create a module `ion_data_model` for storing all the generated code in the output directory 85 | let binding = match args.get_one::("output") { 86 | Some(output_path) => PathBuf::from(output_path), 87 | None => PathBuf::from("./"), 88 | }; 89 | 90 | let output = binding.as_path(); 91 | 92 | // Extract the user provided document authorities/ directories 93 | let authorities: Vec<&String> = args.get_many("authority").unwrap().collect(); 94 | 95 | // Set up document authorities vector 96 | let mut document_authorities: Vec> = vec![]; 97 | args.get_many::("authority") 98 | .unwrap_or_default() 99 | .map(Path::new) 100 | .map(FileSystemDocumentAuthority::new) 101 | .for_each(|a| document_authorities.push(Box::new(a))); 102 | 103 | // Create a new schema system from given document authorities 104 | let mut schema_system = SchemaSystem::new(document_authorities); 105 | 106 | // Generate directories in the output path if the path doesn't exist 107 | if !output.exists() { 108 | fs::create_dir_all(output).unwrap(); 109 | } 110 | 111 | println!("Started generating code..."); 112 | 113 | // generate code based on schema and programming language 114 | match language { 115 | "java" => { 116 | Self::print_java_code_gen_warnings(); 117 | CodeGenerator::::new(output, namespace.unwrap().split('.').map(|s| NamespaceNode::Package(s.to_string())).collect()) 118 | .generate_code_for_authorities(&authorities, &mut schema_system)? 119 | }, 120 | "rust" => { 121 | Self::print_rust_code_gen_warnings(); 122 | CodeGenerator::::new(output) 123 | .generate_code_for_authorities(&authorities, &mut schema_system)? 124 | } 125 | _ => bail!( 126 | "Programming language '{}' is not yet supported. Currently supported targets: 'java', 'rust'", 127 | language 128 | ) 129 | } 130 | 131 | println!("Code generation complete successfully!"); 132 | println!("All the schema files in authority(s) are generated into a flattened namespace, path to generated code: {}", output.display()); 133 | Ok(()) 134 | } 135 | } 136 | 137 | impl GenerateCommand { 138 | // Prints warning messages for Java code generation 139 | fn print_java_code_gen_warnings() { 140 | println!("{}","WARNING: Code generation in Java does not support any `$NOMINAL_ION_TYPES` data type.(For more information: https://amazon-ion.github.io/ion-schema/docs/isl-2-0/spec#built-in-types) Reference issue: https://github.com/amazon-ion/ion-cli/issues/101".yellow().bold()); 141 | println!( 142 | "{}", 143 | "Optional fields in generated code are represented with the wrapper class of that primitive data type and are set to `null` when missing." 144 | .yellow() 145 | .bold() 146 | ); 147 | println!("{}", "When the `writeTo` method is used on an optional field and if the field value is set as null then it would skip serializing that field.".yellow().bold()); 148 | } 149 | 150 | // Prints warning messages for Rust code generation 151 | fn print_rust_code_gen_warnings() { 152 | println!("{}","WARNING: Code generation in Rust does not yet support any `$NOMINAL_ION_TYPES` data type.(For more information: https://amazon-ion.github.io/ion-schema/docs/isl-2-0/spec#built-in-types) Reference issue: https://github.com/amazon-ion/ion-cli/issues/101".yellow().bold()); 153 | println!("{}","Code generation in Rust does not yet support optional/required fields. It does not have any checks added for this on read or write methods. Reference issue: https://github.com/amazon-ion/ion-cli/issues/106".yellow().bold()); 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/result.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::generate::model::{ 2 | EnumBuilderError, ScalarBuilderError, SequenceBuilderError, StructureBuilderError, 3 | WrappedScalarBuilderError, WrappedSequenceBuilderError, 4 | }; 5 | use ion_schema::result::IonSchemaError; 6 | use thiserror::Error; 7 | 8 | /// Represents code generation result 9 | pub type CodeGenResult = Result; 10 | 11 | /// Represents an error found during code generation 12 | #[derive(Debug, Error)] 13 | pub enum CodeGenError { 14 | #[error("{source:?}")] 15 | IonSchemaError { 16 | #[from] 17 | source: IonSchemaError, 18 | }, 19 | #[error("{source:?}")] 20 | IoError { 21 | #[from] 22 | source: std::io::Error, 23 | }, 24 | #[error("{source:?}")] 25 | TeraError { 26 | #[from] 27 | source: tera::Error, 28 | }, 29 | #[error("{description}")] 30 | InvalidDataModel { description: String }, 31 | #[error("{description}")] 32 | DataModelBuilderError { description: String }, 33 | } 34 | 35 | /// A convenience method for creating an CodeGen containing an CodeGenError::InvalidDataModel 36 | /// with the provided description text. 37 | pub fn invalid_abstract_data_type_error>(description: S) -> CodeGenResult { 38 | Err(CodeGenError::InvalidDataModel { 39 | description: description.as_ref().to_string(), 40 | }) 41 | } 42 | 43 | /// A convenience method for creating an CodeGenError::InvalidDataModel 44 | /// with the provided description text. 45 | pub fn invalid_abstract_data_type_raw_error>(description: S) -> CodeGenError { 46 | CodeGenError::InvalidDataModel { 47 | description: description.as_ref().to_string(), 48 | } 49 | } 50 | 51 | impl From for CodeGenError { 52 | fn from(value: WrappedScalarBuilderError) -> Self { 53 | CodeGenError::DataModelBuilderError { 54 | description: value.to_string(), 55 | } 56 | } 57 | } 58 | 59 | impl From for CodeGenError { 60 | fn from(value: ScalarBuilderError) -> Self { 61 | CodeGenError::DataModelBuilderError { 62 | description: value.to_string(), 63 | } 64 | } 65 | } 66 | 67 | impl From for CodeGenError { 68 | fn from(value: SequenceBuilderError) -> Self { 69 | CodeGenError::DataModelBuilderError { 70 | description: value.to_string(), 71 | } 72 | } 73 | } 74 | 75 | impl From for CodeGenError { 76 | fn from(value: WrappedSequenceBuilderError) -> Self { 77 | CodeGenError::DataModelBuilderError { 78 | description: value.to_string(), 79 | } 80 | } 81 | } 82 | 83 | impl From for CodeGenError { 84 | fn from(value: StructureBuilderError) -> Self { 85 | CodeGenError::DataModelBuilderError { 86 | description: value.to_string(), 87 | } 88 | } 89 | } 90 | 91 | impl From for CodeGenError { 92 | fn from(value: EnumBuilderError) -> Self { 93 | CodeGenError::DataModelBuilderError { 94 | description: value.to_string(), 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/java/class.templ: -------------------------------------------------------------------------------- 1 | {# Includes the macros for anonymous types that will be added as child classes #} 2 | {% import "nested_type.templ" as macros %} 3 | {% import "util_macros.templ" as util_macros %} 4 | 5 | {% macro class(model, is_nested) %} 6 | 7 | {% if is_nested == false %} 8 | {% set full_namespace = namespace | map(attribute="Package") | join(sep=".") %} 9 | 10 | package {{ full_namespace }}; 11 | import com.amazon.ion.IonReader; 12 | import com.amazon.ion.IonException; 13 | import com.amazon.ion.IonWriter; 14 | import com.amazon.ion.IonType; 15 | import java.io.IOException; 16 | {% endif %} 17 | 18 | 19 | {# Verify that the abstract data type is a structure and store information for this structure #} 20 | {% set struct_info = model.code_gen_type["Structure"] %} 21 | 22 | {% if is_nested == true %} static {% endif %} class {{ model.name }} { 23 | {% for field_name, field_value in struct_info["fields"] -%} 24 | private {{ field_value.0 | fully_qualified_type_name }} {{ field_name | camel }}; 25 | {% endfor %} 26 | 27 | private {{ model.name }}() {} 28 | 29 | {% for field_name, field_value in struct_info["fields"] -%}public {{ field_value.0 | fully_qualified_type_name }} get{% filter upper_camel %}{{ field_name }}{% endfilter %}() { 30 | return this.{{ field_name | camel }}; 31 | } 32 | {% endfor %} 33 | 34 | {% for field_name, field_val in struct_info["fields"] %} 35 | {% set val = field_val.0 | fully_qualified_type_name %} 36 | public void set{% filter upper_camel %}{{ field_name }}{% endfilter %}({{ val }} {{ field_name | camel }}) { 37 | this.{{ field_name | camel }} = {{ field_name | camel }}; 38 | return; 39 | } 40 | {% endfor %} 41 | 42 | public static class Builder { 43 | {% for field_name, field_val in struct_info["fields"] -%} 44 | {% set propertyName = field_name | camel %} 45 | {% set PropertyType = field_val.0 | fully_qualified_type_name | wrapper_class %} 46 | 47 | private {{ PropertyType }} {{ propertyName }}; 48 | 49 | public Builder {{ propertyName }}({{ PropertyType }} value) { 50 | this.{{ propertyName }} = value; 51 | return this; 52 | } 53 | {% endfor %} 54 | 55 | public {{ model.name }} build() { 56 | {{ model.name }} instance = new {{ model.name }}(); 57 | {% for field_name, field_val in struct_info["fields"] -%} 58 | {% set propertyName = field_name | camel %} 59 | {# field_val.1 is the field occurrence #} 60 | {% if field_val.1 == "Required" %} 61 | if ({{propertyName}} == null) { 62 | throw new IllegalArgumentException("Missing required field {{propertyName}}"); 63 | } 64 | {% endif %} 65 | instance.{{ propertyName }} = {{ propertyName }}; 66 | {% endfor %} 67 | return instance; 68 | } 69 | } 70 | 71 | /** 72 | * Reads a {{ model.name }} from an {@link IonReader}. 73 | * 74 | * This method does not advance the reader at the current level. 75 | * The caller is responsible for positioning the reader on the value to read. 76 | */ 77 | public static {{ model.name }} readFrom(IonReader reader) { 78 | {# Initializes the builder for this class #} 79 | Builder builder = new Builder(); 80 | 81 | {# Reads `Structure` class with multiple fields based on `field.name` #} 82 | reader.stepIn(); 83 | while (reader.hasNext()) { 84 | reader.next(); 85 | String fieldName = reader.getFieldName(); 86 | switch(fieldName) { 87 | {% for field_name, field_val in struct_info["fields"] %} 88 | {% set field_value = field_val.0 | fully_qualified_type_name %} 89 | {% set field_occurrence = field_val.1 %} 90 | {% if field_occurrence == "Optional" %} {% set field_value = field_value | primitive_data_type %} {% endif %} 91 | case "{{ field_name }}": 92 | builder.{{ field_name | camel }}( 93 | {% if field_value | is_built_in_type %} 94 | {% if field_value == "bytes[]" %} 95 | reader.newBytes() 96 | {% else %} 97 | reader.{{ field_value | camel }}Value() 98 | {% endif %} 99 | {% elif field_value is containing("ArrayList") %} 100 | {{ util_macros::read_as_sequence(field_value=field_value,field_name=field_name,type_store=type_store, field_occurrence=field_occurrence) }} 101 | {% else %} 102 | {{ field_value }}.readFrom(reader) 103 | {% endif %}); 104 | break; 105 | {% endfor %} 106 | default: 107 | throw new IonException("Can not read field name:" + fieldName + " for {{ model.name }} as it doesn't exist in the given schema type definition."); 108 | } 109 | } 110 | reader.stepOut(); 111 | 112 | return builder.build(); 113 | } 114 | 115 | /** 116 | * Writes a {{ model.name }} as Ion from an {@link IonWriter}. 117 | * 118 | * This method does not close the writer after writing is complete. 119 | * The caller is responsible for closing the stream associated with the writer. 120 | * This method skips writing a field when it's null. 121 | */ 122 | public void writeTo(IonWriter writer) throws IOException { 123 | {# Writes `Structure` class with multiple fields based on `field.name` as an Ion struct #} 124 | writer.stepIn(IonType.STRUCT); 125 | {% for field_name, field_val in struct_info["fields"] %} 126 | {% set field_value = field_val.0 | fully_qualified_type_name %} 127 | {% set field_occurrence = field_val.1 %} 128 | {% if field_occurrence == "Optional" %} 129 | {% set field_value = field_value | primitive_data_type %} 130 | if (this.{{ field_name | camel }} != null) { 131 | {% endif %} 132 | writer.setFieldName("{{ field_name }}"); 133 | {% if field_value | is_built_in_type == false %} 134 | {% if field_value is containing("ArrayList") %} 135 | {{ util_macros::write_as_sequence(field_value=field_value,field_name=field_name,type_store=type_store) }} 136 | {% else %} 137 | this.{{ field_name | camel }}.writeTo(writer); 138 | {% endif %} 139 | {% else %} 140 | writer.write{{ field_value | replace(from="double", to="float") | replace(from="boolean", to="bool") | upper_camel }}(this.{{ field_name | camel }}); 141 | {% endif %} 142 | {% if field_occurrence == "Optional" %} 143 | } 144 | {% endif %} 145 | {% endfor %} 146 | writer.stepOut(); 147 | } 148 | 149 | {% for inline_type in model.nested_types -%} 150 | {{ macros::nested_type(model=inline_type, is_nested=true) }} 151 | {% endfor -%} 152 | } 153 | {% endmacro model %} 154 | {{ self::class(model=model, is_nested=is_nested) }} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/java/enum.templ: -------------------------------------------------------------------------------- 1 | {% set full_namespace = namespace | map(attribute="Package") | join(sep=".") %} 2 | {% if is_nested == false %} 3 | package {{ full_namespace }}; 4 | import com.amazon.ion.IonReader; 5 | import com.amazon.ion.IonException; 6 | import com.amazon.ion.IonWriter; 7 | import com.amazon.ion.IonType; 8 | import java.io.IOException; 9 | {% endif %} 10 | 11 | {# Verify that the abstract data type is a enum and store information for this enum #} 12 | {% set enum_info = model.code_gen_type["Enum"] %} 13 | 14 | public {% if is_nested == true %} static {% endif %} enum {{ model.name }} { 15 | {% for variant in enum_info["variants"] -%} 16 | {{ variant | snake | upper }}("{{variant}}"), 17 | {% endfor %}; 18 | 19 | private String textValue; 20 | 21 | {{model.name}}(String textValue) { 22 | this.textValue = textValue; 23 | } 24 | 25 | /** 26 | * Writes a {{ model.name }} as Ion from an {@link IonWriter}. 27 | * 28 | * This method does not close the writer after writing is complete. 29 | * The caller is responsible for closing the stream associated with the writer. 30 | */ 31 | public void writeTo(IonWriter writer) throws IOException { 32 | writer.writeSymbol(this.textValue); 33 | } 34 | 35 | /** 36 | * Reads a {{ model.name }} from an {@link IonReader}. 37 | * 38 | * This method does not advance the reader at the current level. 39 | * The caller is responsible for positioning the reader on the value to read. 40 | */ 41 | public static {{ model.name }} readFrom(IonReader reader) { 42 | {# Enums are only supported for symbol types #} 43 | if (reader.getType() != IonType.SYMBOL) { 44 | throw new IonException("Expected symbol, found " + reader.getType() + " while reading {{ model.name }}"); 45 | } 46 | {# Reads given value as a string #} 47 | String value = reader.stringValue(); 48 | switch(value) { 49 | {% for variant in enum_info["variants"] %} 50 | case "{{ variant }}": 51 | return {{ variant | snake | upper }}; 52 | {% endfor %} 53 | default: 54 | throw new IonException(value + "is not a valid value for {{ model.name }}"); 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/java/nested_type.templ: -------------------------------------------------------------------------------- 1 | {% import "util_macros.templ" as util_macros %} 2 | 3 | {# following macro defines an anonymous type as children class for its parent type definition #} 4 | {% macro nested_type(model, is_nested) -%} 5 | {% if model.code_gen_type is containing("Structure")%} 6 | {% include "class.templ" %} 7 | {% elif model.code_gen_type is containing("Enum")%} 8 | {% include "enum.templ" %} 9 | {% endif %} 10 | {% endmacro nested_type -%} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/java/scalar.templ: -------------------------------------------------------------------------------- 1 | {% import "nested_type.templ" as macros %} 2 | 3 | {% macro scalar(model) %} 4 | {% set full_namespace = namespace | map(attribute="Package") | join(sep=".") %} 5 | 6 | package {{ full_namespace }}; 7 | import com.amazon.ion.IonReader; 8 | import com.amazon.ion.IonException; 9 | import com.amazon.ion.IonWriter; 10 | import com.amazon.ion.IonType; 11 | import java.io.IOException; 12 | 13 | {# Verify that the abstract data type is a scalar type and store information for this scalar value #} 14 | {% set scalar_info = model.code_gen_type["WrappedScalar"] %} 15 | {% set base_type = scalar_info["base_type"] | fully_qualified_type_name %} 16 | 17 | class {{ model.name }} { 18 | private {{ base_type }} value; 19 | 20 | public {{ model.name }}() {} 21 | 22 | public {{ base_type }} getValue() { 23 | return this.value; 24 | } 25 | 26 | public void setValue({{ base_type }} value) { 27 | this.value = value; 28 | return; 29 | } 30 | 31 | /** 32 | * Reads a {{ model.name }} from an {@link IonReader}. 33 | * 34 | * This method does not advance the reader at the current level. 35 | * The caller is responsible for positioning the reader on the value to read. 36 | */ 37 | public static {{ model.name }} readFrom(IonReader reader) { 38 | {# Initializes all the fields of this class #} 39 | {{ base_type }} value = 40 | {% if base_type == "boolean" %} 41 | false 42 | {% elif base_type == "int" or base_type == "double" %} 43 | 0 44 | {% else %} 45 | null 46 | {% endif %}; 47 | {# Reads `Value` class with a single field `value` #} 48 | value = {% if base_type | is_built_in_type %} 49 | {% if base_type == "bytes[]" %} 50 | reader.newBytes(); 51 | {% else %} 52 | reader.{{ base_type | camel }}Value(); 53 | {% endif %} 54 | {% else %} 55 | {{ base_type }}.readFrom(reader); 56 | {% endif %} 57 | {{ model.name }} {{ model.name | camel }} = new {{ model.name }}(); 58 | {{ model.name | camel }}.value = value; 59 | 60 | return {{ model.name | camel }}; 61 | } 62 | 63 | /** 64 | * Writes a {{ model.name }} as Ion from an {@link IonWriter}. 65 | * 66 | * This method does not close the writer after writing is complete. 67 | * The caller is responsible for closing the stream associated with the writer. 68 | */ 69 | public void writeTo(IonWriter writer) throws IOException { 70 | {# Writes `Value` class with a single field `value` as an Ion value #} 71 | {% if base_type | is_built_in_type == false %} 72 | this.value.writeTo(writer); 73 | {% else %} 74 | writer.write{{ base_type | replace(from="double", to="float") | replace(from="boolean", to="bool") | upper_camel }}(this.value); 75 | {% endif %} 76 | } 77 | } 78 | {% endmacro %} 79 | {{ self::scalar(model=model) }} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/java/sequence.templ: -------------------------------------------------------------------------------- 1 | {% import "nested_type.templ" as macros %} 2 | 3 | {% macro sequence(model) %} 4 | 5 | {% if is_nested == false %} 6 | {% set full_namespace = namespace | map(attribute="Package") | join(sep=".") %} 7 | 8 | package {{ full_namespace }}; 9 | import com.amazon.ion.IonReader; 10 | import com.amazon.ion.IonException; 11 | import com.amazon.ion.IonWriter; 12 | import com.amazon.ion.IonType; 13 | import java.io.IOException; 14 | {% endif %} 15 | 16 | {# Verify that the abstract data type is a sequence type and store information for this sequence value #} 17 | {% set sequence_info = model.code_gen_type["WrappedSequence"] %} 18 | 19 | class {{ model.name }} { 20 | private java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> value; 21 | 22 | public {{ model.name }}() {} 23 | 24 | public java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> getValue() { 25 | return this.value; 26 | } 27 | 28 | public void setValue(java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> value) { 29 | this.value = value; 30 | return; 31 | } 32 | 33 | /** 34 | * Reads a {{ model.name }} from an {@link IonReader}. 35 | * 36 | * This method does not advance the reader at the current level. 37 | * The caller is responsible for positioning the reader on the value to read. 38 | */ 39 | public static {{ model.name }} readFrom(IonReader reader) { 40 | {# Initializes all the fields of this class #} 41 | java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> value = new java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}>(); 42 | {# Reads `Sequence` class with a single field `value` that is an `ArrayList` #} 43 | if(reader.getType() != IonType.{{ sequence_info["sequence_type"] | upper }}) { 44 | throw new IonException("Expected {{ sequence_info["sequence_type"] }}, found " + reader.getType() + " while reading value."); 45 | } 46 | reader.stepIn(); 47 | {# Iterate through the `ArrayList` and read each element in it based on the data type provided in `sequence_info["sequence_type"]` #} 48 | while (reader.hasNext()) { 49 | reader.next(); 50 | {% if sequence_info["element_type"] |fully_qualified_type_name | is_built_in_type == false %} 51 | value.add({{ sequence_info["element_type"] | fully_qualified_type_name }}.readFrom(reader)); 52 | {% elif sequence_info["element_type"] | fully_qualified_type_name == "bytes[]" %} 53 | value.add(reader.newBytes()); 54 | {% else %} 55 | value.add(reader.{{ sequence_info["element_type"] | fully_qualified_type_name | camel }}Value()); 56 | {% endif %} 57 | } 58 | reader.stepOut(); 59 | {{ model.name }} {{ model.name | camel }} = new {{ model.name }}(); 60 | {{ model.name | camel }}.value = value; 61 | 62 | return {{ model.name | camel }}; 63 | } 64 | 65 | /** 66 | * Writes a {{ model.name }} as Ion from an {@link IonWriter}. 67 | * 68 | * This method does not close the writer after writing is complete. 69 | * The caller is responsible for closing the stream associated with the writer. 70 | */ 71 | public void writeTo(IonWriter writer) throws IOException { 72 | {# Writes `Sequence` class with a single field `value` that is an `ArrayList` as an Ion sequence #} 73 | writer.stepIn(IonType.{{ sequence_info["sequence_type"] | upper }}); 74 | for ({{ sequence_info["element_type"] | fully_qualified_type_name }} value: this.value) { 75 | {% if sequence_info["element_type"] | fully_qualified_type_name | is_built_in_type == false %} 76 | value.writeTo(writer); 77 | {% else %} 78 | writer.write{{ sequence_info["element_type"] | fully_qualified_type_name | replace(from="double", to="float") | replace(from="boolean", to="bool") | upper_camel }}(value); 79 | {% endif %} 80 | } 81 | writer.stepOut(); 82 | } 83 | 84 | {% for inline_type in model.nested_types -%} 85 | {{ macros::nested_type(model=inline_type, is_nested=true) }} 86 | {% endfor -%} 87 | } 88 | {% endmacro %} 89 | {{ self::sequence(model=model) }} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/java/util_macros.templ: -------------------------------------------------------------------------------- 1 | {# following macro defines statements to read a class field as sequence #} 2 | {% macro read_as_sequence(field_name, field_value, type_store, field_occurrence) %} 3 | ((java.util.function.Supplier<{{ field_value }}>) () -> { 4 | {% set field_value_model = type_store[field_value] %} 5 | {{ field_value }} {{ field_name | camel }}List = new {{ field_value }}(); 6 | {# Reads `Sequence` field that is an `ArrayList` #} 7 | if(reader.getType() != IonType.{{ field_value_model.code_gen_type["Sequence"].sequence_type | upper }}) { 8 | throw new IonException("Expected {{ field_value_model.code_gen_type["Sequence"].sequence_type }}, found " + reader.getType() + " while reading {{ field_name | camel }}."); 9 | } 10 | reader.stepIn(); 11 | {# Iterate through the `ArrayList` and read each element in it based on the data type provided in `field.abstract_data_type[Sequence]` #} 12 | while (reader.hasNext()) { 13 | reader.next(); 14 | {% if field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | is_built_in_type == false %} 15 | {{ field_name | camel }}List.add({{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name }}.readFrom(reader)); 16 | {% elif field_value_model.code_gen_type["Sequence"].element_type == "bytes[]" %} 17 | {{ field_name | camel }}List.add(reader.newBytes()); 18 | {% else %} 19 | {{ field_name | camel }}List.add(reader.{{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | camel }}Value()); 20 | {% endif %} 21 | } 22 | reader.stepOut(); 23 | return {{ field_name | camel }}List; 24 | }).get() 25 | {% endmacro %} 26 | {# following macro defines statements to write a class field as sequence #} 27 | {% macro write_as_sequence(field_name, field_value, type_store) %} 28 | {% set field_value_model = type_store[field_value] %} 29 | {# Writes `Sequence` field that is an `ArrayList` as an Ion sequence #} 30 | writer.stepIn(IonType.{{ field_value_model.code_gen_type["Sequence"].sequence_type | upper }}); 31 | for ({{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name }} value: this.{{ field_name |camel }}) { 32 | {% if field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | is_built_in_type == false %} 33 | value.writeTo(writer); 34 | {% else %} 35 | writer.write{{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | replace(from="double", to="float") | replace(from="boolean", to="bool") | upper_camel }}(value); 36 | {% endif %} 37 | } 38 | writer.stepOut(); 39 | {% endmacro %} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/mod.rs: -------------------------------------------------------------------------------- 1 | // This module includes constants that can be used to render templates for generating code. 2 | // Currently, there is no other way to add resources like `.templ` files in cargo binary crate. 3 | // Using these constants allows the binary to access templates through these constants. 4 | 5 | macro_rules! include_template { 6 | ($file:literal) => { 7 | include_str!(concat!( 8 | env!("CARGO_MANIFEST_DIR"), 9 | "/src/bin/ion/commands/generate/templates/", 10 | $file 11 | )) 12 | }; 13 | } 14 | 15 | /// Represents java template constants 16 | pub(crate) mod java { 17 | pub(crate) const CLASS: &str = include_template!("java/class.templ"); 18 | pub(crate) const SCALAR: &str = include_template!("java/scalar.templ"); 19 | pub(crate) const SEQUENCE: &str = include_template!("java/sequence.templ"); 20 | pub(crate) const ENUM: &str = include_template!("java/enum.templ"); 21 | pub(crate) const UTIL_MACROS: &str = include_template!("java/util_macros.templ"); 22 | pub(crate) const NESTED_TYPE: &str = include_template!("java/nested_type.templ"); 23 | } 24 | 25 | /// Represents rust template constants 26 | pub(crate) mod rust { 27 | pub(crate) const STRUCT: &str = include_template!("rust/struct.templ"); 28 | pub(crate) const SCALAR: &str = include_template!("rust/scalar.templ"); 29 | pub(crate) const SEQUENCE: &str = include_template!("rust/sequence.templ"); 30 | pub(crate) const ENUM: &str = include_template!("rust/enum.templ"); 31 | pub(crate) const UTIL_MACROS: &str = include_template!("rust/util_macros.templ"); 32 | pub(crate) const RESULT: &str = include_template!("rust/result.templ"); 33 | pub(crate) const NESTED_TYPE: &str = include_template!("rust/nested_type.templ"); 34 | pub(crate) const IMPORT: &str = include_template!("rust/import.templ"); 35 | } 36 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/enum.templ: -------------------------------------------------------------------------------- 1 | // Enum support is not yet completed for Rust code generation 2 | // This template is just used as placeholder for enums. 3 | 4 | 5 | use {{ model.name | snake }}::{{ model.name }}; 6 | pub mod {{ model.name | snake }} { 7 | use super::*; 8 | #[derive(Debug, Clone, Default)] 9 | pub enum {{ model.name }} { 10 | #[default] 11 | Unit // This is just a placeholder variant for enum generation 12 | } 13 | impl {{ model.name }} { 14 | pub fn read_from(reader: &mut Reader) -> SerdeResult { 15 | todo!("Enums are not supported with code generation yet!") 16 | } 17 | 18 | pub fn write_to(&self, writer: &mut W) -> SerdeResult<()> { 19 | todo!("Enums are not supported with code generation yet!") 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/import.templ: -------------------------------------------------------------------------------- 1 | use ion_rs::{IonResult, IonError, IonReader, Reader, IonWriter, StreamItem}; 2 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/nested_type.templ: -------------------------------------------------------------------------------- 1 | {% import "util_macros.templ" as util_macros %} 2 | 3 | {# following macro defines an anonymous type as children class for its parent type definition #} 4 | {% macro nested_type(model, is_nested) -%} 5 | {% if model.code_gen_type is containing("Structure")%} 6 | {% include "struct.templ" %} 7 | {% elif model.code_gen_type is containing("Enum") %} 8 | {% include "enum.templ" %} 9 | {% endif %} 10 | {% endmacro nested_type -%} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/result.templ: -------------------------------------------------------------------------------- 1 | /// Represents serde result 2 | pub type SerdeResult = Result; 3 | 4 | /// Represents an error found during code generation 5 | #[derive(Debug)] 6 | pub enum SerdeError { 7 | // Represents error found while reading or writing Ion data using Ion reader or writer. 8 | IonError { source: IonError }, 9 | // Represents error found while validating Ion data in `read_from` API for given data model. 10 | ValidationError { description: String }, 11 | } 12 | 13 | /// A convenience method for creating an SerdeError::ValidationError 14 | /// with the provided description text. 15 | pub fn validation_error>(description: S) -> SerdeResult { 16 | Err(SerdeError::ValidationError { 17 | description: description.as_ref().to_string(), 18 | }) 19 | } 20 | 21 | impl From for SerdeError { 22 | fn from(value: IonError) -> Self { 23 | SerdeError::IonError { source: value } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/scalar.templ: -------------------------------------------------------------------------------- 1 | {% import "nested_type.templ" as macros %} 2 | 3 | {# Verify that the abstract data type is a scalar type and store information for this scalar value #} 4 | {% set scalar_info = model.code_gen_type["WrappedScalar"] %} 5 | {% set base_type = scalar_info["base_type"] | fully_qualified_type_name %} 6 | 7 | use {{ model.name | snake }}::{{ model.name }}; 8 | pub mod {{ model.name | snake }} { 9 | use super::*; 10 | 11 | #[derive(Debug, Clone, Default)] 12 | pub struct {{ model.name }} { 13 | value: {{ base_type }}, 14 | } 15 | 16 | impl {{ model.name }} { 17 | pub fn new(value: {{ base_type }}) -> Self { 18 | Self { 19 | value, 20 | } 21 | } 22 | 23 | 24 | pub fn value(&self) -> &{{ base_type }} { 25 | &self.value 26 | } 27 | 28 | 29 | pub fn read_from(reader: &mut Reader) -> SerdeResult { 30 | let mut abstract_data_type = {{ model.name }}::default(); 31 | abstract_data_type.value = {% if base_type | is_built_in_type == false %} 32 | {{ base_type }}::read_from(reader)?; 33 | {% else %} 34 | reader.read_{% if field.source is defined and field.source == "symbol" %}symbol()?.text().unwrap(){% else %}{{ base_type | lower | replace(from="string", to ="str") }}()?{% endif %}{% if base_type| lower == "string" %} .to_string() {% endif %}; 35 | {% endif %} 36 | Ok(abstract_data_type) 37 | } 38 | 39 | pub fn write_to(&self, writer: &mut W) -> SerdeResult<()> { 40 | {% if base_type | is_built_in_type == false %} 41 | self.value.write_to(writer)?; 42 | {% else %} 43 | writer.write_{% if field.source is defined and field.source == "symbol" %}symbol{% else %}{{ base_type | lower }}{% endif %}(self.value.to_owned())?; 44 | {% endif %} 45 | Ok(()) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/sequence.templ: -------------------------------------------------------------------------------- 1 | {% import "nested_type.templ" as macros %} 2 | 3 | {% set sequence_info = model.code_gen_type["WrappedSequence"] %} 4 | 5 | 6 | use {{ model.name | snake }}::{{ model.name }}; 7 | 8 | pub mod {{ model.name | snake }} { 9 | use super::*; 10 | 11 | #[derive(Debug, Clone, Default)] 12 | pub struct {{ model.name }} { 13 | value: Vec<{{ sequence_info["element_type"] | fully_qualified_type_name }}>, 14 | } 15 | 16 | impl {{ model.name }} { 17 | pub fn new(value: Vec<{{ sequence_info["element_type"] | fully_qualified_type_name }}>) -> Self { 18 | Self { 19 | value, 20 | } 21 | } 22 | 23 | 24 | pub fn value(&self) -> &Vec<{{ sequence_info["element_type"] | fully_qualified_type_name }}> { 25 | &self.value 26 | } 27 | 28 | 29 | pub fn read_from(reader: &mut Reader) -> SerdeResult { 30 | let mut abstract_data_type = {{ model.name }}::default(); 31 | 32 | if reader.ion_type() != Some(IonType::{{ sequence_info["sequence_type"] }}) { 33 | return validation_error(format!( 34 | "Expected {{ sequence_info["sequence_type"] }}, found {} while reading {{ model.name }}.", reader.ion_type().unwrap() 35 | )); 36 | } 37 | 38 | reader.step_in()?; 39 | 40 | abstract_data_type.value = { 41 | let mut values = vec![]; 42 | 43 | while reader.next()? != StreamItem::Nothing { 44 | {% if sequence_info["element_type"] | fully_qualified_type_name | is_built_in_type == false %} 45 | values.push({{ sequence_info["element_type"] | fully_qualified_type_name }}::read_from(reader)?); 46 | {% else %} 47 | values.push(reader.read_{% if field.source is defined and field.source == "symbol" %}symbol()?.text().unwrap(){% else %}{{ sequence_info["element_type"] | fully_qualified_type_name | lower | replace(from="string", to ="str") }}()?{% endif %}{% if sequence_info["element_type"] | fully_qualified_type_name | lower== "string" %} .to_string() {% endif %}); 48 | {% endif %} 49 | } 50 | values 51 | }; 52 | reader.step_out()?; 53 | Ok(abstract_data_type) 54 | } 55 | 56 | pub fn write_to(&self, writer: &mut W) -> SerdeResult<()> { 57 | writer.step_in(IonType::{{ sequence_info["sequence_type"] }})?; 58 | for value in &self.value { 59 | {% if sequence_info["element_type"] | fully_qualified_type_name | is_built_in_type == false %} 60 | value.write_to(writer)?; 61 | {% else %} 62 | writer.write_{% if field.source is defined and field.source == "symbol" %}symbol{% else %}{{ sequence_info["element_type"] | fully_qualified_type_name | lower }}{% endif %}(value.to_owned())?; 63 | {% endif %} 64 | } 65 | writer.step_out()?; 66 | Ok(()) 67 | } 68 | } 69 | 70 | 71 | {% for inline_type in model.nested_types -%} 72 | {{ macros::nested_type(model=inline_type, is_nested=true) }} 73 | {% endfor -%} 74 | } 75 | -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/struct.templ: -------------------------------------------------------------------------------- 1 | {# Includes the macros for anonymous types that will be added as child classes #} 2 | {% import "nested_type.templ" as macros %} 3 | {% import "util_macros.templ" as util_macros %} 4 | 5 | {% macro struct(model, is_nested) %} 6 | {% set struct_info = model.code_gen_type["Structure"] %} 7 | 8 | use {{ model.name | snake }}::{{ model.name }}; 9 | pub mod {{ model.name | snake }} { 10 | use super::*; 11 | 12 | #[derive(Debug, Clone, Default)] 13 | pub struct {{ model.name }} { 14 | {% for field_name, field_value in struct_info["fields"] -%} 15 | {{ field_name | snake | indent(first = true) }}: {{ field_value.0 | fully_qualified_type_name }}, 16 | {% endfor %} 17 | } 18 | 19 | impl {{ model.name }} { 20 | pub fn new({% for field_name in struct_info["fields"] | field_names -%}{% set field_value = struct_info["fields"][field_name] %}{{ field_name | snake }}: {{ field_value.0 | fully_qualified_type_name }},{% endfor %}) -> Self { 21 | Self { 22 | {% for field_name, field_value in struct_info["fields"] -%} 23 | {{ field_name | snake }}, 24 | {% endfor %} 25 | } 26 | } 27 | 28 | 29 | {% for field_name, field_value in struct_info["fields"] -%}pub fn {{ field_name | snake }}(&self) -> &{{ field_value.0 | fully_qualified_type_name }} { 30 | &self.{{ field_name | snake }} 31 | } 32 | {% endfor %} 33 | 34 | 35 | pub fn read_from(reader: &mut Reader) -> SerdeResult { 36 | let mut abstract_data_type = {{ model.name }}::default(); 37 | 38 | reader.step_in()?; 39 | while reader.next()? != StreamItem::Nothing { 40 | if let Some(field_name) = reader.field_name()?.text() { 41 | match field_name { 42 | {% for field_name, field_val in struct_info["fields"] -%} 43 | {% set field_value = field_val.0 | fully_qualified_type_name %} 44 | {% if field_value | is_built_in_type == false %} 45 | {% if field_value is containing("Vec") %} 46 | "{{ field_name }}" => { {{ util_macros::read_as_sequence(field_value=field_value,field_name=field_name,type_store=type_store) }} } 47 | {% else %} 48 | "{{ field_name }}" => { abstract_data_type.{{ field_name | snake }} = {{ field_value }}::read_from(reader)?; } 49 | {% endif %} 50 | {% else %} 51 | "{{ field_name }}" => { abstract_data_type.{{ field_name | snake}} = reader.read_{% if field.source is defined and field.source == "symbol" %}symbol()?.text().unwrap(){% else %}{{ field_value | lower | replace(from="string", to ="str") }}()?{% endif %}{% if field_value | lower== "string" %} .to_string() {% endif %}; } 52 | {% endif %} 53 | {% endfor %} 54 | _ => { 55 | {% if abstract_data_type["Structure"] %} 56 | return validation_error( 57 | "Can not read field name:{{ field_name }} for {{ model.name }} as it doesn't exist in the given schema type definition." 58 | ); 59 | {% endif %} 60 | } 61 | } 62 | } 63 | } 64 | reader.step_out()?; 65 | Ok(abstract_data_type) 66 | } 67 | 68 | pub fn write_to(&self, writer: &mut W) -> SerdeResult<()> { 69 | writer.step_in(IonType::Struct)?; 70 | {% for field_name, field_val in struct_info["fields"] %} 71 | {% set field_value = field_val.0 | fully_qualified_type_name %} 72 | writer.set_field_name("{{ field_name }}"); 73 | {% if field_value | is_built_in_type == false %} 74 | {% if field_value is containing("Vec") %} 75 | {{ util_macros::write_as_sequence(field_value=field_value,field_name=field_name,type_store=type_store) }} 76 | {% else %} 77 | self.{{ field_name | snake }}.write_to(writer)?; 78 | {% endif %} 79 | {% else %} 80 | {# TODO: Change the following `to_owned` to only be used when writing i64,f32,f64,bool which require owned value as input #} 81 | writer.write_{% if field.source is defined and field.source == "symbol" %}symbol{% else %}{{ field_value | lower }}{% endif %}(self.{{ field_name | snake }}.to_owned())?; 82 | {% endif %} 83 | {% endfor %} 84 | writer.step_out()?; 85 | Ok(()) 86 | } 87 | } 88 | 89 | {% for inline_type in model.nested_types -%} 90 | {{ macros::nested_type(model=inline_type, is_nested=true) }} 91 | {% endfor -%} 92 | } 93 | {% endmacro struct %} 94 | {{ self::struct(model=model, is_nested=is_nested) }} -------------------------------------------------------------------------------- /src/bin/ion/commands/generate/templates/rust/util_macros.templ: -------------------------------------------------------------------------------- 1 | {# following macro defines statements to read a class field as sequence #} 2 | {% macro read_as_sequence(field_name, field_value, type_store) %} 3 | {% set field_value_model = type_store[field_value] %} 4 | 5 | if reader.ion_type() != Some(IonType::{{ field_value_model.code_gen_type["Sequence"].sequence_type }}) { 6 | return validation_error(format!( 7 | "Expected {{ field_value_model.code_gen_type["Sequence"].sequence_type }}, found {} while reading {{ field_name }}.", reader.ion_type().unwrap() 8 | )); 9 | } 10 | reader.step_in()?; 11 | 12 | abstract_data_type.{{ field_name | snake }} = { 13 | let mut values = vec![]; 14 | 15 | while reader.next()? != StreamItem::Nothing { 16 | {% if field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | is_built_in_type == false %} 17 | values.push({{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name }}::read_from(reader)?); 18 | {% else %} 19 | values.push(reader.read_{% if field.source is defined and field.source == "symbol" %}symbol()?.text().unwrap(){% else %}{{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | lower | replace(from="string", to ="str") }}()?{% endif %}{% if field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | lower== "string" %} .to_string() {% endif %}); 20 | {% endif %} 21 | } 22 | values 23 | }; 24 | reader.step_out()?; 25 | {% endmacro %} 26 | {# following macro defines statements to write a class field as sequence #} 27 | {% macro write_as_sequence(field_name, field_value, type_store) %} 28 | {% set field_value_model = type_store[field_value] %} 29 | writer.step_in(IonType::{{ field_value_model.code_gen_type["Sequence"].sequence_type }}); 30 | for value in &self.{{ field_name | snake }} { 31 | {% if field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | is_built_in_type == false %} 32 | value.write_to(writer)?; 33 | {% else %} 34 | writer.write_{% if field.source is defined and field.source == "symbol" %}symbol{% else %}{{ field_value_model.code_gen_type["Sequence"].element_type | fully_qualified_type_name | lower }}{% endif %}(value.to_owned())?; 35 | {% endif %} 36 | } 37 | writer.step_out()?; 38 | {% endmacro %} -------------------------------------------------------------------------------- /src/bin/ion/commands/hash.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; 2 | use anyhow::Result; 3 | use clap::builder::PossibleValue; 4 | use clap::{value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum}; 5 | use ion_rs::ion_hash::IonHasher; 6 | use ion_rs::*; 7 | use sha2::{Sha256, Sha512}; 8 | use sha3::{Sha3_256, Sha3_512}; 9 | use std::fmt; 10 | use std::io::Write; 11 | 12 | // Macro to eliminate repetitive code for each hash algorithm. 13 | macro_rules! supported_hash_functions { 14 | ($($name:literal => $hash:ident),+$(,)?) => { 15 | #[derive(Copy, Clone, PartialEq, Eq, Default, Debug)] 16 | enum DigestType { 17 | #[default] 18 | $($hash),+ 19 | } 20 | impl DigestType { 21 | const VARIANTS: &'static [DigestType] = &[ 22 | $(DigestType::$hash),+ 23 | ]; 24 | 25 | fn hash_it(&self, element: &Element) -> IonResult> { 26 | match &self { 27 | $(DigestType::$hash => Ok($hash::hash_element(&element)?.to_vec()),)+ 28 | } 29 | } 30 | } 31 | impl ValueEnum for DigestType { 32 | fn value_variants<'a>() -> &'a [Self] { 33 | DigestType::VARIANTS 34 | } 35 | 36 | fn to_possible_value(&self) -> Option { 37 | match self { 38 | $(DigestType::$hash => Some($name.into()),)+ 39 | } 40 | } 41 | } 42 | }; 43 | } 44 | 45 | supported_hash_functions! { 46 | "sha-256" => Sha256, 47 | "sha-512" => Sha512, 48 | "sha3-256" => Sha3_256, 49 | "sha3-512" => Sha3_512, 50 | } 51 | 52 | pub struct HashCommand; 53 | 54 | impl IonCliCommand for HashCommand { 55 | fn name(&self) -> &'static str { 56 | "hash" 57 | } 58 | 59 | fn about(&self) -> &'static str { 60 | "Calculates a hash of Ion values using the Ion Hash algorithm." 61 | } 62 | 63 | fn is_stable(&self) -> bool { 64 | false 65 | } 66 | 67 | fn is_porcelain(&self) -> bool { 68 | false 69 | } 70 | 71 | fn configure_args(&self, command: Command) -> Command { 72 | command 73 | .arg( 74 | Arg::new("hash") 75 | .required(true) 76 | .value_parser(value_parser!(DigestType)), 77 | ) 78 | .with_output() 79 | .with_input() 80 | // TODO: If we want to support other output formats, add flags for them 81 | // and an ArgGroup to ensure only one is selected. 82 | // Default right now is to emit base16 strings of the digest. 83 | .arg( 84 | Arg::new("blob") 85 | .long("blob") 86 | .help("Emit the digest(s) as Ion blob values.") 87 | .action(ArgAction::SetTrue), 88 | ) 89 | } 90 | 91 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 92 | CommandIo::new(args)?.for_each_input(|output, input| { 93 | let mut reader = Reader::new(AnyEncoding, input.into_source())?; 94 | 95 | let hasher = if let Some(hasher) = args.get_one::("hash") { 96 | hasher 97 | } else { 98 | unreachable!("clap ensures that there is a valid argument") 99 | }; 100 | 101 | if args.get_flag("blob") { 102 | let mut writer = Writer::new(v1_0::Text.with_format(TextFormat::Lines), output)?; 103 | for elem in reader.elements() { 104 | let elem = elem?; 105 | let digest = hasher.hash_it(&elem)?; 106 | writer.write_blob(&digest)?; 107 | } 108 | writer.close()?; 109 | } else { 110 | for elem in reader.elements() { 111 | let elem = elem?; 112 | let digest = hasher.hash_it(&elem)?; 113 | let digest_string = digest.iter().fold( 114 | String::with_capacity(digest.len() * 2), 115 | |mut string, byte| { 116 | use fmt::Write; 117 | write!(&mut string, "{:02x}", byte).expect("infallible"); 118 | string 119 | }, 120 | ); 121 | output.write_all(digest_string.as_bytes())?; 122 | output.write_all("\n".as_bytes())?; 123 | } 124 | } 125 | Ok(()) 126 | }) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/bin/ion/commands/head.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::{value_parser, Arg, ArgMatches, Command}; 3 | use ion_rs::{AnyEncoding, Reader}; 4 | 5 | use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; 6 | use crate::transcribe::write_n_as; 7 | 8 | pub struct HeadCommand; 9 | 10 | impl IonCliCommand for HeadCommand { 11 | fn name(&self) -> &'static str { 12 | "head" 13 | } 14 | 15 | fn about(&self) -> &'static str { 16 | "Prints the specified number of top-level values in the input stream." 17 | } 18 | 19 | fn is_stable(&self) -> bool { 20 | true 21 | } 22 | 23 | fn is_porcelain(&self) -> bool { 24 | false 25 | } 26 | 27 | fn configure_args(&self, command: Command) -> Command { 28 | // Same flags as `cat`, but with an added `--values` flag to specify the number of values to 29 | // write. 30 | command 31 | .with_input() 32 | .with_output() 33 | .with_format() 34 | .with_ion_version() 35 | .arg( 36 | Arg::new("values") 37 | .long("values") 38 | .short('n') 39 | .value_parser(value_parser!(usize)) 40 | .allow_negative_numbers(false) 41 | .default_value("10") 42 | .help("Specifies the number of output top-level values."), 43 | ) 44 | } 45 | 46 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 47 | //TODO: Multiple file handling in classic `head` includes a header per file. 48 | // https://github.com/amazon-ion/ion-cli/issues/48 49 | 50 | let num_values = *args.get_one::("values").unwrap(); 51 | 52 | CommandIo::new(args)?.for_each_input(|output, input| { 53 | let mut reader = Reader::new(AnyEncoding, input.into_source())?; 54 | let encoding = *output.encoding(); 55 | let format = *output.format(); 56 | write_n_as(&mut reader, output, encoding, format, num_values)?; 57 | Ok(()) 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/bin/ion/commands/primitive.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::IonCliCommand; 2 | use anyhow::{Context, Result}; 3 | use clap::{Arg, ArgMatches, Command}; 4 | use ion_rs::v1_0::{VarInt, VarUInt}; 5 | 6 | pub struct PrimitiveCommand; 7 | 8 | impl IonCliCommand for PrimitiveCommand { 9 | fn name(&self) -> &'static str { 10 | "primitive" 11 | } 12 | 13 | fn about(&self) -> &'static str { 14 | "Prints the binary representation of an Ion encoding primitive." 15 | } 16 | 17 | fn is_stable(&self) -> bool { 18 | false 19 | } 20 | 21 | fn is_porcelain(&self) -> bool { 22 | true 23 | } 24 | 25 | fn configure_args(&self, command: Command) -> Command { 26 | command 27 | .arg( 28 | Arg::new("type") 29 | .short('t') 30 | .required(true) 31 | .help("The Ion primitive encoding type. (Names are case insensitive.)") 32 | .value_parser(["VarInt", "varint", "VarUInt", "varuint"]), 33 | ) 34 | .arg( 35 | Arg::new("value") 36 | .short('v') 37 | .required(true) 38 | .allow_hyphen_values(true) 39 | .help("The value to encode as the specified primitive."), 40 | ) 41 | } 42 | 43 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 44 | let mut buffer = Vec::new(); 45 | let value_text = args.get_one::("value").unwrap().as_str(); 46 | match args.get_one::("type").unwrap().as_str() { 47 | "varuint" | "VarUInt" => { 48 | let value = integer_from_text(value_text)? as u64; 49 | VarUInt::write_u64(&mut buffer, value).unwrap(); 50 | } 51 | "varint" | "VarInt" => { 52 | let value = integer_from_text(value_text)?; 53 | VarInt::write_i64(&mut buffer, value).unwrap(); 54 | } 55 | unsupported => { 56 | unreachable!( 57 | "clap did not reject unsupported primitive encoding {}", 58 | unsupported 59 | ); 60 | } 61 | } 62 | print!("hex: "); 63 | for byte in buffer.iter() { 64 | // We want the hex bytes to align with the binary bytes that will be printed on the next 65 | // line. Print 6 spaces and a 2-byte hex representation of the byte. 66 | print!(" {:0>2x} ", byte); 67 | } 68 | println!(); 69 | print!("bin: "); 70 | for byte in buffer.iter() { 71 | // Print the binary representation of each byte 72 | print!("{:0>8b} ", byte); 73 | } 74 | println!(); 75 | Ok(()) 76 | } 77 | } 78 | 79 | fn integer_from_text(text: &str) -> Result { 80 | if text.starts_with("0x") { 81 | i64::from_str_radix(text, 16) 82 | .with_context(|| format!("{} is not a valid hexadecimal integer value.", text)) 83 | } else if text.starts_with("0b") { 84 | i64::from_str_radix(text, 2) 85 | .with_context(|| format!("{} is not a valid binary integer value.", text)) 86 | } else { 87 | text.parse::() 88 | .with_context(|| format!("{} is not a valid decimal integer value.", text)) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/bin/ion/commands/schema/check.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::schema::IonSchemaCommandInput; 2 | use crate::commands::IonCliCommand; 3 | use anyhow::Result; 4 | use clap::{Arg, ArgAction, ArgMatches, Command}; 5 | 6 | pub struct CheckCommand; 7 | 8 | impl IonCliCommand for CheckCommand { 9 | fn name(&self) -> &'static str { 10 | "check" 11 | } 12 | 13 | fn about(&self) -> &'static str { 14 | "Loads a schema and checks it for problems." 15 | } 16 | 17 | fn is_stable(&self) -> bool { 18 | false 19 | } 20 | 21 | fn configure_args(&self, command: Command) -> Command { 22 | command.args(IonSchemaCommandInput::schema_args()).arg( 23 | Arg::new("show-debug") 24 | .short('D') 25 | .long("show-debug") 26 | .action(ArgAction::SetTrue), 27 | ) 28 | } 29 | 30 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 31 | let ion_schema_input = IonSchemaCommandInput::read_from_args(args)?; 32 | let schema = ion_schema_input.get_schema(); 33 | if args.get_flag("show-debug") { 34 | println!("Schema: {:#?}", schema); 35 | } 36 | Ok(()) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/bin/ion/commands/schema/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod check; 2 | pub mod validate; 3 | 4 | use crate::commands::command_namespace::IonCliNamespace; 5 | use crate::commands::schema::check::CheckCommand; 6 | use crate::commands::schema::validate::ValidateCommand; 7 | use crate::commands::IonCliCommand; 8 | use anyhow::Context; 9 | use clap::{Arg, ArgAction, ArgMatches, ValueHint}; 10 | use ion_rs::Element; 11 | use ion_schema::authority::{DocumentAuthority, FileSystemDocumentAuthority}; 12 | use ion_schema::schema::Schema; 13 | use ion_schema::system::SchemaSystem; 14 | use ion_schema::types::TypeDefinition; 15 | use std::fs; 16 | use std::path::Path; 17 | use std::sync::Arc; 18 | 19 | pub struct SchemaNamespace; 20 | 21 | impl IonCliNamespace for SchemaNamespace { 22 | fn name(&self) -> &'static str { 23 | "schema" 24 | } 25 | 26 | fn about(&self) -> &'static str { 27 | "The 'schema' command is a namespace for commands that are related to Ion Schema." 28 | } 29 | 30 | fn subcommands(&self) -> Vec> { 31 | vec![ 32 | Box::new(CheckCommand), 33 | Box::new(ValidateCommand), 34 | // TODO: Filter values command? 35 | // TODO: Compare types command? 36 | // TODO: Canonical representation of types command? 37 | ] 38 | } 39 | } 40 | 41 | /// A type that encapsulates the arguments for loading schemas and types. 42 | /// 43 | /// This allows users to specify file authorities, schema files, schema ids, inline schemas, and types. 44 | /// 45 | /// See [CheckCommand] and [ValidateCommand] for example usages. 46 | struct IonSchemaCommandInput { 47 | schema_system: SchemaSystem, 48 | schema: Arc, 49 | type_definition: Option, 50 | } 51 | 52 | impl IonSchemaCommandInput { 53 | fn read_from_args(args: &ArgMatches) -> anyhow::Result { 54 | // Extract the user provided document authorities/ directories 55 | let mut authorities: Vec> = vec![]; 56 | args.get_many::("authority") 57 | .unwrap_or_default() 58 | .map(Path::new) 59 | .map(FileSystemDocumentAuthority::new) 60 | .for_each(|a| authorities.push(Box::new(a))); 61 | 62 | // Create a new schema system from given document authorities 63 | let mut schema_system = SchemaSystem::new(authorities); 64 | 65 | // Load the appropriate schema 66 | let mut empty_schema_version = None; 67 | let mut schema = if args.contains_id("schema-id") { 68 | let schema_id = args.get_one::("schema").unwrap(); 69 | schema_system.load_schema(schema_id)? 70 | } else if args.contains_id("schema-file") { 71 | let file_name = args.get_one::("schema-file").unwrap(); 72 | let content = fs::read(file_name)?; 73 | schema_system.new_schema(&content, "user-provided-schema")? 74 | } else if args.contains_id("schema-text") { 75 | let content = args.get_one::<&str>("schema-text").unwrap(); 76 | schema_system.new_schema(content.as_bytes(), "user-provided-schema")? 77 | } else { 78 | let version = match args.get_one::("empty-schema") { 79 | Some(version) if version == "1.0" => "$ion_schema_1_0", 80 | _ => "$ion_schema_2_0", 81 | }; 82 | empty_schema_version = Some(version); 83 | schema_system 84 | .new_schema(version.as_bytes(), "empty-schema") 85 | .expect("Creating an empty schema should be effectively infallible.") 86 | }; 87 | 88 | // Get the type definition, if the command uses the type-ref arg and a value is provided. 89 | // Clap ensures that if `type` is required, the user must have provided it, so we don't 90 | // have to check the case where the command uses the arg but no value is provided. 91 | let mut type_definition = None; 92 | if let Ok(Some(type_name_or_inline_type)) = args.try_get_one::("type-ref") { 93 | // We allow an inline type when there's an empty schema. 94 | // The easiest way to determine whether this is an inline type or a type name 95 | // is to just try to get it from the schema. If nothing is found, then we'll attempt 96 | // to treat it as an inline type. 97 | type_definition = schema.get_type(type_name_or_inline_type); 98 | 99 | if type_definition.is_none() && empty_schema_version.is_some() { 100 | let version = empty_schema_version.unwrap(); 101 | // There is no convenient way to add a type to an existing schema, so we'll 102 | // construct a new one. 103 | 104 | // Create a symbol element so that ion-rs handle escaping any special characters. 105 | let type_name = Element::symbol(type_name_or_inline_type); 106 | 107 | let new_schema = format!( 108 | r#" 109 | {version} 110 | type::{{ 111 | name: {type_name}, 112 | type: {type_name_or_inline_type} 113 | }} 114 | "# 115 | ); 116 | // And finally update the schema and type. 117 | schema = schema_system.new_schema(new_schema.as_bytes(), "new-schema")?; 118 | type_definition = schema.get_type(type_name_or_inline_type); 119 | } 120 | 121 | // Validate that the user didn't pass in an invalid type 122 | type_definition 123 | .as_ref() 124 | .with_context(|| format!("Type not found {}", type_name_or_inline_type))?; 125 | } 126 | 127 | Ok(IonSchemaCommandInput { 128 | schema_system, 129 | schema, 130 | type_definition, 131 | }) 132 | } 133 | 134 | // If this ever gets used, the `expect` will cause a compiler error so the developer will 135 | // know to come remove this. 136 | #[expect(dead_code)] 137 | fn get_schema_system(&self) -> &SchemaSystem { 138 | &self.schema_system 139 | } 140 | 141 | fn get_schema(&self) -> Arc { 142 | self.schema.clone() 143 | } 144 | 145 | /// Guaranteed to be Some if the command uses the `type-ref` argument and that argument is required. 146 | fn get_type(&self) -> Option<&TypeDefinition> { 147 | self.type_definition.as_ref() 148 | } 149 | 150 | fn type_arg() -> Arg { 151 | Arg::new("type-ref") 152 | .required(true) 153 | .value_name("type") 154 | .help("An ISL type name or, if no schema is specified, an inline type definition.") 155 | } 156 | 157 | fn schema_args() -> Vec { 158 | let schema_options_header = "Selecting a schema"; 159 | let schema_options_group_name = "schema-group"; 160 | vec![ 161 | Arg::new("empty-schema") 162 | .help_heading(schema_options_header) 163 | .group(schema_options_group_name) 164 | .long("empty") 165 | // This is the default if no schema is specified, so we don't need a short flag. 166 | .action(ArgAction::Set) 167 | .value_name("version") 168 | .value_parser(["1.0", "2.0"]) 169 | .default_value("2.0") 170 | .help("An empty schema document for the specified Ion Schema version."), 171 | Arg::new("schema-file") 172 | .help_heading(schema_options_header) 173 | .group(schema_options_group_name) 174 | .long("schema-file") 175 | .short('f') 176 | .action(ArgAction::Set) 177 | .value_hint(ValueHint::FilePath) 178 | .help("A schema file"), 179 | Arg::new("schema-text") 180 | .help_heading(schema_options_header) 181 | .group(schema_options_group_name) 182 | .long("schema-text") 183 | .action(ArgAction::Set) 184 | .help("The Ion text contents of a schema document."), 185 | Arg::new("schema-id") 186 | .help_heading(schema_options_header) 187 | .group(schema_options_group_name) 188 | .long("id") 189 | .requires("authority") 190 | .action(ArgAction::Set) 191 | .help("The ID of a schema to load from one of the configured authorities."), 192 | Arg::new("authority") 193 | .help_heading(schema_options_header) 194 | .long("authority") 195 | .short('A') 196 | .required(false) 197 | .action(ArgAction::Append) 198 | .value_name("directory") 199 | .value_hint(ValueHint::DirPath) 200 | .help( 201 | "The root(s) of the file system authority(s). Authorities are only required if your \ 202 | schema needs to import a type from another schema or if you are loading a schema using \ 203 | the --id option.", 204 | ), 205 | ] 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /src/bin/ion/commands/stats.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; 2 | use anyhow::Result; 3 | use clap::{Arg, ArgMatches, Command}; 4 | use ion_rs::*; 5 | use ion_rs::{AnyEncoding, IonInput, SystemReader, SystemStreamItem}; 6 | use lowcharts::plot; 7 | use std::cmp::max; 8 | 9 | pub struct StatsCommand; 10 | 11 | impl IonCliCommand for StatsCommand { 12 | fn is_porcelain(&self) -> bool { 13 | false 14 | } 15 | 16 | fn name(&self) -> &'static str { 17 | "stats" 18 | } 19 | 20 | fn is_stable(&self) -> bool { 21 | false 22 | } 23 | 24 | fn about(&self) -> &'static str { 25 | "Print statistics about an Ion stream" 26 | } 27 | 28 | fn configure_args(&self, command: Command) -> Command { 29 | command 30 | .long_about("Print the analysis report of the input data stream, including the total number of\n\ 31 | top-level values, their minimum, maximum, and mean sizes, and plot the size distribution of\n\ 32 | the input stream. The report should also include the number of symbol tables in the input\n\ 33 | stream, the total number of different symbols that occurred in the input stream, and the\n\ 34 | maximum depth of the input data stream. Currently, this subcommand only supports data\n\ 35 | analysis on binary Ion data.") 36 | .with_input() 37 | .with_output() 38 | .arg( 39 | Arg::new("count") 40 | .long("count") 41 | .short('n') 42 | .num_args(0) 43 | .help("Emit only the count of items for each supplied stream"), 44 | ) 45 | } 46 | 47 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 48 | CommandIo::new(args)?.for_each_input(|_output, input| { 49 | let mut reader = SystemReader::new(AnyEncoding, input.into_source()); 50 | analyze(&mut reader, &mut std::io::stdout(), args) 51 | }) 52 | } 53 | } 54 | 55 | #[derive(Debug, Clone, PartialEq)] 56 | struct StreamStats { 57 | size_vec: Vec, 58 | symtab_count: i32, 59 | symbols_count: usize, 60 | max_depth: usize, 61 | unparseable_count: usize, 62 | } 63 | 64 | fn analyze( 65 | reader: &mut SystemReader, 66 | mut writer: impl std::io::Write, 67 | args: &ArgMatches, 68 | ) -> Result<()> { 69 | let stats = analyze_data_stream(reader)?; 70 | // Plot a histogram of the above vector, with 4 buckets and a precision 71 | // chosen by library. The number of buckets could be changed as needed. 72 | let options = plot::HistogramOptions { 73 | intervals: 4, 74 | ..Default::default() 75 | }; 76 | let histogram = plot::Histogram::new(&stats.size_vec, options); 77 | 78 | if args.get_flag("count") { 79 | writeln!(writer, "{}", stats.size_vec.len())?; 80 | return Ok(()); 81 | } else { 82 | writeln!( 83 | writer, 84 | "'samples' is the number of top-level values for the input stream." 85 | )?; 86 | writeln!(writer, "The unit of min, max, and avg size is bytes.")?; 87 | writeln!(writer, "{}", histogram)?; 88 | writeln!(writer, "Symbols: {} ", stats.symbols_count)?; 89 | writeln!(writer, "Local symbol tables: {} ", stats.symtab_count)?; 90 | writeln!(writer, "Maximum container depth: {}", stats.max_depth)?; 91 | if stats.unparseable_count > 0 { 92 | writeln!(writer, "Unparseable values: {}", stats.unparseable_count)?; 93 | } 94 | } 95 | 96 | Ok(()) 97 | } 98 | 99 | fn analyze_data_stream( 100 | reader: &mut SystemReader, 101 | ) -> Result { 102 | let mut size_vec: Vec = Vec::new(); 103 | let mut symtab_count = 0; 104 | let mut max_depth = 0; 105 | let mut unparseable_count = 0; 106 | 107 | loop { 108 | let system_result = reader.next_item(); 109 | use SystemStreamItem::*; 110 | 111 | match system_result { 112 | Err(e) => { 113 | unparseable_count += 1; 114 | if matches!(e, IonError::Incomplete(..)) { 115 | break; 116 | } 117 | } 118 | 119 | Ok(item) => match item { 120 | EndOfStream(_) => break, 121 | VersionMarker(_) | EncodingDirective(_) => continue, 122 | SymbolTable(_) => symtab_count += 1, 123 | system_value @ Value(raw_value) => { 124 | let size = system_value 125 | .raw_stream_item() 126 | .map(|v| v.span().bytes().len()) 127 | .unwrap_or(0); // 1.1 values may not have any physical representation 128 | size_vec.push(size as f64); 129 | let current_depth = top_level_max_depth(raw_value)?; 130 | max_depth = max(max_depth, current_depth); 131 | } 132 | // SystemStreamItem is non_exhaustive 133 | unsupported => panic!("Unsupported system stream item: {unsupported:?}"), 134 | }, 135 | } 136 | } 137 | 138 | // Reduce the number of shared symbols. 139 | let version = reader.detected_encoding().version(); 140 | let system_symbols_offset = if version == IonVersion::v1_0 { 141 | version.system_symbol_table().len() 142 | } else { 143 | 0 // 1.1 system symbols are opt-in, it's fair to count them if they are present 144 | }; 145 | 146 | let symbols_count = reader.symbol_table().len() - system_symbols_offset; 147 | 148 | Ok(StreamStats { 149 | size_vec, 150 | symtab_count, 151 | symbols_count, 152 | max_depth, 153 | unparseable_count, 154 | }) 155 | } 156 | 157 | fn top_level_max_depth(value: LazyValue) -> Result { 158 | let mut max_depth = 0; 159 | let mut stack = vec![(value, 0)]; 160 | while let Some((current_value, depth)) = stack.pop() { 161 | max_depth = max(max_depth, depth); 162 | use ValueRef::*; 163 | match current_value.read()? { 164 | Struct(s) => { 165 | for field in s { 166 | stack.push((field?.value(), depth + 1)); 167 | } 168 | } 169 | List(s) => { 170 | for element in s { 171 | stack.push((element?, depth + 1)); 172 | } 173 | } 174 | SExp(s) => { 175 | for element in s { 176 | stack.push((element?, depth + 1)); 177 | } 178 | } 179 | _ => continue, 180 | } 181 | } 182 | Ok(max_depth) 183 | } 184 | 185 | #[test] 186 | fn test_analyze() -> Result<()> { 187 | let expect_out = StreamStats { 188 | // The expected size values are generated from ion inspect. 189 | size_vec: Vec::from([11.0, 16.0, 7.0, 7.0]), 190 | symtab_count: 4, 191 | symbols_count: 8, 192 | max_depth: 2, 193 | unparseable_count: 0, 194 | }; 195 | let test_data: &str = r#" 196 | { 197 | foo: bar, 198 | abc: [123, 456] 199 | } 200 | { 201 | foo: baz, 202 | abc: [42.0, 43e0] 203 | } 204 | { 205 | foo: bar, 206 | test: data 207 | } 208 | { 209 | foo: baz, 210 | type: struct 211 | } 212 | "#; 213 | 214 | let buffer = { 215 | let mut buffer = Vec::new(); 216 | let mut writer = Writer::new(v1_0::Binary, &mut buffer)?; 217 | for element in Reader::new(AnyEncoding, test_data.as_bytes())?.elements() { 218 | writer.write_element(&element.unwrap())?; 219 | writer.flush()?; 220 | } 221 | buffer 222 | }; 223 | let mut reader = SystemReader::new(AnyEncoding, buffer.as_slice()); 224 | let stats = analyze_data_stream(&mut reader)?; 225 | assert_eq!(stats, expect_out); 226 | Ok(()) 227 | } 228 | -------------------------------------------------------------------------------- /src/bin/ion/commands/symtab/filter.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | 3 | use anyhow::{bail, Result}; 4 | use clap::{Arg, ArgAction, ArgMatches, Command}; 5 | use ion_rs::*; 6 | 7 | use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; 8 | use crate::output::CommandOutput; 9 | 10 | pub struct SymtabFilterCommand; 11 | 12 | impl IonCliCommand for SymtabFilterCommand { 13 | fn name(&self) -> &'static str { 14 | "filter" 15 | } 16 | 17 | fn about(&self) -> &'static str { 18 | "Filters user data out of an Ion stream, leaving only the symbol table(s) behind." 19 | } 20 | 21 | fn is_stable(&self) -> bool { 22 | false 23 | } 24 | 25 | fn is_porcelain(&self) -> bool { 26 | false 27 | } 28 | 29 | fn configure_args(&self, command: Command) -> Command { 30 | command.with_input() 31 | .with_output() 32 | .arg(Arg::new("lift") 33 | .long("lift") 34 | .short('l') 35 | .required(false) 36 | .action(ArgAction::SetTrue) 37 | .help("Remove the `$ion_symbol_table` annotation from symtabs, turning them into visible user data") 38 | ) 39 | } 40 | 41 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 42 | let lift_requested = args.get_flag("lift"); 43 | CommandIo::new(args)?.for_each_input(|output, input| { 44 | let mut system_reader = SystemReader::new(AnyEncoding, input.into_source()); 45 | filter_out_user_data(&mut system_reader, output, lift_requested) 46 | }) 47 | } 48 | } 49 | 50 | pub fn filter_out_user_data( 51 | reader: &mut SystemReader, 52 | output: &mut CommandOutput, 53 | lift_requested: bool, 54 | ) -> Result<()> { 55 | loop { 56 | match reader.next_item()? { 57 | SystemStreamItem::VersionMarker(marker) => { 58 | output.write_all(marker.span().bytes())?; 59 | } 60 | SystemStreamItem::SymbolTable(symtab) => { 61 | let Some(raw_value) = symtab.as_value().raw() else { 62 | // This symbol table came from a macro expansion; there are no encoded bytes 63 | // to pass through. 64 | bail!("found an ephemeral symbol table, which is not yet supported") 65 | }; 66 | if lift_requested { 67 | // Only pass through the value portion of the symbol table, stripping off the 68 | // `$ion_symbol_table` annotation. 69 | output.write_all(raw_value.value_span().bytes())?; 70 | } else { 71 | // Pass through the complete symbol table, preserving the `$ion_symbol_table` 72 | // annotation. 73 | output.write_all(raw_value.span().bytes())?; 74 | } 75 | } 76 | SystemStreamItem::Value(_) => continue, 77 | SystemStreamItem::EndOfStream(_) => { 78 | return Ok(()); 79 | } 80 | _ => unreachable!("#[non_exhaustive] enum, current variants covered"), 81 | }; 82 | // If this is a text encoding, then we need delimiting space to separate 83 | // IVMs from their neighboring system stream items. Consider: 84 | // $ion_1_0$ion_1_0 85 | // or 86 | // $ion_symbol_table::{}$ion_1_0$ion_symbol_table::{} 87 | if reader.detected_encoding().is_text() { 88 | output.write_all(b"\n").unwrap() 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/bin/ion/commands/symtab/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::command_namespace::IonCliNamespace; 2 | use crate::commands::symtab::filter::SymtabFilterCommand; 3 | use crate::commands::IonCliCommand; 4 | 5 | pub mod filter; 6 | 7 | pub struct SymtabNamespace; 8 | 9 | impl IonCliNamespace for SymtabNamespace { 10 | fn name(&self) -> &'static str { 11 | "symtab" 12 | } 13 | 14 | fn about(&self) -> &'static str { 15 | "'symtab' is a namespace for commands that operate on symbol tables" 16 | } 17 | 18 | fn subcommands(&self) -> Vec> { 19 | vec![Box::new(SymtabFilterCommand)] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/bin/ion/commands/to/json.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | use std::str::FromStr; 3 | 4 | use anyhow::{Context, Result}; 5 | use clap::{ArgMatches, Command}; 6 | use ion_rs::*; 7 | use serde_json::{Map, Number, Value as JsonValue}; 8 | use zstd::zstd_safe::WriteBuf; 9 | 10 | use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; 11 | use crate::output::CommandOutput; 12 | 13 | pub struct ToJsonCommand; 14 | 15 | impl IonCliCommand for ToJsonCommand { 16 | fn name(&self) -> &'static str { 17 | "json" 18 | } 19 | 20 | fn about(&self) -> &'static str { 21 | "Converts Ion data to JSON." 22 | } 23 | 24 | fn is_stable(&self) -> bool { 25 | false 26 | } 27 | 28 | fn is_porcelain(&self) -> bool { 29 | false 30 | } 31 | 32 | fn configure_args(&self, command: Command) -> Command { 33 | // NOTE: it may be necessary to add format-specific options. For example, a "pretty" option 34 | // would make sense for JSON, but not binary formats like CBOR. 35 | command.with_input().with_output() 36 | } 37 | 38 | fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 39 | CommandIo::new(args)?.for_each_input(|output, input| { 40 | let input_name = input.name().to_owned(); 41 | let mut reader = Reader::new(AnyEncoding, input.into_source()) 42 | .with_context(|| format!("Input file '{}' was not valid Ion.", input_name))?; 43 | convert(&mut reader, output) 44 | }) 45 | } 46 | } 47 | 48 | pub fn convert( 49 | reader: &mut Reader, 50 | output: &mut CommandOutput, 51 | ) -> Result<()> { 52 | const FLUSH_EVERY_N: usize = 100; 53 | let mut value_count = 0usize; 54 | while let Some(value) = reader.next()? { 55 | writeln!(output, "{}", to_json_value(value)?)?; 56 | value_count += 1; 57 | if value_count % FLUSH_EVERY_N == 0 { 58 | output.flush()?; 59 | } 60 | } 61 | Ok(()) 62 | } 63 | 64 | fn to_json_value(value: LazyValue) -> Result { 65 | use ValueRef::*; 66 | let value = match value.read()? { 67 | Null(_) => JsonValue::Null, 68 | Bool(b) => JsonValue::Bool(b), 69 | Int(i) => JsonValue::Number(Number::from(i.expect_i128()?)), 70 | Float(f) if f.is_finite() => JsonValue::Number(Number::from_f64(f).expect("f64 is finite")), 71 | // Special floats like +inf, -inf, and NaN are written as `null` in 72 | // accordance with Ion's JSON down-conversion guidelines. 73 | Float(_f) => JsonValue::Null, 74 | Decimal(d) => { 75 | let mut text = d.to_string().replace('d', "e"); 76 | if text.ends_with('.') { 77 | // If there's a trailing "." with no digits of precision, discard it. JSON's `Number` 78 | // type does not do anything with this information. 79 | let _ = text.pop(); 80 | } 81 | JsonValue::Number( 82 | Number::from_str(text.as_str()) 83 | .with_context(|| format!("{d} could not be turned into a Number"))?, 84 | ) 85 | } 86 | Timestamp(t) => JsonValue::String(t.to_string()), 87 | Symbol(s) => s 88 | .text() 89 | .map(|text| JsonValue::String(text.to_owned())) 90 | .unwrap_or_else(|| JsonValue::Null), 91 | String(s) => JsonValue::String(s.text().to_owned()), 92 | Blob(b) | Clob(b) => { 93 | use base64::{engine::general_purpose as base64_encoder, Engine as _}; 94 | let base64_text = base64_encoder::STANDARD.encode(b.as_slice()); 95 | JsonValue::String(base64_text) 96 | } 97 | SExp(s) => to_json_array(s.iter())?, 98 | List(l) => to_json_array(l.iter())?, 99 | Struct(s) => { 100 | let mut map = Map::new(); 101 | for field in s { 102 | let field = field?; 103 | let name = field.name()?.text().unwrap_or("$0").to_owned(); 104 | let value = to_json_value(field.value())?; 105 | map.insert(name, value); 106 | } 107 | JsonValue::Object(map) 108 | } 109 | }; 110 | Ok(value) 111 | } 112 | 113 | fn to_json_array<'a>( 114 | ion_values: impl IntoIterator>>, 115 | ) -> Result { 116 | let result: Result> = ion_values 117 | .into_iter() 118 | .flat_map(|v| v.map(to_json_value)) 119 | .collect(); 120 | Ok(JsonValue::Array(result?)) 121 | } 122 | -------------------------------------------------------------------------------- /src/bin/ion/commands/to/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::commands::command_namespace::IonCliNamespace; 2 | use crate::commands::IonCliCommand; 3 | 4 | use crate::commands::to::json::ToJsonCommand; 5 | 6 | pub mod json; 7 | 8 | pub struct ToNamespace; 9 | 10 | impl IonCliNamespace for ToNamespace { 11 | fn name(&self) -> &'static str { 12 | "to" 13 | } 14 | 15 | fn about(&self) -> &'static str { 16 | "'to' is a namespace for commands that convert Ion to another data format." 17 | } 18 | 19 | fn subcommands(&self) -> Vec> { 20 | vec![Box::new(ToJsonCommand)] 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/bin/ion/file_writer.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io; 3 | use std::io::{BufWriter, Write}; 4 | use termcolor::{ColorSpec, WriteColor}; 5 | 6 | /// A buffered `io::Write` implementation that implements [`WriteColor`] by reporting that it does 7 | /// not support TTY escape sequences and treating all requests to change or reset the current color 8 | /// as no-ops. 9 | // 10 | // When writing to a file instead of a TTY, we don't want to use `termcolor` escape sequences as 11 | // they would be stored as literal bytes rather than being interpreted. To achieve this, we need an 12 | // `io::Write` implementation that also implements `termcolor`'s `WriteColor` trait. `WriteColor` 13 | // allows the type to specify to whether it supports interpreting escape codes. 14 | // 15 | // We cannot implement `WriteColor` for `BufWriter` directly due to Rust's coherence rules. Our 16 | // crate must own the trait, the implementing type, or both. The `FileWriter` type defined below 17 | // is a simple wrapper around a `BufWriter` that implements both `io::Write` and `termcolor`'s 18 | // `WriteColor` trait. 19 | pub struct FileWriter { 20 | inner: BufWriter, 21 | } 22 | 23 | impl FileWriter { 24 | pub fn new(file: File) -> Self { 25 | Self { 26 | inner: BufWriter::new(file), 27 | } 28 | } 29 | } 30 | 31 | // Delegates all `io::Write` methods to the nested `BufWriter`. 32 | impl Write for FileWriter { 33 | fn write(&mut self, buf: &[u8]) -> io::Result { 34 | self.inner.write(buf) 35 | } 36 | 37 | fn flush(&mut self) -> io::Result<()> { 38 | self.inner.flush() 39 | } 40 | } 41 | 42 | impl WriteColor for FileWriter { 43 | fn supports_color(&self) -> bool { 44 | // FileWriter is never used to write to a TTY, so it does not support escape codes. 45 | false 46 | } 47 | 48 | fn set_color(&mut self, _spec: &ColorSpec) -> io::Result<()> { 49 | // When asked to change the color spec, do nothing. 50 | Ok(()) 51 | } 52 | 53 | fn reset(&mut self) -> io::Result<()> { 54 | // When asked to reset the color spec to the default settings, do nothing. 55 | Ok(()) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/bin/ion/hex_reader.rs: -------------------------------------------------------------------------------- 1 | use ion_rs::{IonInput, IonStream}; 2 | use std::io::{Bytes, Error, ErrorKind, Read}; 3 | 4 | /// Wraps an existing reader in order to reinterpret the content of that reader as a 5 | /// hexadecimal-encoded byte stream. 6 | /// 7 | /// This can read hex digit pairs in the form `0xHH` or `HH` where `H` is a case-insensitive 8 | /// hexadecimal digit. Between pairs, there can be any number of whitespace characters or commas. 9 | /// These are the only accepted characters. 10 | /// 11 | /// If the input contains any unacceptable characters or unpaired hex digits, the `read` function 12 | /// will (upon encountering that character) return `Err`. 13 | pub struct HexReader { 14 | inner: Bytes, 15 | digit_state: DigitState, 16 | } 17 | 18 | #[derive(Eq, PartialEq, Debug)] 19 | enum DigitState { 20 | /// The reader is ready to encounter a hexadecimal-encoded byte. 21 | Empty, 22 | /// The reader has encountered a `0`. This is an ambiguous state where we could be looking at a 23 | /// `0` that is the first in a pair with another hex digit, or it could be the `0` before an `x`. 24 | /// In other words, we're at the start of `0H` or `0xHH`, and we don't yet know which it is. 25 | Zero, 26 | /// The reader has seen `0x`. The next character must be a hex digit, which is the upper nibble 27 | /// of the hex-encoded byte. 28 | ZeroX, 29 | /// The reader has seen either `0xH` or `H`. The next character must be a hex digit, and will 30 | /// form a complete hex-encoded byte. 31 | HasUpperNibble(char), 32 | } 33 | 34 | impl IonInput for HexReader { 35 | type DataSource = IonStream; 36 | 37 | fn into_data_source(self) -> Self::DataSource { 38 | IonStream::new(self) 39 | } 40 | } 41 | 42 | impl From for HexReader { 43 | fn from(value: R) -> Self { 44 | Self { 45 | inner: value.bytes(), 46 | digit_state: DigitState::Empty, 47 | } 48 | } 49 | } 50 | 51 | impl Read for HexReader { 52 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 53 | if buf.is_empty() { 54 | return Ok(0); 55 | } 56 | 57 | let mut bytes_read = 0usize; 58 | 59 | for byte in &mut self.inner { 60 | let c = char::from(byte?); 61 | 62 | use DigitState::*; 63 | match self.digit_state { 64 | Empty if c.is_whitespace() || c == ',' => { /* Ignore these characters */ } 65 | // We've encountered either the first digit or the `0` of `0x`. 66 | Empty if c == '0' => self.digit_state = Zero, 67 | // Now we know that this hex-encoded byte is going to be `0xHH` rather than `0H` 68 | Zero if c == 'x' => self.digit_state = ZeroX, 69 | // Reading the first digit of the hex-encoded byte 70 | Empty | ZeroX if c.is_ascii_hexdigit() => self.digit_state = HasUpperNibble(c), 71 | // Reading the second digit of the hex-encoded byte 72 | Zero if c.is_ascii_hexdigit() => { 73 | // Unwrap is guaranteed not to panic because we've been putting only valid hex 74 | // digit characters in the `digit_buffer` String. 75 | let value = c.to_digit(16).unwrap(); 76 | // This unwrap is guaranteed not to panic because the max it could be is 0x0F 77 | buf[bytes_read] = u8::try_from(value).unwrap(); 78 | bytes_read += 1; 79 | self.digit_state = Empty; 80 | } 81 | HasUpperNibble(c0) if c.is_ascii_hexdigit() => { 82 | // The first unwrap is guaranteed not to panic because we already know that both 83 | // chars are valid hex digits. 84 | // The second unwrap is guaranteed not to panic because the max it could be is 0x0F 85 | let high_nibble: u8 = c0.to_digit(16).unwrap().try_into().unwrap(); 86 | let low_nibble: u8 = c.to_digit(16).unwrap().try_into().unwrap(); 87 | buf[bytes_read] = (high_nibble << 4) + low_nibble; 88 | bytes_read += 1; 89 | self.digit_state = Empty; 90 | } 91 | // Error cases 92 | _ if c.is_whitespace() => { 93 | return Err(Error::new( 94 | ErrorKind::InvalidData, 95 | format!("unexpected whitespace when digit expected: '{c}'"), 96 | )) 97 | } 98 | _ => { 99 | return Err(Error::new( 100 | ErrorKind::InvalidData, 101 | format!("not a valid hexadecimal digit: '{c}'"), 102 | )) 103 | } 104 | } 105 | 106 | if bytes_read == buf.len() { 107 | break; 108 | } 109 | } 110 | 111 | if bytes_read < buf.len() && self.digit_state != DigitState::Empty { 112 | return Err(Error::new( 113 | ErrorKind::InvalidData, 114 | "found an odd number of hex digits", 115 | )); 116 | } 117 | 118 | Ok(bytes_read) 119 | } 120 | } 121 | 122 | #[cfg(test)] 123 | mod tests { 124 | use super::*; 125 | use std::io::Cursor; 126 | 127 | #[test] 128 | fn test_read_hex_digits() { 129 | let hex = "00010203"; 130 | let reader = HexReader::from(Cursor::new(hex)); 131 | let translated_bytes: std::io::Result> = reader.bytes().collect(); 132 | let expected = vec![0u8, 1, 2, 3]; 133 | assert_eq!(expected, translated_bytes.unwrap()) 134 | } 135 | 136 | #[test] 137 | fn test_read_hex_digits_with_whitespace() { 138 | let hex = "00 01\n 02 \t \t\t 03 \r\n04"; 139 | let reader = HexReader::from(Cursor::new(hex)); 140 | let translated_bytes: std::io::Result> = reader.bytes().collect(); 141 | let expected = vec![0u8, 1, 2, 3, 4]; 142 | assert_eq!(expected, translated_bytes.unwrap()) 143 | } 144 | 145 | #[test] 146 | fn test_read_hex_digits_with_leading_0x() { 147 | let hex = "0x00 0x01 0x02 0x03 0x04"; 148 | let reader = HexReader::from(Cursor::new(hex)); 149 | let translated_bytes: std::io::Result> = reader.bytes().collect(); 150 | let expected = vec![0u8, 1, 2, 3, 4]; 151 | assert_eq!(expected, translated_bytes.unwrap()) 152 | } 153 | 154 | #[test] 155 | fn test_read_hex_digits_with_commas() { 156 | let hex = "00,01,02,03,04"; 157 | let reader = HexReader::from(Cursor::new(hex)); 158 | let translated_bytes: std::io::Result> = reader.bytes().collect(); 159 | let expected = vec![0u8, 1, 2, 3, 4]; 160 | assert_eq!(expected, translated_bytes.unwrap()) 161 | } 162 | 163 | #[test] 164 | fn test_read_odd_number_of_hex_digits() { 165 | let hex = "000102030"; 166 | let reader = HexReader::from(Cursor::new(hex)); 167 | let translated_bytes: std::io::Result> = reader.bytes().collect(); 168 | assert!(translated_bytes.is_err()) 169 | } 170 | 171 | #[test] 172 | fn test_read_hex_digits_with_invalid_char() { 173 | let hex = "000102030Q"; 174 | let reader = HexReader::from(Cursor::new(hex)); 175 | let translated_bytes: std::io::Result> = reader.bytes().collect(); 176 | assert!(translated_bytes.is_err()) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/bin/ion/input.rs: -------------------------------------------------------------------------------- 1 | use crate::auto_decompress::{decompress, AutoDecompressingReader}; 2 | use anyhow::Result; 3 | use std::io::{BufReader, Read}; 4 | 5 | // The number of header bytes to inspect with the `infer` crate to detect compression. 6 | const INFER_HEADER_LENGTH: usize = 8; 7 | 8 | /// The compression codec detected at the head of the input stream. 9 | pub enum CompressionDetected { 10 | // Note that `None` may indicate either that compression detection was disabled OR that the 11 | // input stream did not begin with a compression identifier that the Ion CLI supports. 12 | None, 13 | Gzip, 14 | Zstd, 15 | } 16 | 17 | pub struct CommandInput { 18 | source: AutoDecompressingReader, 19 | name: String, 20 | #[allow(dead_code)] 21 | // This field is retained so commands can print debug information about the input source. 22 | // It is not currently used, which generates a compile warning. 23 | compression: CompressionDetected, 24 | } 25 | 26 | impl CommandInput { 27 | /// Performs compression detection on the provided data source and returns a new [`CommandInput`] 28 | /// that will decompress its bytes in a streaming fashion. 29 | pub fn decompress( 30 | name: impl Into, 31 | source: impl Read + 'static, 32 | ) -> Result { 33 | let (compression, decompressed) = decompress(source, INFER_HEADER_LENGTH)?; 34 | Ok(Self { 35 | source: decompressed, 36 | name: name.into(), 37 | compression, 38 | }) 39 | } 40 | 41 | /// Constructs a new [`CommandInput`] that streams data from `source` without attempting to 42 | /// decompress its data. 43 | pub fn without_decompression( 44 | name: impl Into, 45 | source: impl Read + 'static, 46 | ) -> Result { 47 | Ok(Self { 48 | source: BufReader::new(Box::new(source)), 49 | name: name.into(), 50 | compression: CompressionDetected::None, 51 | }) 52 | } 53 | 54 | // TODO: Implement IonInput for mutable references to an impl Read 55 | // For now, creating a `Reader` requires that we hand over the entire BufReader. 56 | // See: https://github.com/amazon-ion/ion-rust/issues/783 57 | pub fn into_source(self) -> AutoDecompressingReader { 58 | self.source 59 | } 60 | 61 | /// Returns either: 62 | /// * the name of the input file that this `CommandInput` represents 63 | /// * the string `"-"` if this `CommandInput` represents STDIN. 64 | pub fn name(&self) -> &str { 65 | &self.name 66 | } 67 | 68 | #[allow(dead_code)] 69 | // This field is retained so commands can print debug information about the input source. 70 | // It is not currently used, which generates a compile warning. 71 | pub fn compression(&self) -> &CompressionDetected { 72 | &self.compression 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/bin/ion/input_grouping.rs: -------------------------------------------------------------------------------- 1 | use clap::{Arg, ArgAction, ArgMatches}; 2 | 3 | /// Flags for determining how a command should group/split its input values. 4 | /// 5 | /// The choices are 6 | /// * `FileHandles` (default) 7 | /// * `Lines` (`-L`) 8 | /// * `TopLevelValues` (`-T`) 9 | /// 10 | /// Default is `FileHandles` because that is the default behavior for commands that do not support 11 | /// these options. 12 | /// 13 | /// To add this to a command: 14 | /// ``` 15 | /// # use clap::Command; 16 | /// fn configure_args(&self, command: Command) -> Command { 17 | /// command.args(InputGrouping::args()) 18 | /// } 19 | /// ``` 20 | /// To read the value in the command: 21 | /// ``` 22 | /// # use clap::ArgMatches; 23 | /// fn run(&self, _command_path: &mut Vec, args: &ArgMatches) -> Result<()> { 24 | /// 25 | /// let grouping = InputGrouping::read_from_args(args); 26 | /// 27 | /// // ... 28 | /// 29 | /// Ok(()) 30 | /// } 31 | /// ``` 32 | #[derive(Copy, Clone)] 33 | pub(crate) enum InputGrouping { 34 | FileHandles, 35 | Lines, 36 | TopLevelValues, 37 | } 38 | 39 | impl InputGrouping { 40 | pub(crate) fn args() -> impl Iterator { 41 | vec![ 42 | Arg::new("group-by-lines") 43 | .group("input-grouping-mode") 44 | .short('L') 45 | .help("Interpret each line as a separate input.") 46 | .action(ArgAction::SetTrue), 47 | Arg::new("group-by-values") 48 | .group("input-grouping-mode") 49 | .short('T') 50 | .help("Interpret each top level value as a separate input.") 51 | .action(ArgAction::SetTrue), 52 | ] 53 | .into_iter() 54 | } 55 | 56 | pub(crate) fn read_from_args(args: &ArgMatches) -> InputGrouping { 57 | if args.get_flag("group-by-lines") { 58 | InputGrouping::Lines 59 | } else if args.get_flag("group-by-values") { 60 | InputGrouping::TopLevelValues 61 | } else { 62 | InputGrouping::FileHandles 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/bin/ion/main.rs: -------------------------------------------------------------------------------- 1 | mod ansi_codes; 2 | mod auto_decompress; 3 | mod commands; 4 | mod file_writer; 5 | mod hex_reader; 6 | mod input; 7 | mod input_grouping; 8 | mod output; 9 | mod transcribe; 10 | 11 | use crate::commands::cat::CatCommand; 12 | use crate::commands::complaint::SucksCommand; 13 | use crate::commands::from::FromNamespace; 14 | use crate::commands::generate::GenerateCommand; 15 | use crate::commands::hash::HashCommand; 16 | use crate::commands::head::HeadCommand; 17 | use crate::commands::inspect::InspectCommand; 18 | use crate::commands::jq::JqCommand; 19 | use crate::commands::primitive::PrimitiveCommand; 20 | use crate::commands::schema::SchemaNamespace; 21 | use crate::commands::stats::StatsCommand; 22 | use crate::commands::symtab::SymtabNamespace; 23 | use crate::commands::to::ToNamespace; 24 | use anyhow::Result; 25 | use commands::{IonCliCommand, IonCliNamespace}; 26 | use ion_rs::IonError; 27 | use std::io::ErrorKind; 28 | 29 | fn main() -> Result<()> { 30 | let root_command = RootCommand; 31 | let args = root_command.clap_command() 32 | .after_help("Having a problem with Ion CLI?\nOpen an issue at https://github.com/amazon-ion/ion-cli/issues/new/choose") 33 | .get_matches(); 34 | let mut command_path: Vec = vec![IonCliNamespace::name(&root_command).to_owned()]; 35 | 36 | if let Err(e) = root_command.run(&mut command_path, &args) { 37 | match e.downcast_ref::() { 38 | // If `ion-cli` is being invoked as part of a pipeline we want to allow the pipeline 39 | // to shut off without printing an error to STDERR. 40 | Some(IonError::Io(error)) if error.source().kind() == ErrorKind::BrokenPipe => { 41 | return Ok(()); 42 | } 43 | _ => return Err(e), 44 | } 45 | }; 46 | 47 | Ok(()) 48 | } 49 | 50 | pub struct RootCommand; 51 | 52 | impl IonCliNamespace for RootCommand { 53 | fn name(&self) -> &'static str { 54 | "ion" 55 | } 56 | 57 | fn about(&self) -> &'static str { 58 | "A collection of tools for working with Ion data." 59 | } 60 | 61 | fn subcommands(&self) -> Vec> { 62 | vec![ 63 | Box::new(CatCommand), 64 | Box::new(FromNamespace), 65 | Box::new(GenerateCommand), 66 | Box::new(HashCommand), 67 | Box::new(HeadCommand), 68 | Box::new(InspectCommand), 69 | Box::new(JqCommand), 70 | Box::new(PrimitiveCommand), 71 | Box::new(SchemaNamespace), 72 | Box::new(SymtabNamespace), 73 | Box::new(ToNamespace), 74 | Box::new(StatsCommand), 75 | Box::new(SucksCommand), 76 | ] 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/bin/ion/output.rs: -------------------------------------------------------------------------------- 1 | use crate::file_writer::FileWriter; 2 | use anyhow::bail; 3 | use ion_rs::{v1_0, v1_1, Format, IonEncoding, Writer}; 4 | use ion_rs::{IonResult, WriteAsIon}; 5 | use std::io; 6 | use std::io::Write; 7 | use syntect::dumps::from_uncompressed_data; 8 | use syntect::easy::HighlightLines; 9 | use syntect::highlighting::Style; 10 | use syntect::parsing::SyntaxSet; 11 | use syntect::util::LinesWithEndings; 12 | use syntect_assets::assets::HighlightingAssets; 13 | use termcolor::{Color, ColorSpec, StandardStreamLock, WriteColor}; 14 | 15 | /// Statically dispatches writes to either an output file or STDOUT while also supporting 16 | /// `termcolor` style escape sequences when the target is a TTY. 17 | pub enum CommandOutput<'a> { 18 | HighlightedOut(HighlightedStreamWriter<'a>, CommandOutputSpec), 19 | StdOut(StandardStreamLock<'a>, CommandOutputSpec), 20 | File(FileWriter, CommandOutputSpec), 21 | } 22 | 23 | pub struct HighlightedStreamWriter<'a> { 24 | assets: HighlightingAssets, 25 | syntaxes: SyntaxSet, 26 | stdout: StandardStreamLock<'a>, 27 | } 28 | 29 | impl<'a> HighlightedStreamWriter<'a> { 30 | pub(crate) fn new(stdout: StandardStreamLock<'a>) -> Self { 31 | // Using syntect-assets for an increased number of supported themes 32 | // Perhaps ideally we'd pull in the assets folder from sharkdp/bat or something 33 | // An older version of that is essentially what syntect-assets is 34 | let assets = HighlightingAssets::from_binary(); 35 | // Switch between .newlines and .nonewlines depending on format? 36 | // Only if we have to. We have a .nonewlines file in assets, but comments in syntect 37 | // lead me to believe that nonewlines mode is buggier and less performant. 38 | // Consider using include_dir here, e.g. include_dir!("$CARGO_MANIFEST_DIR/assets"), 39 | // especially if we decided to go the route of managing themes ourselves 40 | let syntaxes: SyntaxSet = 41 | from_uncompressed_data(include_bytes!("assets/ion.newlines.packdump")) 42 | .expect("Failed to load syntaxes"); 43 | Self { 44 | assets, 45 | syntaxes, 46 | stdout, 47 | } 48 | } 49 | } 50 | 51 | #[allow(non_camel_case_types)] 52 | pub enum CommandOutputWriter<'a, 'b> { 53 | Text_1_0(Writer>), 54 | Binary_1_0(Writer>), 55 | Text_1_1(Writer>), 56 | Binary_1_1(Writer>), 57 | } 58 | 59 | impl CommandOutputWriter<'_, '_> { 60 | pub fn write(&mut self, value: V) -> IonResult<&mut Self> { 61 | match self { 62 | CommandOutputWriter::Text_1_0(w) => w.write(value).map(|_| ())?, 63 | CommandOutputWriter::Binary_1_0(w) => w.write(value).map(|_| ())?, 64 | CommandOutputWriter::Text_1_1(w) => w.write(value).map(|_| ())?, 65 | CommandOutputWriter::Binary_1_1(w) => w.write(value).map(|_| ())?, 66 | } 67 | 68 | Ok(self) 69 | } 70 | 71 | /// Writes bytes of previously encoded values to the output stream. 72 | #[allow(dead_code)] 73 | pub fn flush(&mut self) -> IonResult<()> { 74 | match self { 75 | CommandOutputWriter::Text_1_0(w) => w.flush(), 76 | CommandOutputWriter::Binary_1_0(w) => w.flush(), 77 | CommandOutputWriter::Text_1_1(w) => w.flush(), 78 | CommandOutputWriter::Binary_1_1(w) => w.flush(), 79 | } 80 | } 81 | 82 | pub fn close(self) -> IonResult<()> { 83 | match self { 84 | CommandOutputWriter::Text_1_0(w) => w.close().map(|_| ())?, 85 | CommandOutputWriter::Binary_1_0(w) => w.close().map(|_| ())?, 86 | CommandOutputWriter::Text_1_1(w) => w.close().map(|_| ())?, 87 | CommandOutputWriter::Binary_1_1(w) => w.close().map(|_| ())?, 88 | } 89 | 90 | Ok(()) 91 | } 92 | } 93 | 94 | impl<'a> CommandOutput<'a> { 95 | pub fn spec(&self) -> &CommandOutputSpec { 96 | match self { 97 | CommandOutput::StdOut(_, spec) => spec, 98 | CommandOutput::File(_, spec) => spec, 99 | CommandOutput::HighlightedOut(_, spec) => spec, 100 | } 101 | } 102 | 103 | pub fn format(&self) -> &Format { 104 | &self.spec().format 105 | } 106 | 107 | pub fn encoding(&self) -> &IonEncoding { 108 | &self.spec().encoding 109 | } 110 | 111 | pub fn as_writer<'b>(&'b mut self) -> anyhow::Result> { 112 | let CommandOutputSpec { format, encoding } = *self.spec(); 113 | 114 | Ok(match (encoding, format) { 115 | (IonEncoding::Text_1_0, Format::Text(text_format)) => CommandOutputWriter::Text_1_0( 116 | Writer::new(v1_0::Text.with_format(text_format), self)?, 117 | ), 118 | (IonEncoding::Text_1_1, Format::Text(text_format)) => CommandOutputWriter::Text_1_1( 119 | Writer::new(v1_1::Text.with_format(text_format), self)?, 120 | ), 121 | (IonEncoding::Binary_1_0, Format::Binary) => { 122 | CommandOutputWriter::Binary_1_0(Writer::new(v1_0::Binary, self)?) 123 | } 124 | (IonEncoding::Binary_1_1, Format::Binary) => { 125 | CommandOutputWriter::Binary_1_1(Writer::new(v1_1::Binary, self)?) 126 | } 127 | unrecognized => bail!("unsupported format '{:?}'", unrecognized), 128 | }) 129 | } 130 | } 131 | 132 | #[derive(Debug, Copy, Clone)] 133 | pub struct CommandOutputSpec { 134 | pub format: Format, 135 | pub encoding: IonEncoding, 136 | } 137 | 138 | impl Write for HighlightedStreamWriter<'_> { 139 | fn write(&mut self, buf: &[u8]) -> io::Result { 140 | let output = std::str::from_utf8(buf).unwrap(); 141 | 142 | let ion_syntax = &self.syntaxes.find_syntax_by_name("ion").unwrap(); 143 | // There's a lot to learn from sharkdp/bat the subject of automated light/dark theming, 144 | // see src/theme.rs in: https://github.com/sharkdp/bat/pull/2896 145 | // Here we will hardcode something "dark" until someone complains or sends a patch 146 | let theme = &self.assets.get_theme("Monokai Extended"); //TODO: choose theme somehow 147 | let mut highlighter = HighlightLines::new(ion_syntax, theme); 148 | 149 | for line in LinesWithEndings::from(output) { 150 | let ranges: Vec<(Style, &str)> = 151 | highlighter.highlight_line(line, &self.syntaxes).unwrap(); 152 | for &(ref style, text) in ranges.iter() { 153 | // We won't mess with the background colors 154 | let color = Some(Color::Rgb( 155 | style.foreground.r, 156 | style.foreground.g, 157 | style.foreground.b, 158 | )); 159 | let mut style = ColorSpec::new(); 160 | style.set_fg(color); 161 | self.stdout.set_color(&style)?; 162 | write!(self.stdout, "{}", text)?; 163 | } 164 | } 165 | // If we got here we succeeded in writing all the input bytes, so report that len 166 | Ok(buf.len()) 167 | } 168 | 169 | fn flush(&mut self) -> io::Result<()> { 170 | self.stdout.flush() 171 | } 172 | } 173 | 174 | impl WriteColor for HighlightedStreamWriter<'_> { 175 | fn supports_color(&self) -> bool { 176 | // HighlightedStreamWriter is only used when syntect is managing the color 177 | false 178 | } 179 | 180 | fn set_color(&mut self, _spec: &ColorSpec) -> io::Result<()> { 181 | // When asked to change the color spec, do nothing. 182 | Ok(()) 183 | } 184 | 185 | fn reset(&mut self) -> io::Result<()> { 186 | // When asked to reset the color spec to the default settings, do nothing. 187 | Ok(()) 188 | } 189 | } 190 | 191 | impl Write for CommandOutput<'_> { 192 | fn write(&mut self, buf: &[u8]) -> io::Result { 193 | use CommandOutput::*; 194 | match self { 195 | HighlightedOut(highlighted_writer, ..) => highlighted_writer.write(buf), 196 | StdOut(stdout, ..) => stdout.write(buf), 197 | File(file_writer, ..) => file_writer.write(buf), 198 | } 199 | } 200 | 201 | fn flush(&mut self) -> io::Result<()> { 202 | use CommandOutput::*; 203 | match self { 204 | HighlightedOut(highlighted_writer, ..) => highlighted_writer.flush(), 205 | StdOut(stdout, ..) => stdout.flush(), 206 | File(file_writer, ..) => file_writer.flush(), 207 | } 208 | } 209 | } 210 | 211 | impl WriteColor for CommandOutput<'_> { 212 | fn supports_color(&self) -> bool { 213 | use CommandOutput::*; 214 | match self { 215 | HighlightedOut(highlighted_writer, ..) => highlighted_writer.supports_color(), 216 | StdOut(stdout, ..) => stdout.supports_color(), 217 | File(file_writer, ..) => file_writer.supports_color(), 218 | } 219 | } 220 | 221 | fn set_color(&mut self, spec: &ColorSpec) -> io::Result<()> { 222 | use CommandOutput::*; 223 | match self { 224 | HighlightedOut(highlighted_writer, ..) => highlighted_writer.set_color(spec), 225 | StdOut(stdout, ..) => stdout.set_color(spec), 226 | File(file_writer, ..) => file_writer.set_color(spec), 227 | } 228 | } 229 | 230 | fn reset(&mut self) -> io::Result<()> { 231 | use CommandOutput::*; 232 | match self { 233 | HighlightedOut(highlighted_writer, ..) => highlighted_writer.reset(), 234 | StdOut(stdout, ..) => stdout.reset(), 235 | File(file_writer, ..) => file_writer.reset(), 236 | } 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /src/bin/ion/transcribe.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{bail, Result}; 2 | use ion_rs::*; 3 | use std::io::Write; 4 | 5 | /// Constructs the appropriate writer for the given format, then writes all values from the 6 | /// `Reader` to the new `Writer`. 7 | pub(crate) fn write_all_as( 8 | reader: &mut Reader, 9 | output: &mut impl Write, 10 | encoding: IonEncoding, 11 | format: Format, 12 | ) -> Result { 13 | write_n_as(reader, output, encoding, format, usize::MAX) 14 | } 15 | 16 | /// Constructs the appropriate writer for the given format, then writes up to `count` values from the 17 | /// `Reader` to the new `Writer`. 18 | pub(crate) fn write_n_as( 19 | reader: &mut Reader, 20 | output: &mut impl Write, 21 | encoding: IonEncoding, 22 | format: Format, 23 | count: usize, 24 | ) -> Result { 25 | let written = match (encoding, format) { 26 | (IonEncoding::Text_1_0, Format::Text(text_format)) => { 27 | let mut writer = Writer::new(v1_0::Text.with_format(text_format), output)?; 28 | transcribe_n(reader, &mut writer, count) 29 | } 30 | (IonEncoding::Text_1_1, Format::Text(text_format)) => { 31 | let mut writer = Writer::new(v1_1::Text.with_format(text_format), output)?; 32 | transcribe_n(reader, &mut writer, count) 33 | } 34 | (IonEncoding::Binary_1_0, Format::Binary) => { 35 | let mut writer = Writer::new(v1_0::Binary, output)?; 36 | transcribe_n(reader, &mut writer, count) 37 | } 38 | (IonEncoding::Binary_1_1, Format::Binary) => { 39 | let mut writer = Writer::new(v1_1::Binary, output)?; 40 | transcribe_n(reader, &mut writer, count) 41 | } 42 | unrecognized => bail!("unsupported format '{:?}'", unrecognized), 43 | }?; 44 | Ok(written) 45 | } 46 | 47 | /// Writes up to `count` values from the `Reader` to the provided `Writer`. 48 | fn transcribe_n( 49 | reader: &mut Reader, 50 | writer: &mut Writer, 51 | count: usize, 52 | ) -> Result { 53 | const FLUSH_EVERY_N: usize = 100; 54 | let mut values_since_flush: usize = 0; 55 | let mut index: usize = 0; 56 | 57 | while let Some(value) = reader.next()? { 58 | if index >= count { 59 | break; 60 | } 61 | 62 | writer.write(value)?; 63 | 64 | index += 1; 65 | values_since_flush += 1; 66 | if values_since_flush == FLUSH_EVERY_N { 67 | writer.flush()?; 68 | values_since_flush = 0; 69 | } 70 | } 71 | 72 | writer.flush()?; 73 | Ok(index) 74 | } 75 | -------------------------------------------------------------------------------- /tests/cli.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use assert_cmd::Command; 3 | use ion_rs::Element; 4 | use rstest::*; 5 | use std::fs::File; 6 | use std::io::{Read, Write}; 7 | use std::time::Duration; 8 | use tempfile::TempDir; 9 | 10 | enum FileMode { 11 | /// Use `STDIN` or `STDOUT` 12 | Default, 13 | /// Use a named file 14 | Named, 15 | } 16 | 17 | enum InputCompress { 18 | /// no compression 19 | No, 20 | /// gzip 21 | Gz, 22 | /// zstd 23 | Zst, 24 | } 25 | 26 | struct TestCase> { 27 | /// The text of the ion payload to test 28 | ion_text: S, 29 | /// The expected Ion 30 | expected_ion: Element, 31 | } 32 | 33 | impl From<(&'static str, &'static str)> for TestCase<&'static str> { 34 | /// Simple conversion for static `str` slices into a test case 35 | fn from((ion_text, expected_ion): (&'static str, &'static str)) -> Self { 36 | let expected_ion = Element::read_one(expected_ion.as_bytes()).unwrap(); 37 | Self { 38 | ion_text, 39 | expected_ion, 40 | } 41 | } 42 | } 43 | 44 | #[rstest] 45 | #[case::simple(( 46 | r#" 47 | { 48 | name: "Fido", 49 | 50 | age: years::4, 51 | 52 | birthday: 2012-03-01T, 53 | 54 | toys: [ 55 | ball, 56 | rope, 57 | ], 58 | 59 | weight: pounds::41.2, 60 | 61 | buzz: {{VG8gaW5maW5pdHkuLi4gYW5kIGJleW9uZCE=}}, 62 | } 63 | "#, 64 | r#" 65 | { 66 | name: "Fido", 67 | 68 | age: years::4, 69 | 70 | birthday: 2012-03-01T, 71 | 72 | toys: [ 73 | ball, 74 | rope, 75 | ], 76 | 77 | weight: pounds::41.2, 78 | 79 | buzz: {{VG8gaW5maW5pdHkuLi4gYW5kIGJleW9uZCE=}}, 80 | } 81 | "# 82 | ).into())] 83 | /// Calls the ion CLI binary dump command with a set of arguments the ion-cli is expected to support. 84 | /// This does not verify specific formatting, only basic CLI behavior. 85 | fn run_it>( 86 | #[case] test_case: TestCase, 87 | #[values("", "binary", "text", "pretty")] format_flag: &str, 88 | #[values(FileMode::Default, FileMode::Named)] input_mode: FileMode, 89 | #[values(FileMode::Default, FileMode::Named)] output_mode: FileMode, 90 | #[values(InputCompress::No, InputCompress::Gz, InputCompress::Zst)] 91 | input_compress: InputCompress, 92 | ) -> Result<()> { 93 | let TestCase { 94 | ion_text, 95 | expected_ion, 96 | } = test_case; 97 | 98 | let temp_dir = TempDir::new()?; 99 | let input_path = temp_dir.path().join("INPUT.ion"); 100 | let output_path = temp_dir.path().join("OUTPUT.ion"); 101 | 102 | let mut cmd = Command::cargo_bin("ion")?; 103 | cmd.arg("dump").timeout(Duration::new(5, 0)); 104 | if !format_flag.is_empty() { 105 | cmd.arg("-f"); 106 | cmd.arg(format_flag); 107 | } 108 | match output_mode { 109 | FileMode::Default => { 110 | // do nothing 111 | } 112 | FileMode::Named => { 113 | // tell driver to output to a file 114 | cmd.arg("-o"); 115 | cmd.arg(&output_path); 116 | } 117 | }; 118 | 119 | // prepare input 120 | let input_bytes = match input_compress { 121 | InputCompress::Gz => { 122 | let mut encoder = 123 | flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); 124 | encoder.write_all(ion_text.as_ref().as_bytes())?; 125 | encoder.finish()? 126 | } 127 | InputCompress::Zst => { 128 | let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 1)?; 129 | encoder.write_all(ion_text.as_ref().as_bytes())?; 130 | encoder.finish()? 131 | } 132 | _ => ion_text.as_ref().as_bytes().to_vec(), 133 | }; 134 | 135 | match input_mode { 136 | FileMode::Default => { 137 | // do nothing 138 | cmd.write_stdin(input_bytes); 139 | } 140 | FileMode::Named => { 141 | // dump our test data to input file 142 | let mut input_file = File::create(&input_path)?; 143 | input_file.write_all(&input_bytes)?; 144 | input_file.flush()?; 145 | 146 | // TODO: test multiple input files 147 | 148 | // make this the input for our driver 149 | cmd.arg(input_path.to_str().unwrap()); 150 | } 151 | }; 152 | 153 | let assert = cmd.assert(); 154 | 155 | let actual_ion = match output_mode { 156 | FileMode::Default => { 157 | let output = assert.get_output(); 158 | Element::read_one(&output.stdout)? 159 | } 160 | FileMode::Named => { 161 | let mut output_file = File::open(output_path)?; 162 | let mut output_buffer = vec![]; 163 | output_file.read_to_end(&mut output_buffer)?; 164 | Element::read_one(&output_buffer)? 165 | } 166 | }; 167 | 168 | assert_eq!(expected_ion, actual_ion); 169 | assert.success(); 170 | 171 | Ok(()) 172 | } 173 | 174 | #[rstest] 175 | #[case(0, "")] 176 | #[case(2, "{foo: bar, abc: [123, 456]}\n{foo: baz, abc: [42.0, 4.3e1]}")] 177 | ///Calls ion-cli head with different requested number. Pass the test if the return value equals to the expected value. 178 | fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> Result<()> { 179 | let mut cmd = Command::cargo_bin("ion")?; 180 | let test_data = r#" 181 | { 182 | foo: bar, 183 | abc: [123, 456] 184 | } 185 | { 186 | foo: baz, 187 | abc: [42.0, 43e0] 188 | } 189 | { 190 | foo: bar, 191 | test: data 192 | } 193 | { 194 | foo: baz, 195 | type: struct 196 | } 197 | "#; 198 | let temp_dir = TempDir::new()?; 199 | let input_path = temp_dir.path().join("test.ion"); 200 | let mut input_file = File::create(&input_path)?; 201 | input_file.write_all(test_data.as_bytes())?; 202 | input_file.flush()?; 203 | cmd.args([ 204 | "head", 205 | "--values", 206 | &number.to_string(), 207 | "--format", 208 | "lines", 209 | input_path.to_str().unwrap(), 210 | ]); 211 | let command_assert = cmd.assert(); 212 | let output = command_assert.get_output(); 213 | let stdout = String::from_utf8_lossy(&output.stdout); 214 | assert_eq!( 215 | Element::read_all(stdout.trim_end())?, 216 | Element::read_all(expected_output)? 217 | ); 218 | Ok(()) 219 | } 220 | 221 | mod code_gen_tests { 222 | use super::*; 223 | use std::fs; 224 | 225 | //TODO: Add cargo roundtrip tests once the rust templates are modified based on new code generation model 226 | 227 | #[rstest] 228 | #[case( 229 | "SimpleStruct", 230 | r#" 231 | type::{ 232 | name: simple_struct, 233 | fields: { 234 | name: string, 235 | id: { type: int, occurs: required }, 236 | } 237 | } 238 | "#, 239 | & ["private int id;", "private String name;"], 240 | & ["public String getName() {", "public int getId() {"] 241 | )] 242 | #[case( 243 | "Scalar", 244 | r#" 245 | type::{ 246 | name: scalar, 247 | type: string 248 | } 249 | "#, 250 | & ["private String value;"], 251 | & ["public String getValue() {"] 252 | )] 253 | /// Calls ion-cli generate with different schema file. Pass the test if the return value contains the expected properties and accessors. 254 | fn test_code_generation_in_java( 255 | #[case] test_name: &str, 256 | #[case] test_schema: &str, 257 | #[case] expected_properties: &[&str], 258 | #[case] expected_accessors: &[&str], 259 | ) -> Result<()> { 260 | let mut cmd = Command::cargo_bin("ion")?; 261 | let temp_dir = TempDir::new()?; 262 | let input_schema_path = temp_dir.path().join("test_schema.isl"); 263 | let mut input_schema_file = File::create(input_schema_path)?; 264 | input_schema_file.write_all(test_schema.as_bytes())?; 265 | input_schema_file.flush()?; 266 | cmd.args([ 267 | "-X", 268 | "generate", 269 | "--output", 270 | temp_dir.path().to_str().unwrap(), 271 | "--language", 272 | "java", 273 | "--namespace", 274 | "org.example", 275 | "--authority", 276 | temp_dir.path().to_str().unwrap(), 277 | ]); 278 | let command_assert = cmd.assert(); 279 | let output_file_path = temp_dir.path().join(format!("{}.java", test_name)); 280 | command_assert.success(); 281 | let contents = 282 | fs::read_to_string(output_file_path).expect("Can not read generated code file."); 283 | for expected_property in expected_properties { 284 | assert!(contents.contains(expected_property)); 285 | } 286 | for expected_accessor in expected_accessors { 287 | assert!(contents.contains(expected_accessor)); 288 | } 289 | Ok(()) 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /tests/code-gen-tests.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use assert_cmd::Command; 3 | use rstest::rstest; 4 | use std::fs::File; 5 | use std::io::Write; 6 | use std::path::PathBuf; 7 | use tempfile::TempDir; 8 | 9 | /// Returns a new [PathBuf] instance with the absolute path of the "code-gen-projects" directory. 10 | fn code_gen_projects_path() -> PathBuf { 11 | PathBuf::from_iter([env!("CARGO_MANIFEST_DIR"), "code-gen-projects"]) 12 | } 13 | 14 | #[test] 15 | fn roundtrip_tests_for_generated_code_gradle() -> Result<()> { 16 | // run the gradle project defined under `code-gen-projects`, 17 | // this project runs the code generator in its build process and generates code, 18 | // this project also has some predefined tests for the generated code, 19 | // so simply running the tests on this project builds the project, generates code and runs tests 20 | 21 | // absolute paths for gradle project and executables 22 | let ion_executable = env!("CARGO_BIN_EXE_ion"); 23 | let ion_input = code_gen_projects_path().join("input"); 24 | let test_project_path = code_gen_projects_path().join("java").join("code-gen-demo"); 25 | 26 | let gradle_executable_name = if cfg!(windows) { 27 | "gradlew.bat" 28 | } else { 29 | "gradlew" 30 | }; 31 | 32 | let gradle_executable = test_project_path.join(gradle_executable_name); 33 | 34 | // Clean and Test 35 | let gradle_output = std::process::Command::new(gradle_executable) 36 | .current_dir(test_project_path) 37 | .env("ION_CLI", ion_executable) 38 | .env("ION_INPUT", ion_input) 39 | .arg("clean") 40 | .arg("test") 41 | .output() 42 | .expect("failed to execute Gradle targets 'clean' and 'test'"); 43 | 44 | println!("status: {}", gradle_output.status); 45 | std::io::stdout().write_all(&gradle_output.stdout).unwrap(); 46 | std::io::stderr().write_all(&gradle_output.stderr).unwrap(); 47 | 48 | assert!(gradle_output.status.success()); 49 | Ok(()) 50 | } 51 | 52 | #[test] 53 | fn roundtrip_tests_for_generated_code_cargo() -> Result<()> { 54 | // run the cargo project defined under `code-gen-projects`, 55 | // this project runs the code generator in its build process and generates code, 56 | // this project also has some predefined tests for the generated code, 57 | // so simply running the tests on this project builds the project, generates code and runs tests 58 | 59 | // absolute paths for crate and executables 60 | let ion_executable = env!("CARGO_BIN_EXE_ion"); 61 | let test_project_path = code_gen_projects_path().join("rust").join("code-gen-demo"); 62 | let cargo_executable = env!("CARGO"); 63 | 64 | // Clean 65 | let cargo_clean_output = std::process::Command::new(cargo_executable) 66 | .current_dir(&test_project_path) 67 | .arg("clean") 68 | .output() 69 | .expect("failed to execute 'cargo clean'"); 70 | 71 | println!("Cargo clean status: {}", cargo_clean_output.status); 72 | std::io::stdout() 73 | .write_all(&cargo_clean_output.stdout) 74 | .unwrap(); 75 | std::io::stderr() 76 | .write_all(&cargo_clean_output.stderr) 77 | .unwrap(); 78 | 79 | // Test 80 | let cargo_test_output = std::process::Command::new(cargo_executable) 81 | .current_dir(&test_project_path) 82 | .arg("test") 83 | .env("ION_CLI", ion_executable) 84 | .output() 85 | .expect("failed to execute 'cargo test'"); 86 | 87 | println!("Cargo test status: {}", cargo_test_output.status); 88 | std::io::stdout() 89 | .write_all(&cargo_test_output.stdout) 90 | .unwrap(); 91 | std::io::stderr() 92 | .write_all(&cargo_test_output.stderr) 93 | .unwrap(); 94 | 95 | assert!(cargo_test_output.status.success()); 96 | Ok(()) 97 | } 98 | 99 | //TODO: Add cargo roundtrip tests once the rust templates are modified based on new code generation model 100 | 101 | #[rstest] 102 | #[case::any_element_list( 103 | r#" 104 | type::{ 105 | name: any_element_list, 106 | type: list, // this doesn't specify the type for elements in the list with `element` constraint 107 | } 108 | "#, 109 | )] 110 | #[case::any_sequence_type( 111 | r#" 112 | type::{ 113 | name: any_sequence_type, 114 | element: int, // this doesn't specify the type of sequence with `type` constraint 115 | } 116 | "# 117 | )] 118 | // Currently any struct type is not supported, it requires having a `fields` constraint 119 | #[case::any_struct_type( 120 | r#" 121 | type::{ 122 | name: any_struct_type, 123 | type: struct, // this doesn't specify `fields` of the struct 124 | } 125 | "# 126 | )] 127 | /// Calls ion-cli generate with different unsupported schema types. Verify that `generate` subcommand returns an error for these schema types. 128 | fn test_unsupported_schema_types_failures(#[case] test_schema: &str) -> Result<()> { 129 | let mut cmd = Command::cargo_bin("ion")?; 130 | let temp_dir = TempDir::new()?; 131 | let input_schema_path = temp_dir.path().join("test_schema.isl"); 132 | let mut input_schema_file = File::create(input_schema_path)?; 133 | input_schema_file.write_all(test_schema.as_bytes())?; 134 | input_schema_file.flush()?; 135 | cmd.args([ 136 | "-X", 137 | "generate", 138 | "--schema", 139 | "test_schema.isl", 140 | "--output", 141 | temp_dir.path().to_str().unwrap(), 142 | "--language", 143 | "java", 144 | "--namespace", 145 | "org.example", 146 | "--directory", 147 | temp_dir.path().to_str().unwrap(), 148 | ]); 149 | let command_assert = cmd.assert(); 150 | // Code generation process should return an error for unsupported schema types 151 | command_assert.failure(); 152 | Ok(()) 153 | } 154 | --------------------------------------------------------------------------------