├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── codeql-analysis.yml │ └── go.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── cmd └── ion-go │ ├── eventwriter.go │ ├── main.go │ ├── nopwriter.go │ ├── process.go │ ├── schema.go │ └── util.go ├── go.mod ├── go.sum ├── internal └── version.go ├── ion ├── binaryreader.go ├── binaryreader_test.go ├── binarywriter.go ├── binarywriter_test.go ├── bits.go ├── bits_test.go ├── bitstream.go ├── bitstream_test.go ├── buf.go ├── buf_test.go ├── catalog.go ├── catalog_test.go ├── cmp_test.go ├── consts.go ├── ctx.go ├── decimal.go ├── decimal_test.go ├── err.go ├── fields.go ├── integration_test.go ├── marshal.go ├── marshal_test.go ├── reader.go ├── reader_test.go ├── readlocalsymboltable.go ├── readlocalsymboltable_test.go ├── skipper.go ├── skipper_test.go ├── symboltable.go ├── symboltable_test.go ├── symboltoken.go ├── symboltoken_test.go ├── textreader.go ├── textreader_test.go ├── textutils.go ├── textutils_test.go ├── textwriter.go ├── textwriter_test.go ├── timestamp.go ├── timestamp_test.go ├── tokenizer.go ├── tokenizer_test.go ├── type.go ├── type_test.go ├── unmarshal.go ├── unmarshal_test.go └── writer.go └── scripts ├── build.sh └── install.sh /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Issue #, if available: 4 | 5 | Description of changes: 6 | 7 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 8 | 9 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '30 4 * * 4' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v2 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v1 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v1 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v1 71 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go Build 2 | on: [push, pull_request] 3 | jobs: 4 | 5 | build: 6 | name: Build 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 1.13 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.13 14 | id: go 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v3 18 | with: 19 | submodules: recursive 20 | 21 | - name: Get dependencies 22 | run: | 23 | go get -v -t -d ./... 24 | if [ -f Gopkg.toml ]; then 25 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 26 | dep ensure 27 | fi 28 | 29 | - name: Build 30 | run: go build -v ./... 31 | 32 | - name: Test with coverage 33 | run: go test -v ./... -coverprofile coverage.txt 34 | 35 | - name: Upload Coverage report to CodeCov 36 | uses: codecov/codecov-action@v1 37 | with: 38 | token: ${{secrets.CODECOV_TOKEN}} 39 | file: ./coverage.txt 40 | 41 | lint: 42 | name: Lint 43 | runs-on: ubuntu-latest 44 | steps: 45 | - name: Setup Go 46 | uses: actions/setup-go@v3 47 | with: 48 | go-version: "^1.13" 49 | 50 | - name: Install goimports 51 | run: go install golang.org/x/tools/cmd/goimports@latest 52 | 53 | - name: Checkout code 54 | uses: actions/checkout@v3 55 | 56 | # goimports provides a superset of gofmt functionality 57 | - run: goimports -w . 58 | 59 | - name: Run go mod tidy on all modules 60 | run: go mod tidy 61 | 62 | # If there are any diffs from goimports or go mod tidy, fail. 63 | - name: Verify no changes from goimports and go mod tidy. 64 | run: | 65 | if [ -n "$(git status --porcelain)" ]; then 66 | # Show the files that failed to pass the check. 67 | echo 'Lint check failed:' 68 | git diff --minimal --compact-summary 69 | echo 'To fix this check, run "goimports -w . && go mod tidy"' 70 | exit 1 71 | fi 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /ion-go 2 | /cmd/ion-go/ion-go 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ion-tests"] 2 | path = ion-tests 3 | url = https://github.com/amazon-ion/ion-tests.git 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | test: 3 | go test ./... 4 | 5 | .PHONY: build 6 | build: 7 | ./scripts/build.sh 8 | 9 | .PHONY: install 10 | install: 11 | ./scripts/install.sh 12 | 13 | .PHONY: clean 14 | clean: 15 | -rm ion-go 16 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | (https://github.com/fernomac/ion-go/blob/master/NOTICE) 4 | Amazon Ion Go 5 | Copyright 2019 David Murray 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon Ion Go 2 | 3 | [![Build Status](https://github.com/amazon-ion/ion-go/workflows/Go%20Build/badge.svg)](https://github.com/amazon-ion/ion-go/actions?query=workflow%3A%22Go+Build%22) 4 | [![license](https://img.shields.io/hexpm/l/plug.svg)](https://github.com/amazon-ion/ion-go/blob/master/LICENSE) 5 | [![docs](https://img.shields.io/badge/docs-api-green.svg)](https://pkg.go.dev/github.com/amazon-ion/ion-go?tab=doc) 6 | 7 | Amazon Ion ( https://amazon-ion.github.io/ion-docs/ ) library for Go 8 | 9 | [Ion Cookbook](https://amazon-ion.github.io/ion-docs/guides/cookbook.html) demonstrates code samples for some simple Amazon Ion use cases. 10 | 11 | This package is based on work from David Murray ([fernomac](https://github.com/fernomac/)) on https://github.com/fernomac/ion-go. 12 | The Ion team greatly appreciates David's contributions to the Ion community. 13 | 14 | 15 | ## Users 16 | 17 | Here are some projects that use the Ion Go library 18 | 19 | * [Restish](https://rest.sh/): "...a CLI for interacting with REST-ish HTTP APIs with some nice features built-in" 20 | 21 | 22 | We'll be happy to add you to our list, send us a pull request. 23 | 24 | 25 | ## Git Setup 26 | 27 | 28 | This repository contains a [git submodule](https://git-scm.com/docs/git-submodule) 29 | called `ion-tests`, which holds test data used by `ion-go`'s unit tests. 30 | 31 | The easiest way to clone the `ion-go` repository and initialize its `ion-tests` 32 | submodule is to run the following command. 33 | 34 | ``` 35 | $ git clone --recursive https://github.com/amazon-ion/ion-go.git ion-go 36 | ``` 37 | 38 | Alternatively, the submodule may be initialized independent of the clone 39 | by running the following commands: 40 | 41 | ``` 42 | $ git submodule init 43 | $ git submodule update 44 | ``` 45 | 46 | ## Development 47 | 48 | This package uses [Go Modules](https://github.com/golang/go/wiki/Modules) to model 49 | its dependencies. 50 | 51 | Assuming the `go` command is in your path, building the module can be done as: 52 | 53 | ``` 54 | $ go build -v ./... 55 | ``` 56 | 57 | Running all the tests can be executed with: 58 | 59 | ``` 60 | $ go test -v ./... 61 | ``` 62 | 63 | We use [`goimports`](https://pkg.go.dev/golang.org/x/tools/cmd/goimports?tab=doc) to format 64 | our imports and files in general. Running this before commit is advised: 65 | 66 | ``` 67 | $ goimports -w . 68 | ``` 69 | 70 | It is recommended that you hook this in your favorite IDE (`Tools` > `File Watchers` in Goland, for example). 71 | 72 | ## Usage 73 | 74 | Import `github.com/amazon-ion/ion-go/ion` and you're off to the races. 75 | 76 | ### Marshaling and Unmarshaling 77 | 78 | Similar to GoLang's built-in [json](https://golang.org/pkg/encoding/json/) package, 79 | you can marshal and unmarshal Go types to Ion. Marshaling requires you to specify 80 | whether you'd like text or binary Ion. Unmarshaling is smart enough to do the right 81 | thing. Both follow the style of json name tags, and `Marshal` honors `omitempty`. 82 | 83 | ```Go 84 | type T struct { 85 | A string 86 | B struct { 87 | RenamedC int `ion:"C"` 88 | D []int `ion:",omitempty"` 89 | } 90 | } 91 | 92 | func main() { 93 | t := T{} 94 | 95 | err := ion.Unmarshal([]byte(`{A:"Ion!",B:{C:2,D:[3,4]}}`), &t) 96 | if err != nil { 97 | panic(err) 98 | } 99 | fmt.Printf("--- t:\n%v\n\n", t) 100 | 101 | text, err := ion.MarshalText(&t) 102 | if err != nil { 103 | panic(err) 104 | } 105 | fmt.Printf("--- text:\n%s\n\n", string(text)) 106 | 107 | binary, err := ion.MarshalBinary(&t) 108 | if err != nil { 109 | panic(err) 110 | } 111 | fmt.Printf("--- binary:\n%X\n\n", binary) 112 | } 113 | ``` 114 | 115 | In order to Marshal/Unmarshal Ion values with annotation, we use a Go struct with two fields, 116 | 117 | 1. one field of type `[]string` and tagged with `ion:",annotation"`. 118 | 2. the other field with appropriate type and optional tag to hold our Ion value. For instance, 119 | to Marshal `age::20`, it must be in a struct as below: 120 | ```GO 121 | type foo struct { 122 | Value interface{} 123 | AnyName []string `ion:",annotations"` 124 | } 125 | data := foo{20, []string{"age"}} 126 | val, err := ion.MarshalText(data) 127 | if err != nil { 128 | panic(err) 129 | } 130 | fmt.Println("Ion text: ", string(val)) // Ion text: age::20 131 | ``` 132 | 133 | And to Unmarshal the same data, we can do as shown below: 134 | ```Go 135 | type foo struct { 136 | Value interface{} 137 | AnyName []string `ion:",annotations"` 138 | } 139 | var val foo 140 | err := ion.UnmarshalString("age::20", &val) 141 | if err != nil { 142 | panic(err) 143 | } 144 | fmt.Printf("Val = %+v\n", val) // Val = {Value:20 AnyName:[age]} 145 | ``` 146 | 147 | 148 | ### Encoding and Decoding 149 | 150 | To read or write multiple values at once, use an `Encoder` or `Decoder`: 151 | 152 | ```Go 153 | func main() { 154 | dec := ion.NewTextDecoder(os.Stdin) 155 | enc := ion.NewBinaryEncoder(os.Stdout) 156 | 157 | for { 158 | // Decode one Ion whole value from stdin. 159 | val, err := dec.Decode() 160 | if err == ion.ErrNoInput { 161 | break 162 | } else if err != nil { 163 | panic(err) 164 | } 165 | 166 | // Encode it to stdout. 167 | if err := enc.Encode(val); err != nil { 168 | panic(err) 169 | } 170 | } 171 | 172 | if err := enc.Finish(); err != nil { 173 | panic(err) 174 | } 175 | } 176 | ``` 177 | 178 | ### Reading and Writing 179 | 180 | For low-level streaming read and write access, use a `Reader` or `Writer`. 181 | The following example shows how to create a reader, read values from that reader, 182 | and write those values out using a writer: 183 | 184 | ```Go 185 | func writeFromReaderToWriter(reader Reader, writer Writer) { 186 | for reader.Next() { 187 | name := reader.FieldName() 188 | if name != nil { 189 | err := writer.FieldName(*name) 190 | if err != nil { 191 | panic(err) 192 | } 193 | } 194 | 195 | an := reader.Annotations() 196 | if len(an) > 0 { 197 | err := writer.Annotations(an...) 198 | if err != nil { 199 | panic(err) 200 | } 201 | } 202 | 203 | currentType := reader.Type() 204 | if reader.IsNull() { 205 | err := writer.WriteNullType(currentType) 206 | if err != nil { 207 | panic(err) 208 | } 209 | continue 210 | } 211 | 212 | switch currentType { 213 | case BoolType: 214 | val, err := reader.BoolValue() 215 | if err != nil { 216 | panic("Something went wrong while reading a Boolean value: " + err.Error()) 217 | } 218 | err = writer.WriteBool(val) 219 | if err != nil { 220 | panic("Something went wrong while writing a Boolean value: " + err.Error()) 221 | } 222 | 223 | case StringType: 224 | val, err := reader.StringValue() 225 | if err != nil { 226 | panic("Something went wrong while reading a String value: " + err.Error()) 227 | } 228 | err = writer.WriteString(val) 229 | if err != nil { 230 | panic("Something went wrong while writing a String value: " + err.Error()) 231 | } 232 | 233 | case StructType: 234 | err := reader.StepIn() 235 | if err != nil { 236 | panic(err) 237 | } 238 | err = writer.BeginStruct() 239 | if err != nil { 240 | panic(err) 241 | } 242 | writeFromReaderToWriter(reader, writer) 243 | err = reader.StepOut() 244 | if err != nil { 245 | panic(err) 246 | } 247 | err = writer.EndStruct() 248 | if err != nil { 249 | panic(err) 250 | } 251 | default: 252 | panic("This is an example, only taking in Bool, String and Struct") 253 | } 254 | } 255 | 256 | if reader.Err() != nil { 257 | panic(reader.Err().Error()) 258 | } 259 | } 260 | 261 | func main() { 262 | reader := NewReaderString("foo::{name:\"bar\", complete:false}") 263 | str := strings.Builder{} 264 | writer := NewTextWriter(&str) 265 | 266 | writeFromReaderToWriter(reader, writer) 267 | err := writer.Finish() 268 | if err != nil { 269 | panic(err) 270 | } 271 | fmt.Println(str.String()) 272 | } 273 | ``` 274 | 275 | ### Symbol Tables 276 | 277 | By default, when writing binary Ion, a local symbol table is built as you write 278 | values (which are buffered in memory until you call `Finish` so the symbol table 279 | can be written out first). You can optionally provide one or more 280 | `SharedSymbolTable`s to the writer, which it will reference as needed rather 281 | than directly including those symbols in the local symbol table. 282 | 283 | ```Go 284 | type Item struct { 285 | ID string `ion:"id"` 286 | Name string `ion:"name"` 287 | Description string `ion:"description"` 288 | } 289 | 290 | var ItemSharedSymbols = ion.NewSharedSymbolTable("item", 1, []string{ 291 | "item", 292 | "id", 293 | "name", 294 | "description", 295 | }) 296 | 297 | type SpicyItem struct { 298 | Item 299 | Spiciness int `ion:"spiciness"` 300 | } 301 | 302 | func WriteSpicyItemsTo(out io.Writer, items []SpicyItem) error { 303 | writer := ion.NewBinaryWriter(out, ItemSharedSymbols) 304 | 305 | for _, item := range items { 306 | writer.Annotation("item") 307 | if err := ion.EncodeTo(writer, item); err != nil { 308 | return err 309 | } 310 | } 311 | 312 | return writer.Finish() 313 | } 314 | ``` 315 | 316 | You can alternatively provide the writer with a complete, pre-built local symbol table. 317 | This allows values to be written without buffering, however any attempt to write a 318 | symbol that is not included in the symbol table will result in an error: 319 | 320 | ```Go 321 | func WriteItemsToLST(out io.Writer, items []SpicyItem) error { 322 | lst := ion.NewLocalSymbolTable([]SharedSymbolTable{ItemSharedSymbols}, []string{ 323 | "spiciness", 324 | }) 325 | 326 | writer := ion.NewBinaryWriterLST(out, lst) 327 | 328 | for _, item := range items { 329 | writer.Annotation("item") 330 | if err := ion.EncodeTo(writer, item); err != nil { 331 | return err 332 | } 333 | } 334 | 335 | return writer.Finish() 336 | } 337 | ``` 338 | 339 | When reading binary Ion, shared symbol tables are provided by a `Catalog`. A basic 340 | catalog can be constructed by calling `NewCatalog`; a smarter implementation may 341 | load shared symbol tables from a database on demand. 342 | 343 | ```Go 344 | 345 | func ReadItemsFrom(in io.Reader) ([]Item, error) { 346 | item := Item{} 347 | items := []Item{} 348 | 349 | cat := ion.NewCatalog(ItemSharedSymbols) 350 | dec := ion.NewDecoder(ion.NewReaderCat(in, cat)) 351 | 352 | for { 353 | err := dec.DecodeTo(&item) 354 | if err == ion.ErrNoInput { 355 | return items, nil 356 | } 357 | if err != nil { 358 | return nil, err 359 | } 360 | 361 | items = append(items, item) 362 | } 363 | } 364 | ``` 365 | ### License 366 | 367 | This library is licensed under the Apache 2.0 License. 368 | -------------------------------------------------------------------------------- /cmd/ion-go/eventwriter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "io" 21 | "math/big" 22 | "strings" 23 | 24 | "github.com/amazon-ion/ion-go/ion" 25 | ) 26 | 27 | type eventwriter struct { 28 | enc *ion.Encoder 29 | 30 | depth int 31 | fieldname *string 32 | annotations []ion.SymbolToken 33 | inStruct map[int]bool 34 | } 35 | 36 | // NewEventWriter creates an ion.Writer that writes out a sequence 37 | // of ion-test-driver events. 38 | func NewEventWriter(out io.Writer) ion.Writer { 39 | w := ion.NewTextWriter(out) 40 | w.WriteSymbol(ion.NewSymbolTokenFromString("$ion_event_stream")) 41 | 42 | return &eventwriter{enc: ion.NewEncoder(w)} 43 | } 44 | 45 | func (e *eventwriter) FieldName(val ion.SymbolToken) error { 46 | if val.Text == nil { 47 | sid := fmt.Sprintf("$%v", val.LocalSID) 48 | e.fieldname = &sid 49 | return nil 50 | } 51 | 52 | e.fieldname = val.Text 53 | return nil 54 | } 55 | 56 | func (e *eventwriter) Annotation(val ion.SymbolToken) error { 57 | e.annotations = append(e.annotations, val) 58 | return nil 59 | } 60 | 61 | func (e *eventwriter) Annotations(values ...ion.SymbolToken) error { 62 | e.annotations = append(e.annotations, values...) 63 | return nil 64 | } 65 | 66 | func (e *eventwriter) WriteNull() error { 67 | return e.write(event{ 68 | EventType: scalar, 69 | IonType: iontype(ion.NullType), 70 | ValueText: "null", 71 | }) 72 | } 73 | 74 | func (e *eventwriter) WriteNullType(val ion.Type) error { 75 | return e.write(event{ 76 | EventType: scalar, 77 | IonType: iontype(val), 78 | ValueText: "null." + val.String(), 79 | }) 80 | } 81 | 82 | func (e *eventwriter) WriteBool(val bool) error { 83 | return e.write(event{ 84 | EventType: scalar, 85 | IonType: iontype(ion.BoolType), 86 | ValueText: stringify(val), 87 | }) 88 | } 89 | 90 | func (e *eventwriter) WriteInt(val int64) error { 91 | return e.write(event{ 92 | EventType: scalar, 93 | IonType: iontype(ion.IntType), 94 | ValueText: stringify(val), 95 | }) 96 | } 97 | 98 | func (e *eventwriter) WriteUint(val uint64) error { 99 | return e.write(event{ 100 | EventType: scalar, 101 | IonType: iontype(ion.IntType), 102 | ValueText: stringify(val), 103 | }) 104 | } 105 | 106 | func (e *eventwriter) WriteBigInt(val *big.Int) error { 107 | return e.write(event{ 108 | EventType: scalar, 109 | IonType: iontype(ion.IntType), 110 | ValueText: stringify(val), 111 | }) 112 | } 113 | 114 | func (e *eventwriter) WriteFloat(val float64) error { 115 | return e.write(event{ 116 | EventType: scalar, 117 | IonType: iontype(ion.FloatType), 118 | ValueText: stringify(val), 119 | }) 120 | } 121 | 122 | func (e *eventwriter) WriteDecimal(val *ion.Decimal) error { 123 | return e.write(event{ 124 | EventType: scalar, 125 | IonType: iontype(ion.DecimalType), 126 | ValueText: stringify(val), 127 | }) 128 | } 129 | 130 | func (e *eventwriter) WriteTimestamp(val ion.Timestamp) error { 131 | return e.write(event{ 132 | EventType: scalar, 133 | IonType: iontype(ion.TimestampType), 134 | ValueText: stringify(val), 135 | }) 136 | } 137 | 138 | func (e *eventwriter) WriteSymbol(val ion.SymbolToken) error { 139 | return e.write(event{ 140 | EventType: scalar, 141 | IonType: iontype(ion.SymbolType), 142 | ValueText: symbolify(val), 143 | }) 144 | } 145 | 146 | func (e *eventwriter) WriteSymbolFromString(val string) error { 147 | return e.write(event{ 148 | EventType: scalar, 149 | IonType: iontype(ion.SymbolType), 150 | ValueText: stringify(val), 151 | }) 152 | } 153 | 154 | func (e *eventwriter) WriteString(val string) error { 155 | return e.write(event{ 156 | EventType: scalar, 157 | IonType: iontype(ion.StringType), 158 | ValueText: stringify(val), 159 | }) 160 | } 161 | 162 | func (e *eventwriter) WriteClob(val []byte) error { 163 | return e.write(event{ 164 | EventType: scalar, 165 | IonType: iontype(ion.ClobType), 166 | ValueText: clobify(val), 167 | }) 168 | } 169 | 170 | func (e *eventwriter) WriteBlob(val []byte) error { 171 | return e.write(event{ 172 | EventType: scalar, 173 | IonType: iontype(ion.BlobType), 174 | ValueText: stringify(val), 175 | }) 176 | } 177 | 178 | func (e *eventwriter) BeginList() error { 179 | err := e.write(event{ 180 | EventType: containerStart, 181 | IonType: iontype(ion.ListType), 182 | }) 183 | if err != nil { 184 | return err 185 | } 186 | e.depth++ 187 | return nil 188 | } 189 | 190 | func (e *eventwriter) EndList() error { 191 | e.depth-- 192 | return e.write(event{ 193 | EventType: containerEnd, 194 | IonType: iontype(ion.ListType), 195 | }) 196 | } 197 | 198 | func (e *eventwriter) BeginSexp() error { 199 | err := e.write(event{ 200 | EventType: containerStart, 201 | IonType: iontype(ion.SexpType), 202 | }) 203 | if err != nil { 204 | return err 205 | } 206 | e.depth++ 207 | return nil 208 | } 209 | 210 | func (e *eventwriter) EndSexp() error { 211 | e.depth-- 212 | return e.write(event{ 213 | EventType: containerEnd, 214 | IonType: iontype(ion.SexpType), 215 | }) 216 | } 217 | 218 | func (e *eventwriter) BeginStruct() error { 219 | err := e.write(event{ 220 | EventType: containerStart, 221 | IonType: iontype(ion.StructType), 222 | }) 223 | if err != nil { 224 | return err 225 | } 226 | e.depth++ 227 | e.inStruct[e.depth] = true 228 | return nil 229 | } 230 | 231 | func (e *eventwriter) EndStruct() error { 232 | e.inStruct[e.depth] = false 233 | e.depth-- 234 | return e.write(event{ 235 | EventType: containerEnd, 236 | IonType: iontype(ion.StructType), 237 | }) 238 | } 239 | 240 | func (e *eventwriter) Finish() error { 241 | if err := e.write(event{EventType: streamEnd}); err != nil { 242 | return err 243 | } 244 | return e.enc.Finish() 245 | } 246 | 247 | func (e *eventwriter) IsInStruct() bool { 248 | return e.inStruct[e.depth] == true 249 | } 250 | 251 | func stringify(val interface{}) string { 252 | bs, err := ion.MarshalText(val) 253 | if err != nil { 254 | panic(err) 255 | } 256 | return string(bs) 257 | } 258 | 259 | func symbolify(val ion.SymbolToken) string { 260 | buf := strings.Builder{} 261 | w := ion.NewTextWriterOpts(&buf, ion.TextWriterQuietFinish) 262 | 263 | w.WriteSymbol(val) 264 | if err := w.Finish(); err != nil { 265 | panic(err) 266 | } 267 | 268 | return buf.String() 269 | } 270 | 271 | func clobify(val []byte) string { 272 | buf := strings.Builder{} 273 | w := ion.NewTextWriterOpts(&buf, ion.TextWriterQuietFinish) 274 | 275 | w.WriteClob(val) 276 | if err := w.Finish(); err != nil { 277 | panic(err) 278 | } 279 | 280 | return buf.String() 281 | } 282 | 283 | func (e *eventwriter) write(ev event) error { 284 | name := e.fieldname 285 | e.fieldname = nil 286 | annos := e.annotations 287 | e.annotations = nil 288 | 289 | if name != nil { 290 | ev.FieldName = &ion.SymbolToken{Text: name} 291 | } 292 | 293 | if len(annos) > 0 { 294 | asyms := make([]ion.SymbolToken, len(annos)) 295 | for i, a := range annos { 296 | asyms[i] = a 297 | } 298 | ev.Annotations = asyms 299 | } 300 | 301 | ev.Depth = e.depth 302 | 303 | return e.enc.Encode(ev) 304 | } 305 | -------------------------------------------------------------------------------- /cmd/ion-go/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "errors" 20 | "fmt" 21 | "os" 22 | 23 | "github.com/amazon-ion/ion-go/internal" 24 | "github.com/amazon-ion/ion-go/ion" 25 | ) 26 | 27 | // main is the main entry point for ion-go. 28 | func main() { 29 | if len(os.Args) <= 1 { 30 | printHelp() 31 | return 32 | } 33 | 34 | var err error 35 | 36 | switch os.Args[1] { 37 | case "help", "--help", "-h": 38 | printHelp() 39 | 40 | case "version", "--version", "-v": 41 | err = printVersion() 42 | 43 | case "process": 44 | err = process(os.Args[2:]) 45 | 46 | default: 47 | err = errors.New("unrecognized command \"" + os.Args[1] + "\"") 48 | } 49 | 50 | if err != nil { 51 | fmt.Println(err.Error()) 52 | printHelp() 53 | } 54 | } 55 | 56 | // printHelp prints the help message for the program. 57 | func printHelp() { 58 | fmt.Println("Usage:") 59 | fmt.Println(" ion-go help") 60 | fmt.Println(" ion-go version") 61 | fmt.Println(" ion-go process [args]") 62 | fmt.Println(" ion-go compare [args]") 63 | fmt.Println(" ion-go extract [args]") 64 | fmt.Println() 65 | fmt.Println("Commands:") 66 | fmt.Println(" help Prints this help message.") 67 | fmt.Println(" version Prints version information about this tool.") 68 | fmt.Println(" extract Extracts symbols from the given inputs into a shared symbol table.") 69 | fmt.Println(" compare Compares all inputs against all other inputs and writes out a ComparisonReport.") 70 | fmt.Println(" process Reads the input file(s) and re-writes the contents in the specified format.") 71 | } 72 | 73 | // printVersion prints (in ion) the version info for this tool. 74 | func printVersion() error { 75 | w := ion.NewTextWriterOpts(os.Stdout, ion.TextWriterPretty) 76 | 77 | if err := w.BeginStruct(); err != nil { 78 | return err 79 | } 80 | { 81 | if err := w.FieldName(ion.NewSymbolTokenFromString("version")); err != nil { 82 | return err 83 | } 84 | if err := w.WriteString(internal.GitCommit); err != nil { 85 | return err 86 | } 87 | 88 | if err := w.FieldName(ion.NewSymbolTokenFromString("build_time")); err != nil { 89 | return err 90 | } 91 | 92 | buildtime, err := ion.NewTimestampFromStr(internal.BuildTime, ion.TimestampPrecisionSecond, ion.TimezoneUTC) 93 | if err == nil { 94 | if err := w.WriteTimestamp(buildtime); err != nil { 95 | return err 96 | } 97 | } else if err := w.WriteString("unknown-buildtime"); err != nil { 98 | return err 99 | } 100 | } 101 | if err := w.EndStruct(); err != nil { 102 | return err 103 | } 104 | 105 | if err := w.Finish(); err != nil { 106 | panic(err) 107 | } 108 | 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /cmd/ion-go/nopwriter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "math/big" 20 | 21 | "github.com/amazon-ion/ion-go/ion" 22 | ) 23 | 24 | type nopwriter struct{} 25 | 26 | // NewNopWriter returns a no-op Ion writer. 27 | func NewNopWriter() ion.Writer { 28 | return nopwriter{} 29 | } 30 | 31 | func (nopwriter) FieldName(ion.SymbolToken) error { 32 | return nil 33 | } 34 | 35 | func (nopwriter) Annotation(ion.SymbolToken) error { 36 | return nil 37 | } 38 | 39 | func (nopwriter) Annotations(...ion.SymbolToken) error { 40 | return nil 41 | } 42 | 43 | func (nopwriter) WriteNull() error { 44 | return nil 45 | } 46 | 47 | func (nopwriter) WriteNullType(ion.Type) error { 48 | return nil 49 | } 50 | 51 | func (nopwriter) WriteBool(bool) error { 52 | return nil 53 | } 54 | 55 | func (nopwriter) WriteInt(int64) error { 56 | return nil 57 | } 58 | 59 | func (nopwriter) WriteUint(uint64) error { 60 | return nil 61 | } 62 | 63 | func (nopwriter) WriteBigInt(*big.Int) error { 64 | return nil 65 | } 66 | 67 | func (nopwriter) WriteFloat(float64) error { 68 | return nil 69 | } 70 | 71 | func (nopwriter) WriteDecimal(*ion.Decimal) error { 72 | return nil 73 | } 74 | 75 | func (nopwriter) WriteTimestamp(ion.Timestamp) error { 76 | return nil 77 | } 78 | 79 | func (nopwriter) WriteSymbol(ion.SymbolToken) error { 80 | return nil 81 | } 82 | 83 | func (nopwriter) WriteSymbolFromString(string) error { 84 | return nil 85 | } 86 | 87 | func (nopwriter) WriteString(string) error { 88 | return nil 89 | } 90 | 91 | func (nopwriter) WriteClob([]byte) error { 92 | return nil 93 | } 94 | 95 | func (nopwriter) WriteBlob([]byte) error { 96 | return nil 97 | } 98 | 99 | func (nopwriter) BeginList() error { 100 | return nil 101 | } 102 | 103 | func (nopwriter) EndList() error { 104 | return nil 105 | } 106 | 107 | func (nopwriter) BeginSexp() error { 108 | return nil 109 | } 110 | 111 | func (nopwriter) EndSexp() error { 112 | return nil 113 | } 114 | 115 | func (nopwriter) BeginStruct() error { 116 | return nil 117 | } 118 | 119 | func (nopwriter) EndStruct() error { 120 | return nil 121 | } 122 | 123 | func (nopwriter) Finish() error { 124 | return nil 125 | } 126 | 127 | func (nopwriter) IsInStruct() bool { 128 | return false 129 | } 130 | -------------------------------------------------------------------------------- /cmd/ion-go/process.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "errors" 20 | "fmt" 21 | "io" 22 | "strings" 23 | 24 | "github.com/amazon-ion/ion-go/ion" 25 | ) 26 | 27 | // process reads the specified input file(s) and re-writes the contents in the 28 | // specified format. 29 | func process(args []string) error { 30 | p, err := newProcessor(args) 31 | if err != nil { 32 | return err 33 | } 34 | return p.run() 35 | } 36 | 37 | type processor struct { 38 | infs []string 39 | outf string 40 | errf string 41 | 42 | format string 43 | 44 | out ion.Writer 45 | err *ErrorReport 46 | loc string 47 | idx int 48 | } 49 | 50 | func newProcessor(args []string) (*processor, error) { 51 | ret := &processor{} 52 | 53 | i := 0 54 | for ; i < len(args); i++ { 55 | arg := args[i] 56 | if !strings.HasPrefix(arg, "-") { 57 | break 58 | } 59 | if arg == "-" || arg == "--" { 60 | i++ 61 | break 62 | } 63 | 64 | switch arg { 65 | case "-o", "--output": 66 | i++ 67 | if i >= len(args) { 68 | return nil, errors.New("no output file specified") 69 | } 70 | ret.outf = args[i] 71 | 72 | case "-f", "--output-format": 73 | i++ 74 | if i >= len(args) { 75 | return nil, errors.New("no output format specified") 76 | } 77 | ret.format = args[i] 78 | 79 | case "-e", "--error-report": 80 | i++ 81 | if i >= len(args) { 82 | return nil, errors.New("no error report file specified") 83 | } 84 | ret.errf = args[i] 85 | 86 | // https://github.com/amazon-ion/ion-go/issues/121 87 | 88 | default: 89 | return nil, errors.New("unrecognized option \"" + arg + "\"") 90 | } 91 | } 92 | 93 | // Any remaining args are input files. 94 | for ; i < len(args); i++ { 95 | ret.infs = append(ret.infs, args[i]) 96 | } 97 | 98 | return ret, nil 99 | } 100 | 101 | func (p *processor) run() (deferredErr error) { 102 | outf, err := OpenOutput(p.outf) 103 | if err != nil { 104 | return err 105 | } 106 | defer func() { 107 | closeError := outf.Close() 108 | if err == nil { 109 | deferredErr = closeError 110 | } else { 111 | deferredErr = err 112 | } 113 | }() 114 | 115 | switch p.format { 116 | case "", "pretty": 117 | p.out = ion.NewTextWriterOpts(outf, ion.TextWriterPretty) 118 | case "text": 119 | p.out = ion.NewTextWriter(outf) 120 | case "binary": 121 | p.out = ion.NewBinaryWriter(outf) 122 | case "events": 123 | p.out = NewEventWriter(outf) 124 | case "none": 125 | p.out = NewNopWriter() 126 | default: 127 | err = errors.New("unrecognized output format \"" + p.format + "\"") 128 | return err 129 | } 130 | 131 | errf, err := OpenError(p.errf) 132 | if err != nil { 133 | return err 134 | } 135 | defer func() { 136 | finishError := errf.Close() 137 | if err == nil { 138 | deferredErr = finishError 139 | } else { 140 | deferredErr = err 141 | } 142 | }() 143 | 144 | p.err = NewErrorReport(errf) 145 | defer func() { 146 | finishError := p.err.Finish() 147 | if err == nil { 148 | deferredErr = finishError 149 | } else { 150 | deferredErr = err 151 | } 152 | }() 153 | 154 | if len(p.infs) == 0 { 155 | p.processStdin() 156 | return nil 157 | } 158 | 159 | err = p.processFiles() 160 | if err != nil { 161 | return err 162 | } 163 | 164 | return err 165 | } 166 | 167 | func (p *processor) processStdin() { 168 | p.loc = "stdin" 169 | p.processReader(stdin{}) 170 | p.loc = "" 171 | 172 | if err := p.out.Finish(); err != nil { 173 | p.error(write, err) 174 | } 175 | } 176 | 177 | func (p *processor) processFiles() error { 178 | for _, inf := range p.infs { 179 | if err := p.processFile(inf); err != nil { 180 | return err 181 | } 182 | } 183 | 184 | if err := p.out.Finish(); err != nil { 185 | p.error(write, err) 186 | } 187 | 188 | return nil 189 | } 190 | 191 | func (p *processor) processFile(in string) (err error) { 192 | f, err := OpenInput(in) 193 | if err != nil { 194 | return err 195 | } 196 | 197 | defer func() { 198 | err = f.Close() 199 | }() 200 | 201 | p.loc = in 202 | p.processReader(f) 203 | p.loc = "" 204 | 205 | return nil 206 | } 207 | 208 | func (p *processor) processReader(in io.Reader) { 209 | // We intentionally ignore the returned error; it's been written 210 | // to p.err, and only gets returned to short-circuit further execution. 211 | p.process(ion.NewReader(in)) 212 | } 213 | 214 | func (p *processor) process(in ion.Reader) error { 215 | var err error 216 | 217 | for in.Next() { 218 | p.idx++ 219 | name, e := in.FieldName() 220 | if e != nil { 221 | return p.error(read, err) 222 | } 223 | if name != nil { 224 | if err = p.out.FieldName(*name); err != nil { 225 | return p.error(write, err) 226 | } 227 | } 228 | 229 | annos, err := in.Annotations() 230 | if err != nil { 231 | return p.error(read, err) 232 | } 233 | if len(annos) > 0 { 234 | if err = p.out.Annotations(annos...); err != nil { 235 | return p.error(write, err) 236 | } 237 | } 238 | 239 | switch in.Type() { 240 | case ion.NullType: 241 | err = p.out.WriteNull() 242 | 243 | case ion.BoolType: 244 | val, err := in.BoolValue() 245 | if err != nil { 246 | return p.error(read, err) 247 | } 248 | err = p.out.WriteBool(*val) 249 | 250 | case ion.IntType: 251 | size, err := in.IntSize() 252 | if err != nil { 253 | return p.error(read, err) 254 | } 255 | 256 | switch size { 257 | case ion.Int32: 258 | val, err := in.IntValue() 259 | if err != nil { 260 | return p.error(read, err) 261 | } 262 | err = p.out.WriteInt(int64(*val)) 263 | 264 | case ion.Int64: 265 | val, err := in.Int64Value() 266 | if err != nil { 267 | return p.error(read, err) 268 | } 269 | err = p.out.WriteInt(*val) 270 | 271 | case ion.BigInt: 272 | val, err := in.BigIntValue() 273 | if err != nil { 274 | return p.error(read, err) 275 | } 276 | err = p.out.WriteBigInt(val) 277 | 278 | default: 279 | panic(fmt.Sprintf("bad int size: %v", size)) 280 | } 281 | 282 | case ion.FloatType: 283 | val, err := in.FloatValue() 284 | if err != nil { 285 | return p.error(read, err) 286 | } 287 | err = p.out.WriteFloat(*val) 288 | 289 | case ion.DecimalType: 290 | val, err := in.DecimalValue() 291 | if err != nil { 292 | return p.error(read, err) 293 | } 294 | err = p.out.WriteDecimal(val) 295 | 296 | case ion.TimestampType: 297 | val, err := in.TimestampValue() 298 | if err != nil { 299 | return p.error(read, err) 300 | } 301 | err = p.out.WriteTimestamp(*val) 302 | 303 | case ion.SymbolType: 304 | val, err := in.SymbolValue() 305 | if err != nil { 306 | return p.error(read, err) 307 | } 308 | if val != nil { 309 | err = p.out.WriteSymbol(*val) 310 | } 311 | 312 | case ion.StringType: 313 | val, err := in.StringValue() 314 | if err != nil { 315 | return p.error(read, err) 316 | } 317 | if val != nil { 318 | err = p.out.WriteString(*val) 319 | } 320 | 321 | case ion.ClobType: 322 | val, err := in.ByteValue() 323 | if err != nil { 324 | return p.error(read, err) 325 | } 326 | err = p.out.WriteClob(val) 327 | 328 | case ion.BlobType: 329 | val, err := in.ByteValue() 330 | if err != nil { 331 | return p.error(read, err) 332 | } 333 | err = p.out.WriteBlob(val) 334 | 335 | case ion.ListType: 336 | if err := in.StepIn(); err != nil { 337 | return p.error(read, err) 338 | } 339 | if err := p.out.BeginList(); err != nil { 340 | return p.error(write, err) 341 | } 342 | if err := p.process(in); err != nil { 343 | return err 344 | } 345 | p.idx++ 346 | if err := in.StepOut(); err != nil { 347 | return p.error(read, err) 348 | } 349 | err = p.out.EndList() 350 | 351 | case ion.SexpType: 352 | if err := in.StepIn(); err != nil { 353 | return p.error(read, err) 354 | } 355 | if err := p.out.BeginSexp(); err != nil { 356 | return p.error(write, err) 357 | } 358 | if err := p.process(in); err != nil { 359 | return err 360 | } 361 | p.idx++ 362 | if err := in.StepOut(); err != nil { 363 | return p.error(read, err) 364 | } 365 | err = p.out.EndSexp() 366 | 367 | case ion.StructType: 368 | if err := in.StepIn(); err != nil { 369 | return p.error(read, err) 370 | } 371 | if err := p.out.BeginStruct(); err != nil { 372 | return p.error(write, err) 373 | } 374 | if err := p.process(in); err != nil { 375 | return err 376 | } 377 | p.idx++ 378 | if err := in.StepOut(); err != nil { 379 | return p.error(read, err) 380 | } 381 | err = p.out.EndStruct() 382 | 383 | default: 384 | panic(fmt.Sprintf("bad ion type: %v", in.Type())) 385 | } 386 | 387 | if err != nil { 388 | return p.error(write, err) 389 | } 390 | } 391 | 392 | if err := in.Err(); err != nil { 393 | return p.error(read, err) 394 | } 395 | return nil 396 | } 397 | 398 | func (p *processor) error(typ errortype, err error) error { 399 | p.err.Append(typ, err.Error(), p.loc, p.idx) 400 | return err 401 | } 402 | -------------------------------------------------------------------------------- /cmd/ion-go/schema.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "strings" 21 | 22 | "github.com/amazon-ion/ion-go/ion" 23 | ) 24 | 25 | type importdescriptor struct { 26 | ImportName string `ion:"import_name"` 27 | Version int `ion:"version"` 28 | MaxID int `ion:"max_id"` 29 | } 30 | 31 | type eventtype uint8 32 | 33 | const ( 34 | containerStart eventtype = iota 35 | containerEnd 36 | scalar 37 | symbolTable 38 | streamEnd 39 | ) 40 | 41 | func (e eventtype) String() string { 42 | switch e { 43 | case containerStart: 44 | return "CONTAINER_START" 45 | case containerEnd: 46 | return "CONTAINER_END" 47 | case scalar: 48 | return "SCALAR" 49 | case symbolTable: 50 | return "SYMBOL_TABLE" 51 | case streamEnd: 52 | return "STREAM_END" 53 | default: 54 | panic(fmt.Sprintf("unknown eventtype %d", e)) 55 | } 56 | } 57 | 58 | func (e eventtype) MarshalIon(w ion.Writer) error { 59 | return w.WriteSymbolFromString(e.String()) 60 | } 61 | 62 | type iontype ion.Type 63 | 64 | func (i iontype) MarshalIon(w ion.Writer) error { 65 | return w.WriteSymbolFromString(strings.ToUpper(ion.Type(i).String())) 66 | } 67 | 68 | // event describes an Ion processing event. 69 | type event struct { 70 | EventType eventtype `ion:"event_type"` 71 | IonType iontype `ion:"ion_type,omitempty"` 72 | FieldName *ion.SymbolToken `ion:"field_name,omitempty"` 73 | Annotations []ion.SymbolToken `ion:"annotations,omitempty"` 74 | ValueText string `ion:"value_text,omitempty"` 75 | ValueBinary []int `ion:"value_binary,omitempty"` 76 | Imports []importdescriptor `ion:"imports,omitempty"` 77 | Depth int `ion:"depth"` 78 | } 79 | 80 | type errortype uint8 81 | 82 | const ( 83 | read errortype = iota 84 | write 85 | state 86 | ) 87 | 88 | func (e errortype) String() string { 89 | switch e { 90 | case read: 91 | return "READ" 92 | case write: 93 | return "WRITE" 94 | case state: 95 | return "STATE" 96 | default: 97 | panic(fmt.Sprintf("unknown errortype %d", e)) 98 | } 99 | } 100 | 101 | func (e errortype) MarshalIon(w ion.Writer) error { 102 | return w.WriteSymbolFromString(e.String()) 103 | } 104 | 105 | // errordescription describes an error during Ion processing. 106 | type errordescription struct { 107 | ErrorType errortype `ion:"error_type"` 108 | Message string `ion:"message"` 109 | Location string `ion:"location"` 110 | Index int `ion:"event_index"` 111 | } 112 | -------------------------------------------------------------------------------- /cmd/ion-go/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package main 17 | 18 | import ( 19 | "io" 20 | "os" 21 | 22 | "github.com/amazon-ion/ion-go/ion" 23 | ) 24 | 25 | type stdin struct{} 26 | 27 | func (stdin) Read(bs []byte) (int, error) { return os.Stdin.Read(bs) } 28 | func (stdin) Close() error { return nil } 29 | 30 | // OpenInput opens an input stream. 31 | func OpenInput(in string) (io.ReadCloser, error) { 32 | r, err := os.Open(in) 33 | if err != nil { 34 | return nil, err 35 | } 36 | return r, nil 37 | } 38 | 39 | type uncloseable struct { 40 | w io.Writer 41 | } 42 | 43 | func (u uncloseable) Write(bs []byte) (int, error) { 44 | return u.w.Write(bs) 45 | } 46 | 47 | func (u uncloseable) Close() error { 48 | return nil 49 | } 50 | 51 | // OpenOutput opens the output stream. 52 | func OpenOutput(outf string) (io.WriteCloser, error) { 53 | if outf == "" { 54 | return uncloseable{os.Stdout}, nil 55 | } 56 | return os.OpenFile(outf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644) 57 | } 58 | 59 | // OpenError opens the error stream. 60 | func OpenError(errf string) (io.WriteCloser, error) { 61 | if errf == "" { 62 | return uncloseable{os.Stderr}, nil 63 | } 64 | return os.OpenFile(errf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644) 65 | } 66 | 67 | // ErrorReport is a (serialized) report of errors that occur during processing. 68 | type ErrorReport struct { 69 | w *ion.Encoder 70 | } 71 | 72 | // NewErrorReport creates a new ErrorReport. 73 | func NewErrorReport(w io.Writer) *ErrorReport { 74 | return &ErrorReport{ 75 | w: ion.NewTextEncoder(w), 76 | } 77 | } 78 | 79 | // Append appends an error to this report. 80 | func (r *ErrorReport) Append(typ errortype, msg, loc string, idx int) { 81 | if err := r.w.Encode(errordescription{typ, msg, loc, idx}); err != nil { 82 | panic(err) 83 | } 84 | } 85 | 86 | // Finish finishes writing this report. 87 | func (r *ErrorReport) Finish() error { 88 | return r.w.Finish() 89 | } 90 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/amazon-ion/ion-go 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/google/go-cmp v0.5.0 7 | github.com/stretchr/testify v1.6.1 8 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= 4 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 8 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 9 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 10 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 11 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 12 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 16 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 17 | -------------------------------------------------------------------------------- /internal/version.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | // Build-time constants injected with ldflags. 4 | var ( 5 | GitCommit = "unknown-commit" 6 | BuildTime = "unknown-buildtime" 7 | ) 8 | -------------------------------------------------------------------------------- /ion/binaryreader.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "bufio" 20 | "fmt" 21 | ) 22 | 23 | // A binaryReader reads binary Ion. 24 | type binaryReader struct { 25 | reader 26 | 27 | bits bitstream 28 | cat Catalog 29 | resetPos uint64 30 | } 31 | 32 | func newBinaryReaderBuf(in *bufio.Reader, cat Catalog) Reader { 33 | r := &binaryReader{ 34 | cat: cat, 35 | } 36 | r.bits.Init(in) 37 | return r 38 | } 39 | 40 | // Reset causes the binary reader to start reading from the given input bytes 41 | // while skipping most of the initialization steps needed to prepare the 42 | // reader. Reset is most commonly called with the same bytes as the reader 43 | // was originally created with (e.g. via NewReaderBytes) as an optimization 44 | // when the same data needs to be read multiple times. 45 | // 46 | // While it is possible to call Reset with different input bytes, the Reader 47 | // will only work correctly if the new bytes contain the exact same binary 48 | // version marker and local symbols as the original input. If there are any 49 | // doubts whether this is the case, it is instead recommended to create a 50 | // new Reader using NewReaderBytes (or NewReaderCat) instead. Attempting to 51 | // reuse a binaryReader with inconsistent input bytes will cause the reader 52 | // to return errors, misappropriate values to unrelated or non-existent 53 | // attributes, or incorrectly parse data values. 54 | // 55 | // This API is experimental and should be considered unstable. 56 | // See https://github.com/amazon-ion/ion-go/pull/196 57 | func (r *binaryReader) Reset(in []byte) error { 58 | if r.resetPos == invalidReset { 59 | return &UsageError{"binaryReader.Reset", "cannot reset when multiple local symbol tables found"} 60 | } 61 | r.annotations = nil 62 | r.valueType = NoType 63 | r.value = nil 64 | r.err = nil 65 | r.eof = false 66 | r.bits = bitstream{} 67 | r.bits.InitBytes(in[r.resetPos:]) 68 | return nil 69 | } 70 | 71 | const invalidReset uint64 = 1<<64 - 1 72 | 73 | // Next moves the reader to the next value. 74 | func (r *binaryReader) Next() bool { 75 | if r.eof || r.err != nil { 76 | return false 77 | } 78 | 79 | r.clear() 80 | 81 | done := false 82 | for !done { 83 | done, r.err = r.next() 84 | if r.err != nil { 85 | return false 86 | } 87 | } 88 | 89 | return !r.eof 90 | } 91 | 92 | // Next consumes the next raw value from the stream, returning true if it 93 | // represents a user-facing value and false if it does not. 94 | func (r *binaryReader) next() (bool, error) { 95 | if err := r.bits.Next(); err != nil { 96 | return false, err 97 | } 98 | 99 | code := r.bits.Code() 100 | switch code { 101 | case bitcodeEOF: 102 | r.eof = true 103 | return true, nil 104 | 105 | case bitcodeBVM: 106 | err := r.readBVM() 107 | return false, err 108 | 109 | case bitcodeFieldID: 110 | err := r.readFieldName() 111 | return false, err 112 | 113 | case bitcodeAnnotation: 114 | err := r.readAnnotations() 115 | return false, err 116 | 117 | case bitcodeNull: 118 | if !r.bits.IsNull() { 119 | // NOP padding; skip it and keep going. 120 | err := r.bits.SkipValue() 121 | return false, err 122 | } 123 | r.valueType = NullType 124 | return true, nil 125 | 126 | case bitcodeFalse, bitcodeTrue: 127 | r.valueType = BoolType 128 | if !r.bits.IsNull() { 129 | r.value = r.bits.Code() == bitcodeTrue 130 | } 131 | return true, nil 132 | 133 | case bitcodeInt, bitcodeNegInt: 134 | r.valueType = IntType 135 | if !r.bits.IsNull() { 136 | val, err := r.bits.ReadInt() 137 | if err != nil { 138 | return false, err 139 | } 140 | r.value = val 141 | } 142 | return true, nil 143 | 144 | case bitcodeFloat: 145 | r.valueType = FloatType 146 | if !r.bits.IsNull() { 147 | val, err := r.bits.ReadFloat() 148 | if err != nil { 149 | return false, err 150 | } 151 | r.value = val 152 | } 153 | return true, nil 154 | 155 | case bitcodeDecimal: 156 | r.valueType = DecimalType 157 | if !r.bits.IsNull() { 158 | val, err := r.bits.ReadDecimal() 159 | if err != nil { 160 | return false, err 161 | } 162 | r.value = val 163 | } 164 | return true, nil 165 | 166 | case bitcodeTimestamp: 167 | r.valueType = TimestampType 168 | if !r.bits.IsNull() { 169 | val, err := r.bits.ReadTimestamp() 170 | if err != nil { 171 | return false, err 172 | } 173 | r.value = val 174 | } 175 | return true, nil 176 | 177 | case bitcodeSymbol: 178 | if !r.bits.IsNull() { 179 | id, err := r.bits.ReadSymbolID() 180 | if err != nil { 181 | return false, err 182 | } 183 | st, err := NewSymbolTokenBySID(r.SymbolTable(), int64(id)) 184 | if err != nil { 185 | return false, err 186 | } 187 | r.value = &st 188 | } 189 | r.valueType = SymbolType 190 | return true, nil 191 | 192 | case bitcodeString: 193 | r.valueType = StringType 194 | if !r.bits.IsNull() { 195 | val, err := r.bits.ReadString() 196 | if err != nil { 197 | return false, err 198 | } 199 | r.value = val 200 | } 201 | return true, nil 202 | 203 | case bitcodeClob: 204 | r.valueType = ClobType 205 | if !r.bits.IsNull() { 206 | val, err := r.bits.ReadBytes() 207 | if err != nil { 208 | return false, err 209 | } 210 | r.value = val 211 | } 212 | return true, nil 213 | 214 | case bitcodeBlob: 215 | r.valueType = BlobType 216 | if !r.bits.IsNull() { 217 | val, err := r.bits.ReadBytes() 218 | if err != nil { 219 | return false, err 220 | } 221 | r.value = val 222 | } 223 | return true, nil 224 | 225 | case bitcodeList: 226 | r.valueType = ListType 227 | if !r.bits.IsNull() { 228 | r.value = ListType 229 | } 230 | return true, nil 231 | 232 | case bitcodeSexp: 233 | r.valueType = SexpType 234 | if !r.bits.IsNull() { 235 | r.value = SexpType 236 | } 237 | return true, nil 238 | 239 | case bitcodeStruct: 240 | r.valueType = StructType 241 | if !r.bits.IsNull() { 242 | r.value = StructType 243 | } 244 | 245 | // If it's a local symbol table, install it and keep going. 246 | if r.ctx.peek() == ctxAtTopLevel && isIonSymbolTable(r.annotations) { 247 | if r.IsNull() { 248 | r.clear() 249 | r.lst = V1SystemSymbolTable 250 | return false, nil 251 | } 252 | st, err := readLocalSymbolTable(r, r.cat) 253 | if err == nil { 254 | r.lst = st 255 | if r.resetPos == 0 { 256 | r.resetPos = r.bits.pos 257 | } else { 258 | r.resetPos = invalidReset 259 | } 260 | return false, nil 261 | } 262 | return false, err 263 | } 264 | 265 | return true, nil 266 | } 267 | panic(fmt.Sprintf("invalid bitcode %v", code)) 268 | } 269 | 270 | func isIonSymbolTable(as []SymbolToken) bool { 271 | return len(as) > 0 && as[0].Text != nil && *as[0].Text == "$ion_symbol_table" 272 | } 273 | 274 | // ReadBVM reads a BVM, validates it, and resets the local symbol table. 275 | func (r *binaryReader) readBVM() error { 276 | major, minor, err := r.bits.ReadBVM() 277 | if err != nil { 278 | return err 279 | } 280 | 281 | switch major { 282 | case 1: 283 | switch minor { 284 | case 0: 285 | r.lst = V1SystemSymbolTable 286 | return nil 287 | } 288 | } 289 | 290 | return &UnsupportedVersionError{ 291 | int(major), 292 | int(minor), 293 | r.bits.Pos() - 4, 294 | } 295 | } 296 | 297 | // ReadFieldName reads and resolves a field name. 298 | func (r *binaryReader) readFieldName() error { 299 | id, err := r.bits.ReadFieldID() 300 | if err != nil { 301 | return err 302 | } 303 | 304 | st, err := NewSymbolTokenBySID(r.SymbolTable(), int64(id)) 305 | if err != nil { 306 | return err 307 | } 308 | 309 | r.fieldName = &st 310 | return nil 311 | } 312 | 313 | // ReadAnnotations reads and resolves a set of annotations. 314 | func (r *binaryReader) readAnnotations() error { 315 | as, err := r.bits.ReadAnnotations(r.SymbolTable()) 316 | if err != nil { 317 | return err 318 | } 319 | 320 | r.annotations = as 321 | 322 | return nil 323 | } 324 | 325 | // StepIn steps in to a container-type value 326 | func (r *binaryReader) StepIn() error { 327 | if r.err != nil { 328 | return r.err 329 | } 330 | 331 | if r.valueType != ListType && r.valueType != SexpType && r.valueType != StructType { 332 | return &UsageError{"Reader.StepIn", fmt.Sprintf("cannot step in to a %v", r.valueType)} 333 | } 334 | if r.value == nil { 335 | return &UsageError{"Reader.StepIn", "cannot step in to a null container"} 336 | } 337 | 338 | r.ctx.push(containerTypeToCtx(r.valueType)) 339 | r.clear() 340 | r.bits.StepIn() 341 | 342 | return nil 343 | } 344 | 345 | // StepOut steps out of a container-type value. 346 | func (r *binaryReader) StepOut() error { 347 | if r.err != nil { 348 | return r.err 349 | } 350 | if r.ctx.peek() == ctxAtTopLevel { 351 | return &UsageError{"Reader.StepOut", "cannot step out of top-level datagram"} 352 | } 353 | 354 | if err := r.bits.StepOut(); err != nil { 355 | return err 356 | } 357 | 358 | r.clear() 359 | r.ctx.pop() 360 | r.eof = false 361 | 362 | return nil 363 | } 364 | -------------------------------------------------------------------------------- /ion/bits.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "math/big" 20 | ) 21 | 22 | // uintLen pre-calculates the length, in bytes, of the given uint value. 23 | func uintLen(v uint64) uint64 { 24 | length := uint64(1) 25 | v >>= 8 26 | 27 | for v > 0 { 28 | length++ 29 | v >>= 8 30 | } 31 | 32 | return length 33 | } 34 | 35 | // appendUint appends a uint value to the given slice. The reader is 36 | // expected to know how many bytes the value takes up. 37 | func appendUint(b []byte, v uint64) []byte { 38 | var buf [8]byte 39 | 40 | i := 7 41 | buf[i] = byte(v & 0xFF) 42 | v >>= 8 43 | 44 | for v > 0 { 45 | i-- 46 | buf[i] = byte(v & 0xFF) 47 | v >>= 8 48 | } 49 | 50 | return append(b, buf[i:]...) 51 | } 52 | 53 | // intLen pre-calculates the length, in bytes, of the given int value. 54 | func intLen(n int64) uint64 { 55 | if n == 0 { 56 | return 0 57 | } 58 | 59 | mag := uint64(n) 60 | if n < 0 { 61 | mag = uint64(-n) 62 | } 63 | 64 | length := uintLen(mag) 65 | 66 | // If the high bit is a one, we need an extra byte to store the sign bit. 67 | hb := mag >> ((length - 1) * 8) 68 | if hb&0x80 != 0 { 69 | length++ 70 | } 71 | 72 | return length 73 | } 74 | 75 | // appendInt appends a (signed) int to the given slice. The reader is 76 | // expected to know how many bytes the value takes up. 77 | func appendInt(b []byte, n int64) []byte { 78 | if n == 0 { 79 | return b 80 | } 81 | 82 | neg := false 83 | mag := uint64(n) 84 | 85 | if n < 0 { 86 | neg = true 87 | mag = uint64(-n) 88 | } 89 | 90 | var buf [8]byte 91 | bits := buf[:0] 92 | bits = appendUint(bits, mag) 93 | 94 | if bits[0]&0x80 == 0 { 95 | // We've got space we can use for the sign bit. 96 | if neg { 97 | bits[0] ^= 0x80 98 | } 99 | } else { 100 | // We need to add more space. 101 | bit := byte(0) 102 | if neg { 103 | bit = 0x80 104 | } 105 | b = append(b, bit) 106 | } 107 | 108 | return append(b, bits...) 109 | } 110 | 111 | // bigIntLen pre-calculates the length, in bytes, of the given big.Int value. 112 | func bigIntLen(v *big.Int) uint64 { 113 | if v.Sign() == 0 { 114 | return 0 115 | } 116 | 117 | bitl := v.BitLen() 118 | bytel := bitl / 8 119 | 120 | // Either bitl is evenly divisible by 8, in which case we need another 121 | // byte for the sign bit, or its not in which case we need to round up 122 | // (but will then have room for the sign bit). 123 | return uint64(bytel) + 1 124 | } 125 | 126 | // appendBigInt appends a (signed) big.Int to the given slice. The reader is 127 | // expected to know how many bytes the value takes up. 128 | func appendBigInt(b []byte, v *big.Int) []byte { 129 | sign := v.Sign() 130 | if sign == 0 { 131 | return b 132 | } 133 | 134 | bits := v.Bytes() 135 | 136 | if bits[0]&0x80 == 0 { 137 | // We've got space we can use for the sign bit. 138 | if sign < 0 { 139 | bits[0] ^= 0x80 140 | } 141 | } else { 142 | // We need to add more space. 143 | bit := byte(0) 144 | if sign < 0 { 145 | bit = 0x80 146 | } 147 | b = append(b, bit) 148 | } 149 | 150 | return append(b, bits...) 151 | } 152 | 153 | // varUintLen pre-calculates the length, in bytes, of the given varUint value. 154 | func varUintLen(v uint64) uint64 { 155 | length := uint64(1) 156 | v >>= 7 157 | 158 | for v > 0 { 159 | length++ 160 | v >>= 7 161 | } 162 | 163 | return length 164 | } 165 | 166 | // appendVarUint appends a variable-length-encoded uint to the given slice. 167 | // Each byte stores seven bits of value; the high bit is a flag marking the 168 | // last byte of the value. 169 | func appendVarUint(b []byte, v uint64) []byte { 170 | var buf [10]byte 171 | 172 | i := 9 173 | buf[i] = 0x80 | byte(v&0x7F) 174 | v >>= 7 175 | 176 | for v > 0 { 177 | i-- 178 | buf[i] = byte(v & 0x7F) 179 | v >>= 7 180 | } 181 | 182 | return append(b, buf[i:]...) 183 | } 184 | 185 | // varIntLen pre-calculates the length, in bytes, of the given varInt value. 186 | func varIntLen(v int64) uint64 { 187 | mag := uint64(v) 188 | if v < 0 { 189 | mag = uint64(-v) 190 | } 191 | 192 | // Reserve one extra bit of the first byte for sign. 193 | length := uint64(1) 194 | mag >>= 6 195 | 196 | for mag > 0 { 197 | length++ 198 | mag >>= 7 199 | } 200 | 201 | return length 202 | } 203 | 204 | // appendVarInt appends a variable-length-encoded int to the given slice. 205 | // Most bytes store seven bits of value; the high bit is a flag marking the 206 | // last byte of the value. The first byte additionally stores a sign bit. 207 | func appendVarInt(b []byte, v int64) []byte { 208 | var buf [10]byte 209 | 210 | signbit := byte(0) 211 | mag := uint64(v) 212 | if v < 0 { 213 | signbit = 0x40 214 | mag = uint64(-v) 215 | } 216 | 217 | next := mag >> 6 218 | if next == 0 { 219 | // The whole thing fits in one byte. 220 | return append(b, 0x80|signbit|byte(mag&0x3F)) 221 | } 222 | 223 | i := 9 224 | buf[i] = 0x80 | byte(mag&0x7F) 225 | mag >>= 7 226 | next = mag >> 6 227 | 228 | for next > 0 { 229 | i-- 230 | buf[i] = byte(mag & 0x7F) 231 | mag >>= 7 232 | next = mag >> 6 233 | } 234 | 235 | i-- 236 | buf[i] = signbit | byte(mag&0x3F) 237 | 238 | return append(b, buf[i:]...) 239 | } 240 | 241 | // tagLen pre-calculates the length, in bytes, of a tag. 242 | func tagLen(length uint64) uint64 { 243 | if length < 0x0E { 244 | return 1 245 | } 246 | return 1 + varUintLen(length) 247 | } 248 | 249 | // appendTag appends a code+len tag to the given slice. 250 | func appendTag(b []byte, code byte, length uint64) []byte { 251 | if length < 0x0E { 252 | // Short form, with length embedded in the code byte. 253 | return append(b, code|byte(length)) 254 | } 255 | 256 | // Long form, with separate length. 257 | b = append(b, code|0x0E) 258 | return appendVarUint(b, length) 259 | } 260 | 261 | // timestampLen pre-calculates the length, in bytes, of the given timestamp value. 262 | func timestampLen(offset int, utc Timestamp) uint64 { 263 | var ret uint64 264 | 265 | if utc.kind == TimezoneUnspecified { 266 | ret = 1 267 | } else { 268 | ret = varIntLen(int64(offset)) 269 | } 270 | 271 | // We expect at least Year precision. 272 | ret += varUintLen(uint64(utc.dateTime.Year())) 273 | 274 | // Month, day, hour, minute, and second are all guaranteed to be one byte. 275 | switch utc.precision { 276 | case TimestampPrecisionMonth: 277 | ret++ 278 | case TimestampPrecisionDay: 279 | ret += 2 280 | case TimestampPrecisionMinute: 281 | // Hour and Minute combined 282 | ret += 4 283 | case TimestampPrecisionSecond, TimestampPrecisionNanosecond: 284 | ret += 5 285 | } 286 | 287 | if utc.precision == TimestampPrecisionNanosecond && utc.numFractionalSeconds > 0 { 288 | ret++ // For fractional seconds precision indicator 289 | 290 | ns := utc.TruncatedNanoseconds() 291 | if ns > 0 { 292 | ret += intLen(int64(ns)) 293 | } 294 | } 295 | 296 | return ret 297 | } 298 | 299 | // appendTimestamp appends a timestamp value 300 | func appendTimestamp(b []byte, offset int, utc Timestamp) []byte { 301 | if utc.kind == TimezoneUnspecified { 302 | // Unknown offset 303 | b = append(b, 0xC0) 304 | } else { 305 | b = appendVarInt(b, int64(offset)) 306 | } 307 | 308 | // We expect at least Year precision. 309 | b = appendVarUint(b, uint64(utc.dateTime.Year())) 310 | 311 | switch utc.precision { 312 | case TimestampPrecisionMonth: 313 | b = appendVarUint(b, uint64(utc.dateTime.Month())) 314 | case TimestampPrecisionDay: 315 | b = appendVarUint(b, uint64(utc.dateTime.Month())) 316 | b = appendVarUint(b, uint64(utc.dateTime.Day())) 317 | case TimestampPrecisionMinute: 318 | b = appendVarUint(b, uint64(utc.dateTime.Month())) 319 | b = appendVarUint(b, uint64(utc.dateTime.Day())) 320 | 321 | // The hour and minute is considered as a single component. 322 | b = appendVarUint(b, uint64(utc.dateTime.Hour())) 323 | b = appendVarUint(b, uint64(utc.dateTime.Minute())) 324 | case TimestampPrecisionSecond, TimestampPrecisionNanosecond: 325 | b = appendVarUint(b, uint64(utc.dateTime.Month())) 326 | b = appendVarUint(b, uint64(utc.dateTime.Day())) 327 | 328 | // The hour and minute is considered as a single component. 329 | b = appendVarUint(b, uint64(utc.dateTime.Hour())) 330 | b = appendVarUint(b, uint64(utc.dateTime.Minute())) 331 | b = appendVarUint(b, uint64(utc.dateTime.Second())) 332 | } 333 | 334 | if utc.precision == TimestampPrecisionNanosecond && utc.numFractionalSeconds > 0 { 335 | b = append(b, utc.numFractionalSeconds|0xC0) 336 | 337 | ns := utc.TruncatedNanoseconds() 338 | if ns > 0 { 339 | b = appendInt(b, int64(ns)) 340 | } 341 | } 342 | 343 | return b 344 | } 345 | -------------------------------------------------------------------------------- /ion/bits_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "bytes" 20 | "fmt" 21 | "math" 22 | "math/big" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/assert" 27 | ) 28 | 29 | func TestAppendUint(t *testing.T) { 30 | test := func(val uint64, elen uint64, ebits []byte) { 31 | t.Run(fmt.Sprintf("%x", val), func(t *testing.T) { 32 | length := uintLen(val) 33 | assert.Equal(t, elen, length) 34 | 35 | bits := appendUint(nil, val) 36 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 37 | }) 38 | } 39 | 40 | test(0, 1, []byte{0}) 41 | test(0xFF, 1, []byte{0xFF}) 42 | test(0x1FF, 2, []byte{0x01, 0xFF}) 43 | test(math.MaxUint64, 8, []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}) 44 | } 45 | 46 | func TestAppendInt(t *testing.T) { 47 | test := func(val int64, elen uint64, ebits []byte) { 48 | t.Run(fmt.Sprintf("%x", val), func(t *testing.T) { 49 | length := intLen(val) 50 | assert.Equal(t, elen, length) 51 | 52 | bits := appendInt(nil, val) 53 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 54 | }) 55 | } 56 | 57 | test(0, 0, []byte{}) 58 | test(0x7F, 1, []byte{0x7F}) 59 | test(-0x7F, 1, []byte{0xFF}) 60 | 61 | test(0xFF, 2, []byte{0x00, 0xFF}) 62 | test(-0xFF, 2, []byte{0x80, 0xFF}) 63 | 64 | test(0x7FFF, 2, []byte{0x7F, 0xFF}) 65 | test(-0x7FFF, 2, []byte{0xFF, 0xFF}) 66 | 67 | test(math.MaxInt64, 8, []byte{0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}) 68 | test(-math.MaxInt64, 8, []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}) 69 | test(math.MinInt64, 9, []byte{0x80, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}) 70 | } 71 | 72 | func TestAppendBigInt(t *testing.T) { 73 | test := func(val *big.Int, elen uint64, ebits []byte) { 74 | t.Run(fmt.Sprintf("%x", val), func(t *testing.T) { 75 | length := bigIntLen(val) 76 | assert.Equal(t, elen, length) 77 | 78 | bits := appendBigInt(nil, val) 79 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 80 | }) 81 | } 82 | 83 | test(big.NewInt(0), 0, []byte{}) 84 | test(big.NewInt(0x7F), 1, []byte{0x7F}) 85 | test(big.NewInt(-0x7F), 1, []byte{0xFF}) 86 | 87 | test(big.NewInt(0xFF), 2, []byte{0x00, 0xFF}) 88 | test(big.NewInt(-0xFF), 2, []byte{0x80, 0xFF}) 89 | 90 | test(big.NewInt(0x7FFF), 2, []byte{0x7F, 0xFF}) 91 | test(big.NewInt(-0x7FFF), 2, []byte{0xFF, 0xFF}) 92 | } 93 | 94 | func TestAppendVarUint(t *testing.T) { 95 | test := func(val uint64, elen uint64, ebits []byte) { 96 | t.Run(fmt.Sprintf("%x", val), func(t *testing.T) { 97 | length := varUintLen(val) 98 | assert.Equal(t, elen, length) 99 | 100 | bits := appendVarUint(nil, val) 101 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 102 | }) 103 | } 104 | 105 | test(0, 1, []byte{0x80}) 106 | test(0x7F, 1, []byte{0xFF}) 107 | test(0xFF, 2, []byte{0x01, 0xFF}) 108 | test(0x1FF, 2, []byte{0x03, 0xFF}) 109 | test(0x3FFF, 2, []byte{0x7F, 0xFF}) 110 | test(0x7FFF, 3, []byte{0x01, 0x7F, 0xFF}) 111 | test(0x7FFFFFFFFFFFFFFF, 9, []byte{0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 112 | test(0xFFFFFFFFFFFFFFFF, 10, []byte{0x01, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 113 | } 114 | 115 | func TestAppendVarInt(t *testing.T) { 116 | test := func(val int64, elen uint64, ebits []byte) { 117 | t.Run(fmt.Sprintf("%x", val), func(t *testing.T) { 118 | length := varIntLen(val) 119 | assert.Equal(t, elen, length) 120 | 121 | bits := appendVarInt(nil, val) 122 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 123 | }) 124 | } 125 | 126 | test(0, 1, []byte{0x80}) 127 | 128 | test(0x3F, 1, []byte{0xBF}) // 1011 1111 129 | test(-0x3F, 1, []byte{0xFF}) 130 | 131 | test(0x7F, 2, []byte{0x00, 0xFF}) 132 | test(-0x7F, 2, []byte{0x40, 0xFF}) 133 | 134 | test(0x1FFF, 2, []byte{0x3F, 0xFF}) 135 | test(-0x1FFF, 2, []byte{0x7F, 0xFF}) 136 | 137 | test(0x3FFF, 3, []byte{0x00, 0x7F, 0xFF}) 138 | test(-0x3FFF, 3, []byte{0x40, 0x7F, 0xFF}) 139 | 140 | test(0x3FFFFFFFFFFFFFFF, 9, []byte{0x3F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 141 | test(-0x3FFFFFFFFFFFFFFF, 9, []byte{0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 142 | 143 | test(math.MaxInt64, 10, []byte{0x00, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 144 | test(-math.MaxInt64, 10, []byte{0x40, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 145 | test(math.MinInt64, 10, []byte{0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}) 146 | } 147 | 148 | func TestAppendTag(t *testing.T) { 149 | test := func(code byte, vlen uint64, elen uint64, ebits []byte) { 150 | t.Run(fmt.Sprintf("(%x,%v)", code, vlen), func(t *testing.T) { 151 | length := tagLen(vlen) 152 | assert.Equal(t, elen, length) 153 | 154 | bits := appendTag(nil, code, vlen) 155 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 156 | }) 157 | } 158 | 159 | test(0x20, 1, 1, []byte{0x21}) 160 | test(0x30, 0x0D, 1, []byte{0x3D}) 161 | test(0x40, 0x0E, 2, []byte{0x4E, 0x8E}) 162 | test(0x50, math.MaxInt64, 10, []byte{0x5E, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0xFF}) 163 | } 164 | 165 | func TestAppendTimestamp(t *testing.T) { 166 | test := func(val Timestamp, elen uint64, ebits []byte) { 167 | t.Run(fmt.Sprintf("%x", val.dateTime), func(t *testing.T) { 168 | _, offset := val.dateTime.Zone() 169 | offset /= 60 170 | val.dateTime = val.dateTime.In(time.UTC) 171 | 172 | length := timestampLen(offset, val) 173 | assert.Equal(t, elen, length) 174 | 175 | bits := appendTimestamp(nil, offset, val) 176 | assert.True(t, bytes.Equal(bits, ebits), "expected %v, got %v", fmtbytes(ebits), fmtbytes(bits)) 177 | }) 178 | } 179 | 180 | nowish, _ := NewTimestampFromStr("2019-08-04T18:15:43.863494+10:00", TimestampPrecisionNanosecond, TimezoneLocal) 181 | 182 | test(NewDateTimestamp(time.Time{}, TimestampPrecisionSecond), 7, []byte{0xC0, 0x81, 0x81, 0x81, 0x80, 0x80, 0x80}) 183 | test(nowish, 13, []byte{ 184 | 0x04, 0xD8, // offset: +600 minutes (+10:00) 185 | 0x0F, 0xE3, // year: 2019 186 | 0x88, // month: 8 187 | 0x84, // day: 4 188 | 0x88, // hour: 8 utc (18 local) 189 | 0x8F, // minute: 15 190 | 0xAB, // second: 43 191 | 0xC6, // exp: 6 precision units 192 | 0x0D, 0x2D, 0x06, // nsec: 863494 193 | }) 194 | } 195 | -------------------------------------------------------------------------------- /ion/bitstream_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestBitstream(t *testing.T) { 26 | ion := []byte{ 27 | 0xE0, 0x01, 0x00, 0xEA, // $ion_1_0 28 | 0xEE, 0x9F, 0x81, 0x83, 0xDE, 0x9B, // $ion_symbol_table::{ 29 | 0x86, 0xBE, 0x8E, // imports:[ 30 | 0xDD, // { 31 | 0x84, 0x85, 'b', 'o', 'g', 'u', 's', // name: "bogus" 32 | 0x85, 0x21, 0x2A, // version: 42 33 | 0x88, 0x21, 0x64, // max_id: 100 34 | // }] 35 | 0x87, 0xB8, // symbols: [ 36 | 0x83, 'f', 'o', 'o', // "foo" 37 | 0x83, 'b', 'a', 'r', // "bar" 38 | // ] 39 | // } 40 | 0xD0, // {} 41 | 0xEA, 0x81, 0xEE, 0xD7, // foo::{ 42 | 0x84, 0xE3, 0x81, 0xEF, 0x0F, // name:bar::null, 43 | 0x88, 0x20, // max_id:0 44 | // } 45 | } 46 | 47 | b := bitstream{} 48 | b.InitBytes(ion) 49 | 50 | next := func(code bitcode, null bool, length uint64) { 51 | require.NoError(t, b.Next()) 52 | assert.Equal(t, code, b.Code()) 53 | assert.Equal(t, null, b.IsNull()) 54 | assert.Equal(t, length, b.Len()) 55 | } 56 | 57 | fieldid := func(eid uint64) { 58 | id, err := b.ReadFieldID() 59 | require.NoError(t, err) 60 | assert.Equal(t, eid, id) 61 | } 62 | 63 | next(bitcodeBVM, false, 3) 64 | maj, min, err := b.ReadBVM() 65 | require.NoError(t, err) 66 | if maj != 1 && min != 0 { 67 | t.Errorf("expected $ion_1.0, got $ion_%v.%v", maj, min) 68 | } 69 | 70 | next(bitcodeAnnotation, false, 31) 71 | as, err := b.ReadAnnotations(V1SystemSymbolTable) 72 | require.NoError(t, err) 73 | if len(as) != 1 || as[0].LocalSID != 3 { // $ion_symbol_table 74 | t.Errorf("expected [3], got %v", as) 75 | } 76 | 77 | next(bitcodeStruct, false, 27) 78 | b.StepIn() 79 | { 80 | next(bitcodeFieldID, false, 0) 81 | fieldid(6) // imports 82 | 83 | next(bitcodeList, false, 14) 84 | b.StepIn() 85 | { 86 | next(bitcodeStruct, false, 13) 87 | } 88 | require.NoError(t, b.StepOut()) 89 | 90 | next(bitcodeFieldID, false, 0) 91 | fieldid(7) // symbols 92 | 93 | next(bitcodeList, false, 8) 94 | next(bitcodeEOF, false, 0) 95 | } 96 | require.NoError(t, b.StepOut()) 97 | 98 | next(bitcodeStruct, false, 0) 99 | next(bitcodeAnnotation, false, 10) 100 | next(bitcodeEOF, false, 0) 101 | next(bitcodeEOF, false, 0) 102 | } 103 | 104 | func TestBitcodeString(t *testing.T) { 105 | for i := bitcodeNone; i <= bitcodeAnnotation+1; i++ { 106 | assert.NotEmpty(t, i.String(), "expected non-empty string for bitcode %v", uint8(i)) 107 | } 108 | } 109 | 110 | func TestBinaryReadTimestamp(t *testing.T) { 111 | test := func(ion []byte, expectedValue string, expectedPrecision TimestampPrecision, expectedKind TimezoneKind) { 112 | t.Run(expectedValue, func(t *testing.T) { 113 | b := bitstream{} 114 | b.InitBytes(ion) 115 | assert.NoError(t, b.Next()) 116 | 117 | val, err := b.ReadTimestamp() 118 | require.NoError(t, err) 119 | 120 | expectedTimestamp, err := NewTimestampFromStr(expectedValue, expectedPrecision, expectedKind) 121 | require.NoError(t, err) 122 | 123 | assert.True(t, val.Equal(expectedTimestamp), "expected %v, got %v", expectedTimestamp, val) 124 | }) 125 | } 126 | 127 | test([]byte{ 128 | 0x63, 129 | 0x80, // offset 0 130 | 0x0F, 0xD0, // year: 2000 131 | }, "2000T", TimestampPrecisionYear, TimezoneUnspecified) 132 | 133 | test([]byte{ 134 | 0x64, 135 | 0x80, // offset 0 136 | 0x0F, 0xD0, // year: 2000 137 | 0x85, // month: 5 138 | }, "2000-05T", TimestampPrecisionMonth, TimezoneUnspecified) 139 | 140 | test([]byte{ 141 | 0x65, 142 | 0x80, // offset 0 143 | 0x0F, 0xD0, // year: 2000 144 | 0x85, // month: 5 145 | 0x86, // day: 6 146 | }, "2000-05-06T", TimestampPrecisionDay, TimezoneUnspecified) 147 | 148 | test([]byte{ 149 | 0x67, 150 | 0x80, // offset 0 151 | 0x0F, 0xD0, // year: 2000 152 | 0x85, // month: 5 153 | 0x86, // day: 6 154 | 0x87, // hour: 7 155 | 0x88, // minute: 8 156 | }, "2000-05-06T07:08Z", TimestampPrecisionMinute, TimezoneUTC) 157 | 158 | test([]byte{ 159 | 0x68, 160 | 0x80, // offset 0 161 | 0x0F, 0xD0, // year: 2000 162 | 0x85, // month: 5 163 | 0x86, // day: 6 164 | 0x87, // hour: 7 165 | 0x88, // minute: 8 166 | 0x89, // second: 9 167 | }, "2000-05-06T07:08:09Z", TimestampPrecisionSecond, TimezoneUTC) 168 | 169 | test([]byte{ 170 | 0x6A, 171 | 0x80, // offset 0 172 | 0x0F, 0xD0, // year: 2000 173 | 0x81, // month: 1 174 | 0x81, // day: 1 175 | 0x80, // hour: 0 176 | 0x80, // minute: 0 177 | 0x80, // second: 0 178 | 0x80, // 0 precision units 179 | 0x00, // 0 180 | }, "2000-01-01T00:00:00Z", TimestampPrecisionSecond, TimezoneUTC) 181 | 182 | test([]byte{ 183 | 0x69, 184 | 0x80, // offset 0 185 | 0x0F, 0xD0, // year: 2000 186 | 0x81, // month: 1 187 | 0x81, // day: 1 188 | 0x80, // hour: 0 189 | 0x80, // minute: 0 190 | 0x80, // second: 0 191 | 0xC2, // 2 precision units 192 | }, "2000-01-01T00:00:00.00Z", TimestampPrecisionNanosecond, TimezoneUTC) 193 | 194 | test([]byte{ 195 | 0x6A, 196 | 0x80, // offset 0 197 | 0x0F, 0xD0, // year: 2000 198 | 0x85, // month: 5 199 | 0x86, // day: 6 200 | 0x87, // hour: 7 201 | 0x88, // minute: 8 202 | 0x89, // second: 9 203 | 0xC3, // 3 precision units 204 | 0x64, // 100 205 | }, "2000-05-06T07:08:09.100Z", TimestampPrecisionNanosecond, TimezoneUTC) 206 | 207 | test([]byte{ 208 | 0x6C, 209 | 0x80, // offset 0 210 | 0x0F, 0xD0, // year: 2000 211 | 0x85, // month: 5 212 | 0x86, // day: 6 213 | 0x87, // hour: 7 214 | 0x88, // minute: 8 215 | 0x89, // second: 9 216 | 0xC6, // 6 precision units 217 | 0x01, 0x87, 0x04, // 100100 218 | }, "2000-05-06T07:08:09.100100Z", TimestampPrecisionNanosecond, TimezoneUTC) 219 | 220 | test([]byte{ 221 | 0x6C, 222 | 0x88, // offset +8 223 | 0x0F, 0xD0, // year: 2000 224 | 0x85, // month: 5 225 | 0x86, // day: 6 226 | 0x87, // hour: 7 227 | 0x88, // minute: 8 utc (16 local) 228 | 0x89, // second: 9 229 | 0xC6, // 6 precision units 230 | 0x01, 0x87, 0x04, // 100100 231 | }, "2000-05-06T07:16:09.100100+00:08", TimestampPrecisionNanosecond, TimezoneLocal) 232 | 233 | // Test >9 fractional seconds. 234 | test([]byte{ 235 | 0x6A, 236 | 0x80, // offset 0 237 | 0x0F, 0xD0, // year: 2000 238 | 0x81, // month: 1 239 | 0x81, // day: 1 240 | 0x80, // hour: 0 241 | 0x80, // minute: 0 242 | 0x80, // second: 0 243 | 0xCA, // 10 precision units 244 | 0x2C, // 44 245 | }, "2000-01-01T00:00:00.000000004Z", TimestampPrecisionNanosecond, TimezoneUTC) 246 | 247 | test([]byte{ 248 | 0x6A, 249 | 0x80, // offset 0 250 | 0x0F, 0xD0, // year: 2000 251 | 0x81, // month: 1 252 | 0x81, // day: 1 253 | 0x80, // hour: 0 254 | 0x80, // minute: 0 255 | 0x80, // second: 0 256 | 0xCA, // 10 precision units 257 | 0x2D, // 45 258 | }, "2000-01-01T00:00:00.000000005Z", TimestampPrecisionNanosecond, TimezoneUTC) 259 | 260 | test([]byte{ 261 | 0x6A, 262 | 0x80, // offset 0 263 | 0x0F, 0xD0, // year: 2000 264 | 0x81, // month: 1 265 | 0x81, // day: 1 266 | 0x80, // hour: 0 267 | 0x80, // minute: 0 268 | 0x80, // second: 0 269 | 0xCA, // 10 precision units 270 | 0x2E, // 46 271 | }, "2000-01-01T00:00:00.000000005Z", TimestampPrecisionNanosecond, TimezoneUTC) 272 | 273 | test([]byte{ 274 | 0x6E, 275 | 0x8E, 276 | 0x80, // offset 0 277 | 0x0F, 0xD0, // year: 2000 278 | 0x8C, // month: 12 279 | 0x9F, // day: 31 280 | 0x97, // hour: 23 281 | 0xBB, // minute: 59 282 | 0xBB, // second: 59 283 | 0xCA, // 10 precision units 284 | 0x02, 0x54, 0x0B, 0xE3, 0xFF, // 9999999999 285 | }, "2001-01-01T00:00:00.000000000Z", TimestampPrecisionNanosecond, TimezoneUTC) 286 | } 287 | -------------------------------------------------------------------------------- /ion/buf.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "io" 20 | ) 21 | 22 | // Writing binary ion is a bit tricky: values are preceded by their length, 23 | // which can be hard to predict until we've actually written out the value. 24 | // To make matters worse, we can't predict the length of the /length/ ahead 25 | // of time in order to reserve space for it, because it uses a variable-length 26 | // encoding. To avoid copying bytes around all over the place, we write into 27 | // an in-memory tree structure, which we then blast out to the actual io.Writer 28 | // once all the relevant lengths are known. 29 | 30 | // A bufnode is a node in the partially-serialized tree. 31 | type bufnode interface { 32 | Len() uint64 33 | EmitTo(w io.Writer) error 34 | } 35 | 36 | // A bufseq is a bufnode that's also an appendable sequence of bufnodes. 37 | type bufseq interface { 38 | bufnode 39 | Append(n bufnode) 40 | } 41 | 42 | var _ bufnode = atom([]byte{}) 43 | var _ bufseq = &datagram{} 44 | var _ bufseq = &container{} 45 | 46 | // An atom is a value that has been fully serialized and can be emitted directly. 47 | type atom []byte 48 | 49 | func (a atom) Len() uint64 { 50 | return uint64(len(a)) 51 | } 52 | 53 | func (a atom) EmitTo(w io.Writer) error { 54 | _, err := w.Write(a) 55 | return err 56 | } 57 | 58 | // A datagram is a sequence of nodes that will be emitted one 59 | // after another. Most notably, used to buffer top-level values 60 | // when we haven't yet finalized the local symbol table. 61 | type datagram struct { 62 | len uint64 63 | children []bufnode 64 | } 65 | 66 | func (d *datagram) Append(n bufnode) { 67 | d.len += n.Len() 68 | d.children = append(d.children, n) 69 | } 70 | 71 | func (d *datagram) Len() uint64 { 72 | return d.len 73 | } 74 | 75 | func (d *datagram) EmitTo(w io.Writer) error { 76 | for _, child := range d.children { 77 | if err := child.EmitTo(w); err != nil { 78 | return err 79 | } 80 | } 81 | 82 | return nil 83 | } 84 | 85 | // A container is a datagram that's preceded by a code+length tag. 86 | type container struct { 87 | code byte 88 | datagram 89 | } 90 | 91 | func (c *container) Len() uint64 { 92 | if c.len < 0x0E { 93 | return c.len + 1 94 | } 95 | return c.len + (varUintLen(c.len) + 1) 96 | } 97 | 98 | func (c *container) EmitTo(w io.Writer) error { 99 | var arr [11]byte 100 | buf := arr[:0] 101 | buf = appendTag(buf, c.code, c.len) 102 | 103 | if _, err := w.Write(buf); err != nil { 104 | return err 105 | } 106 | return c.datagram.EmitTo(w) 107 | } 108 | 109 | // A bufstack is a stack of bufseqs, more or less matching the 110 | // stack of BeginList/Sexp/Struct calls made on a binaryWriter. 111 | // The top of the stack is the sequence we're currently writing 112 | // values into; when it's popped off, it will be appended to the 113 | // bufseq below it. 114 | type bufstack struct { 115 | arr []bufseq 116 | } 117 | 118 | func (s *bufstack) peek() bufseq { 119 | if len(s.arr) == 0 { 120 | return nil 121 | } 122 | return s.arr[len(s.arr)-1] 123 | } 124 | 125 | func (s *bufstack) push(b bufseq) { 126 | s.arr = append(s.arr, b) 127 | } 128 | 129 | func (s *bufstack) pop() { 130 | if len(s.arr) == 0 { 131 | panic("pop called on an empty stack") 132 | } 133 | s.arr = s.arr[:len(s.arr)-1] 134 | } 135 | -------------------------------------------------------------------------------- /ion/buf_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "bytes" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestBufnode(t *testing.T) { 27 | root := container{code: 0xE0} 28 | root.Append(atom([]byte{0x81, 0x83})) 29 | { 30 | symtab := &container{code: 0xD0} 31 | { 32 | symtab.Append(atom([]byte{0x86})) // varUint(6) 33 | { 34 | imps := &container{code: 0xB0} 35 | { 36 | imp0 := &container{code: 0xD0} 37 | { 38 | imp0.Append(atom([]byte{0x84})) // varUint(4) 39 | imp0.Append(atom([]byte{0x85, 'b', 'o', 'g', 'u', 's'})) 40 | imp0.Append(atom([]byte{0x85})) // varUint(5) 41 | imp0.Append(atom([]byte{0x21, 0x2A})) 42 | imp0.Append(atom([]byte{0x88})) // varUint(8) 43 | imp0.Append(atom([]byte{0x21, 0x64})) 44 | } 45 | imps.Append(imp0) 46 | } 47 | symtab.Append(imps) 48 | } 49 | 50 | symtab.Append(atom([]byte{0x87})) // varUint(7) 51 | { 52 | syms := &container{code: 0xB0} 53 | { 54 | syms.Append(atom([]byte{0x83, 'f', 'o', 'o'})) 55 | syms.Append(atom([]byte{0x83, 'b', 'a', 'r'})) 56 | } 57 | symtab.Append(syms) 58 | } 59 | } 60 | root.Append(symtab) 61 | } 62 | 63 | buf := bytes.Buffer{} 64 | 65 | require.NoError(t, root.EmitTo(&buf)) 66 | 67 | val := buf.Bytes() 68 | eval := []byte{ 69 | // $ion_symbol_table::{ 70 | 0xEE, 0x9F, 0x81, 0x83, 0xDE, 0x9B, 71 | // imports:[ 72 | 0x86, 0xBE, 0x8E, 73 | // { 74 | 0xDD, 75 | // name: "bogus" 76 | 0x84, 0x85, 'b', 'o', 'g', 'u', 's', 77 | // version: 42 78 | 0x85, 0x21, 0x2A, 79 | // max_id: 100 80 | 0x88, 0x21, 0x64, 81 | // } 82 | // ], 83 | // symbols:[ 84 | 0x87, 0xB8, 85 | // "foo", 86 | 0x83, 'f', 'o', 'o', 87 | // "bar" 88 | 0x83, 'b', 'a', 'r', 89 | // ] 90 | // } 91 | } 92 | 93 | assert.True(t, bytes.Equal(val, eval), "expected %v, got %v", fmtbytes(eval), fmtbytes(val)) 94 | } 95 | -------------------------------------------------------------------------------- /ion/catalog.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "bytes" 20 | "fmt" 21 | "io" 22 | "strings" 23 | ) 24 | 25 | // A Catalog provides access to shared symbol tables. 26 | type Catalog interface { 27 | FindExact(name string, version int) SharedSymbolTable 28 | FindLatest(name string) SharedSymbolTable 29 | } 30 | 31 | // A basicCatalog wraps an in-memory collection of shared symbol tables. 32 | type basicCatalog struct { 33 | ssts map[string]SharedSymbolTable 34 | latest map[string]SharedSymbolTable 35 | } 36 | 37 | // NewCatalog creates a new basic catalog containing the given symbol tables. 38 | func NewCatalog(ssts ...SharedSymbolTable) Catalog { 39 | cat := &basicCatalog{ 40 | ssts: make(map[string]SharedSymbolTable), 41 | latest: make(map[string]SharedSymbolTable), 42 | } 43 | for _, sst := range ssts { 44 | cat.add(sst) 45 | } 46 | return cat 47 | } 48 | 49 | // Add adds a shared symbol table to the catalog. 50 | func (c *basicCatalog) add(sst SharedSymbolTable) { 51 | key := fmt.Sprintf("%v/%v", sst.Name(), sst.Version()) 52 | c.ssts[key] = sst 53 | 54 | cur, ok := c.latest[sst.Name()] 55 | if !ok || sst.Version() > cur.Version() { 56 | c.latest[sst.Name()] = sst 57 | } 58 | } 59 | 60 | // FindExact attempts to find a shared symbol table with the given name and version. 61 | func (c *basicCatalog) FindExact(name string, version int) SharedSymbolTable { 62 | key := fmt.Sprintf("%v/%v", name, version) 63 | return c.ssts[key] 64 | } 65 | 66 | // FindLatest finds the shared symbol table with the given name and largest version. 67 | func (c *basicCatalog) FindLatest(name string) SharedSymbolTable { 68 | return c.latest[name] 69 | } 70 | 71 | // A System is a reader factory wrapping a catalog. 72 | type System struct { 73 | Catalog Catalog 74 | } 75 | 76 | // NewReader creates a new reader using this system's catalog. 77 | func (s System) NewReader(in io.Reader) Reader { 78 | return NewReaderCat(in, s.Catalog) 79 | } 80 | 81 | // NewReaderString creates a new reader using this system's catalog. 82 | func (s System) NewReaderString(in string) Reader { 83 | return NewReaderCat(strings.NewReader(in), s.Catalog) 84 | } 85 | 86 | // NewReaderBytes creates a new reader using this system's catalog. 87 | func (s System) NewReaderBytes(in []byte) Reader { 88 | return NewReaderCat(bytes.NewReader(in), s.Catalog) 89 | } 90 | 91 | // Unmarshal unmarshals Ion data using this system's catalog. 92 | func (s System) Unmarshal(data []byte, v interface{}) error { 93 | r := s.NewReaderBytes(data) 94 | d := NewDecoder(r) 95 | return d.DecodeTo(v) 96 | } 97 | 98 | // UnmarshalString unmarshals Ion data using this system's catalog. 99 | func (s System) UnmarshalString(data string, v interface{}) error { 100 | r := s.NewReaderString(data) 101 | d := NewDecoder(r) 102 | return d.DecodeTo(v) 103 | } 104 | -------------------------------------------------------------------------------- /ion/catalog_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "bytes" 20 | "fmt" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | type Item struct { 28 | ID int `ion:"id"` 29 | Name string `ion:"name"` 30 | Description string `ion:"description"` 31 | } 32 | 33 | func TestCatalog(t *testing.T) { 34 | sst := NewSharedSymbolTable("item", 1, []string{ 35 | "item", 36 | "id", 37 | "name", 38 | "description", 39 | }) 40 | 41 | buf := bytes.Buffer{} 42 | out := NewBinaryWriter(&buf, sst) 43 | 44 | for i := 0; i < 10; i++ { 45 | assert.NoError(t, out.Annotation(NewSymbolTokenFromString("item"))) 46 | assert.NoError(t, 47 | MarshalTo(out, &Item{ 48 | ID: i, 49 | Name: fmt.Sprintf("Item %v", i), 50 | Description: fmt.Sprintf("The %vth test item", i), 51 | })) 52 | } 53 | require.NoError(t, out.Finish()) 54 | 55 | bs := buf.Bytes() 56 | 57 | sys := System{Catalog: NewCatalog(sst)} 58 | in := sys.NewReaderBytes(bs) 59 | 60 | i := 0 61 | for ; ; i++ { 62 | item := Item{} 63 | err := UnmarshalFrom(in, &item) 64 | if err == ErrNoInput { 65 | break 66 | } 67 | require.NoError(t, err) 68 | 69 | assert.Equal(t, i, item.ID) 70 | } 71 | 72 | assert.Equal(t, 10, i) 73 | } 74 | -------------------------------------------------------------------------------- /ion/cmp_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "math" 20 | "reflect" 21 | 22 | "github.com/google/go-cmp/cmp" 23 | "github.com/google/go-cmp/cmp/cmpopts" 24 | ) 25 | 26 | type ionEqual interface { 27 | eq(other ionEqual) bool 28 | } 29 | 30 | type ionFloat struct{ float64 } 31 | type ionDecimal struct{ *Decimal } 32 | type ionTimestamp struct{ Timestamp } 33 | type ionSymbol struct{ *SymbolToken } 34 | 35 | func (thisFloat ionFloat) eq(other ionEqual) bool { 36 | float1 := thisFloat.float64 37 | float2 := other.(ionFloat).float64 38 | 39 | return math.Signbit(float1) == math.Signbit(float2) && 40 | cmp.Equal(float1, float2, cmpopts.EquateNaNs()) 41 | } 42 | 43 | func (thisDecimal ionDecimal) eq(other ionEqual) bool { 44 | if val, ok := other.(ionDecimal); ok { 45 | if thisDecimal.scale != val.scale || thisDecimal.isNegZero != val.isNegZero { 46 | return false 47 | } 48 | return thisDecimal.Decimal.Equal(val.Decimal) 49 | } 50 | return false 51 | } 52 | 53 | func (thisTimestamp ionTimestamp) eq(other ionEqual) bool { 54 | if val, ok := other.(ionTimestamp); ok { 55 | return thisTimestamp.Equal(val.Timestamp) 56 | } 57 | return false 58 | } 59 | 60 | func (thisSymbol ionSymbol) eq(other ionEqual) bool { 61 | if val, ok := other.(ionSymbol); ok { 62 | return thisSymbol.SymbolToken.Equal(val.SymbolToken) 63 | } 64 | return false 65 | } 66 | 67 | func cmpAnnotations(thisAnnotations, otherAnnotations []SymbolToken) bool { 68 | if len(thisAnnotations) == 0 && len(otherAnnotations) == 0 { 69 | return true 70 | } 71 | 72 | if len(thisAnnotations) != len(otherAnnotations) { 73 | return false 74 | } 75 | 76 | res := false 77 | for idx, this := range thisAnnotations { 78 | other := otherAnnotations[idx] 79 | res = this.Equal(&other) 80 | 81 | if !res { 82 | return false 83 | } 84 | } 85 | return res 86 | } 87 | 88 | func cmpFloats(thisValue, otherValue interface{}) bool { 89 | if !haveSameTypes(thisValue, otherValue) { 90 | return false 91 | } 92 | 93 | switch val := thisValue.(type) { 94 | case float64: 95 | thisFloat := ionFloat{val} 96 | return thisFloat.eq(ionFloat{otherValue.(float64)}) 97 | case nil: 98 | return otherValue == nil 99 | default: 100 | return false 101 | } 102 | } 103 | 104 | func cmpDecimals(thisValue, otherValue interface{}) bool { 105 | if !haveSameTypes(thisValue, otherValue) { 106 | return false 107 | } 108 | 109 | switch val := thisValue.(type) { 110 | case *Decimal: 111 | thisDecimal := ionDecimal{val} 112 | return thisDecimal.eq(ionDecimal{otherValue.(*Decimal)}) 113 | case nil: 114 | return otherValue == nil 115 | default: 116 | return false 117 | } 118 | } 119 | 120 | func cmpTimestamps(thisValue, otherValue interface{}) bool { 121 | if !haveSameTypes(thisValue, otherValue) { 122 | return false 123 | } 124 | 125 | switch val := thisValue.(type) { 126 | case Timestamp: 127 | thisTimestamp := ionTimestamp{val} 128 | return thisTimestamp.eq(ionTimestamp{otherValue.(Timestamp)}) 129 | case nil: 130 | return otherValue == nil 131 | default: 132 | return false 133 | } 134 | } 135 | 136 | func cmpSymbols(thisValue, otherValue interface{}) bool { 137 | if thisValue == nil || otherValue == nil { 138 | return thisValue == nil && otherValue == nil 139 | } 140 | 141 | if val1, ok := thisValue.(SymbolToken); ok { 142 | if val2, ok := otherValue.(SymbolToken); ok { 143 | return val1.Equal(&val2) 144 | } 145 | } else if val1, ok := thisValue.(*SymbolToken); ok { 146 | if val2, ok := otherValue.(*SymbolToken); ok { 147 | return val1.Equal(val2) 148 | } 149 | } 150 | 151 | return false 152 | } 153 | 154 | func cmpValueSlices(thisValues, otherValues []interface{}) bool { 155 | if len(thisValues) == 0 && len(otherValues) == 0 { 156 | return true 157 | } 158 | 159 | if len(thisValues) != len(otherValues) { 160 | return false 161 | } 162 | 163 | res := false 164 | for idx, this := range thisValues { 165 | other := otherValues[idx] 166 | 167 | if !haveSameTypes(this, other) { 168 | return false 169 | } 170 | 171 | thisItem := getContainersType(this) 172 | otherItem := getContainersType(other) 173 | res = containersEquality(thisItem, otherItem) 174 | 175 | if !res { 176 | return false 177 | } 178 | } 179 | return res 180 | } 181 | 182 | func cmpStruct(thisValues, otherValues []interface{}) bool { 183 | if len(thisValues) == 0 && len(otherValues) == 0 { 184 | return true 185 | } 186 | 187 | if len(thisValues) != len(otherValues) { 188 | return false 189 | } 190 | 191 | var res bool 192 | var checked []int 193 | for _, this := range thisValues { 194 | res = false 195 | var thisItem = getContainersType(this) 196 | for i := 0; i < len(otherValues); i++ { 197 | if contains(checked, i) { 198 | continue 199 | } 200 | if !haveSameTypes(this, otherValues[i]) { 201 | continue 202 | } else { 203 | otherItem := getContainersType(otherValues[i]) 204 | res = containersEquality(thisItem, otherItem) 205 | if res { 206 | if !contains(checked, i) { 207 | checked = append(checked, i) 208 | } 209 | break 210 | } 211 | } 212 | } 213 | } 214 | if len(otherValues) != len(checked) { 215 | return false 216 | } 217 | 218 | return res 219 | } 220 | 221 | func haveSameTypes(this, other interface{}) bool { 222 | return reflect.TypeOf(this) == reflect.TypeOf(other) 223 | } 224 | 225 | func getContainersType(in interface{}) interface{} { 226 | switch in := in.(type) { 227 | case *string: 228 | return in 229 | case nil: 230 | return nil 231 | default: 232 | return in.(ionItem) 233 | } 234 | } 235 | 236 | func contains(list []int, idx int) bool { 237 | for _, num := range list { 238 | if num == idx { 239 | return true 240 | } 241 | } 242 | return false 243 | } 244 | 245 | // non-null containers have ionItems inside them 246 | func containersEquality(this, other interface{}) bool { 247 | switch this.(type) { 248 | case nil: 249 | return other == nil 250 | default: 251 | otherItem := other.(ionItem) 252 | thisItem := this.(ionItem) 253 | if thisItem.fieldName.Equal(&otherItem.fieldName) && thisItem.equal(otherItem) { 254 | return true 255 | } 256 | } 257 | return false 258 | } 259 | -------------------------------------------------------------------------------- /ion/consts.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "math/big" 20 | "reflect" 21 | "time" 22 | ) 23 | 24 | var binaryNulls = func() []byte { 25 | ret := make([]byte, StructType+1) 26 | ret[NoType] = 0x0F 27 | ret[NullType] = 0x0F 28 | ret[BoolType] = 0x1F 29 | ret[IntType] = 0x2F 30 | ret[FloatType] = 0x4F 31 | ret[DecimalType] = 0x5F 32 | ret[TimestampType] = 0x6F 33 | ret[SymbolType] = 0x7F 34 | ret[StringType] = 0x8F 35 | ret[ClobType] = 0x9F 36 | ret[BlobType] = 0xAF 37 | ret[ListType] = 0xBF 38 | ret[SexpType] = 0xCF 39 | ret[StructType] = 0xDF 40 | return ret 41 | }() 42 | 43 | var textNulls = func() []string { 44 | ret := make([]string, StructType+1) 45 | ret[NoType] = "null" 46 | ret[NullType] = "null.null" 47 | ret[BoolType] = "null.bool" 48 | ret[IntType] = "null.int" 49 | ret[FloatType] = "null.float" 50 | ret[DecimalType] = "null.decimal" 51 | ret[TimestampType] = "null.timestamp" 52 | ret[SymbolType] = "null.symbol" 53 | ret[StringType] = "null.string" 54 | ret[ClobType] = "null.clob" 55 | ret[BlobType] = "null.blob" 56 | ret[ListType] = "null.list" 57 | ret[SexpType] = "null.sexp" 58 | ret[StructType] = "null.struct" 59 | return ret 60 | }() 61 | 62 | var hexChars = []byte{ 63 | '0', '1', '2', '3', '4', '5', '6', '7', 64 | '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 65 | } 66 | 67 | var timestampType = reflect.TypeOf(Timestamp{}) 68 | var nativeTimeType = reflect.TypeOf(time.Time{}) 69 | var decimalType = reflect.TypeOf(Decimal{}) 70 | var bigIntType = reflect.TypeOf(big.Int{}) 71 | var symbolType = reflect.TypeOf(SymbolToken{}) 72 | -------------------------------------------------------------------------------- /ion/ctx.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import "fmt" 19 | 20 | // ctx is the current reader or writer context. 21 | type ctx uint8 22 | 23 | const ( 24 | ctxAtTopLevel ctx = iota 25 | ctxInStruct 26 | ctxInList 27 | ctxInSexp 28 | ) 29 | 30 | func ctxToContainerType(c ctx) Type { 31 | switch c { 32 | case ctxInList: 33 | return ListType 34 | case ctxInSexp: 35 | return SexpType 36 | case ctxInStruct: 37 | return StructType 38 | default: 39 | return NoType 40 | } 41 | } 42 | 43 | func containerTypeToCtx(t Type) ctx { 44 | switch t { 45 | case ListType: 46 | return ctxInList 47 | case SexpType: 48 | return ctxInSexp 49 | case StructType: 50 | return ctxInStruct 51 | default: 52 | panic(fmt.Sprintf("type %v is not a container type", t)) 53 | } 54 | } 55 | 56 | // ctxstack is a context stack. 57 | type ctxstack struct { 58 | arr []ctx 59 | } 60 | 61 | // peek returns the current context. 62 | func (c *ctxstack) peek() ctx { 63 | if len(c.arr) == 0 { 64 | return ctxAtTopLevel 65 | } 66 | return c.arr[len(c.arr)-1] 67 | } 68 | 69 | // push pushes a new context onto the stack. 70 | func (c *ctxstack) push(ctx ctx) { 71 | c.arr = append(c.arr, ctx) 72 | } 73 | 74 | // pop pops the top context off the stack. 75 | func (c *ctxstack) pop() { 76 | if len(c.arr) == 0 { 77 | panic("pop called at top level") 78 | } 79 | c.arr = c.arr[:len(c.arr)-1] 80 | } 81 | -------------------------------------------------------------------------------- /ion/decimal.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "fmt" 20 | "math" 21 | "math/big" 22 | "strconv" 23 | "strings" 24 | ) 25 | 26 | // A ParseError is returned if ParseDecimal is called with a parameter that 27 | // cannot be parsed as a Decimal. 28 | type ParseError struct { 29 | Num string 30 | Msg string 31 | } 32 | 33 | func (e *ParseError) Error() string { 34 | return fmt.Sprintf("ion: ParseDecimal(%v): %v", e.Num, e.Msg) 35 | } 36 | 37 | // https://github.com/amazon-ion/ion-go/issues/119 38 | 39 | // Decimal is an arbitrary-precision decimal value. 40 | type Decimal struct { 41 | n *big.Int 42 | scale int32 43 | isNegZero bool 44 | } 45 | 46 | // NewDecimal creates a new decimal whose value is equal to n * 10^exp. 47 | func NewDecimal(n *big.Int, exp int32, negZero bool) *Decimal { 48 | return &Decimal{ 49 | n: n, 50 | scale: -exp, 51 | isNegZero: negZero, 52 | } 53 | } 54 | 55 | // NewDecimalInt creates a new decimal whose value is equal to n. 56 | func NewDecimalInt(n int64) *Decimal { 57 | return NewDecimal(big.NewInt(n), 0, false) 58 | } 59 | 60 | // MustParseDecimal parses the given string into a decimal object, 61 | // panicking on error. 62 | func MustParseDecimal(in string) *Decimal { 63 | d, err := ParseDecimal(in) 64 | if err != nil { 65 | panic(err) 66 | } 67 | return d 68 | } 69 | 70 | // ParseDecimal parses the given string into a decimal object, 71 | // returning an error on failure. 72 | func ParseDecimal(in string) (*Decimal, error) { 73 | if len(in) == 0 { 74 | return nil, &ParseError{in, "empty string"} 75 | } 76 | 77 | exponent := int32(0) 78 | 79 | d := strings.IndexAny(in, "Dd") 80 | if d != -1 { 81 | // There's an explicit exponent. 82 | exp := in[d+1:] 83 | if len(exp) == 0 { 84 | return nil, &ParseError{in, "unexpected end of input after d"} 85 | } 86 | 87 | tmp, err := strconv.ParseInt(exp, 10, 32) 88 | if err != nil { 89 | return nil, &ParseError{in, err.Error()} 90 | } 91 | 92 | exponent = int32(tmp) 93 | in = in[:d] 94 | } 95 | 96 | d = strings.Index(in, ".") 97 | if d != -1 { 98 | // There's zero or more decimal places. 99 | ipart := in[:d] 100 | fpart := in[d+1:] 101 | 102 | exponent -= int32(len(fpart)) 103 | in = ipart + fpart 104 | } 105 | 106 | n, ok := new(big.Int).SetString(in, 10) 107 | if !ok { 108 | // Unfortunately this is all we get? 109 | return nil, &ParseError{in, "cannot parse coefficient"} 110 | } 111 | 112 | isNegZero := n.Sign() == 0 && len(in) > 0 && in[0] == '-' 113 | 114 | return NewDecimal(n, exponent, isNegZero), nil 115 | } 116 | 117 | // CoEx returns this decimal's coefficient and exponent. 118 | func (d *Decimal) CoEx() (*big.Int, int32) { 119 | return d.n, -d.scale 120 | } 121 | 122 | // Abs returns the absolute value of this Decimal. 123 | func (d *Decimal) Abs() *Decimal { 124 | return &Decimal{ 125 | n: new(big.Int).Abs(d.n), 126 | scale: d.scale, 127 | } 128 | } 129 | 130 | // Add returns the result of adding this Decimal to another Decimal. 131 | func (d *Decimal) Add(o *Decimal) *Decimal { 132 | // a*10^x + b*10^y = (a*10^(x-y) + b) * 10^y 133 | dd, oo := rescale(d, o) 134 | return &Decimal{ 135 | n: new(big.Int).Add(dd.n, oo.n), 136 | scale: dd.scale, 137 | } 138 | } 139 | 140 | // Sub returns the result of substrating another Decimal from this Decimal. 141 | func (d *Decimal) Sub(o *Decimal) *Decimal { 142 | dd, oo := rescale(d, o) 143 | return &Decimal{ 144 | n: new(big.Int).Sub(dd.n, oo.n), 145 | scale: dd.scale, 146 | } 147 | } 148 | 149 | // Neg returns the negative of this Decimal. 150 | func (d *Decimal) Neg() *Decimal { 151 | return &Decimal{ 152 | n: new(big.Int).Neg(d.n), 153 | scale: d.scale, 154 | } 155 | } 156 | 157 | // Mul multiplies two decimals and returns the result. 158 | func (d *Decimal) Mul(o *Decimal) *Decimal { 159 | // a*10^x * b*10^y = (a*b) * 10^(x+y) 160 | scale := int64(d.scale) + int64(o.scale) 161 | if scale > math.MaxInt32 || scale < math.MinInt32 { 162 | panic("exponent out of bounds") 163 | } 164 | 165 | return &Decimal{ 166 | n: new(big.Int).Mul(d.n, o.n), 167 | scale: int32(scale), 168 | } 169 | } 170 | 171 | // ShiftL returns a new decimal shifted the given number of decimal 172 | // places to the left. It's a computationally-cheap way to compute 173 | // d * 10^shift. 174 | func (d *Decimal) ShiftL(shift int) *Decimal { 175 | scale := int64(d.scale) - int64(shift) 176 | if scale > math.MaxInt32 || scale < math.MinInt32 { 177 | panic("exponent out of bounds") 178 | } 179 | 180 | return &Decimal{ 181 | n: d.n, 182 | scale: int32(scale), 183 | } 184 | } 185 | 186 | // ShiftR returns a new decimal shifted the given number of decimal 187 | // places to the right. It's a computationally-cheap way to compute 188 | // d / 10^shift. 189 | func (d *Decimal) ShiftR(shift int) *Decimal { 190 | scale := int64(d.scale) + int64(shift) 191 | if scale > math.MaxInt32 || scale < math.MinInt32 { 192 | panic("exponent out of bounds") 193 | } 194 | 195 | return &Decimal{ 196 | n: d.n, 197 | scale: int32(scale), 198 | } 199 | } 200 | 201 | // https://github.com/amazon-ion/ion-go/issues/118 202 | 203 | // Sign returns -1 if the value is less than 0, 0 if it is equal to zero, 204 | // and +1 if it is greater than zero. 205 | func (d *Decimal) Sign() int { 206 | return d.n.Sign() 207 | } 208 | 209 | // Cmp compares two decimals, returning -1 if d is smaller, +1 if d is 210 | // larger, and 0 if they are equal (ignoring precision). 211 | func (d *Decimal) Cmp(o *Decimal) int { 212 | dd, oo := rescale(d, o) 213 | return dd.n.Cmp(oo.n) 214 | } 215 | 216 | // Equal determines if two decimals are equal (discounting precision, 217 | // at least for now). 218 | func (d *Decimal) Equal(o *Decimal) bool { 219 | return d.Cmp(o) == 0 220 | } 221 | 222 | func rescale(a, b *Decimal) (*Decimal, *Decimal) { 223 | if a.scale < b.scale { 224 | return a.upscale(b.scale), b 225 | } else if a.scale > b.scale { 226 | return a, b.upscale(a.scale) 227 | } else { 228 | return a, b 229 | } 230 | } 231 | 232 | // Make 'n' bigger by making 'scale' smaller, since we know we can 233 | // do that. (1d100 -> 10d99). Makes comparisons and math easier, at the 234 | // expense of more storage space. Technically speaking implies adding 235 | // more precision, but we're not tracking that too closely. 236 | func (d *Decimal) upscale(scale int32) *Decimal { 237 | diff := int64(scale) - int64(d.scale) 238 | if diff < 0 { 239 | panic("can't upscale to a smaller scale") 240 | } 241 | 242 | pow := new(big.Int).Exp(big.NewInt(10), big.NewInt(diff), nil) 243 | n := new(big.Int).Mul(d.n, pow) 244 | 245 | return &Decimal{ 246 | n: n, 247 | scale: scale, 248 | } 249 | } 250 | 251 | // Check to upscale a decimal which means to make 'n' bigger by making 'scale' smaller. 252 | // Makes comparisons and math easier, at the expense of more storage space. 253 | func (d *Decimal) checkToUpscale() (*Decimal, error) { 254 | if d.scale < 0 { 255 | // Don't even bother trying this with numbers that *definitely* too big to represent 256 | // as an int64, because upscale(0) will consume a bunch of memory. 257 | if d.scale < -20 { 258 | return d, &strconv.NumError{ 259 | Func: "ParseInt", 260 | Num: d.String(), 261 | Err: strconv.ErrRange, 262 | } 263 | } 264 | return d.upscale(0), nil 265 | } 266 | return d, nil 267 | } 268 | 269 | // Trunc attempts to truncate this decimal to an int64, dropping any fractional bits. 270 | func (d *Decimal) trunc() (int64, error) { 271 | ud, err := d.checkToUpscale() 272 | if err != nil { 273 | return 0, err 274 | } 275 | str := ud.n.String() 276 | 277 | truncateTo := len(str) - int(ud.scale) 278 | if truncateTo <= 0 { 279 | return 0, nil 280 | } 281 | 282 | return strconv.ParseInt(str[:truncateTo], 10, 64) 283 | } 284 | 285 | // Round attempts to truncate this decimal to an int64, rounding any fractional bits. 286 | func (d *Decimal) round() (int64, error) { 287 | ud, err := d.checkToUpscale() 288 | if err != nil { 289 | return 0, err 290 | } 291 | 292 | floatValue := float64(ud.n.Int64()) / math.Pow10(int(ud.scale)) 293 | roundedValue := math.Round(floatValue) 294 | return int64(roundedValue), nil 295 | } 296 | 297 | // Truncate returns a new decimal, truncated to the given number of 298 | // decimal digits of precision. It does not round, so 19.Truncate(1) 299 | // = 1d1. 300 | func (d *Decimal) Truncate(precision int) *Decimal { 301 | if precision <= 0 { 302 | panic("precision must be positive") 303 | } 304 | 305 | // Is there a better way to calculate precision? It really 306 | // seems like there should be... 307 | 308 | str := d.n.String() 309 | if str[0] == '-' { 310 | // Cheating a bit. 311 | precision++ 312 | } 313 | 314 | diff := len(str) - precision 315 | if diff <= 0 { 316 | // Already small enough, nothing to truncate. 317 | return d 318 | } 319 | 320 | // Lazy man's division by a power of 10. 321 | n, ok := new(big.Int).SetString(str[:precision], 10) 322 | if !ok { 323 | // Should never happen, since we started with a valid int. 324 | panic("failed to parse integer") 325 | } 326 | 327 | scale := int64(d.scale) - int64(diff) 328 | if scale < math.MinInt32 { 329 | panic("exponent out of range") 330 | } 331 | 332 | return &Decimal{ 333 | n: n, 334 | scale: int32(scale), 335 | } 336 | } 337 | 338 | // String formats the decimal as a string in Ion text format. 339 | func (d *Decimal) String() string { 340 | switch { 341 | case d.scale == 0: 342 | // Value is an unscaled integer. Just mark it as a decimal. 343 | if d.isNegZero { 344 | return "-0." 345 | } 346 | return d.n.String() + "." 347 | 348 | case d.scale < 0: 349 | // Value is a upscaled integer, nn'd'ss 350 | if d.isNegZero { 351 | return "-0d" + fmt.Sprintf("%d", -d.scale) 352 | } 353 | return d.n.String() + "d" + fmt.Sprintf("%d", -d.scale) 354 | 355 | default: 356 | // Value is a downscaled integer nn.nn('d'-ss)? 357 | var str string 358 | if d.isNegZero { 359 | str = "-0" 360 | } else { 361 | str = d.n.String() 362 | } 363 | 364 | idx := len(str) - int(d.scale) 365 | 366 | prefix := 1 367 | if len(str) > 0 && str[0] == '-' { 368 | // Account for leading '-'. 369 | prefix++ 370 | } 371 | 372 | if idx >= prefix { 373 | // Put the decimal point in the middle, no exponent. 374 | return str[:idx] + "." + str[idx:] 375 | } 376 | 377 | // Put the decimal point at the beginning and 378 | // add a (negative) exponent. 379 | b := strings.Builder{} 380 | b.WriteString(str[:prefix]) 381 | 382 | if len(str) > prefix { 383 | b.WriteString(".") 384 | b.WriteString(str[prefix:]) 385 | } 386 | 387 | b.WriteString("d") 388 | b.WriteString(fmt.Sprintf("%d", idx-prefix)) 389 | 390 | return b.String() 391 | } 392 | } 393 | 394 | // UnmarshalJSON implements the json.Unmarshaler interface. 395 | func (d *Decimal) UnmarshalJSON(decimalBytes []byte) error { 396 | str := string(decimalBytes) 397 | if str == "null" { 398 | return nil 399 | } 400 | str = strings.Replace(str, "E", "D", 1) 401 | str = strings.Replace(str, "e", "d", 1) 402 | parsed, err := ParseDecimal(str) 403 | if err != nil { 404 | return fmt.Errorf("error unmarshalling decimal '%s': %w", str, err) 405 | } 406 | *d = *parsed 407 | return nil 408 | } 409 | 410 | // MarshalJSON implements the json.Marshaler interface. 411 | func (d *Decimal) MarshalJSON() ([]byte, error) { 412 | absN := new(big.Int).Abs(d.n).String() 413 | scale := int(-d.scale) 414 | sign := d.n.Sign() 415 | 416 | var str string 417 | if scale == 0 { 418 | str = absN 419 | } else if scale > 0 { 420 | // add zeroes to the right 421 | str = absN + strings.Repeat("0", scale) 422 | } else { 423 | // add zeroes to the left 424 | absScale := -scale 425 | nLen := len(absN) 426 | 427 | if absScale >= nLen { 428 | str = "0." + strings.Repeat("0", absScale-nLen) + absN 429 | } else { 430 | str = absN[:nLen-absScale] + "." + absN[nLen-absScale:] 431 | } 432 | str = strings.TrimRight(str, "0") 433 | str = strings.TrimSuffix(str, ".") 434 | } 435 | 436 | if sign == -1 { 437 | str = "-" + str 438 | } 439 | return []byte(str), nil 440 | } 441 | -------------------------------------------------------------------------------- /ion/decimal_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "encoding/json" 20 | "fmt" 21 | "math/big" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/assert" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestDecimalToString(t *testing.T) { 29 | test := func(n int64, scale int32, expected string) { 30 | t.Run(expected, func(t *testing.T) { 31 | d := Decimal{ 32 | n: big.NewInt(n), 33 | scale: scale, 34 | } 35 | actual := d.String() 36 | assert.Equal(t, expected, actual) 37 | }) 38 | } 39 | 40 | test(0, 0, "0.") 41 | test(0, -1, "0d1") 42 | test(0, 1, "0d-1") 43 | 44 | test(1, 0, "1.") 45 | test(1, -1, "1d1") 46 | test(1, 1, "1d-1") 47 | 48 | test(-1, 0, "-1.") 49 | test(-1, -1, "-1d1") 50 | test(-1, 1, "-1d-1") 51 | 52 | test(123, 0, "123.") 53 | test(-456, 0, "-456.") 54 | 55 | test(123, -5, "123d5") 56 | test(-456, -5, "-456d5") 57 | 58 | test(123, 1, "12.3") 59 | test(123, 2, "1.23") 60 | test(123, 3, "1.23d-1") 61 | test(123, 4, "1.23d-2") 62 | 63 | test(-456, 1, "-45.6") 64 | test(-456, 2, "-4.56") 65 | test(-456, 3, "-4.56d-1") 66 | test(-456, 4, "-4.56d-2") 67 | } 68 | 69 | func TestParseDecimal(t *testing.T) { 70 | test := func(in string, n *big.Int, scale int32) { 71 | t.Run(in, func(t *testing.T) { 72 | d, err := ParseDecimal(in) 73 | require.NoError(t, err) 74 | 75 | assert.True(t, n.Cmp(d.n) == 0, "wrong n; expected %v, got %v", n, d.n) 76 | assert.Equal(t, scale, d.scale) 77 | }) 78 | } 79 | 80 | test("0", big.NewInt(0), 0) 81 | test("-0", big.NewInt(0), 0) 82 | test("0D0", big.NewInt(0), 0) 83 | test("-0d-1", big.NewInt(0), 1) 84 | 85 | test("1.", big.NewInt(1), 0) 86 | test("1.0", big.NewInt(10), 1) 87 | test("0.123", big.NewInt(123), 3) 88 | 89 | test("1d0", big.NewInt(1), 0) 90 | test("1d1", big.NewInt(1), -1) 91 | test("1d+1", big.NewInt(1), -1) 92 | test("1d-1", big.NewInt(1), 1) 93 | 94 | test("-0.12d4", big.NewInt(-12), -2) 95 | } 96 | 97 | func absF(d *Decimal) *Decimal { return d.Abs() } 98 | func negF(d *Decimal) *Decimal { return d.Neg() } 99 | 100 | type unaryOp struct { 101 | sym string 102 | fun func(d *Decimal) *Decimal 103 | } 104 | 105 | var abs = &unaryOp{"abs", absF} 106 | var neg = &unaryOp{"neg", negF} 107 | 108 | func testUnaryOp(t *testing.T, a, e string, op *unaryOp) { 109 | t.Run(op.sym+"("+a+")="+e, func(t *testing.T) { 110 | aa, _ := ParseDecimal(a) 111 | ee, _ := ParseDecimal(e) 112 | actual := op.fun(aa) 113 | assert.True(t, actual.Equal(ee), "expected %v, got %v", ee, actual) 114 | }) 115 | } 116 | 117 | func TestAbs(t *testing.T) { 118 | test := func(a, e string) { 119 | testUnaryOp(t, a, e, abs) 120 | } 121 | 122 | test("0", "0") 123 | test("1d100", "1d100") 124 | test("-1d100", "1d100") 125 | test("1.2d-3", "1.2d-3") 126 | test("-1.2d-3", "1.2d-3") 127 | } 128 | 129 | func TestNeg(t *testing.T) { 130 | test := func(a, e string) { 131 | testUnaryOp(t, a, e, neg) 132 | } 133 | 134 | test("0", "0") 135 | test("1d100", "-1d100") 136 | test("-1d100", "1d100") 137 | test("1.2d-3", "-1.2d-3") 138 | test("-1.2d-3", "1.2d-3") 139 | } 140 | 141 | func TestTrunc(t *testing.T) { 142 | test := func(a string, eval int64) { 143 | t.Run(fmt.Sprintf("trunc(%v)=%v", a, eval), func(t *testing.T) { 144 | aa := MustParseDecimal(a) 145 | val, err := aa.trunc() 146 | require.NoError(t, err) 147 | assert.Equal(t, eval, val) 148 | }) 149 | } 150 | 151 | test("0.", 0) 152 | test("0.01", 0) 153 | test("1.", 1) 154 | test("-1.", -1) 155 | test("1.01", 1) 156 | test("-1.01", -1) 157 | test("101", 101) 158 | test("1d3", 1000) 159 | } 160 | 161 | func TestRound(t *testing.T) { 162 | test := func(a string, eval int64) { 163 | t.Run(fmt.Sprintf("trunc(%v)=%v", a, eval), func(t *testing.T) { 164 | aa := MustParseDecimal(a) 165 | val, err := aa.round() 166 | require.NoError(t, err) 167 | assert.Equal(t, eval, val) 168 | }) 169 | } 170 | 171 | test("0.", 0) 172 | test("0.01", 0) 173 | test("1.", 1) 174 | test("-1.", -1) 175 | test("1.01", 1) 176 | test("-1.01", -1) 177 | test("1.4", 1) 178 | test("1.5", 2) 179 | test("1.6", 2) 180 | test("0.4", 0) 181 | test("0.5", 1) 182 | test("0.9999999999", 1) 183 | test("0.099", 0) 184 | test("101", 101) 185 | test("1d3", 1000) 186 | } 187 | 188 | func addF(a, b *Decimal) *Decimal { return a.Add(b) } 189 | func subF(a, b *Decimal) *Decimal { return a.Sub(b) } 190 | func mulF(a, b *Decimal) *Decimal { return a.Mul(b) } 191 | 192 | type binop struct { 193 | sym string 194 | fun func(a, b *Decimal) *Decimal 195 | } 196 | 197 | func TestShiftL(t *testing.T) { 198 | test := func(a string, b int, e string) { 199 | aa, _ := ParseDecimal(a) 200 | ee, _ := ParseDecimal(e) 201 | actual := aa.ShiftL(b) 202 | assert.True(t, actual.Equal(ee), "expected %v, got %v", ee, actual) 203 | } 204 | 205 | test("0", 10, "0") 206 | test("1", 0, "1") 207 | test("123", 1, "1230") 208 | test("123", 100, "123d100") 209 | test("1.23d-100", 102, "123") 210 | } 211 | 212 | func TestShiftR(t *testing.T) { 213 | test := func(a string, b int, e string) { 214 | aa, _ := ParseDecimal(a) 215 | ee, _ := ParseDecimal(e) 216 | actual := aa.ShiftR(b) 217 | assert.True(t, actual.Equal(ee), "expected %v, got %v", ee, actual) 218 | } 219 | 220 | test("0", 10, "0") 221 | test("1", 0, "1") 222 | test("123", 1, "12.3") 223 | test("123", 100, "1.23d-98") 224 | test("1.23d100", 98, "123") 225 | } 226 | 227 | var add = &binop{"+", addF} 228 | var sub = &binop{"-", subF} 229 | var mul = &binop{"*", mulF} 230 | 231 | func testBinaryOp(t *testing.T, a, b, e string, op *binop) { 232 | t.Run(a+op.sym+b+"="+e, func(t *testing.T) { 233 | aa, _ := ParseDecimal(a) 234 | bb, _ := ParseDecimal(b) 235 | ee, _ := ParseDecimal(e) 236 | 237 | actual := op.fun(aa, bb) 238 | assert.True(t, actual.Equal(ee), "expected %v, got %v", ee, actual) 239 | }) 240 | } 241 | 242 | func TestAdd(t *testing.T) { 243 | test := func(a, b, e string) { 244 | testBinaryOp(t, a, b, e, add) 245 | } 246 | 247 | test("1", "0", "1") 248 | test("1", "1", "2") 249 | test("1", "0.1", "1.1") 250 | test("0.3", "0.06", "0.36") 251 | test("1", "100", "101") 252 | test("1d100", "1d98", "101d98") 253 | test("1d-100", "1d-98", "1.01d-98") 254 | } 255 | 256 | func TestSub(t *testing.T) { 257 | test := func(a, b, e string) { 258 | testBinaryOp(t, a, b, e, sub) 259 | } 260 | 261 | test("1", "0", "1") 262 | test("1", "1", "0") 263 | test("1", "0.1", "0.9") 264 | test("0.3", "0.06", "0.24") 265 | test("1", "100", "-99") 266 | test("1d100", "1d98", "99d98") 267 | test("1d-100", "1d-98", "-99d-100") 268 | } 269 | 270 | func TestMul(t *testing.T) { 271 | test := func(a, b, e string) { 272 | testBinaryOp(t, a, b, e, mul) 273 | } 274 | 275 | test("1", "0", "0") 276 | test("1", "1", "1") 277 | test("2", "-1", "-2") 278 | test("7", "6", "42") 279 | test("10", "0.3", "3") 280 | test("3d100", "2d50", "6d150") 281 | test("3d-100", "2d-50", "6d-150") 282 | test("2d100", "4d-98", "8d2") 283 | } 284 | 285 | func TestTruncate(t *testing.T) { 286 | test := func(a string, p int, expected string) { 287 | t.Run(fmt.Sprintf("trunc(%v,%v)", a, p), func(t *testing.T) { 288 | aa := MustParseDecimal(a) 289 | actual := aa.Truncate(p).String() 290 | assert.Equal(t, expected, actual) 291 | }) 292 | } 293 | 294 | test("1", 1, "1.") 295 | test("1", 10, "1.") 296 | test("10", 1, "1d1") 297 | test("1999", 1, "1d3") 298 | test("1.2345", 3, "1.23") 299 | test("100d100", 2, "10d101") 300 | test("1.2345d-100", 2, "1.2d-100") 301 | } 302 | 303 | func TestCmp(t *testing.T) { 304 | test := func(a, b string, expected int) { 305 | t.Run("("+a+","+b+")", func(t *testing.T) { 306 | ad, _ := ParseDecimal(a) 307 | bd, _ := ParseDecimal(b) 308 | actual := ad.Cmp(bd) 309 | assert.Equal(t, expected, actual) 310 | }) 311 | } 312 | 313 | test("0", "0", 0) 314 | test("0", "1", -1) 315 | test("0", "-1", 1) 316 | 317 | test("1d2", "100", 0) 318 | test("100", "1d2", 0) 319 | test("1d2", "10", 1) 320 | test("10", "1d2", -1) 321 | 322 | test("0.01", "1d-2", 0) 323 | test("1d-2", "0.01", 0) 324 | test("0.01", "1d-3", 1) 325 | test("1d-3", "0.01", -1) 326 | } 327 | 328 | func TestUpscale(t *testing.T) { 329 | d, _ := ParseDecimal("1d1") 330 | actual := d.upscale(4).String() 331 | assert.Equal(t, "10.0000", actual) 332 | } 333 | 334 | func TestMarshalJSON(t *testing.T) { 335 | test := func(a string, expected string) { 336 | t.Run("("+a+")", func(t *testing.T) { 337 | ad, err := ParseDecimal(a) 338 | require.NoError(t, err) 339 | 340 | am, err := ad.MarshalJSON() 341 | require.NoError(t, err) 342 | 343 | assert.Equal(t, []byte(expected), am) 344 | }) 345 | } 346 | test("123000", "123000") 347 | 348 | test("1.01", "1.01") 349 | test("0.01", "0.01") 350 | test("0.0", "0") 351 | test("0.123456789012345678901234567890", "0.12345678901234567890123456789") // Trims trailing zeros 352 | test("123456789012345678901234567890.123456789012345678901234567890", "123456789012345678901234567890.12345678901234567890123456789") // Trims trailing zeros 353 | 354 | test("1d-2", "0.01") 355 | test("1d-3", "0.001") 356 | test("1d2", "100") 357 | 358 | test("-1d3", "-1000") 359 | test("-1d-3", "-0.001") 360 | test("-0.0", "0") 361 | test("-0.1", "-0.1") 362 | } 363 | 364 | func TestUnmarshalJSON(t *testing.T) { 365 | test := func(a string, expected string) { 366 | t.Run("("+a+")", func(t *testing.T) { 367 | expectedDec := MustParseDecimal(expected) 368 | 369 | var r struct { 370 | D *Decimal `json:"d"` 371 | } 372 | err := json.Unmarshal([]byte(`{"d":`+a+`}`), &r) 373 | require.NoError(t, err) 374 | 375 | assert.Truef(t, expectedDec.Equal(r.D), "expected %v, got %v", expected, r.D) 376 | }) 377 | } 378 | 379 | test("123000", "123000") 380 | test("123.1", "123.1") 381 | test("123.10", "123.1") 382 | test("-123000", "-123000") 383 | test("-123.1", "-123.1") 384 | test("-123.10", "-123.1") 385 | 386 | test("1e+2", "100") 387 | test("1e2", "100") 388 | test("1E2", "100") 389 | test("1E+2", "100") 390 | 391 | test("-1e+2", "-100") 392 | test("-1e2", "-100") 393 | test("-1E2", "-100") 394 | test("-1E+2", "-100") 395 | 396 | test("1e-2", "0.01") 397 | test("1E-2", "0.01") 398 | 399 | test("-1e-2", "-0.01") 400 | test("-1E-2", "-0.01") 401 | } 402 | -------------------------------------------------------------------------------- /ion/err.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import "fmt" 19 | 20 | // A UsageError is returned when you use a Reader or Writer in an inappropriate way. 21 | type UsageError struct { 22 | API string 23 | Msg string 24 | } 25 | 26 | func (e *UsageError) Error() string { 27 | return fmt.Sprintf("ion: usage error in %v: %v", e.API, e.Msg) 28 | } 29 | 30 | // An IOError is returned when there is an error reading from or writing to an 31 | // underlying io.Reader or io.Writer. 32 | type IOError struct { 33 | Err error 34 | } 35 | 36 | func (e *IOError) Error() string { 37 | return fmt.Sprintf("ion: i/o error: %v", e.Err) 38 | } 39 | 40 | // A SyntaxError is returned when a Reader encounters invalid input for which no more 41 | // specific error type is defined. 42 | type SyntaxError struct { 43 | Msg string 44 | Offset uint64 45 | } 46 | 47 | func (e *SyntaxError) Error() string { 48 | return fmt.Sprintf("ion: syntax error: %v (offset %v)", e.Msg, e.Offset) 49 | } 50 | 51 | // An UnexpectedEOFError is returned when a Reader unexpectedly encounters an 52 | // io.EOF error. 53 | type UnexpectedEOFError struct { 54 | Offset uint64 55 | } 56 | 57 | func (e *UnexpectedEOFError) Error() string { 58 | return fmt.Sprintf("ion: unexpected end of input (offset %v)", e.Offset) 59 | } 60 | 61 | // An UnsupportedVersionError is returned when a Reader encounters a binary version 62 | // marker with a version that this library does not understand. 63 | type UnsupportedVersionError struct { 64 | Major int 65 | Minor int 66 | Offset uint64 67 | } 68 | 69 | func (e *UnsupportedVersionError) Error() string { 70 | return fmt.Sprintf("ion: unsupported version %v.%v (offset %v)", e.Major, e.Minor, e.Offset) 71 | } 72 | 73 | // An InvalidTagByteError is returned when a binary Reader encounters an invalid 74 | // tag byte. 75 | type InvalidTagByteError struct { 76 | Byte byte 77 | Offset uint64 78 | } 79 | 80 | func (e *InvalidTagByteError) Error() string { 81 | return fmt.Sprintf("ion: invalid tag byte 0x%02X (offset %v)", e.Byte, e.Offset) 82 | } 83 | 84 | // An UnexpectedRuneError is returned when a text Reader encounters an unexpected rune. 85 | type UnexpectedRuneError struct { 86 | Rune rune 87 | Offset uint64 88 | } 89 | 90 | func (e *UnexpectedRuneError) Error() string { 91 | return fmt.Sprintf("ion: unexpected rune %q (offset %v)", e.Rune, e.Offset) 92 | } 93 | 94 | // An UnexpectedTokenError is returned when a text Reader encounters an unexpected 95 | // token. 96 | type UnexpectedTokenError struct { 97 | Token string 98 | Offset uint64 99 | } 100 | 101 | func (e *UnexpectedTokenError) Error() string { 102 | return fmt.Sprintf("ion: unexpected token '%v' (offset %v)", e.Token, e.Offset) 103 | } 104 | -------------------------------------------------------------------------------- /ion/fields.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "fmt" 20 | "reflect" 21 | "strings" 22 | ) 23 | 24 | // A field is a reflectively-accessed field of a struct type. 25 | type field struct { 26 | name string 27 | typ reflect.Type 28 | path []int 29 | omitEmpty bool 30 | hint Type 31 | annotations bool 32 | } 33 | 34 | func (f *field) setopts(opts string) { 35 | for opts != "" { 36 | var o string 37 | 38 | i := strings.Index(opts, ",") 39 | if i >= 0 { 40 | o, opts = opts[:i], opts[i+1:] 41 | } else { 42 | o, opts = opts, "" 43 | } 44 | 45 | switch o { 46 | case "omitempty": 47 | f.omitEmpty = true 48 | case "symbol": 49 | f.hint = SymbolType 50 | case "clob": 51 | f.hint = ClobType 52 | case "sexp": 53 | f.hint = SexpType 54 | case "annotations": 55 | f.annotations = true 56 | } 57 | } 58 | } 59 | 60 | // A fielder maps out the fields of a type. 61 | type fielder struct { 62 | fields []field 63 | index map[string]bool 64 | } 65 | 66 | // FieldsFor returns the fields of the given struct type. 67 | // https://github.com/amazon-ion/ion-go/issues/117 68 | func fieldsFor(t reflect.Type) []field { 69 | fldr := fielder{index: map[string]bool{}} 70 | fldr.inspect(t, nil) 71 | return fldr.fields 72 | } 73 | 74 | // Inspect recursively inspects a type to determine all of its fields. 75 | func (f *fielder) inspect(t reflect.Type, path []int) { 76 | for i := 0; i < t.NumField(); i++ { 77 | sf := t.Field(i) 78 | if !visible(&sf) { 79 | // Skip non-visible fields. 80 | continue 81 | } 82 | 83 | tag := sf.Tag.Get("ion") 84 | if tag == "-" { 85 | // Skip fields that are explicitly hidden by tag. 86 | continue 87 | } 88 | name, opts := parseIonTag(tag) 89 | 90 | newpath := make([]int, len(path)+1) 91 | copy(newpath, path) 92 | newpath[len(path)] = i 93 | 94 | ft := sf.Type 95 | if ft.Name() == "" && ft.Kind() == reflect.Ptr { 96 | ft = ft.Elem() 97 | } 98 | 99 | if name == "" && sf.Anonymous && ft.Kind() == reflect.Struct { 100 | // Dig in to the embedded struct. 101 | f.inspect(ft, newpath) 102 | } else { 103 | // Add this named field. 104 | if name == "" { 105 | name = sf.Name 106 | } 107 | 108 | if f.index[name] { 109 | panic(fmt.Sprintf("too many fields named %v", name)) 110 | } 111 | f.index[name] = true 112 | 113 | field := field{ 114 | name: name, 115 | typ: ft, 116 | path: newpath, 117 | } 118 | field.setopts(opts) 119 | 120 | f.fields = append(f.fields, field) 121 | } 122 | } 123 | } 124 | 125 | // Visible returns true if the given StructField should show up in the output. 126 | func visible(sf *reflect.StructField) bool { 127 | exported := sf.PkgPath == "" 128 | if sf.Anonymous { 129 | t := sf.Type 130 | if t.Kind() == reflect.Ptr { 131 | t = t.Elem() 132 | } 133 | if t.Kind() == reflect.Struct { 134 | // Fields of embedded structs are visible even if the struct type itself is not. 135 | return true 136 | } 137 | } 138 | return exported 139 | } 140 | 141 | // ParseIonTag parses a `ion:"..."` field tag, returning the name and opts. 142 | func parseIonTag(tag string) (string, string) { 143 | if idx := strings.Index(tag, ","); idx != -1 { 144 | // Ignore additional Ion options, at least for now. 145 | return tag[:idx], tag[idx+1:] 146 | } 147 | return tag, "" 148 | } 149 | -------------------------------------------------------------------------------- /ion/marshal_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "bytes" 20 | "math" 21 | "math/big" 22 | "strings" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/assert" 27 | "github.com/stretchr/testify/require" 28 | ) 29 | 30 | func TestMarshalText(t *testing.T) { 31 | test := func(v interface{}, eval string) { 32 | t.Run(eval, func(t *testing.T) { 33 | val, err := MarshalText(v) 34 | require.NoError(t, err) 35 | assert.Equal(t, eval, string(val)) 36 | }) 37 | } 38 | 39 | test(nil, "null") 40 | test(true, "true") 41 | test(false, "false") 42 | 43 | test(byte(42), "42") 44 | test(-42, "-42") 45 | test(uint64(math.MaxUint64), "18446744073709551615") 46 | bigIntPos, _ := new(big.Int).SetString("123456789012345678901234567890", 10) 47 | test(bigIntPos, "123456789012345678901234567890") 48 | bigIntNeg, _ := new(big.Int).SetString("-123456789012345678901234567890", 10) 49 | test(bigIntNeg, "-123456789012345678901234567890") 50 | test(math.MinInt64, "-9223372036854775808") 51 | 52 | test(42.0, "4.2e+1") 53 | test(math.Inf(1), "+inf") 54 | test(math.Inf(-1), "-inf") 55 | test(math.NaN(), "nan") 56 | 57 | test(MustParseDecimal("1.20"), "1.20") 58 | test(NewTimestamp(time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC), TimestampPrecisionSecond, TimezoneUTC), 59 | "2010-01-01T00:00:00Z") 60 | test(time.Date(2010, 1, 1, 0, 0, 0, 1, time.UTC), "2010-01-01T00:00:00.000000001Z") 61 | test(time.Date(2010, 1, 1, 0, 0, 0, 770000000, time.UTC), "2010-01-01T00:00:00.770000000Z") 62 | loc, _ := time.LoadLocation("EST") 63 | test(time.Date(2010, 1, 1, 0, 0, 0, 0, loc), "2010-01-01T00:00:00.000000000-05:00") 64 | loc = time.FixedZone("UTC+8", 8*60*60) 65 | test(time.Date(2010, 1, 1, 0, 0, 0, 0, loc), "2010-01-01T00:00:00.000000000+08:00") 66 | 67 | test("hello\tworld", "\"hello\\tworld\"") 68 | 69 | test(struct{ A, B int }{42, 0}, "{A:42,B:0}") 70 | test(struct { 71 | A int `ion:"val,ignoreme"` 72 | B int `ion:"-"` 73 | C int `ion:",omitempty"` 74 | d int 75 | }{42, 0, 0, 0}, "{val:42}") 76 | 77 | test(struct{ V interface{} }{}, "{V:null}") 78 | test(struct{ V interface{} }{"42"}, "{V:\"42\"}") 79 | 80 | fortyTwo := 42 81 | 82 | test(struct{ V *int }{}, "{V:null}") 83 | test(struct{ V *int }{&fortyTwo}, "{V:42}") 84 | 85 | test(map[string]int{"b": 2, "a": 1}, "{a:1,b:2}") 86 | 87 | test(struct{ V []int }{}, "{V:null}") 88 | test(struct{ V []int }{[]int{4, 2}}, "{V:[4,2]}") 89 | 90 | test(struct{ V []byte }{}, "{V:null}") 91 | test(struct{ V []byte }{[]byte{4, 2}}, "{V:{{BAI=}}}") 92 | 93 | test(struct{ V [2]byte }{[2]byte{4, 2}}, "{V:[4,2]}") 94 | } 95 | 96 | func TestMarshalBinary(t *testing.T) { 97 | test := func(v interface{}, name string, eval []byte) { 98 | t.Run(name, func(t *testing.T) { 99 | val, err := MarshalBinary(v) 100 | require.NoError(t, err) 101 | assert.True(t, bytes.Equal(val, eval), "expected '%v', got '%v'", fmtbytes(eval), fmtbytes(val)) 102 | }) 103 | } 104 | 105 | // Null. 106 | test(nil, "null", prefixIVM([]byte{0x0F})) 107 | 108 | // Boolean. 109 | test(true, "boolean", prefixIVM([]byte{0x11})) // true 110 | 111 | // Positive Integer. 112 | test(32767, "integer", prefixIVM([]byte{0x22, 0x7F, 0xFF})) // 32767 113 | 114 | // Negative Integer. 115 | test(-32767, "negative integer", prefixIVM([]byte{0x32, 0x7F, 0xFF})) // -32767 116 | 117 | // Float32 valid type. Go treats floats as float64 by default, unless specified. 118 | // Explicitly cast number to be of float32 and ensure type is handled. This should not be an unknown type. 119 | test(float32(math.MaxFloat32), "float32 valid type", prefixIVM([]byte{0x44, 0x7F, 0x7F, 0xFF, 0xFF})) // 3.40282346638528859811704183484516925440e+38 120 | 121 | // Float32. Ensure number can be represented losslessly as a float32 by testing that byte length is 5. 122 | // This should not be represented as a float64. 123 | test(math.MaxFloat32, "float32", prefixIVM([]byte{0x44, 0x7F, 0x7F, 0xFF, 0xFF})) // 3.40282346638528859811704183484516925440e+38 124 | 125 | // Float 64. 126 | test(math.MaxFloat64, "float64", prefixIVM([]byte{0x48, 0x7F, 0xEF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})) // 1.797693134862315708145274237317043567981e+308 127 | 128 | // Decimal. 129 | test(MustParseDecimal("0d-63"), "decimal", prefixIVM([]byte{0x51, 0xFF})) // 0d-63 130 | 131 | // String. 132 | test("abc", "string", prefixIVM([]byte{0x83, 'a', 'b', 'c'})) // "abc" 133 | 134 | // Blob. 135 | test([]byte{97, 98, 99}, "blob", prefixIVM([]byte{0xA3, 'a', 'b', 'c'})) 136 | 137 | // List. 138 | test([]int{2, 3, 4}, "list", prefixIVM([]byte{0xB6, 0x21, 0x02, 0x21, 0x03, 0x21, 0x04})) 139 | 140 | // Struct. 141 | test(struct{ A, B int }{42, 0}, "{A:42,B:0}", prefixIVM([]byte{ 142 | 0xE9, 0x81, 0x83, 0xD6, 0x87, 0xB4, 0x81, 'A', 0x81, 'B', 143 | 0xD5, 144 | 0x8A, 0x21, 0x2A, 145 | 0x8B, 0x20, 146 | })) 147 | 148 | test(time.Date(2010, 1, 1, 0, 0, 0, 1, time.UTC), "time with 1 nanosecond", prefixIVM([]byte{ 149 | 0x6A, 0x80, 0x0F, 0xDA, 0x81, 0x81, 0x80, 0x80, 0x80, 150 | 0xC9, // exponent: 9 151 | 0x01, // coefficient: 1 152 | })) 153 | 154 | test(time.Date(2010, 1, 1, 0, 0, 0, 5000, time.UTC), "time with 5000 nanoseconds", prefixIVM([]byte{ 155 | 0x6B, 0x80, 0x0F, 0xDA, 0x81, 0x81, 0x80, 0x80, 0x80, 156 | 0xC9, // exponent: 9 157 | 0x13, 0x88, // coefficient: 5000 158 | })) 159 | } 160 | 161 | func prefixIVM(data []byte) []byte { 162 | prefix := []byte{0xE0, 0x01, 0x00, 0xEA} // $ion_1_0 163 | return append(prefix, data...) 164 | } 165 | 166 | func TestMarshalBinaryLST(t *testing.T) { 167 | lsta := NewLocalSymbolTable(nil, nil) 168 | lstb := NewLocalSymbolTable(nil, []string{ 169 | "A", "B", 170 | }) 171 | 172 | test := func(v interface{}, name string, lst SymbolTable, eval []byte) { 173 | t.Run(name, func(t *testing.T) { 174 | val, err := MarshalBinaryLST(v, lst) 175 | require.NoError(t, err) 176 | assert.True(t, bytes.Equal(val, eval), "expected '%v', got '%v'", fmtbytes(eval), fmtbytes(val)) 177 | }) 178 | } 179 | 180 | test(nil, "null", lsta, []byte{0xE0, 0x01, 0x00, 0xEA, 0x0F}) 181 | test(struct{ A, B int }{42, 0}, "{A:42,B:0}", lstb, []byte{ 182 | 0xE0, 0x01, 0x00, 0xEA, 183 | 0xE9, 0x81, 0x83, 0xD6, 0x87, 0xB4, 0x81, 'A', 0x81, 'B', 184 | 0xD5, 185 | 0x8A, 0x21, 0x2A, 186 | 0x8B, 0x20, 187 | }) 188 | } 189 | 190 | func TestMarshalNestedStructs(t *testing.T) { 191 | type gp struct { 192 | A int `ion:"a"` 193 | } 194 | 195 | type gp2 struct { 196 | B int `ion:"b"` 197 | } 198 | 199 | type parent struct { 200 | gp 201 | *gp2 202 | C int `ion:"c"` 203 | } 204 | 205 | type root struct { 206 | parent 207 | D int `ion:"d"` 208 | } 209 | 210 | v := root{ 211 | parent: parent{ 212 | gp: gp{ 213 | A: 1, 214 | }, 215 | gp2: &gp2{ 216 | B: 2, 217 | }, 218 | C: 3, 219 | }, 220 | D: 4, 221 | } 222 | 223 | val, err := MarshalText(v) 224 | require.NoError(t, err) 225 | 226 | eval := "{a:1,b:2,c:3,d:4}" 227 | assert.Equal(t, eval, string(val)) 228 | } 229 | 230 | func TestMarshalHints(t *testing.T) { 231 | type hints struct { 232 | String string `ion:"str,omitempty,string"` 233 | Symbol string `ion:"sym,omitempty,symbol"` 234 | Strings []string `ion:"strs,string"` 235 | Symbols []string `ion:"syms,symbol"` 236 | StrMap map[string]string `ion:"strm"` 237 | SymMap map[string]string `ion:"symm,symbol"` 238 | Blob []byte `ion:"bl,blob,omitempty"` 239 | Clob []byte `ion:"cl,clob,omitempty"` 240 | Sexp []int `ion:"sx,sexp"` 241 | } 242 | 243 | v := hints{ 244 | String: "string", 245 | Symbol: "symbol", 246 | Strings: []string{"a", "b"}, 247 | Symbols: []string{"c", "d"}, 248 | StrMap: map[string]string{"a": "b"}, 249 | SymMap: map[string]string{"c": "d"}, 250 | Blob: []byte("blob"), 251 | Clob: []byte("clob"), 252 | Sexp: []int{1, 2, 3}, 253 | } 254 | 255 | val, err := MarshalText(v) 256 | require.NoError(t, err) 257 | 258 | eval := `{` + 259 | `str:"string",` + 260 | `sym:symbol,` + 261 | `strs:["a","b"],` + 262 | `syms:[c,d],` + 263 | `strm:{a:"b"},` + 264 | `symm:{c:d},` + 265 | `bl:{{YmxvYg==}},` + 266 | `cl:{{"clob"}},` + 267 | `sx:(1 2 3)` + 268 | `}` 269 | 270 | assert.Equal(t, eval, string(val)) 271 | } 272 | 273 | type marshalMe uint8 274 | 275 | var _ Marshaler = marshalMe(0) 276 | 277 | const ( 278 | one marshalMe = iota 279 | two 280 | three 281 | four 282 | ) 283 | 284 | func (m marshalMe) String() string { 285 | switch m { 286 | case one: 287 | return "ONE" 288 | case two: 289 | return "TWO" 290 | case three: 291 | return "THREE" 292 | case four: 293 | return "FOUR" 294 | default: 295 | panic("unexpected value") 296 | } 297 | } 298 | 299 | func (m marshalMe) MarshalIon(w Writer) error { 300 | return w.WriteSymbolFromString(m.String()) 301 | } 302 | 303 | func TestMarshalCustomMarshaler(t *testing.T) { 304 | buf := strings.Builder{} 305 | enc := NewTextEncoder(&buf) 306 | 307 | require.NoError(t, enc.Encode(one)) 308 | require.NoError(t, enc.EncodeAs([]marshalMe{two, three}, SexpType)) 309 | 310 | v := struct { 311 | Num marshalMe `ion:"num"` 312 | }{four} 313 | require.NoError(t, enc.Encode(v)) 314 | require.NoError(t, enc.Finish()) 315 | 316 | val := buf.String() 317 | eval := "ONE\n(TWO THREE)\n{num:FOUR}\n" 318 | 319 | assert.Equal(t, eval, val) 320 | } 321 | 322 | func TestMarshalValuesWithAnnotation(t *testing.T) { 323 | test := func(v interface{}, testName, eval string) { 324 | t.Run(testName, func(t *testing.T) { 325 | val, err := MarshalText(v) 326 | require.NoError(t, err) 327 | assert.Equal(t, eval, string(val)) 328 | }) 329 | } 330 | 331 | type foo struct { 332 | Value interface{} 333 | AnyName []SymbolToken `ion:",annotations"` 334 | } 335 | 336 | buildValue := func(val interface{}) foo { 337 | return foo{val, []SymbolToken{NewSymbolTokenFromString("symbols or string"), NewSymbolTokenFromString("annotations")}} 338 | } 339 | 340 | test(buildValue(nil), "null", "'symbols or string'::annotations::null") 341 | test(buildValue(true), "bool", "'symbols or string'::annotations::true") 342 | test(buildValue(5), "int", "'symbols or string'::annotations::5") 343 | test(buildValue(float32(math.MaxFloat32)), "float", "'symbols or string'::annotations::3.4028234663852886e+38") 344 | test(buildValue(MustParseDecimal("1.2")), "decimal", "'symbols or string'::annotations::1.2") 345 | test(buildValue(NewTimestamp(time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC), TimestampPrecisionSecond, TimezoneUTC)), 346 | "timestamp", "'symbols or string'::annotations::2000-01-02T03:04:05Z") 347 | test(buildValue("stringValue"), "string", "'symbols or string'::annotations::\"stringValue\"") 348 | test(buildValue([]byte{4, 2}), "blob", "'symbols or string'::annotations::{{BAI=}}") 349 | test(buildValue([]int{3, 5, 7}), "list", "'symbols or string'::annotations::[3,5,7]") 350 | test(buildValue(map[string]int{"b": 2, "a": 1}), "struct", "'symbols or string'::annotations::{a:1,b:2}") 351 | } 352 | -------------------------------------------------------------------------------- /ion/reader_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "io/ioutil" 20 | "os" 21 | "path/filepath" 22 | "strings" 23 | "testing" 24 | 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | var blacklist = map[string]bool{ 29 | "../ion-tests/iontestdata/good/emptyAnnotatedInt.10n": true, 30 | "../ion-tests/iontestdata/good/subfieldVarUInt32bit.ion": true, 31 | "../ion-tests/iontestdata/good/utf16.ion": true, 32 | "../ion-tests/iontestdata/good/utf32.ion": true, 33 | "../ion-tests/iontestdata/good/whitespace.ion": true, 34 | "../ion-tests/iontestdata/good/item1.10n": true, 35 | "../ion-tests/iontestdata/good/typecodes/T7-large.10n": true, 36 | } 37 | 38 | type drainfunc func(t *testing.T, r Reader, f string) 39 | 40 | func TestDecodeFiles(t *testing.T) { 41 | testReadDir(t, "../ion-tests/iontestdata/good", func(t *testing.T, r Reader, f string) { 42 | d := NewDecoder(r) 43 | for { 44 | v, err := d.Decode() 45 | if err == ErrNoInput { 46 | break 47 | } 48 | require.NoError(t, err) 49 | _ = v 50 | } 51 | }) 52 | } 53 | 54 | func testReadDir(t *testing.T, path string, d drainfunc) { 55 | files, err := ioutil.ReadDir(path) 56 | require.NoError(t, err) 57 | 58 | for _, file := range files { 59 | fp := filepath.Join(path, file.Name()) 60 | if file.IsDir() { 61 | testReadDir(t, fp, d) 62 | } else { 63 | t.Run(fp, func(t *testing.T) { 64 | testReadFile(t, fp, d) 65 | }) 66 | } 67 | } 68 | } 69 | 70 | func testReadFile(t *testing.T, path string, d drainfunc) { 71 | if _, ok := blacklist[path]; ok { 72 | return 73 | } 74 | if strings.HasSuffix(path, "md") { 75 | return 76 | } 77 | 78 | file, err := os.Open(path) 79 | require.NoError(t, err) 80 | 81 | r := NewReader(file) 82 | 83 | d(t, r, path) 84 | 85 | require.NoError(t, file.Close()) 86 | } 87 | -------------------------------------------------------------------------------- /ion/readlocalsymboltable.go: -------------------------------------------------------------------------------- 1 | package ion 2 | 3 | import "fmt" 4 | 5 | // ReadLocalSymbolTable reads and installs a new local symbol table. 6 | func readLocalSymbolTable(r Reader, cat Catalog) (SymbolTable, error) { 7 | if err := r.StepIn(); err != nil { 8 | return nil, err 9 | } 10 | 11 | var imps []SharedSymbolTable 12 | var syms []string 13 | 14 | foundImport := false 15 | foundLocals := false 16 | 17 | for r.Next() { 18 | var err error 19 | fieldName, err := r.FieldName() 20 | if err != nil { 21 | return nil, err 22 | } 23 | if fieldName == nil || fieldName.Text == nil { 24 | return nil, fmt.Errorf("ion: field name is nil") 25 | } 26 | 27 | switch *fieldName.Text { 28 | case "symbols": 29 | if foundLocals { 30 | return nil, fmt.Errorf("ion: multiple symbol fields found within a single local symbol table") 31 | } 32 | foundLocals = true 33 | syms, err = readSymbols(r) 34 | case "imports": 35 | if foundImport { 36 | return nil, fmt.Errorf("ion: multiple imports fields found within a single local symbol table") 37 | } 38 | foundImport = true 39 | imps, err = readImports(r, cat) 40 | } 41 | if err != nil { 42 | return nil, err 43 | } 44 | } 45 | 46 | if err := r.StepOut(); err != nil { 47 | return nil, err 48 | } 49 | 50 | return NewLocalSymbolTable(imps, syms), nil 51 | } 52 | 53 | // ReadImports reads the imports field of a local symbol table. 54 | func readImports(r Reader, cat Catalog) ([]SharedSymbolTable, error) { 55 | if r.Type() == SymbolType { 56 | val, err := r.SymbolValue() 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | if val.LocalSID == 3 { 62 | // Special case that imports the current local symbol table. 63 | if r.SymbolTable() == nil || r.SymbolTable() == V1SystemSymbolTable { 64 | return nil, nil 65 | } 66 | 67 | imps := r.SymbolTable().Imports() 68 | lsst := NewSharedSymbolTable("", 0, r.SymbolTable().Symbols()) 69 | return append(imps, lsst), nil 70 | } 71 | } 72 | 73 | if r.Type() != ListType || r.IsNull() { 74 | return nil, nil 75 | } 76 | if err := r.StepIn(); err != nil { 77 | return nil, err 78 | } 79 | 80 | var imps []SharedSymbolTable 81 | for r.Next() { 82 | imp, err := readImport(r, cat) 83 | if err != nil { 84 | return nil, err 85 | } 86 | if imp != nil { 87 | imps = append(imps, imp) 88 | } 89 | } 90 | 91 | err := r.StepOut() 92 | return imps, err 93 | } 94 | 95 | // ReadImport reads an import definition. 96 | func readImport(r Reader, cat Catalog) (SharedSymbolTable, error) { 97 | if r.Type() != StructType || r.IsNull() { 98 | return nil, nil 99 | } 100 | if err := r.StepIn(); err != nil { 101 | return nil, err 102 | } 103 | 104 | name := "" 105 | version := -1 106 | maxID := int64(-1) 107 | 108 | for r.Next() { 109 | fieldName, err := r.FieldName() 110 | if err != nil { 111 | return nil, err 112 | } 113 | if fieldName == nil || fieldName.Text == nil { 114 | return nil, fmt.Errorf("ion: field name is nil") 115 | } 116 | 117 | switch *fieldName.Text { 118 | case "name": 119 | if r.Type() == StringType { 120 | val, err := r.StringValue() 121 | if err != nil { 122 | return nil, err 123 | } 124 | name = *val 125 | } 126 | case "version": 127 | if r.Type() == IntType { 128 | val, err := r.IntValue() 129 | if err != nil { 130 | return nil, err 131 | } 132 | version = *val 133 | } 134 | case "max_id": 135 | if r.Type() == IntType { 136 | if r.IsNull() { 137 | return nil, fmt.Errorf("ion: max id is null") 138 | } 139 | i, err := r.Int64Value() 140 | if err != nil { 141 | return nil, err 142 | } 143 | maxID = *i 144 | } 145 | } 146 | } 147 | 148 | if err := r.StepOut(); err != nil { 149 | return nil, err 150 | } 151 | 152 | if name == "" || name == "$ion" { 153 | return nil, nil 154 | } 155 | if version < 1 { 156 | version = 1 157 | } 158 | 159 | var imp SharedSymbolTable 160 | if cat != nil { 161 | imp = cat.FindExact(name, version) 162 | if imp == nil { 163 | imp = cat.FindLatest(name) 164 | } 165 | } 166 | 167 | if maxID < 0 { 168 | if imp == nil || version != imp.Version() { 169 | return nil, fmt.Errorf("ion: import of shared table %v/%v lacks a valid max_id, but an exact "+ 170 | "match was not found in the catalog", name, version) 171 | } 172 | maxID = int64(imp.MaxID()) 173 | } 174 | 175 | if imp == nil { 176 | imp = &bogusSST{ 177 | name: name, 178 | version: version, 179 | maxID: uint64(maxID), 180 | } 181 | } else { 182 | imp = imp.Adjust(uint64(maxID)) 183 | } 184 | 185 | return imp, nil 186 | } 187 | 188 | // ReadSymbols reads the symbols from a symbol table. 189 | func readSymbols(r Reader) ([]string, error) { 190 | if r.Type() != ListType { 191 | return nil, nil 192 | } 193 | if err := r.StepIn(); err != nil { 194 | return nil, err 195 | } 196 | 197 | var syms []string 198 | for r.Next() { 199 | if r.Type() == StringType { 200 | sym, err := r.StringValue() 201 | 202 | if err != nil { 203 | return nil, err 204 | } 205 | if sym != nil { 206 | syms = append(syms, *sym) 207 | } else { 208 | syms = append(syms, "") 209 | } 210 | } else { 211 | syms = append(syms, "") 212 | } 213 | } 214 | 215 | err := r.StepOut() 216 | return syms, err 217 | } 218 | -------------------------------------------------------------------------------- /ion/readlocalsymboltable_test.go: -------------------------------------------------------------------------------- 1 | package ion 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestLocalSymbolTableAppend(t *testing.T) { 10 | text := `$ion_symbol_table:: 11 | { 12 | symbols:[ "s1", "s2" ] 13 | } 14 | $ion_symbol_table:: 15 | { 16 | imports: $ion_symbol_table, 17 | symbols:[ "s3", "s4", "s5" ] 18 | } 19 | null` 20 | 21 | r := NewReaderString(text) 22 | assert.True(t, r.Next()) 23 | st := r.SymbolTable() 24 | 25 | imports := st.Imports() 26 | systemTable := imports[0] 27 | systemMaxID := systemTable.MaxID() 28 | 29 | checkSymbol(t, "s1", systemMaxID+1, st) 30 | checkSymbol(t, "s2", systemMaxID+2, st) 31 | checkSymbol(t, "s3", systemMaxID+3, st) 32 | checkSymbol(t, "s4", systemMaxID+4, st) 33 | checkSymbol(t, "s5", systemMaxID+5, st) 34 | checkUnknownSymbolText(t, "unknown", st) 35 | checkUnknownSymbolID(t, 33, st) 36 | } 37 | 38 | func TestLocalSymbolTableMultiAppend(t *testing.T) { 39 | text := `$ion_symbol_table:: 40 | { 41 | symbols:[ "s1", "s2" ] 42 | } 43 | $ion_symbol_table:: 44 | { 45 | imports: $ion_symbol_table, 46 | symbols:[ "s3" ] 47 | } 48 | $ion_symbol_table:: 49 | { 50 | imports: $ion_symbol_table, 51 | symbols:[ "s4", "s5" ] 52 | } 53 | $ion_symbol_table:: 54 | { 55 | imports: $ion_symbol_table, 56 | symbols:[ "s6" ] 57 | } 58 | null` 59 | 60 | r := NewReaderString(text) 61 | assert.True(t, r.Next()) 62 | st := r.SymbolTable() 63 | 64 | imports := st.Imports() 65 | systemTable := imports[0] 66 | systemMaxID := systemTable.MaxID() 67 | 68 | checkSymbol(t, "s1", systemMaxID+1, st) 69 | checkSymbol(t, "s2", systemMaxID+2, st) 70 | checkSymbol(t, "s3", systemMaxID+3, st) 71 | checkSymbol(t, "s4", systemMaxID+4, st) 72 | checkSymbol(t, "s5", systemMaxID+5, st) 73 | checkSymbol(t, "s6", systemMaxID+6, st) 74 | 75 | checkUnknownSymbolText(t, "unknown", st) 76 | checkUnknownSymbolID(t, 33, st) 77 | } 78 | 79 | func TestLocalSymbolTableAppendEmptyList(t *testing.T) { 80 | original := `$ion_symbol_table:: 81 | { 82 | symbols:[ "s1" ] 83 | }` 84 | 85 | appended := original + 86 | `$ion_symbol_table:: 87 | { 88 | imports: $ion_symbol_table, 89 | symbols:[] 90 | } 91 | null` 92 | 93 | r := NewReaderString(original + "null") 94 | assert.True(t, r.Next()) 95 | ost := r.SymbolTable() 96 | 97 | originalSymbol := ost.Find("s1") 98 | 99 | r = NewReaderString(appended) 100 | assert.True(t, r.Next()) 101 | ast := r.SymbolTable() 102 | appendedSymbol := ast.Find("s1") 103 | 104 | assert.Equal(t, originalSymbol.LocalSID, appendedSymbol.LocalSID, 105 | "Original symbol SID: %v,did not match Appended symbol SID: %v", originalSymbol.LocalSID, appendedSymbol.LocalSID) 106 | } 107 | 108 | func TestLocalSymbolTableAppendNonUnique(t *testing.T) { 109 | text := `$ion_symbol_table:: 110 | { 111 | symbols:[ "foo" ] 112 | } 113 | $10 114 | $ion_symbol_table:: 115 | { 116 | imports: $ion_symbol_table, 117 | symbols:[ "foo", "bar" ] 118 | } 119 | $11 120 | $12` 121 | 122 | r := NewReaderString(text) 123 | assert.True(t, r.Next()) 124 | assert.True(t, r.Next()) 125 | st := r.SymbolTable() 126 | systemMaxID := getSystemMaxID(st) 127 | 128 | checkSymbol(t, "foo", systemMaxID+1, st) 129 | checkSymbol(t, "foo", systemMaxID+2, st) 130 | checkSymbol(t, "bar", systemMaxID+3, st) 131 | } 132 | 133 | func TestLocalSymbolTableAppendOutOfBounds(t *testing.T) { 134 | text := `$ion_symbol_table:: 135 | { 136 | symbols:[ "foo" ] 137 | } 138 | $10 139 | $ion_symbol_table:: 140 | { 141 | imports: $ion_symbol_table, 142 | symbols:[ "foo" ] 143 | } 144 | $11 145 | $12` 146 | 147 | r := NewReaderString(text) 148 | assert.True(t, r.Next()) 149 | assert.True(t, r.Next()) 150 | st := r.SymbolTable() 151 | systemMaxID := getSystemMaxID(st) 152 | 153 | checkSymbol(t, "foo", systemMaxID+1, st) 154 | checkSymbol(t, "foo", systemMaxID+2, st) 155 | checkUnknownSymbolID(t, systemMaxID+3, st) 156 | } 157 | 158 | func getSystemMaxID(st SymbolTable) uint64 { 159 | imports := st.Imports() 160 | systemTable := imports[0] 161 | return systemTable.MaxID() 162 | } 163 | 164 | func checkSymbol(t *testing.T, eval string, SID uint64, st SymbolTable) { 165 | val, ok := st.FindByID(SID) 166 | assert.True(t, ok, "Failed on checking symbol. Symbol table could not find symbol given the ID: %v", SID) 167 | assert.Equal(t, eval, val, "Failed on checking symbol. Symbol table returned the symbol name: %v, did not match the expected name: %v", val, eval) 168 | } 169 | 170 | func checkUnknownSymbolText(t *testing.T, val string, st SymbolTable) { 171 | symbolToken := st.Find(val) 172 | assert.Nil(t, symbolToken, "Failed on checking unknown symbol. Symbol table found symbol given the name: %v", val) 173 | _, ok := st.FindByName(val) 174 | assert.False(t, ok, "Failed on checking unknown symbol. Symbol table found symbol given the name: %v", val) 175 | } 176 | 177 | func checkUnknownSymbolID(t *testing.T, val uint64, st SymbolTable) { 178 | _, ok := st.FindByID(val) 179 | assert.False(t, ok, "Failed on checking unknown symbol. Symbol table found symbol given the SID: %v", val) 180 | } 181 | -------------------------------------------------------------------------------- /ion/skipper_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestSkipNumber(t *testing.T) { 26 | test, testErr := testSkip(t, (*tokenizer).skipNumber) 27 | 28 | test("", -1) 29 | test("0", -1) 30 | test("-1234567890,", ',') 31 | test("1.2 ", ' ') 32 | test("1d45\n", '\n') 33 | test("1.4e-12//", '/') 34 | 35 | testErr("1.2d3d", "ion: unexpected rune 'd' (offset 5)") 36 | } 37 | 38 | func TestSkipBinary(t *testing.T) { 39 | test, testErr := testSkip(t, (*tokenizer).skipBinary) 40 | 41 | test("0b0", -1) 42 | test("-0b10 ", ' ') 43 | test("0b010101,", ',') 44 | 45 | testErr("0b2", "ion: unexpected rune '2' (offset 2)") 46 | } 47 | 48 | func TestSkipHex(t *testing.T) { 49 | test, testErr := testSkip(t, (*tokenizer).skipHex) 50 | 51 | test("0x0", -1) 52 | test("-0x0F ", ' ') 53 | test("0x1234567890abcdefABCDEF,", ',') 54 | 55 | testErr("0x0G", "ion: unexpected rune 'G' (offset 3)") 56 | } 57 | 58 | func TestSkipTimestamp(t *testing.T) { 59 | test, testErr := testSkip(t, (*tokenizer).skipTimestamp) 60 | 61 | test("2001T", -1) 62 | test("2001-01T,", ',') 63 | test("2001-01-02}", '}') 64 | test("2001-01-02T ", ' ') 65 | test("2001-01-02T+00:00\t", '\t') 66 | test("2001-01-02T-00:00\n", '\n') 67 | test("2001-01-02T03:04+00:00 ", ' ') 68 | test("2001-01-02T03:04-00:00 ", ' ') 69 | test("2001-01-02T03:04Z ", ' ') 70 | test("2001-01-02T03:04z ", ' ') 71 | test("2001-01-02T03:04:05Z ", ' ') 72 | test("2001-01-02T03:04:05+00:00 ", ' ') 73 | test("2001-01-02T03:04:05.666Z ", ' ') 74 | test("2001-01-02T03:04:05.666666z ", ' ') 75 | 76 | testErr("", "ion: unexpected end of input (offset 0)") 77 | testErr("2001", "ion: unexpected end of input (offset 4)") 78 | testErr("2001z", "ion: unexpected rune 'z' (offset 4)") 79 | testErr("20011", "ion: unexpected rune '1' (offset 4)") 80 | testErr("2001-0", "ion: unexpected end of input (offset 6)") 81 | testErr("2001-01", "ion: unexpected end of input (offset 7)") 82 | testErr("2001-01-02Tz", "ion: unexpected rune 'z' (offset 11)") 83 | testErr("2001-01-02T03", "ion: unexpected end of input (offset 13)") 84 | testErr("2001-01-02T03z", "ion: unexpected rune 'z' (offset 13)") 85 | testErr("2001-01-02T03:04x ", "ion: unexpected rune 'x' (offset 16)") 86 | testErr("2001-01-02T03:04:05x ", "ion: unexpected rune 'x' (offset 19)") 87 | } 88 | 89 | func TestSkipSymbol(t *testing.T) { 90 | test, _ := testSkip(t, (*tokenizer).skipSymbol) 91 | 92 | test("f", -1) 93 | test("foo:", ':') 94 | test("foo,", ',') 95 | test("foo ", ' ') 96 | test("foo\n", '\n') 97 | test("foo]", ']') 98 | test("foo}", '}') 99 | test("foo)", ')') 100 | test("foo\\n", '\\') 101 | } 102 | 103 | func TestSkipSymbolQuoted(t *testing.T) { 104 | test, testErr := testSkip(t, (*tokenizer).skipSymbolQuoted) 105 | 106 | test("'", -1) 107 | test("foo',", ',') 108 | test("foo\\'bar':", ':') 109 | test("foo\\\nbar',", ',') 110 | 111 | testErr("foo", "ion: unexpected end of input (offset 3)") 112 | testErr("foo\n", "ion: unexpected rune '\\n' (offset 3)") 113 | } 114 | 115 | func TestSkipSymbolOperator(t *testing.T) { 116 | test, _ := testSkip(t, (*tokenizer).skipSymbolOperator) 117 | 118 | test("+", -1) 119 | test("++", -1) 120 | test("+= ", ' ') 121 | test("%b", 'b') 122 | } 123 | 124 | func TestSkipString(t *testing.T) { 125 | test, testErr := testSkip(t, (*tokenizer).skipString) 126 | 127 | test("\"", -1) 128 | test("\",", ',') 129 | test("foo\\\"bar\"], \"\"", ']') 130 | test("foo\\\nbar\" \t\t\t", ' ') 131 | 132 | testErr("foobar", "ion: unexpected end of input (offset 6)") 133 | testErr("foobar\n", "ion: unexpected rune '\\n' (offset 6)") 134 | } 135 | 136 | func TestSkipLongString(t *testing.T) { 137 | test, _ := testSkip(t, (*tokenizer).skipLongString) 138 | 139 | test("'''", -1) 140 | test("''',", ',') 141 | test("abc''',", ',') 142 | test("abc''' }", '}') 143 | test("abc''' /*more*/ '''def'''\t//more\r\n]", ']') 144 | } 145 | 146 | func TestSkipBlob(t *testing.T) { 147 | test, testErr := testSkip(t, (*tokenizer).skipBlob) 148 | 149 | test("}}", -1) 150 | test("oogboog}},{{}}", ',') 151 | test("'''not encoded'''}}\n", '\n') 152 | 153 | testErr("", "ion: unexpected end of input (offset 1)") 154 | testErr("oogboog", "ion: unexpected end of input (offset 7)") 155 | testErr("oogboog}", "ion: unexpected end of input (offset 8)") 156 | testErr("oog}{boog", "ion: unexpected rune '{' (offset 4)") 157 | } 158 | 159 | func TestSkipList(t *testing.T) { 160 | test, testErr := testSkip(t, (*tokenizer).skipList) 161 | 162 | test("]", -1) 163 | test("[]],", ',') 164 | test("[123, \"]\", ']']] ", ' ') 165 | 166 | testErr("abc, def, ", "ion: unexpected end of input (offset 10)") 167 | } 168 | 169 | type skipFunc func(*tokenizer) (int, error) 170 | type skipTestFunc func(string, int) 171 | type skipTestErrFunc func(string, string) 172 | 173 | func testSkip(t *testing.T, f skipFunc) (skipTestFunc, skipTestErrFunc) { 174 | test := func(str string, ec int) { 175 | t.Run(str, func(t *testing.T) { 176 | tok := tokenizeString(str) 177 | c, err := f(tok) 178 | require.NoError(t, err) 179 | assert.Equal(t, ec, c) 180 | }) 181 | } 182 | testErr := func(str string, e string) { 183 | t.Run(str, func(t *testing.T) { 184 | tok := tokenizeString(str) 185 | _, err := f(tok) 186 | require.Error(t, err) 187 | assert.Equal(t, e, err.Error()) 188 | }) 189 | } 190 | return test, testErr 191 | } 192 | -------------------------------------------------------------------------------- /ion/symboltable_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "fmt" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestSharedSymbolTable(t *testing.T) { 27 | st := NewSharedSymbolTable("test", 2, []string{ 28 | "abc", 29 | "def", 30 | "foo'bar", 31 | "null", 32 | "def", 33 | "ghi", 34 | }) 35 | 36 | assert.Equal(t, "test", st.Name()) 37 | assert.Equal(t, 2, st.Version()) 38 | assert.Equal(t, 6, int(st.MaxID())) 39 | 40 | testFindByName(t, st, "def", 2) 41 | testFindByName(t, st, "null", 4) 42 | testFindByName(t, st, "bogus", 0) 43 | 44 | testFindByID(t, st, 0, "") 45 | testFindByID(t, st, 2, "def") 46 | testFindByID(t, st, 4, "null") 47 | testFindByID(t, st, 7, "") 48 | 49 | testFindSymbolToken(t, st, "def", NewSymbolTokenFromString("def")) 50 | testFindSymbolToken(t, st, "foo'bar", NewSymbolTokenFromString("foo'bar")) 51 | 52 | testString(t, st, `$ion_shared_symbol_table::{name:"test",version:2,symbols:["abc","def","foo'bar","null","def","ghi"]}`) 53 | } 54 | 55 | func TestLocalSymbolTable(t *testing.T) { 56 | st := NewLocalSymbolTable(nil, []string{"foo", "bar"}) 57 | 58 | assert.Equal(t, 11, int(st.MaxID())) 59 | 60 | testFindByName(t, st, "$ion", 1) 61 | testFindByName(t, st, "foo", 10) 62 | testFindByName(t, st, "bar", 11) 63 | testFindByName(t, st, "bogus", 0) 64 | 65 | testFindByID(t, st, 0, "") 66 | testFindByID(t, st, 1, "$ion") 67 | testFindByID(t, st, 10, "foo") 68 | testFindByID(t, st, 11, "bar") 69 | testFindByID(t, st, 12, "") 70 | 71 | testFindSymbolToken(t, st, "foo", NewSymbolTokenFromString("foo")) 72 | testFindSymbolToken(t, st, "bar", NewSymbolTokenFromString("bar")) 73 | testFindSymbolToken(t, st, "$ion", NewSymbolTokenFromString("$ion")) 74 | 75 | testString(t, st, `$ion_symbol_table::{symbols:["foo","bar"]}`) 76 | } 77 | 78 | func TestLocalSymbolTableWithImports(t *testing.T) { 79 | shared := NewSharedSymbolTable("shared", 1, []string{ 80 | "foo", 81 | "bar", 82 | }) 83 | imports := []SharedSymbolTable{shared} 84 | 85 | st := NewLocalSymbolTable(imports, []string{ 86 | "foo2", 87 | "bar2", 88 | }) 89 | 90 | assert.Equal(t, 13, int(st.MaxID())) 91 | 92 | testFindByName(t, st, "$ion", 1) 93 | testFindByName(t, st, "$ion_shared_symbol_table", 9) 94 | testFindByName(t, st, "foo", 10) 95 | testFindByName(t, st, "bar", 11) 96 | testFindByName(t, st, "foo2", 12) 97 | testFindByName(t, st, "bar2", 13) 98 | testFindByName(t, st, "bogus", 0) 99 | 100 | testFindByID(t, st, 0, "") 101 | testFindByID(t, st, 1, "$ion") 102 | testFindByID(t, st, 9, "$ion_shared_symbol_table") 103 | testFindByID(t, st, 10, "foo") 104 | testFindByID(t, st, 11, "bar") 105 | testFindByID(t, st, 12, "foo2") 106 | testFindByID(t, st, 13, "bar2") 107 | testFindByID(t, st, 14, "") 108 | 109 | testFindSymbolToken(t, st, "foo", NewSymbolTokenFromString("foo")) 110 | testFindSymbolToken(t, st, "bar", NewSymbolTokenFromString("bar")) 111 | testFindSymbolToken(t, st, "foo2", NewSymbolTokenFromString("foo2")) 112 | testFindSymbolToken(t, st, "bar2", NewSymbolTokenFromString("bar2")) 113 | 114 | testString(t, st, `$ion_symbol_table::{imports:[{name:"shared",version:1,max_id:2}],symbols:["foo2","bar2"]}`) 115 | } 116 | 117 | func TestSymbolTableBuilder(t *testing.T) { 118 | b := NewSymbolTableBuilder() 119 | 120 | id, ok := b.Add("name") 121 | assert.False(t, ok, "Add(name) returned true") 122 | assert.Equal(t, 4, int(id), "Add(name) returned %v", id) 123 | 124 | id, ok = b.Add("foo") 125 | assert.True(t, ok, "Add(foo) returned false") 126 | assert.Equal(t, 10, int(id), "Add(foo) returned %v", id) 127 | 128 | id, ok = b.Add("foo") 129 | assert.False(t, ok, "Second Add(foo) returned true") 130 | assert.Equal(t, 10, int(id), "Second Add(foo) returned %v", id) 131 | 132 | st := b.Build() 133 | assert.Equal(t, 10, int(st.MaxID()), "maxid returned %v", st.MaxID()) 134 | 135 | testFindByName(t, st, "$ion", 1) 136 | testFindByName(t, st, "foo", 10) 137 | testFindByName(t, st, "bogus", 0) 138 | 139 | testFindByID(t, st, 1, "$ion") 140 | testFindByID(t, st, 10, "foo") 141 | testFindByID(t, st, 11, "") 142 | } 143 | 144 | func testFindByName(t *testing.T, st SymbolTable, sym string, expected uint64) { 145 | t.Run("FindByName("+sym+")", func(t *testing.T) { 146 | actual, ok := st.FindByName(sym) 147 | if expected == 0 { 148 | require.False(t, ok) 149 | } else { 150 | require.True(t, ok) 151 | assert.Equal(t, expected, actual) 152 | } 153 | }) 154 | } 155 | 156 | func testFindByID(t *testing.T, st SymbolTable, id uint64, expected string) { 157 | t.Run(fmt.Sprintf("FindByID(%v)", id), func(t *testing.T) { 158 | actual, ok := st.FindByID(id) 159 | if expected == "" { 160 | require.False(t, ok) 161 | } else { 162 | require.True(t, ok) 163 | assert.Equal(t, expected, actual) 164 | } 165 | }) 166 | } 167 | 168 | func testFindSymbolToken(t *testing.T, st SymbolTable, sym string, expected SymbolToken) { 169 | t.Run("Find("+sym+")", func(t *testing.T) { 170 | actual := st.Find(sym) 171 | require.NotNil(t, actual) 172 | 173 | assert.True(t, actual.Equal(&expected), "expected %v, got %v", expected, actual) 174 | }) 175 | } 176 | 177 | func testString(t *testing.T, st SymbolTable, expected string) { 178 | t.Run("String()", func(t *testing.T) { 179 | actual := st.String() 180 | assert.Equal(t, expected, actual) 181 | }) 182 | } 183 | 184 | func newString(value string) *string { 185 | return &value 186 | } 187 | -------------------------------------------------------------------------------- /ion/symboltoken.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "fmt" 20 | "strconv" 21 | ) 22 | 23 | const ( 24 | // SymbolIDUnknown is placeholder for when a symbol token has no symbol ID. 25 | SymbolIDUnknown = -1 26 | ) 27 | 28 | // ImportSource is a reference to a SID within a shared symbol table. 29 | type ImportSource struct { 30 | // The name of the shared symbol table that this token refers to. 31 | Table string 32 | 33 | // The ID of the interned symbol text within the shared SymbolTable. 34 | // This must be greater or equal to 1. 35 | SID int64 36 | } 37 | 38 | func newSource(table string, sid int64) *ImportSource { 39 | value := ImportSource{ 40 | Table: table, 41 | SID: sid, 42 | } 43 | return &value 44 | } 45 | 46 | // Equal figures out if two import sources are equal for each component. 47 | func (is *ImportSource) Equal(o *ImportSource) bool { 48 | return is.Table == o.Table && is.SID == o.SID 49 | } 50 | 51 | // SymbolToken is the representation for annotations, field names, and the textual content of Ion symbol values. 52 | // The `nil` value for SymbolToken is the SID `$0`. 53 | type SymbolToken struct { 54 | // The string text of the token or nil if unknown. 55 | Text *string 56 | // Local symbol ID associated with the token. 57 | LocalSID int64 58 | // The shared symbol table location that this token came from, or nil if undefined. 59 | Source *ImportSource 60 | } 61 | 62 | func (st *SymbolToken) String() string { 63 | text := "nil" 64 | if st.Text != nil { 65 | text = fmt.Sprintf("%q", *st.Text) 66 | } 67 | 68 | source := "nil" 69 | if st.Source != nil { 70 | source = fmt.Sprintf("{%q %d}", st.Source.Table, st.Source.SID) 71 | } 72 | 73 | return fmt.Sprintf("{%s %d %s}", text, st.LocalSID, source) 74 | } 75 | 76 | // Equal figures out if two symbol tokens are equivalent. 77 | func (st *SymbolToken) Equal(o *SymbolToken) bool { 78 | if st.Text == nil && o.Text == nil { 79 | if st.Source == nil && o.Source == nil { 80 | return true 81 | } 82 | if st.Source != nil && o.Source != nil { 83 | return st.Source.Equal(o.Source) 84 | } 85 | return false 86 | } 87 | 88 | if st.Text != nil && o.Text != nil { 89 | return *st.Text == *o.Text 90 | } 91 | return false 92 | } 93 | 94 | // Parses text of the form '$n' for some integer n. 95 | func symbolIdentifier(symbolText string) (int64, bool) { 96 | if len(symbolText) > 1 && symbolText[0] == '$' { 97 | if sid, err := strconv.Atoi(symbolText[1:]); err == nil { 98 | return int64(sid), true 99 | } 100 | } 101 | 102 | return SymbolIDUnknown, false 103 | } 104 | 105 | // NewSymbolTokenFromString returns a Symbol Token with the given text value and undefined SID and Source. 106 | func NewSymbolTokenFromString(text string) SymbolToken { 107 | return SymbolToken{Text: &text, LocalSID: SymbolIDUnknown} 108 | } 109 | 110 | func newSymbolTokenPtrFromString(text string) *SymbolToken { 111 | return &SymbolToken{Text: &text, LocalSID: SymbolIDUnknown} 112 | } 113 | 114 | // NewSymbolTokenBySID will check and return a symbol token if the given id exists in a symbol table, 115 | // otherwise return a new symbol token. 116 | func NewSymbolTokenBySID(symbolTable SymbolTable, sid int64) (SymbolToken, error) { 117 | if sid < 0 || uint64(sid) > symbolTable.MaxID() { 118 | return SymbolToken{}, fmt.Errorf("ion: Symbol token not found for SID '%v' in symbol table %v", sid, symbolTable) 119 | } 120 | 121 | text, ok := symbolTable.FindByID(uint64(sid)) 122 | if !ok { 123 | return SymbolToken{LocalSID: sid}, nil 124 | } 125 | 126 | return SymbolToken{Text: &text, LocalSID: sid}, nil 127 | } 128 | 129 | // NewSymbolToken will check and return a symbol token if it exists in a symbol table, 130 | // otherwise return a new symbol token. 131 | func NewSymbolToken(symbolTable SymbolTable, text string) (SymbolToken, error) { 132 | if symbolTable == nil { 133 | return SymbolToken{}, fmt.Errorf("ion: invalid symbol table") 134 | } 135 | 136 | sid, ok := symbolTable.FindByName(text) 137 | if !ok { 138 | return SymbolToken{Text: &text, LocalSID: SymbolIDUnknown}, nil 139 | } 140 | 141 | return SymbolToken{Text: &text, LocalSID: int64(sid)}, nil 142 | } 143 | 144 | // NewSymbolTokens will check and return a list of symbol tokens if they exists in a symbol table, 145 | // otherwise return a list of new symbol tokens. 146 | func NewSymbolTokens(symbolTable SymbolTable, textVals []string) ([]SymbolToken, error) { 147 | var tokens []SymbolToken 148 | for _, text := range textVals { 149 | token, err := NewSymbolToken(symbolTable, text) 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | tokens = append(tokens, token) 155 | } 156 | 157 | return tokens, nil 158 | } 159 | 160 | func newSymbolToken(symbolTable SymbolTable, text string) (SymbolToken, error) { 161 | if sid, ok := symbolIdentifier(text); ok { 162 | return NewSymbolTokenBySID(symbolTable, sid) 163 | } 164 | return NewSymbolToken(symbolTable, text) 165 | } 166 | -------------------------------------------------------------------------------- /ion/symboltoken_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "fmt" 20 | "testing" 21 | 22 | "github.com/google/go-cmp/cmp" 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | var symbolTokenEqualsTestData = []struct { 27 | text1 *string 28 | sid1 int64 29 | source1 *ImportSource 30 | text2 *string 31 | sid2 int64 32 | source2 *ImportSource 33 | equals bool 34 | }{ 35 | {nil, 123, nil, nil, 123, nil, true}, 36 | {nil, 123, newSource("table", 1), nil, 456, newSource("table", 1), true}, 37 | {newString("text1"), 123, nil, newString("text1"), 123, nil, true}, 38 | {newString("text2"), 123, newSource("table", 1), newString("text2"), 456, newSource("table", 1), true}, 39 | {nil, 123, newSource("table", 1), nil, 123, nil, false}, 40 | {nil, 123, nil, newString("text1"), 456, nil, false}, 41 | {nil, 123, newSource("table", 1), nil, 123, newSource("table2", 1), false}, 42 | {nil, 123, newSource("table", 1), nil, 456, newSource("table", 2), false}, 43 | {newString("text2"), 123, nil, newString("text3"), 123, nil, false}, 44 | } 45 | 46 | func TestSymbolTokenEqualsOperator(t *testing.T) { 47 | for _, testData := range symbolTokenEqualsTestData { 48 | st1 := SymbolToken{Text: testData.text1, LocalSID: testData.sid1, Source: testData.source1} 49 | st2 := SymbolToken{Text: testData.text2, LocalSID: testData.sid2, Source: testData.source2} 50 | 51 | if testData.equals { 52 | assert.True(t, st1.Equal(&st2)) 53 | } else { 54 | assert.False(t, st1.Equal(&st2)) 55 | } 56 | } 57 | } 58 | 59 | // Make sure SymbolToken conforms to Stringer 60 | var _ fmt.Stringer = &SymbolToken{} 61 | 62 | func TestSymbolToken_String(t *testing.T) { 63 | cases := []struct { 64 | desc string 65 | token SymbolToken 66 | expected string 67 | }{ 68 | { 69 | desc: "Text and SID", 70 | token: SymbolToken{ 71 | Text: newString("hello"), 72 | LocalSID: 10, 73 | Source: nil, 74 | }, 75 | expected: `{"hello" 10 nil}`, 76 | }, 77 | { 78 | desc: "nil Text", 79 | token: SymbolToken{ 80 | Text: nil, 81 | LocalSID: 11, 82 | Source: nil, 83 | }, 84 | expected: `{nil 11 nil}`, 85 | }, 86 | { 87 | desc: "Text and SID with Import", 88 | token: SymbolToken{ 89 | Text: newString("world"), 90 | LocalSID: 12, 91 | Source: newSource("foobar", 3), 92 | }, 93 | expected: `{"world" 12 {"foobar" 3}}`, 94 | }, 95 | } 96 | 97 | for _, c := range cases { 98 | t.Run(c.desc, func(t *testing.T) { 99 | diff := cmp.Diff(c.expected, c.token.String()) 100 | assert.Empty(t, diff, "Token String() differs (-expected, +actual):\n%s", diff) 101 | }) 102 | } 103 | } 104 | 105 | func TestNewImportSource(t *testing.T) { 106 | is := newSource("table", 1) 107 | assert.Equal(t, "table", is.Table) 108 | assert.Equal(t, int64(1), is.SID) 109 | } 110 | 111 | var importSourceEqualsTestData = []struct { 112 | text1 string 113 | sid1 int64 114 | text2 string 115 | sid2 int64 116 | equals bool 117 | }{ 118 | {"text1", 123, "text1", 123, true}, 119 | {"text2", 456, "text2", 456, true}, 120 | {"text1", 123, "text1", 456, false}, 121 | {"text2", 456, "text3", 456, false}, 122 | } 123 | 124 | func TestImportSourceEqualsOperator(t *testing.T) { 125 | for _, testData := range importSourceEqualsTestData { 126 | is1 := newSource(testData.text1, testData.sid1) 127 | is2 := newSource(testData.text2, testData.sid2) 128 | 129 | if testData.equals { 130 | assert.True(t, is1.Equal(is2)) 131 | } else { 132 | assert.False(t, is1.Equal(is2)) 133 | } 134 | } 135 | } 136 | 137 | func TestNewSymbolTokenThatAlreadyExistInSymbolTable(t *testing.T) { 138 | expectedSymbolToken := NewSymbolTokenFromString("$ion") 139 | 140 | actualSymbolToken, err := NewSymbolToken(V1SystemSymbolTable, "$ion") 141 | assert.NoError(t, err, "expected NewSymbolToken() to execute without errors") 142 | 143 | assert.True(t, actualSymbolToken.Equal(&expectedSymbolToken), "expected %v, got %v", expectedSymbolToken, actualSymbolToken) 144 | } 145 | 146 | func TestNewSymbolTokenThatDoesNotExistInSymbolTable(t *testing.T) { 147 | expectedSymbolToken := NewSymbolTokenFromString("newToken") 148 | 149 | actualSymbolToken, err := NewSymbolToken(V1SystemSymbolTable, "newToken") 150 | assert.NoError(t, err, "expected NewSymbolToken() to execute without errors") 151 | 152 | assert.True(t, actualSymbolToken.Equal(&expectedSymbolToken), "expected %v, got %v", expectedSymbolToken, actualSymbolToken) 153 | } 154 | 155 | func TestNewSymbolTokensThatAlreadyExistInSymbolTable(t *testing.T) { 156 | expectedSymbolTokens := []SymbolToken{ 157 | NewSymbolTokenFromString("$ion"), 158 | NewSymbolTokenFromString("$ion_1_0")} 159 | 160 | actualSymbolTokens, err := NewSymbolTokens(V1SystemSymbolTable, []string{"$ion", "$ion_1_0"}) 161 | assert.NoError(t, err, "expected NewSymbolTokens() to execute without errors") 162 | 163 | for index, actualSymbolToken := range actualSymbolTokens { 164 | assert.True(t, actualSymbolToken.Equal(&expectedSymbolTokens[index]), "expected %v, got %v", &expectedSymbolTokens[index], actualSymbolToken) 165 | } 166 | } 167 | 168 | func TestNewSymbolTokensThatDoNotExistInSymbolTable(t *testing.T) { 169 | expectedSymbolTokens := []SymbolToken{ 170 | NewSymbolTokenFromString("newToken1"), 171 | NewSymbolTokenFromString("newToken2")} 172 | 173 | actualSymbolTokens, err := NewSymbolTokens(V1SystemSymbolTable, []string{"newToken1", "newToken2"}) 174 | assert.NoError(t, err, "expected NewSymbolTokens() to execute without errors") 175 | 176 | for index, actualSymbolToken := range actualSymbolTokens { 177 | assert.True(t, actualSymbolToken.Equal(&expectedSymbolTokens[index]), "expected %v, got %v", &expectedSymbolTokens[index], actualSymbolToken) 178 | } 179 | } 180 | 181 | func TestSymbolIdentifier(t *testing.T) { 182 | test := func(sym string, expectedSID int64, expectedOK bool) { 183 | t.Run(sym, func(t *testing.T) { 184 | sid, ok := symbolIdentifier(sym) 185 | assert.Equal(t, expectedOK, ok) 186 | 187 | if expectedOK { 188 | assert.Equal(t, expectedSID, sid) 189 | } 190 | }) 191 | } 192 | 193 | test("", SymbolIDUnknown, false) 194 | test("1", SymbolIDUnknown, false) 195 | test("a", SymbolIDUnknown, false) 196 | test("$", SymbolIDUnknown, false) 197 | test("$1", 1, true) 198 | test("$1234567890", 1234567890, true) 199 | test("$a", SymbolIDUnknown, false) 200 | test("$1234a567890", SymbolIDUnknown, false) 201 | } 202 | -------------------------------------------------------------------------------- /ion/textutils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "fmt" 20 | "io" 21 | "math/big" 22 | "strconv" 23 | "strings" 24 | ) 25 | 26 | // Does this symbol need to be quoted in text form? 27 | func symbolNeedsQuoting(sym string) bool { 28 | switch sym { 29 | case "", "null", "true", "false", "nan": 30 | return true 31 | } 32 | 33 | if !isIdentifierStart(int(sym[0])) { 34 | return true 35 | } 36 | 37 | for i := 1; i < len(sym); i++ { 38 | if !isIdentifierPart(int(sym[i])) { 39 | return true 40 | } 41 | } 42 | 43 | return false 44 | } 45 | 46 | // Is this a valid first character for an identifier? 47 | func isIdentifierStart(c int) bool { 48 | if c >= 'a' && c <= 'z' { 49 | return true 50 | } 51 | if c >= 'A' && c <= 'Z' { 52 | return true 53 | } 54 | if c == '_' || c == '$' { 55 | return true 56 | } 57 | return false 58 | } 59 | 60 | // Is this a valid character for later in an identifier? 61 | func isIdentifierPart(c int) bool { 62 | return isIdentifierStart(c) || isDigit(c) 63 | } 64 | 65 | // Is this a valid hex digit? 66 | func isHexDigit(c int) bool { 67 | if isDigit(c) { 68 | return true 69 | } 70 | if c >= 'a' && c <= 'f' { 71 | return true 72 | } 73 | if c >= 'A' && c <= 'F' { 74 | return true 75 | } 76 | return false 77 | } 78 | 79 | // Is this a digit? 80 | func isDigit(c int) bool { 81 | return c >= '0' && c <= '9' 82 | } 83 | 84 | // Is this a valid part of an operator symbol? 85 | func isOperatorChar(c int) bool { 86 | switch c { 87 | case '!', '#', '%', '&', '*', '+', '-', '.', '/', ';', '<', '=', 88 | '>', '?', '@', '^', '`', '|', '~': 89 | return true 90 | default: 91 | return false 92 | } 93 | } 94 | 95 | // Does this character mark the end of a normal (unquoted) value? Does 96 | // *not* check for the start of a comment, because that requires two 97 | // characters. Use tokenizer.isStopChar(c) or check for it yourself. 98 | func isStopChar(c int) bool { 99 | switch c { 100 | case -1, '{', '}', '[', ']', '(', ')', ',', '"', '\'', 101 | ' ', '\t', '\n', '\r': 102 | return true 103 | default: 104 | return false 105 | } 106 | } 107 | 108 | // Is this character whitespace? 109 | func isWhitespace(c int) bool { 110 | switch c { 111 | case ' ', '\t', '\n', '\r': 112 | return true 113 | } 114 | return false 115 | } 116 | 117 | // Formats a float64 in Ion text style. 118 | func formatFloat(val float64) string { 119 | str := strconv.FormatFloat(val, 'e', -1, 64) 120 | 121 | // Ion uses lower case for special values. 122 | switch str { 123 | case "NaN": 124 | return "nan" 125 | case "+Inf": 126 | return "+inf" 127 | case "-Inf": 128 | return "-inf" 129 | } 130 | 131 | idx := strings.Index(str, "e") 132 | if idx < 0 { 133 | // We need to add an 'e' or it will get interpreted as an Ion decimal. 134 | str += "e0" 135 | } else if idx+2 < len(str) && str[idx+2] == '0' { 136 | // FormatFloat returns exponents with a leading ±0 in some cases; strip it. 137 | str = str[:idx+2] + str[idx+3:] 138 | } 139 | 140 | return str 141 | } 142 | 143 | // Write the given symbol out. 144 | func writeSymbol(val interface{}, out io.Writer) error { 145 | token := val.(SymbolToken) 146 | 147 | var text string 148 | if token.Text != nil { 149 | text = *token.Text 150 | 151 | if _, ok := symbolIdentifier(text); ok { 152 | // Wrap text value in single quotes if the symbol's text is a symbol identifier 153 | // (ie. of form $n for some integer n) 154 | // This is done to distinguish from actual symbol table mappings. 155 | text = fmt.Sprintf("'%v'", text) 156 | return writeRawString(text, out) 157 | } 158 | } else if token.LocalSID != SymbolIDUnknown { 159 | text = fmt.Sprintf("$%v", token.LocalSID) 160 | } else { 161 | return fmt.Errorf("ion: invalid symbol token") 162 | } 163 | 164 | return writeSymbolFromString(text, out) 165 | } 166 | 167 | // Write the given symbol out, quoting and encoding if necessary. 168 | func writeSymbolFromString(val interface{}, out io.Writer) error { 169 | sym := val.(string) 170 | 171 | if symbolNeedsQuoting(sym) { 172 | if err := writeRawChar('\'', out); err != nil { 173 | return err 174 | } 175 | if err := writeEscapedSymbol(sym, out); err != nil { 176 | return err 177 | } 178 | return writeRawChar('\'', out) 179 | } 180 | return writeRawString(sym, out) 181 | } 182 | 183 | // Write the given symbol out, escaping any characters that need escaping. 184 | func writeEscapedSymbol(sym string, out io.Writer) error { 185 | for i := 0; i < len(sym); i++ { 186 | c := sym[i] 187 | if c < 32 || c == '\\' || c == '\'' { 188 | if err := writeEscapedChar(c, out); err != nil { 189 | return err 190 | } 191 | } else { 192 | if err := writeRawChar(c, out); err != nil { 193 | return err 194 | } 195 | } 196 | } 197 | return nil 198 | } 199 | 200 | // Write the given string out, escaping any characters that need escaping. 201 | func writeEscapedString(str string, out io.Writer) error { 202 | for i := 0; i < len(str); i++ { 203 | c := str[i] 204 | if c < 32 || c == '\\' || c == '"' { 205 | if err := writeEscapedChar(c, out); err != nil { 206 | return err 207 | } 208 | } else { 209 | if err := writeRawChar(c, out); err != nil { 210 | return err 211 | } 212 | } 213 | } 214 | return nil 215 | } 216 | 217 | // Write out the given character in escaped form. 218 | func writeEscapedChar(c byte, out io.Writer) error { 219 | switch c { 220 | case 0: 221 | return writeRawString("\\0", out) 222 | case '\a': 223 | return writeRawString("\\a", out) 224 | case '\b': 225 | return writeRawString("\\b", out) 226 | case '\t': 227 | return writeRawString("\\t", out) 228 | case '\n': 229 | return writeRawString("\\n", out) 230 | case '\f': 231 | return writeRawString("\\f", out) 232 | case '\r': 233 | return writeRawString("\\r", out) 234 | case '\v': 235 | return writeRawString("\\v", out) 236 | case '\'': 237 | return writeRawString("\\'", out) 238 | case '"': 239 | return writeRawString("\\\"", out) 240 | case '\\': 241 | return writeRawString("\\\\", out) 242 | default: 243 | buf := []byte{'\\', 'x', hexChars[(c>>4)&0xF], hexChars[c&0xF]} 244 | return writeRawChars(buf, out) 245 | } 246 | } 247 | 248 | // Write out the given raw string. 249 | func writeRawString(val interface{}, out io.Writer) error { 250 | s := val.(string) 251 | 252 | _, err := out.Write([]byte(s)) 253 | return err 254 | } 255 | 256 | // Write out the given raw character sequence. 257 | func writeRawChars(cs []byte, out io.Writer) error { 258 | _, err := out.Write(cs) 259 | return err 260 | } 261 | 262 | // Write out the given raw character. 263 | func writeRawChar(c byte, out io.Writer) error { 264 | _, err := out.Write([]byte{c}) 265 | return err 266 | } 267 | 268 | func parseFloat(str string) (float64, error) { 269 | val, err := strconv.ParseFloat(str, 64) 270 | if err != nil { 271 | if ne, ok := err.(*strconv.NumError); ok { 272 | if ne.Err == strconv.ErrRange { 273 | // Ignore me, val will be +-inf which is fine. 274 | return val, nil 275 | } 276 | } 277 | } 278 | return val, err 279 | } 280 | 281 | func parseDecimal(str string) (*Decimal, error) { 282 | return ParseDecimal(str) 283 | } 284 | 285 | func parseInt(str string, radix int) (interface{}, error) { 286 | digits := str 287 | 288 | switch radix { 289 | case 10: 290 | // All set. 291 | 292 | case 2, 16: 293 | neg := false 294 | if digits[0] == '-' { 295 | neg = true 296 | digits = digits[1:] 297 | } 298 | 299 | // Skip over the '0x' prefix. 300 | digits = digits[2:] 301 | if neg { 302 | digits = "-" + digits 303 | } 304 | 305 | default: 306 | panic("unsupported radix") 307 | } 308 | 309 | i, err := strconv.ParseInt(digits, radix, 64) 310 | if err == nil { 311 | return i, nil 312 | } 313 | if err.(*strconv.NumError).Err != strconv.ErrRange { 314 | return nil, err 315 | } 316 | 317 | bi, ok := (&big.Int{}).SetString(digits, radix) 318 | if !ok { 319 | return nil, &strconv.NumError{ 320 | Func: "ParseInt", 321 | Num: str, 322 | Err: strconv.ErrSyntax, 323 | } 324 | } 325 | 326 | return bi, nil 327 | } 328 | 329 | func parseTimestamp(val string) (Timestamp, error) { 330 | return ParseTimestamp(val) 331 | } 332 | -------------------------------------------------------------------------------- /ion/textutils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "strings" 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestParseTimestamp(t *testing.T) { 28 | test := func(str string, eval string, expectedPrecision TimestampPrecision, expectedKind TimezoneKind, expectedFractionSeconds uint8) { 29 | t.Run(str, func(t *testing.T) { 30 | val, err := parseTimestamp(str) 31 | require.NoError(t, err) 32 | 33 | et, err := time.Parse(time.RFC3339Nano, eval) 34 | require.NoError(t, err) 35 | 36 | expectedTimestamp := NewTimestampWithFractionalSeconds(et, expectedPrecision, expectedKind, expectedFractionSeconds) 37 | 38 | assert.True(t, val.Equal(expectedTimestamp), "expected %v, got %v", expectedTimestamp, val) 39 | }) 40 | } 41 | 42 | test("1234T", "1234-01-01T00:00:00Z", TimestampPrecisionYear, TimezoneUnspecified, 0) 43 | test("1234-05T", "1234-05-01T00:00:00Z", TimestampPrecisionMonth, TimezoneUnspecified, 0) 44 | test("1234-05-06", "1234-05-06T00:00:00Z", TimestampPrecisionDay, TimezoneUnspecified, 0) 45 | test("1234-05-06T", "1234-05-06T00:00:00Z", TimestampPrecisionDay, TimezoneUnspecified, 0) 46 | test("1234-05-06T07:08Z", "1234-05-06T07:08:00Z", TimestampPrecisionMinute, TimezoneUTC, 0) 47 | test("1234-05-06T07:08:09Z", "1234-05-06T07:08:09Z", TimestampPrecisionSecond, TimezoneUTC, 0) 48 | test("1234-05-06T07:08:09.100Z", "1234-05-06T07:08:09.100Z", TimestampPrecisionNanosecond, TimezoneUTC, 3) 49 | test("1234-05-06T07:08:09.100100Z", "1234-05-06T07:08:09.100100Z", TimestampPrecisionNanosecond, TimezoneUTC, 6) 50 | 51 | // Test rounding of >=9 fractional seconds. 52 | test("1234-05-06T07:08:09.000100100Z", "1234-05-06T07:08:09.000100100Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 53 | test("1234-05-06T07:08:09.100100100Z", "1234-05-06T07:08:09.100100100Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 54 | test("1234-05-06T07:08:09.00010010044Z", "1234-05-06T07:08:09.000100100Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 55 | test("1234-05-06T07:08:09.00010010055Z", "1234-05-06T07:08:09.000100101Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 56 | test("1234-05-06T07:08:09.00010010099Z", "1234-05-06T07:08:09.000100101Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 57 | test("1234-05-06T07:08:09.99999999999Z", "1234-05-06T07:08:10.000000000Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 58 | test("1234-12-31T23:59:59.99999999999Z", "1235-01-01T00:00:00.000000000Z", TimestampPrecisionNanosecond, TimezoneUTC, 9) 59 | test("1234-05-06T07:08:09.000100100+09:10", "1234-05-06T07:08:09.000100100+09:10", TimestampPrecisionNanosecond, TimezoneLocal, 9) 60 | test("1234-05-06T07:08:09.100100100-10:11", "1234-05-06T07:08:09.100100100-10:11", TimestampPrecisionNanosecond, TimezoneLocal, 9) 61 | test("1234-05-06T07:08:09.00010010044+09:10", "1234-05-06T07:08:09.000100100+09:10", TimestampPrecisionNanosecond, TimezoneLocal, 9) 62 | test("1234-05-06T07:08:09.00010010055-10:11", "1234-05-06T07:08:09.000100101-10:11", TimestampPrecisionNanosecond, TimezoneLocal, 9) 63 | test("1234-05-06T07:08:09.00010010099+09:10", "1234-05-06T07:08:09.000100101+09:10", TimestampPrecisionNanosecond, TimezoneLocal, 9) 64 | test("1234-05-06T07:08:09.99999999999-10:11", "1234-05-06T07:08:10.000000000-10:11", TimestampPrecisionNanosecond, TimezoneLocal, 9) 65 | test("1234-12-31T23:59:59.99999999999+09:10", "1235-01-01T00:00:00.000000000+09:10", TimestampPrecisionNanosecond, TimezoneLocal, 9) 66 | 67 | test("1234-05-06T07:08+09:10", "1234-05-06T07:08:00+09:10", TimestampPrecisionMinute, TimezoneLocal, 0) 68 | test("1234-05-06T07:08:09-10:11", "1234-05-06T07:08:09-10:11", TimestampPrecisionSecond, TimezoneLocal, 0) 69 | } 70 | 71 | func TestWriteSymbol(t *testing.T) { 72 | test := func(sym, expected string) { 73 | t.Run(expected, func(t *testing.T) { 74 | buf := strings.Builder{} 75 | require.NoError(t, writeSymbolFromString(sym, &buf)) 76 | actual := buf.String() 77 | assert.Equal(t, expected, actual) 78 | }) 79 | } 80 | 81 | test("", "''") 82 | test("null", "'null'") 83 | test("null.null", "'null.null'") 84 | 85 | test("basic", "basic") 86 | test("_basic_", "_basic_") 87 | test("$basic$", "$basic$") 88 | test("$123", "$123") 89 | 90 | test("123", "'123'") 91 | test("abc'def", "'abc\\'def'") 92 | test("abc\"def", "'abc\"def'") 93 | } 94 | 95 | func TestSymbolNeedsQuoting(t *testing.T) { 96 | test := func(sym string, expected bool) { 97 | t.Run(sym, func(t *testing.T) { 98 | actual := symbolNeedsQuoting(sym) 99 | assert.Equal(t, expected, actual) 100 | }) 101 | } 102 | 103 | test("", true) 104 | test("null", true) 105 | test("true", true) 106 | test("false", true) 107 | test("nan", true) 108 | 109 | test("basic", false) 110 | test("_basic_", false) 111 | test("basic$123", false) 112 | test("$", false) 113 | test("$basic", false) 114 | test("$123", false) 115 | 116 | test("123", true) 117 | test("abc.def", true) 118 | test("abc,def", true) 119 | test("abc:def", true) 120 | test("abc{def", true) 121 | test("abc}def", true) 122 | test("abc[def", true) 123 | test("abc]def", true) 124 | test("abc'def", true) 125 | test("abc\"def", true) 126 | } 127 | 128 | func TestWriteEscapedSymbol(t *testing.T) { 129 | test := func(sym, expected string) { 130 | t.Run(expected, func(t *testing.T) { 131 | buf := strings.Builder{} 132 | require.NoError(t, writeEscapedSymbol(sym, &buf)) 133 | actual := buf.String() 134 | assert.Equal(t, expected, actual) 135 | }) 136 | } 137 | 138 | test("basic", "basic") 139 | test("\"basic\"", "\"basic\"") 140 | test("o'clock", "o\\'clock") 141 | test("c:\\", "c:\\\\") 142 | } 143 | 144 | func TestWriteEscapedChar(t *testing.T) { 145 | test := func(c byte, expected string) { 146 | t.Run(expected, func(t *testing.T) { 147 | buf := strings.Builder{} 148 | require.NoError(t, writeEscapedChar(c, &buf)) 149 | actual := buf.String() 150 | assert.Equal(t, expected, actual) 151 | }) 152 | } 153 | 154 | test(0, "\\0") 155 | test('\n', "\\n") 156 | test(1, "\\x01") 157 | test('\xFF', "\\xFF") 158 | } 159 | -------------------------------------------------------------------------------- /ion/type.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import "fmt" 19 | 20 | // A Type represents the type of an Ion Value. 21 | type Type uint8 22 | 23 | const ( 24 | // NoType is returned by a Reader that is not currently pointing at a value. 25 | NoType Type = iota 26 | 27 | // NullType is the type of the (unqualified) Ion null value. 28 | NullType 29 | 30 | // BoolType is the type of an Ion boolean, true or false. 31 | BoolType 32 | 33 | // IntType is the type of a signed Ion integer of arbitrary size. 34 | IntType 35 | 36 | // FloatType is the type of a fixed-precision Ion floating-point value. 37 | FloatType 38 | 39 | // DecimalType is the type of an arbitrary-precision Ion decimal value. 40 | DecimalType 41 | 42 | // TimestampType is the type of an arbitrary-precision Ion timestamp. 43 | TimestampType 44 | 45 | // SymbolType is the type of an Ion symbol, mapped to an integer ID by a SymbolTable 46 | // to (potentially) save space. 47 | SymbolType 48 | 49 | // StringType is the type of a non-symbol Unicode string, represented directly. 50 | StringType 51 | 52 | // ClobType is the type of a character large object. Like a BlobType, it stores an 53 | // arbitrary sequence of bytes, but it represents them in text form as an escaped-ASCII 54 | // string rather than a base64-encoded string. 55 | ClobType 56 | 57 | // BlobType is the type of a binary large object; a sequence of arbitrary bytes. 58 | BlobType 59 | 60 | // ListType is the type of a list, recursively containing zero or more Ion values. 61 | ListType 62 | 63 | // SexpType is the type of an s-expression. Like a ListType, it contains a sequence 64 | // of zero or more Ion values, but with a lisp-like syntax when encoded as text. 65 | SexpType 66 | 67 | // StructType is the type of a structure, recursively containing a sequence of named 68 | // (by an Ion symbol) Ion values. 69 | StructType 70 | ) 71 | 72 | // String implements fmt.Stringer for Type. 73 | func (t Type) String() string { 74 | switch t { 75 | case NoType: 76 | return "" 77 | case NullType: 78 | return "null" 79 | case BoolType: 80 | return "bool" 81 | case IntType: 82 | return "int" 83 | case FloatType: 84 | return "float" 85 | case DecimalType: 86 | return "decimal" 87 | case TimestampType: 88 | return "timestamp" 89 | case StringType: 90 | return "string" 91 | case SymbolType: 92 | return "symbol" 93 | case BlobType: 94 | return "blob" 95 | case ClobType: 96 | return "clob" 97 | case StructType: 98 | return "struct" 99 | case ListType: 100 | return "list" 101 | case SexpType: 102 | return "sexp" 103 | default: 104 | return fmt.Sprintf("", uint8(t)) 105 | } 106 | } 107 | 108 | // IsScalar determines if the type is a scalar type 109 | func IsScalar(t Type) bool { 110 | return NullType <= t && t <= BlobType 111 | } 112 | 113 | // IsContainer determines if the type is a container type 114 | func IsContainer(t Type) bool { 115 | return ListType <= t && t <= StructType 116 | } 117 | 118 | // IntSize represents the size of an integer. 119 | type IntSize uint8 120 | 121 | const ( 122 | // NullInt is the size of null.int and other things that aren't actually ints. 123 | NullInt IntSize = iota 124 | // Int32 is the size of an Ion integer that can be losslessly stored in an int32. 125 | Int32 126 | // Int64 is the size of an Ion integer that can be losslessly stored in an int64. 127 | Int64 128 | // BigInt is the size of an Ion integer that can only be losslessly stored in a big.Int. 129 | BigInt 130 | ) 131 | 132 | // String implements fmt.Stringer for IntSize. 133 | func (i IntSize) String() string { 134 | switch i { 135 | case NullInt: 136 | return "null.int" 137 | case Int32: 138 | return "int32" 139 | case Int64: 140 | return "int64" 141 | case BigInt: 142 | return "big.Int" 143 | default: 144 | return fmt.Sprintf("", uint8(i)) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /ion/type_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestTypeToString(t *testing.T) { 25 | for i := NoType; i <= StructType+1; i++ { 26 | assert.NotEmpty(t, i.String(), "expected a non-empty string for type %v", uint8(i)) 27 | } 28 | } 29 | 30 | func TestIntSizeToString(t *testing.T) { 31 | for i := NullInt; i <= BigInt+1; i++ { 32 | assert.NotEmpty(t, i.String(), "expected a non-empty string for type %v", uint8(i)) 33 | } 34 | } 35 | 36 | func TestIsScalar(t *testing.T) { 37 | scalarTypes := []Type{NullType, BoolType, IntType, FloatType, DecimalType, 38 | TimestampType, SymbolType, StringType, ClobType, BlobType} 39 | 40 | for _, ionType := range scalarTypes { 41 | assert.True(t, IsScalar(ionType)) 42 | } 43 | 44 | nonScalarTypes := []Type{NoType, ListType, SexpType, StructType} 45 | 46 | for _, ionType := range nonScalarTypes { 47 | assert.False(t, IsScalar(ionType)) 48 | } 49 | } 50 | 51 | func TestIsContainer(t *testing.T) { 52 | containerTypes := []Type{ListType, SexpType, StructType} 53 | 54 | for _, ionType := range containerTypes { 55 | assert.True(t, IsContainer(ionType)) 56 | } 57 | 58 | nonContainerTypes := []Type{NoType, NullType, BoolType, IntType, FloatType, DecimalType, 59 | TimestampType, SymbolType, StringType, ClobType, BlobType} 60 | 61 | for _, ionType := range nonContainerTypes { 62 | assert.False(t, IsContainer(ionType)) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /ion/writer.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ion 17 | 18 | import ( 19 | "io" 20 | "math/big" 21 | ) 22 | 23 | // A Writer writes a stream of Ion values. 24 | // 25 | // The various Write methods write atomic values to the current output stream. The 26 | // Begin methods begin writing a list, sexp, or struct respectively. Subsequent 27 | // calls to Write will write values inside of the container until a matching 28 | // End method is called. 29 | // 30 | // var w Writer 31 | // w.BeginSexp() 32 | // { 33 | // w.WriteInt(1) 34 | // w.WriteSymbolFromString("+") 35 | // w.WriteInt(1) 36 | // } 37 | // w.EndSexp() 38 | // 39 | // When writing values inside a struct, the FieldName method must be called before 40 | // each value to set the value's field name. The Annotation method may likewise 41 | // be called before writing any value to add an annotation to the value. 42 | // 43 | // var w Writer 44 | // w.Annotation("user") 45 | // w.BeginStruct() 46 | // { 47 | // w.FieldName("id") 48 | // w.WriteString("foo") 49 | // w.FieldName("name") 50 | // w.WriteString("bar") 51 | // } 52 | // w.EndStruct() 53 | // 54 | // When you're done writing values, you should call Finish to ensure everything has 55 | // been flushed from in-memory buffers. While individual methods all return an error 56 | // on failure, implementations will remember any errors, no-op subsequent calls, and 57 | // return the previous error. This lets you keep code a bit cleaner by only checking 58 | // the return value of the final method call (generally Finish). 59 | // 60 | // var w Writer 61 | // writeSomeStuff(w) 62 | // if err := w.Finish(); err != nil { 63 | // return err 64 | // } 65 | type Writer interface { 66 | // FieldName sets the field name for the next value written. 67 | FieldName(val SymbolToken) error 68 | 69 | // Annotation adds a single annotation to the next value written. 70 | Annotation(val SymbolToken) error 71 | 72 | // Annotations adds multiple annotations to the next value written. 73 | Annotations(values ...SymbolToken) error 74 | 75 | // WriteNull writes an untyped null value. 76 | WriteNull() error 77 | 78 | // WriteNullType writes a null value with a type qualifier, e.g. null.bool. 79 | WriteNullType(t Type) error 80 | 81 | // WriteBool writes a boolean value. 82 | WriteBool(val bool) error 83 | 84 | // WriteInt writes an integer value. 85 | WriteInt(val int64) error 86 | 87 | // WriteUint writes an unsigned integer value. 88 | WriteUint(val uint64) error 89 | 90 | // WriteBigInt writes a big integer value. 91 | WriteBigInt(val *big.Int) error 92 | 93 | // WriteFloat writes a floating-point value. 94 | WriteFloat(val float64) error 95 | 96 | // WriteDecimal writes an arbitrary-precision decimal value. 97 | WriteDecimal(val *Decimal) error 98 | 99 | // WriteTimestamp writes a timestamp value. 100 | WriteTimestamp(val Timestamp) error 101 | 102 | // WriteSymbol writes a symbol value given a SymbolToken. 103 | WriteSymbol(val SymbolToken) error 104 | 105 | // WriteSymbolFromString writes a symbol value given a string. 106 | WriteSymbolFromString(val string) error 107 | 108 | // WriteString writes a string value. 109 | WriteString(val string) error 110 | 111 | // WriteClob writes a clob value. 112 | WriteClob(val []byte) error 113 | 114 | // WriteBlob writes a blob value. 115 | WriteBlob(val []byte) error 116 | 117 | // BeginList begins writing a list value. 118 | BeginList() error 119 | 120 | // EndList finishes writing a list value. 121 | EndList() error 122 | 123 | // BeginSexp begins writing an s-expression value. 124 | BeginSexp() error 125 | 126 | // EndSexp finishes writing an s-expression value. 127 | EndSexp() error 128 | 129 | // BeginStruct begins writing a struct value. 130 | BeginStruct() error 131 | 132 | // EndStruct finishes writing a struct value. 133 | EndStruct() error 134 | 135 | // Finish finishes writing values and flushes any buffered data. 136 | Finish() error 137 | 138 | // IsInStruct indicates if we are currently writing a struct or not. 139 | IsInStruct() bool 140 | } 141 | 142 | // A writer holds shared stuff for all writers. 143 | type writer struct { 144 | out io.Writer 145 | ctx ctxstack 146 | err error 147 | 148 | fieldName *SymbolToken 149 | annotations []SymbolToken 150 | } 151 | 152 | // FieldName sets the field name symbol for the next value written. 153 | // It may only be called while writing a struct. 154 | func (w *writer) FieldName(val SymbolToken) error { 155 | if w.err != nil { 156 | return w.err 157 | } 158 | if !w.IsInStruct() { 159 | w.err = &UsageError{"Writer.FieldName", "called when not writing a struct"} 160 | return w.err 161 | } 162 | 163 | w.fieldName = &val 164 | return nil 165 | } 166 | 167 | // Annotation adds an annotation to the next value written. 168 | func (w *writer) Annotation(val SymbolToken) error { 169 | if w.err != nil { 170 | return w.err 171 | } 172 | 173 | w.annotations = append(w.annotations, val) 174 | 175 | return nil 176 | } 177 | 178 | // Annotations adds one or more annotations to the next value written. 179 | func (w *writer) Annotations(values ...SymbolToken) error { 180 | if w.err != nil { 181 | return w.err 182 | } 183 | 184 | w.annotations = append(w.annotations, values...) 185 | 186 | return nil 187 | } 188 | 189 | // IsInStruct returns true if we're currently writing a struct. 190 | func (w *writer) IsInStruct() bool { 191 | return w.ctx.peek() == ctxInStruct 192 | } 193 | 194 | // Clear clears field name and annotations after writing a value. 195 | func (w *writer) clear() { 196 | w.fieldName = nil 197 | w.annotations = nil 198 | } 199 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | export COMMIT=$(git rev-parse --short HEAD 2> /dev/null) 6 | export BUILDTIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") 7 | 8 | export LDFLAGS="\ 9 | -X \"github.com/amazon-ion/ion-go/internal.GitCommit=${COMMIT}\" \ 10 | -X \"github.com/amazon-ion/ion-go/internal.BuildTime=${BUILDTIME}\" \ 11 | ${LDFLAGS:-}" 12 | 13 | go build -o ion-go --ldflags "${LDFLAGS}" ./cmd/ion-go 14 | -------------------------------------------------------------------------------- /scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | export COMMIT=$(git rev-parse --short HEAD 2> /dev/null) 6 | export BUILDTIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") 7 | 8 | export LDFLAGS="\ 9 | -X \"github.com/amazon-ion/ion-go/internal.GitCommit=${COMMIT}\" \ 10 | -X \"github.com/amazon-ion/ion-go/internal.BuildTime=${BUILDTIME}\" \ 11 | ${LDFLAGS:-}" 12 | 13 | go install --ldflags "${LDFLAGS}" ./cmd/ion-go 14 | --------------------------------------------------------------------------------