├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── .gitignore ├── ARCHITECTURE.md ├── LICENSE.md ├── Makefile ├── README.md ├── REPORTING-BUGS.md ├── TESTING.md ├── benchmark_cbe_test.go ├── bugreport ├── bugreport.cte ├── bugreport_test.go └── templates │ ├── incorrectly_allowed.cte │ └── incorrectly_rejected.cte ├── builder ├── builder.go ├── builder_array.go ├── builder_bool.go ├── builder_custom.go ├── builder_edge.go ├── builder_event_rcv.go ├── builder_ignore.go ├── builder_interface.go ├── builder_map.go ├── builder_marker.go ├── builder_node.go ├── builder_numeric.go ├── builder_ptr.go ├── builder_record.go ├── builder_record_type.go ├── builder_rid.go ├── builder_slice.go ├── builder_struct.go ├── builder_test.go ├── builder_time.go ├── builder_top_level.go ├── builder_typed_array.go ├── context.go ├── conversions.go ├── generated-do-not-edit.go ├── reference_filler.go ├── session.go └── testhelpers_test.go ├── cbe ├── cbe_test.go ├── common.go ├── decoder.go ├── decoder_reader.go ├── encoder.go ├── encoder_writer.go └── marshal.go ├── ce ├── api.go ├── arrays.go ├── ce_test.go ├── decoder.go ├── encoder.go ├── events │ └── events.go ├── marshaler.go └── unmarshaler.go ├── codegen ├── .gitignore ├── README.md ├── antlr │ ├── antlr-4.12.0-complete.jar │ └── antlr.go ├── builder │ └── builder.go ├── chars │ └── chars.go ├── common │ └── common.go ├── cte │ ├── .gitignore │ ├── CTELexer.g4 │ ├── CTEParser.g4 │ ├── README.md │ ├── antlr.go │ └── main.go ├── datatypes │ └── datatypes.go ├── go.mod ├── go.sum ├── main.go ├── rules │ └── rules.go ├── test │ ├── .gitignore │ ├── CEEventLexer.g4 │ ├── CEEventParser.g4 │ ├── antlr.go │ ├── events.go │ └── main.go └── tests │ ├── common.go │ ├── generated-do-not-edit.go │ ├── golang_generators.go │ ├── main.go │ ├── templates.go │ └── testfile_generators.go ├── configuration ├── builder.go ├── configuration.go ├── debug.go ├── decoder.go ├── encoder.go ├── iterator.go ├── marshal.go └── rules.go ├── conversions └── conversions.go ├── cte ├── cte_test.go ├── decoder.go ├── encoder.go ├── encoder_array.go ├── encoder_context.go ├── encoder_decorators.go ├── encoder_writer.go ├── escapes.go ├── marshal.go ├── parser.go └── parser │ ├── CTELexer.interp │ ├── CTELexer.tokens │ ├── CTEParser.interp │ ├── CTEParser.tokens │ ├── cte_lexer.go │ ├── cte_parser.go │ ├── cteparser_base_listener.go │ └── cteparser_listener.go ├── custom_build_iter_test.go ├── demonstration_test.go ├── docs.go ├── go.mod ├── go.sum ├── internal ├── arrays │ ├── arrays.go │ ├── arrays_impurego.go │ ├── arrays_purego.go │ └── arrays_test.go ├── chars │ ├── chars.go │ └── generated-do-not-edit.go └── common │ ├── common.go │ ├── consts.go │ ├── go_tags.go │ ├── golang_1_12_pre.go │ └── golang_1_12_up.go ├── iterator ├── context.go ├── iterator.go ├── iterator_root.go ├── iterator_test.go ├── iterators.go ├── session.go └── testhelpers_test.go ├── nullevent └── null_event_receiver.go ├── rules ├── context.go ├── context_array.go ├── generated-do-not-edit.go ├── rules.go ├── rules_array.go ├── rules_containers.go ├── rules_event_rcv.go ├── rules_marker_ref.go ├── rules_other.go ├── rules_test.go └── testhelpers_test.go ├── test ├── conversions.go ├── event.go ├── event_parser │ ├── event_parser.go │ └── parser │ │ ├── CEEventLexer.interp │ │ ├── CEEventLexer.tokens │ │ ├── CEEventParser.interp │ │ ├── CEEventParser.tokens │ │ ├── ceevent_lexer.go │ │ ├── ceevent_parser.go │ │ ├── ceeventparser_base_listener.go │ │ └── ceeventparser_listener.go ├── event_receiver.go ├── events.go ├── generated-do-not-edit.go ├── test_helpers.go ├── test_runner │ ├── common.go │ ├── test_base.go │ ├── test_must_fail.go │ ├── test_must_succeed.go │ ├── test_runner.go │ ├── test_suite.go │ └── test_unit.go └── util.go ├── tests ├── .gitignore ├── README.md ├── snippet_check │ ├── .gitignore │ ├── README.md │ └── main.go ├── suites │ ├── examples │ │ ├── ce-specification-examples.cte │ │ └── website-examples.cte │ ├── general │ │ ├── arrays.cte │ │ ├── comments.cte │ │ ├── containers.cte │ │ ├── cte-complex.cte │ │ ├── cte-spacing.cte │ │ ├── float.cte │ │ ├── integer.cte │ │ ├── other.cte │ │ ├── strings.cte │ │ ├── time.cte │ │ └── version.cte │ ├── generated │ │ ├── array-float16-generated.cte │ │ ├── array-float32-generated.cte │ │ ├── array-float64-generated.cte │ │ ├── array-int16-generated.cte │ │ ├── array-int32-generated.cte │ │ ├── array-int64-generated.cte │ │ ├── array-int8-generated.cte │ │ ├── array-uint16-generated.cte │ │ ├── array-uint32-generated.cte │ │ ├── array-uint64-generated.cte │ │ ├── array-uint8-generated.cte │ │ ├── cte-header-generated.cte │ │ ├── edge-generated.cte │ │ ├── list-generated.cte │ │ ├── map-generated.cte │ │ ├── node-generated.cte │ │ ├── record-generated.cte │ │ ├── rules-bab-generated.cte │ │ ├── rules-baf16-generated.cte │ │ ├── rules-baf32-generated.cte │ │ ├── rules-baf64-generated.cte │ │ ├── rules-bai16-generated.cte │ │ ├── rules-bai32-generated.cte │ │ ├── rules-bai64-generated.cte │ │ ├── rules-bai8-generated.cte │ │ ├── rules-bau-generated.cte │ │ ├── rules-bau16-generated.cte │ │ ├── rules-bau32-generated.cte │ │ ├── rules-bau64-generated.cte │ │ ├── rules-bau8-generated.cte │ │ ├── rules-bcb-generated.cte │ │ ├── rules-bct-generated.cte │ │ ├── rules-bmedia-generated.cte │ │ ├── rules-brid-generated.cte │ │ ├── rules-bs-generated.cte │ │ └── tlo-generated.cte │ ├── issues │ │ └── github-issues.cte │ └── template.cte └── tests_test.go ├── types └── types.go └── version └── version.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Test Case** 14 | 15 | 19 | 20 | ```cte 21 | { 22 | "tests" = [ 23 | { 24 | "name" = "My Bug Report" 25 | "cte" = "\.%%%% 26 | c0 27 | { 28 | "a" = true 29 | } 30 | %%%%" 31 | "cbe" = @u8x[83 00 79 81 61 7d 7b] 32 | "events" = [ 33 | "bd" "v 0" "m" 34 | "s a" "tt" 35 | "e" "ed" 36 | ] 37 | "from" = ["t" "b" "e"] 38 | "to" = ["t" "b" "e"] 39 | } 40 | ] 41 | } 42 | ``` 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.out 2 | *.test 3 | .idea 4 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | License 2 | ======= 3 | 4 | License Type: MIT 5 | 6 | Online Reference: https://opensource.org/licenses/MIT 7 | 8 | 9 | License 10 | ------- 11 | 12 | Copyright 2019 Karl Stenerud 13 | 14 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean test testnormal testpurego test32bit cover benchmark compare lint 2 | 3 | clean: 4 | go clean 5 | rm -f *.out *.test 6 | 7 | test: testnormal testpurego test32bit 8 | 9 | testnormal: 10 | $(info ** Running Tests (normal)) 11 | go test -coverprofile=coverage.out ./... 12 | 13 | testpurego: 14 | $(info ** Running Tests (purego)) 15 | go test -tags purego ./... 16 | 17 | test32bit: 18 | $(info ** Running Tests (32-bit)) 19 | GOARCH=386 go test ./... 20 | 21 | cover: 22 | $(info To see coverage: go tool cover -html=coverage.out) 23 | go test -coverprofile=coverage.out ./... 24 | 25 | benchmark: 26 | $(info To see profile: go tool pprof cpuprofile.out) 27 | go test -run BenchmarkCBE* -bench BenchmarkCBE* -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 28 | 29 | benchmarkCBEMarshal: 30 | $(info To see profile: go tool pprof cpuprofile.out) 31 | go test -run BenchmarkCBEMarshal -bench BenchmarkCBEMarshal -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 32 | 33 | benchmarkCTEMarshal: 34 | $(info To see profile: go tool pprof cpuprofile.out) 35 | go test -run BenchmarkCTEMarshal -bench BenchmarkCTEMarshal -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 36 | 37 | benchmarkJSONMarshal: 38 | $(info To see profile: go tool pprof cpuprofile.out) 39 | go test -run BenchmarkJSONMarshal -bench BenchmarkJSONMarshal -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 40 | 41 | benchmarkCBEUnmarshal: 42 | $(info To see profile: go tool pprof cpuprofile.out) 43 | go test -run BenchmarkCBEUnmarshalNoRules -bench BenchmarkCBEUnmarshalNoRules -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 44 | 45 | benchmarkCTEUnmarshal: 46 | $(info To see profile: go tool pprof cpuprofile.out) 47 | go test -run BenchmarkCTEUnmarshalNoRules -bench BenchmarkCTEUnmarshalNoRules -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 48 | 49 | benchmarkCBEUnmarshalRules: 50 | $(info To see profile: go tool pprof cpuprofile.out) 51 | go test -run BenchmarkCBEUnmarshalRules -bench BenchmarkCBEUnmarshalRules -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 52 | 53 | benchmarkJSONUnmarshal: 54 | $(info To see profile: go tool pprof cpuprofile.out) 55 | go test -run BenchmarkJSONUnmarshalNoRules -bench BenchmarkJSONUnmarshalNoRules -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 56 | 57 | benchmarkCTEDecode: 58 | $(info To see profile: go tool pprof cpuprofile.out) 59 | go test -run BenchmarkCTEDecode -bench BenchmarkCTEDecode -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 60 | 61 | benchmarkRules: 62 | $(info To see profile: go tool pprof cpuprofile.out) 63 | go test -run BenchmarkRules -bench BenchmarkRules -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 64 | 65 | benchmarkBuilder: 66 | $(info To see profile: go tool pprof cpuprofile.out) 67 | go test -run BenchmarkBuilder -bench BenchmarkBuilder -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 68 | 69 | benchmarkIterator: 70 | $(info To see profile: go tool pprof cpuprofile.out) 71 | go test -run BenchmarkIterator -bench BenchmarkIterator -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out 72 | 73 | compare: 74 | go test -run Benchmark* -bench Benchmark* 75 | 76 | lint: 77 | golint | grep -v "don't use an underscore in package name" | grep -v "imported but not used" || true 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Concise Encoding Go Library 2 | =========================== 3 | 4 | This is the official reference implementation of [concise encoding](https://github.com/kstenerud/concise-encoding). 5 | 6 | Concise Encoding is the secure data format for a modern world, solving these problems: 7 | 8 | ### Security 9 | 10 | State actors, criminal organizations and mercenaries are now actively hacking governments, companies and individuals to steal secrets, plant malware, and hold your data hostage. 11 | 12 | The existing ad-hoc data formats are [too loosely defined to be secure](ce-structure.md#attack-vectors), and can't be fixed because they're not versioned. 13 | 14 | **Concise Encoding is designed for security, and is versioned so that it can be updated to handle new threats**. 15 | 16 | ### Efficiency AND ease-of-use 17 | 18 | We send so much data now that efficiency is critical, but switching to binary means giving up the ease of text formats. 19 | 20 | **Concise Encoding gives you ease and efficiency with its 1:1 compatible [text](cte-specification.md) and [binary](cbe-specification.md) formats. `Edit text. Transmit binary.`** 21 | 22 | ### Types 23 | 24 | Lack of types forces everyone to add extra encoding steps to send their data, which is buggy, reduces compatibility, and opens new security holes. 25 | 26 | **Concise Encoding supports all of the common types natively**. 27 | 28 | 29 | 30 | Prerelease 31 | ---------- 32 | 33 | Please note that this library is still officially in the prerelease stage. The most common functionality is in-place and tested, but there are still some parts to do: 34 | 35 | * Extremely large values aren't tested well 36 | * Everything marked with a `TODO` in the code 37 | 38 | ### Prerelease Document Version 39 | 40 | The official Concise Encoding version during pre-release is `0`. Please use this version in your documents so as not to cause issues with old, potentially incompatible documents once the final release occurs. After release, document version 0 will be retired permanently, and considered invalid (there will be no backwards compatibility with the pre-release). 41 | 42 | The pre-release library will also accept version `1` even though it's technically not correct, but will always emit version `0`. 43 | 44 | 45 | 46 | Example Tool 47 | ------------ 48 | 49 | Click [here](https://github.com/kstenerud/enctool) for a simple data encoding format conversion tool that uses this library. 50 | 51 | 52 | 53 | Library Usage 54 | ------------- 55 | 56 | https://play.golang.org/p/6_pD6CQVLuN (If play.golang.org times out, just retry) 57 | 58 | ```golang 59 | package main 60 | 61 | import ( 62 | "bytes" 63 | "fmt" 64 | "time" 65 | 66 | "github.com/kstenerud/go-concise-encoding/ce" 67 | ) 68 | 69 | func demonstrateCTEMarshal() { 70 | dict := map[interface{}]interface{}{ 71 | "a key": 2.5, 72 | 900: time.Date(2020, time.Month(1), 15, 13, 41, 0, 599000, time.UTC), 73 | } 74 | var buffer bytes.Buffer 75 | err := ce.MarshalCTE(dict, &buffer, configuration.New()) 76 | if err != nil { 77 | fmt.Printf("Error marshaling CTE: %v\n", err) 78 | return 79 | } 80 | fmt.Printf("Marshaled CTE: %v\n", string(buffer.Bytes())) 81 | // Prints: Marshaled CTE: c1 {"a key"=2.5 900=2020-01-15/13:41:00.000599} 82 | } 83 | 84 | func demonstrateCTEUnmarshal() { 85 | data := bytes.NewBuffer([]byte(`c1 {"a key"=2.5 900=2020-01-15/13:41:00.000599}`)) 86 | 87 | value, err := ce.UnmarshalCTE(data, nil, configuration.New()) 88 | if err != nil { 89 | fmt.Printf("Error unmarshaling CTE: %v\n", err) 90 | return 91 | } 92 | fmt.Printf("Unmarshaled CTE: %v\n", value) 93 | // Prints: Unmarshaled CTE: map[a key:2.5 900:2020-01-15/13:41:00.000599] 94 | } 95 | 96 | func demonstrateCBEMarshal() { 97 | dict := map[interface{}]interface{}{ 98 | "a key": 2.5, 99 | 900: time.Date(2020, time.Month(1), 15, 13, 41, 0, 599000, time.UTC), 100 | } 101 | var buffer bytes.Buffer 102 | err := ce.MarshalCBE(dict, &buffer, configuration.New()) 103 | if err != nil { 104 | fmt.Printf("Error marshaling CBE: %v\n", err) 105 | return 106 | } 107 | fmt.Printf("Marshaled CBE: %v\n", buffer.Bytes()) 108 | // Prints: Marshaled CBE: [3 1 121 133 97 32 107 101 121 112 0 0 32 64 106 132 3 155 188 18 0 32 109 47 80 0 123] 109 | } 110 | 111 | func demonstrateCBEUnmarshal() { 112 | data := bytes.NewBuffer([]byte{0x03, 0x01, 0x79, 0x85, 0x61, 0x20, 0x6b, 0x65, 113 | 0x79, 0x70, 0x00, 0x00, 0x20, 0x40, 0x6a, 0x84, 0x03, 0x9b, 0xbc, 0x12, 114 | 0x00, 0x20, 0x6d, 0x2f, 0x50, 0x00, 0x7b}) 115 | 116 | value, err := ce.UnmarshalCBE(data, nil, configuration.New()) 117 | if err != nil { 118 | fmt.Printf("Error unmarshaling CBE: %v\n", err) 119 | return 120 | } 121 | fmt.Printf("Unmarshaled CBE: %v\n", value) 122 | // Prints: Unmarshaled CBE: map[a key:2.5 900:2020-01-15/13:41:00.000599] 123 | } 124 | 125 | func main() { 126 | demonstrateCTEMarshal() 127 | demonstrateCTEUnmarshal() 128 | demonstrateCBEMarshal() 129 | demonstrateCBEUnmarshal() 130 | } 131 | ``` 132 | 133 | 134 | Architecture 135 | ------------ 136 | 137 | The architecture is designed to keep things as simple as possible for this reference implementation. 138 | 139 | See [ARCHITECTURE.md](ARCHITECTURE.md) 140 | -------------------------------------------------------------------------------- /REPORTING-BUGS.md: -------------------------------------------------------------------------------- 1 | Bug Reporting 2 | ============= 3 | 4 | When this library is misbehaving, the quickest and easiest way to describe the problem is to demonstrate it in action. This is where Concise Encoding [unit tests](TESTING.md) shine, as they allow you to provide an input and specify what the output should look like. 5 | 6 | The unit tests have more knobs than are needed for your average bug report, so the [bugreport](bugreport) directory helps you build your bug report repro case with minimal fuss. 7 | 8 | 9 | 10 | Contents 11 | -------- 12 | 13 | * [Quick Example](#quick-example) 14 | * [Templates](#templates) 15 | * [Building your bug report](#building-your-bug-report) 16 | 17 | 18 | 19 | Quick Example 20 | ------------- 21 | 22 | Here's a quick example of what a unit test looks like: 23 | 24 | ```cte 25 | c0 26 | { 27 | "ceTests" = [ 28 | { 29 | "name" = "My Bug Report" 30 | "cte" = "\.%%%% 31 | c0 32 | { 33 | "a" = true 34 | } 35 | %%%%" 36 | "cbe" = @u8x[81 00 79 81 61 7d 7b] 37 | "from" = ["t"] 38 | "to" = ["b"] 39 | } 40 | ] 41 | } 42 | ``` 43 | 44 | This unit test attempts to decode a CTE document (in this case `c0 {"a"=true}`), validates that it is a properly formed document, converts it to CBE, and compares it to the expected CBE document [`81 00 79 81 61 7d 7b`]. 45 | 46 | ### Fields: 47 | 48 | * **name**: The name of this test (call it whatever you want). 49 | * **cte**: A CTE document (encoded using a [verbatim sequence](https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#verbatim-sequence) with the sentinel `%%%%` - change it if your data contains this character sequence). 50 | * **cbe**: A CBE document. 51 | * **from**: Specifies what field(s) to read from as input (in this case "t" meaning text, i.e. the contents of "cte"). 52 | * **to**: Specifies what field(s) to compare to as output (in this case "b" meaning binary, i.e. the contents of "cbe"). 53 | 54 | ### Notes: 55 | 56 | * After trimming leading and trailing whitespace, the expected CTE document is byte compared with the actual output, so it **must** match the pretty printing of the library in order to be considered a match. 57 | * There are also other fields that may be used, and more extensive `"from"` and `"to"` field combinations that are possible. For more information, please read the [unit test documentation](TESTING.md). 58 | 59 | 60 | 61 | Templates 62 | --------- 63 | 64 | Although the [unit test documentation](TESTING.md) describes fully how these unit tests work, the impatient can usually get the job done with a template. 65 | 66 | The [templates](bugreport/templates) directory contains templates for the kinds of bugs most likely to be encountered: 67 | 68 | | Situation | Template | Notes | 69 | | ----------------------------- | -------- | ---------------------------------------------------------------------- | 70 | | CBE is output incorrectly | [here](bugreport/templates/cbe_output_incorrect.cte) | "cbe" contains what the output should look like | 71 | | CTE is output incorrectly | [here](bugreport/templates/cte_output_incorrect.cte) | "cte" contains what the output should look like | 72 | | CBE is decoded incorrectly | [here](bugreport/templates/cbe_decoded_incorrectly.cte) | "cte" or "events" contains what should be generated from the CBE data | 73 | | CTE is decoded incorrectly | [here](bugreport/templates/cte_decoded_incorrectly.cte) | "cbe" or "events" contains what should be generated from the CTE data | 74 | | Document wrongfully rejected | [here](bugreport/templates/doc_wrongfully_rejected.cte) | "cbe" or "cte" contains the document that is being wrongfully rejected | 75 | | Document wrongfully allowed | [here](bugreport/templates/doc_wrongfully_allowed.cte) | "cbe" or "cte" contains the document that is being wrongfully allowed | 76 | 77 | For common issues, overwrite [bugreport.cte](bugreport/bugreport.cte) with one of the [templates](bugreport/templates), fill in your CTE and CBE data (and possibly events), and run the bugreport test to verify failure. 78 | 79 | 80 | 81 | Building your bug report 82 | ------------------------ 83 | 84 | - Update to the latest go-concise-encoding to make sure the problem hasn't already been fixed. 85 | 86 | - Modify [bugreport.cte](bugreport/bugreport.cte) (or copy over one of the [templates](bugreport/templates)) to include the input and output that demonstrates the issue you're encountering. 87 | 88 | - run `go test` inside the [bugreport](bugreport) directory and observe the test failing in the way you expect it to. 89 | 90 | - Include the contents of bugreport.cte in your [bug report](https://github.com/kstenerud/go-concise-encoding/issues). 91 | -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | Concise Encoding Codec Test System 2 | ================================== 3 | 4 | All codec tests are written as portable CTE test files so that the same tests can be run against multiple implementations (different languages, different platforms, etc). 5 | 6 | The tests are kept in the [`suites`](suites) directory, including [template.cte](tests/suites/template.cte) (which contains a full description of the test file structure and what everything's for). 7 | 8 | 9 | Running the Tests 10 | ----------------- 11 | 12 | [`tests/tests_test.go`](tests/tests_test.go) will run all tests in the [`tests/suites`](tests/suites) directory (and all subdirs). They'll be automatically run if you invoke `go test ./...`. 13 | 14 | To run a specific test, import the test runner and give it a test suite to run: 15 | 16 | ```go 17 | package xyz 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/kstenerud/go-concise-encoding/test/test_runner" 23 | ) 24 | 25 | func TestSomething(t *testing.T) { 26 | test_runner.RunCEUnitTests(t, "path/to/something.cte") 27 | } 28 | ``` 29 | 30 | 31 | Running tests on your own implementation 32 | ---------------------------------------- 33 | 34 | You will need to have at least the following types implemented to be able to load a basic test file: 35 | 36 | - String 37 | - Integer 38 | - Boolean 39 | - List 40 | - Map 41 | - Uint8 Array 42 | 43 | The event shorthand system will require a bit more work to implement. See the code in the [test](test) directory for an example. There's also an Antlr grammar ([lexer](codegen/test/CEEventLexer.g4), [parser](codegen/test/CEEventParser.g4)) to parse the event shorthand. 44 | 45 | If you're building a CBE codec and don't yet have a CTE decoder, you could convert the test files from CTE to CBE using [enctool](https://github.com/kstenerud/enctool) and then load them via your CBE decoder. 46 | -------------------------------------------------------------------------------- /bugreport/bugreport.cte: -------------------------------------------------------------------------------- 1 | c0 2 | /** 3 | * Unit test to demonstrate a bug in the codec (for use in bug reports). 4 | * 5 | * For information about the "events" field, see: 6 | * https://github.com/kstenerud/go-concise-encoding/blob/master/TESTING.md 7 | * 8 | * Note: The "cte" field is using a verbatim sequence: 9 | * https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#verbatim-sequence 10 | * Make sure the end-of-sequence identifier is unique enough that your 11 | * CTE document doesn't also contain it! 12 | * 13 | * Instructions: 14 | * 15 | * - Make sure your git repo is up to date. 16 | * - Modify this document to craft a test that demonstrates a bug in the codec. 17 | * - From the "bugreport" dir, run the bugreport test (`go test`) and verify that the test fails. 18 | * - Paste the contents of bugreport.cte into your bug report. 19 | * 20 | * Note: There are unit test templates in the "templates" dir for common classes of bugs. 21 | */ 22 | { 23 | "type" = { 24 | "identifier" = "ce-test" 25 | "version" = 1 26 | } 27 | "ceversion" = 0 28 | "tests" = [ 29 | { 30 | "name" = "My Bug Report" 31 | "mustSucceed" = [ 32 | { 33 | "cte" = "\.%%%% 34 | { 35 | "a" = true 36 | } 37 | %%%%" 38 | "cbe" = @u8x[99 81 61 79 9b] 39 | "events" = [ 40 | "m" 41 | "s=a" "b=true" 42 | "e" 43 | ] 44 | } 45 | ] 46 | "mustFail" = [ 47 | 48 | ] 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /bugreport/bugreport_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package bugreport 22 | 23 | import ( 24 | "testing" 25 | 26 | "github.com/kstenerud/go-concise-encoding/test/test_runner" 27 | ) 28 | 29 | func TestCE(t *testing.T) { 30 | test_runner.RunTests(t, "bugreport.cte") 31 | } 32 | -------------------------------------------------------------------------------- /bugreport/templates/incorrectly_allowed.cte: -------------------------------------------------------------------------------- 1 | c0 2 | /** 3 | * Template: Demonstrate a bug where the codec wrongfully allows an invalid document. 4 | * 5 | * For information about "events", see https://github.com/kstenerud/go-concise-encoding/blob/master/TESTING.md 6 | * 7 | * Note: The "cte" field is using a verbatim sequence: 8 | * https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#verbatim-sequence 9 | * Make sure the end-of-sequence identifier is unique enough that your CTE document doesn't also contain it! 10 | * 11 | * Instructions: 12 | * 13 | * - Make sure your git repo is up to date. 14 | * - Copy this template and overwrite "bugreport.cte" in the "bugreport" dir. 15 | * - Fill in either the "cbe" or "cte" field with the invalid document that is being allowed. 16 | * - Delete whichever of the two fields you're not using. 17 | * - Set the "from" field to either "b" or "t" depending on whether you're inputting CBE or CTE. 18 | * - From the "bugreport" dir, run the bugreport test (`go test`) and verify that the test fails. 19 | * - Paste the contents of bugreport.cte into your bug report. 20 | */ 21 | { 22 | "type" = { 23 | "identifier" = "ce-test" 24 | "version" = 1 25 | } 26 | "ceversion" = 0 27 | "tests" = [ 28 | { 29 | "name" = "CBE is wrongfully allowed" 30 | "mustFail" = [ 31 | { 32 | "cbe" = @u8x[99 81 61 79 9b 9b] 33 | } 34 | ] 35 | } 36 | 37 | { 38 | "name" = "CTE is wrongfully allowed" 39 | "mustFail" = [ 40 | { 41 | "cte" = "\.%%%% 42 | { 43 | "a" = ttrue 44 | } 45 | %%%%" 46 | } 47 | ] 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /bugreport/templates/incorrectly_rejected.cte: -------------------------------------------------------------------------------- 1 | c0 2 | /** 3 | * Template: Demonstrate a bug where the codec incorrectly rejects a document (CBE and/or CTE). 4 | * 5 | * For information about "events", see https://github.com/kstenerud/go-concise-encoding/blob/master/TESTING.md 6 | * 7 | * Note: The "cte" field is using a verbatim sequence: 8 | * https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#verbatim-sequence 9 | * Make sure the end-of-sequence identifier is unique enough that your CTE document doesn't also contain it! 10 | * 11 | * Instructions: 12 | * 13 | * - Make sure your git repo is up to date. 14 | * - Copy this template and overwrite "bugreport.cte" in the "bugreport" dir. 15 | * - Fill in any/all of the "cbe", "cte", and "events" fields to specify what should be converted to what. 16 | * - Set the "lossyCBE", "lossyCTE", lossyEvents" if you need to suppress checking one or more of the output types. 17 | * - From the "bugreport" dir, run the bugreport test (`go test`) and verify that the test fails. 18 | * - Paste the contents of bugreport.cte into your bug report. 19 | */ 20 | { 21 | "type" = { 22 | "identifier" = "ce-test" 23 | "version" = 1 24 | } 25 | "ceversion" = 0 26 | "tests" = [ 27 | { 28 | "name" = "Incorrectly rejected document" 29 | "mustSucceed" = [ 30 | { 31 | "cbe" = @u8x[99 81 61 79 9b] 32 | "cte" = "\.%%%% 33 | { 34 | "a" = true 35 | } 36 | %%%%" 37 | "events" = [ 38 | "m" 39 | "s=a" "b=true" 40 | "e" 41 | ] 42 | } 43 | ] 44 | } 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /builder/builder_bool.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package builder 22 | 23 | import ( 24 | "reflect" 25 | ) 26 | 27 | type boolBuilder struct{} 28 | 29 | var globalBoolBuilder = &boolBuilder{} 30 | 31 | func generateBoolBuilder(ctx *Context) Builder { return globalBoolBuilder } 32 | func (_this *boolBuilder) String() string { return reflect.TypeOf(_this).String() } 33 | 34 | func (_this *boolBuilder) BuildFromBool(ctx *Context, value bool, dst reflect.Value) reflect.Value { 35 | dst.SetBool(value) 36 | return dst 37 | } 38 | 39 | func (_this *boolBuilder) BuildArtificiallyEndContainer(ctx *Context) { 40 | } 41 | -------------------------------------------------------------------------------- /builder/builder_custom.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package builder 22 | 23 | import ( 24 | "reflect" 25 | ) 26 | 27 | type customBuilder struct{} 28 | 29 | var globalCustomBuilder = &customBuilder{} 30 | 31 | func generateCustomBuilder(ctx *Context) Builder { return globalCustomBuilder } 32 | func (_this *customBuilder) String() string { return reflect.TypeOf(_this).String() } 33 | 34 | func (_this *customBuilder) BuildFromCustomBinary(ctx *Context, customType uint64, data []byte, dst reflect.Value) reflect.Value { 35 | ctx.TryBuildFromCustomBinary(_this, customType, data, dst) 36 | 37 | return dst 38 | } 39 | 40 | func (_this *customBuilder) BuildFromCustomText(ctx *Context, customType uint64, data string, dst reflect.Value) reflect.Value { 41 | ctx.TryBuildFromCustomText(_this, customType, data, dst) 42 | return dst 43 | } 44 | 45 | func (_this *customBuilder) BuildArtificiallyEndContainer(ctx *Context) { 46 | } 47 | -------------------------------------------------------------------------------- /builder/builder_record_type.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package builder 22 | 23 | import ( 24 | "math/big" 25 | "reflect" 26 | 27 | "github.com/cockroachdb/apd/v2" 28 | compact_float "github.com/kstenerud/go-compact-float" 29 | compact_time "github.com/kstenerud/go-compact-time" 30 | "github.com/kstenerud/go-concise-encoding/ce/events" 31 | ) 32 | 33 | type recordTypeBuilder struct{} 34 | 35 | var globalRecordTypeBuilder = &recordTypeBuilder{} 36 | 37 | func generateRecordTypeBuilder(ctx *Context) Builder { return globalRecordTypeBuilder } 38 | func (_this *recordTypeBuilder) String() string { return reflect.TypeOf(_this).String() } 39 | 40 | func (_this *recordTypeBuilder) BuildFromBool(ctx *Context, value bool, dst reflect.Value) reflect.Value { 41 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 42 | builder.BuildFromBool(c, value, unusedValue) 43 | }) 44 | return dst 45 | } 46 | func (_this *recordTypeBuilder) BuildFromInt(ctx *Context, value int64, dst reflect.Value) reflect.Value { 47 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 48 | builder.BuildFromInt(c, value, unusedValue) 49 | }) 50 | return dst 51 | } 52 | func (_this *recordTypeBuilder) BuildFromUint(ctx *Context, value uint64, dst reflect.Value) reflect.Value { 53 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 54 | builder.BuildFromUint(c, value, unusedValue) 55 | }) 56 | return dst 57 | } 58 | func (_this *recordTypeBuilder) BuildFromBigInt(ctx *Context, value *big.Int, dst reflect.Value) reflect.Value { 59 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 60 | builder.BuildFromBigInt(c, value, unusedValue) 61 | }) 62 | return dst 63 | } 64 | func (_this *recordTypeBuilder) BuildFromFloat(ctx *Context, value float64, dst reflect.Value) reflect.Value { 65 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 66 | builder.BuildFromFloat(c, value, unusedValue) 67 | }) 68 | return dst 69 | } 70 | func (_this *recordTypeBuilder) BuildFromBigFloat(ctx *Context, value *big.Float, dst reflect.Value) reflect.Value { 71 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 72 | builder.BuildFromBigFloat(c, value, unusedValue) 73 | }) 74 | return dst 75 | } 76 | func (_this *recordTypeBuilder) BuildFromDecimalFloat(ctx *Context, value compact_float.DFloat, dst reflect.Value) reflect.Value { 77 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 78 | builder.BuildFromDecimalFloat(c, value, unusedValue) 79 | }) 80 | return dst 81 | } 82 | func (_this *recordTypeBuilder) BuildFromBigDecimalFloat(ctx *Context, value *apd.Decimal, dst reflect.Value) reflect.Value { 83 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 84 | builder.BuildFromBigDecimalFloat(c, value, unusedValue) 85 | }) 86 | return dst 87 | } 88 | func (_this *recordTypeBuilder) BuildFromUID(ctx *Context, value []byte, dst reflect.Value) reflect.Value { 89 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 90 | builder.BuildFromUID(c, value, unusedValue) 91 | }) 92 | return dst 93 | } 94 | func (_this *recordTypeBuilder) BuildFromArray(ctx *Context, arrayType events.ArrayType, value []byte, dst reflect.Value) reflect.Value { 95 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 96 | builder.BuildFromArray(c, arrayType, value, unusedValue) 97 | }) 98 | return dst 99 | } 100 | func (_this *recordTypeBuilder) BuildFromStringlikeArray(ctx *Context, arrayType events.ArrayType, value string, dst reflect.Value) reflect.Value { 101 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 102 | builder.BuildFromStringlikeArray(c, arrayType, value, unusedValue) 103 | }) 104 | return dst 105 | } 106 | func (_this *recordTypeBuilder) BuildFromTime(ctx *Context, value compact_time.Time, dst reflect.Value) reflect.Value { 107 | ctx.AddRecordTypeKey(func(c *Context, builder Builder) { 108 | builder.BuildFromTime(c, value, unusedValue) 109 | }) 110 | return dst 111 | } 112 | func (_this *recordTypeBuilder) BuildEndContainer(ctx *Context) { 113 | ctx.EndRecordType() 114 | } 115 | 116 | func (_this *recordTypeBuilder) BuildArtificiallyEndContainer(ctx *Context) { 117 | _this.BuildEndContainer(ctx) 118 | } 119 | -------------------------------------------------------------------------------- /builder/builder_rid.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package builder 22 | 23 | import ( 24 | "reflect" 25 | 26 | "github.com/kstenerud/go-concise-encoding/ce/events" 27 | ) 28 | 29 | type ridBuilder struct{} 30 | 31 | var globalRidBuilder = &ridBuilder{} 32 | 33 | func generateUrlBuilder(ctx *Context) Builder { return globalRidBuilder } 34 | func (_this *ridBuilder) String() string { return reflect.TypeOf(_this).String() } 35 | 36 | func (_this *ridBuilder) BuildFromArray(ctx *Context, arrayType events.ArrayType, value []byte, dst reflect.Value) reflect.Value { 37 | switch arrayType { 38 | case events.ArrayTypeResourceID: 39 | setRIDFromString(string(value), dst) 40 | default: 41 | PanicBadEvent(_this, "BuildFromArray(%v)", arrayType) 42 | } 43 | return dst 44 | } 45 | 46 | func (_this *ridBuilder) BuildFromStringlikeArray(ctx *Context, arrayType events.ArrayType, value string, dst reflect.Value) reflect.Value { 47 | switch arrayType { 48 | case events.ArrayTypeResourceID: 49 | setRIDFromString(value, dst) 50 | default: 51 | PanicBadEvent(_this, "BuildFromStringlikeArray(%v)", arrayType) 52 | } 53 | return dst 54 | } 55 | 56 | func (_this *ridBuilder) BuildArtificiallyEndContainer(ctx *Context) { 57 | } 58 | 59 | // ============================================================================ 60 | 61 | type pRidBuilder struct{} 62 | 63 | var globalPRidBuilder = &pRidBuilder{} 64 | 65 | func generatePRidBuilder(ctx *Context) Builder { return globalPRidBuilder } 66 | func (_this *pRidBuilder) String() string { return reflect.TypeOf(_this).String() } 67 | 68 | func (_this *pRidBuilder) BuildFromNull(ctx *Context, dst reflect.Value) reflect.Value { 69 | dst.Set(reflect.Zero(dst.Type())) 70 | return dst 71 | } 72 | 73 | func (_this *pRidBuilder) BuildFromArray(ctx *Context, arrayType events.ArrayType, value []byte, dst reflect.Value) reflect.Value { 74 | switch arrayType { 75 | case events.ArrayTypeResourceID: 76 | setPRIDFromString(string(value), dst) 77 | default: 78 | PanicBadEvent(_this, "BuildFromArray(%v)", arrayType) 79 | } 80 | return dst 81 | } 82 | 83 | func (_this *pRidBuilder) BuildFromStringlikeArray(ctx *Context, arrayType events.ArrayType, value string, dst reflect.Value) reflect.Value { 84 | switch arrayType { 85 | case events.ArrayTypeResourceID: 86 | setPRIDFromString(value, dst) 87 | default: 88 | PanicBadEvent(_this, "BuildFromStringlikeArray(%v)", arrayType) 89 | } 90 | return dst 91 | } 92 | 93 | func (_this *pRidBuilder) BuildArtificiallyEndContainer(ctx *Context) { 94 | } 95 | -------------------------------------------------------------------------------- /builder/builder_time.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package builder 22 | 23 | import ( 24 | "reflect" 25 | 26 | compact_time "github.com/kstenerud/go-compact-time" 27 | ) 28 | 29 | // Go Time 30 | 31 | var globalTimeBuilder = &timeBuilder{} 32 | 33 | type timeBuilder struct{} 34 | 35 | func generateTimeBuilder(ctx *Context) Builder { return globalTimeBuilder } 36 | func (_this *timeBuilder) String() string { return reflect.TypeOf(_this).String() } 37 | 38 | func (_this *timeBuilder) BuildFromTime(ctx *Context, value compact_time.Time, dst reflect.Value) reflect.Value { 39 | v, err := value.AsGoTime() 40 | if err != nil { 41 | panic(err) 42 | } 43 | dst.Set(reflect.ValueOf(v)) 44 | return dst 45 | } 46 | 47 | func (_this *timeBuilder) BuildArtificiallyEndContainer(ctx *Context) { 48 | } 49 | 50 | // ============================================================================ 51 | 52 | var globalCompactTimeBuilder = &compactTimeBuilder{} 53 | 54 | type compactTimeBuilder struct{} 55 | 56 | func generateCompactTimeBuilder(ctx *Context) Builder { return globalCompactTimeBuilder } 57 | func (_this *compactTimeBuilder) String() string { return reflect.TypeOf(_this).String() } 58 | 59 | func (_this *compactTimeBuilder) BuildFromNull(ctx *Context, dst reflect.Value) reflect.Value { 60 | dst.Set(reflect.ValueOf(compact_time.Time{})) 61 | return dst 62 | } 63 | 64 | func (_this *compactTimeBuilder) BuildFromTime(ctx *Context, value compact_time.Time, dst reflect.Value) reflect.Value { 65 | dst.Set(reflect.ValueOf(value)) 66 | return dst 67 | } 68 | 69 | func (_this *compactTimeBuilder) BuildArtificiallyEndContainer(ctx *Context) { 70 | } 71 | 72 | // ============================================================================ 73 | 74 | var globalPCompactTimeBuilder = &pCompactTimeBuilder{} 75 | 76 | type pCompactTimeBuilder struct{} 77 | 78 | func generatePCompactTimeBuilder(ctx *Context) Builder { return &pCompactTimeBuilder{} } 79 | func (_this *pCompactTimeBuilder) String() string { return reflect.TypeOf(_this).String() } 80 | 81 | func (_this *pCompactTimeBuilder) BuildFromNull(ctx *Context, dst reflect.Value) reflect.Value { 82 | dst.Set(reflect.ValueOf(compact_time.ZeroDate())) 83 | return dst 84 | } 85 | 86 | func (_this *pCompactTimeBuilder) BuildFromTime(ctx *Context, value compact_time.Time, dst reflect.Value) reflect.Value { 87 | dst.Set(reflect.ValueOf(value)) 88 | return dst 89 | } 90 | 91 | func (_this *pCompactTimeBuilder) BuildArtificiallyEndContainer(ctx *Context) { 92 | } 93 | -------------------------------------------------------------------------------- /builder/reference_filler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package builder 22 | 23 | import ( 24 | "reflect" 25 | ) 26 | 27 | // TODO: Also keep a map of the costs of the references (bytes, depth, etc) (in rules) 28 | 29 | // ReferenceFiller tracks markers and references encountered in a document, 30 | // filling out references when their corresponding markers are found. 31 | type ReferenceFiller struct { 32 | markedValues map[string]reflect.Value 33 | unresolvedReferences map[string][]func(reflect.Value) 34 | } 35 | 36 | // Create and initialize a new ReferenceFiller 37 | func NewReferenceFiller() *ReferenceFiller { 38 | _this := new(ReferenceFiller) 39 | _this.Init() 40 | return _this 41 | } 42 | 43 | // Initialize an existing ReferenceFiller 44 | func (_this *ReferenceFiller) Init() { 45 | _this.markedValues = make(map[string]reflect.Value) 46 | _this.unresolvedReferences = make(map[string][]func(reflect.Value)) 47 | } 48 | 49 | // Notify that a new marker has been found. 50 | func (_this *ReferenceFiller) NotifyMarker(id []byte, value reflect.Value) { 51 | idAsString := string(id) 52 | _this.markedValues[idAsString] = value 53 | if setters, ok := _this.unresolvedReferences[idAsString]; ok { 54 | for _, setter := range setters { 55 | setter(value) 56 | } 57 | delete(_this.unresolvedReferences, idAsString) 58 | } 59 | } 60 | 61 | // Notify that a new reference has been found. valueSetter will be called when 62 | // the marker with lookingForID is found (possibly immediately if it already has 63 | // been found). 64 | func (_this *ReferenceFiller) NotifyLocalReference(lookingForID []byte, valueSetter func(value reflect.Value)) { 65 | idAsString := string(lookingForID) 66 | if value, ok := _this.markedValues[idAsString]; ok { 67 | valueSetter(value) 68 | return 69 | } 70 | _this.unresolvedReferences[idAsString] = append(_this.unresolvedReferences[idAsString], valueSetter) 71 | } 72 | -------------------------------------------------------------------------------- /ce/arrays.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package ce 22 | 23 | import ( 24 | "github.com/kstenerud/go-concise-encoding/internal/arrays" 25 | ) 26 | 27 | // These functions are used to decode multibyte array elements coming from OnArrayData events. 28 | // If you're using lower level APIs like decoders, you'll need these. 29 | 30 | // Copy byte slice data, interpreting as little endian data of larger types. 31 | func BytesToInt8Slice(data []byte) []int8 { return arrays.BytesToInt8Slice(data) } 32 | 33 | func BytesToUint16Slice(data []byte) []uint16 { return arrays.BytesToUint16Slice(data) } 34 | 35 | func BytesToInt16Slice(data []byte) []int16 { return arrays.BytesToInt16Slice(data) } 36 | 37 | func BytesToUint32Slice(data []byte) []uint32 { return arrays.BytesToUint32Slice(data) } 38 | 39 | func BytesToInt32Slice(data []byte) []int32 { return arrays.BytesToInt32Slice(data) } 40 | 41 | func BytesToFloat32Slice(data []byte) []float32 { return arrays.BytesToFloat32Slice(data) } 42 | 43 | func BytesToUint64Slice(data []byte) []uint64 { return arrays.BytesToUint64Slice(data) } 44 | 45 | func BytesToInt64Slice(data []byte) []int64 { return arrays.BytesToInt64Slice(data) } 46 | 47 | func BytesToFloat64Slice(data []byte) []float64 { return arrays.BytesToFloat64Slice(data) } 48 | 49 | // Reinterpret slices as byte slices. On little endian machines, these are zero 50 | // cost. On big endian machines, these will allocate space and copy the data. 51 | // Warning: Do not write to or store the resulting byte slice! The contents 52 | // should be considered volatile and likely to change if the source 53 | // slice changes. 54 | func Int8SliceAsBytes(data []int8) []byte { return arrays.Int8SliceAsBytes(data) } 55 | 56 | func Uint16SliceAsBytes(data []uint16) []byte { return arrays.Uint16SliceAsBytes(data) } 57 | 58 | func Int16SliceAsBytes(data []int16) []byte { return arrays.Int16SliceAsBytes(data) } 59 | 60 | func Uint32SliceAsBytes(data []uint32) []byte { return arrays.Uint32SliceAsBytes(data) } 61 | 62 | func Int32SliceAsBytes(data []int32) []byte { return arrays.Int32SliceAsBytes(data) } 63 | 64 | func Float32SliceAsBytes(data []float32) []byte { return arrays.Float32SliceAsBytes(data) } 65 | 66 | func Uint64SliceAsBytes(data []uint64) []byte { return arrays.Uint64SliceAsBytes(data) } 67 | 68 | func Int64SliceAsBytes(data []int64) []byte { return arrays.Int64SliceAsBytes(data) } 69 | 70 | func Float64SliceAsBytes(data []float64) []byte { return arrays.Float64SliceAsBytes(data) } 71 | -------------------------------------------------------------------------------- /ce/ce_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package ce 22 | 23 | import ( 24 | "bytes" 25 | "strings" 26 | "testing" 27 | 28 | "github.com/kstenerud/go-concise-encoding/builder" 29 | "github.com/kstenerud/go-concise-encoding/configuration" 30 | "github.com/kstenerud/go-describe" 31 | "github.com/kstenerud/go-equivalence" 32 | ) 33 | 34 | func TestCEUnmarshalCTE(t *testing.T) { 35 | document := "c0 [1 2 3]" 36 | expected := []interface{}{1, 2, 3} 37 | 38 | config := configuration.New() 39 | actual, err := UnmarshalFromCEDocument([]byte(document), nil, config) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | if !equivalence.IsEquivalent(expected, actual) { 44 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 45 | } 46 | 47 | stream := strings.NewReader(document) 48 | actual, err = UnmarshalCE(stream, nil, config) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | if !equivalence.IsEquivalent(expected, actual) { 53 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 54 | } 55 | } 56 | 57 | func TestCEDecodeCTE(t *testing.T) { 58 | document := "c1 [1 2 3]" 59 | expected := []interface{}{1, 2, 3} 60 | 61 | config := configuration.New() 62 | decoder := NewCEDecoder(config) 63 | 64 | builderSession := builder.NewSession(nil, config) 65 | builder := builder.NewBuilderEventReceiver(builderSession, nil, config) 66 | err := decoder.DecodeDocument([]byte(document), builder) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | actual := builder.GetBuiltObject() 71 | if !equivalence.IsEquivalent(expected, actual) { 72 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 73 | } 74 | 75 | stream := strings.NewReader(document) 76 | err = decoder.Decode(stream, builder) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | actual = builder.GetBuiltObject() 81 | if !equivalence.IsEquivalent(expected, actual) { 82 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 83 | } 84 | } 85 | 86 | func TestCEUnmarshalCBE(t *testing.T) { 87 | document := []byte{0x81, 0x00, 0x9a, 0x01, 0x02, 0x03, 0x9b} 88 | expected := []interface{}{1, 2, 3} 89 | 90 | config := configuration.New() 91 | actual, err := UnmarshalFromCEDocument([]byte(document), nil, config) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | if !equivalence.IsEquivalent(expected, actual) { 96 | t.Errorf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 97 | } 98 | 99 | stream := bytes.NewBuffer(document) 100 | actual, err = UnmarshalCE(stream, nil, config) 101 | if err != nil { 102 | t.Fatal(err) 103 | } 104 | if !equivalence.IsEquivalent(expected, actual) { 105 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 106 | } 107 | } 108 | 109 | func TestCEDecodeCBE(t *testing.T) { 110 | document := []byte{0x81, 0x00, 0x9a, 0x01, 0x02, 0x03, 0x9b} 111 | expected := []interface{}{1, 2, 3} 112 | 113 | config := configuration.New() 114 | decoder := NewCEDecoder(config) 115 | 116 | builderSession := builder.NewSession(nil, config) 117 | builder := builder.NewBuilderEventReceiver(builderSession, nil, config) 118 | err := decoder.DecodeDocument([]byte(document), builder) 119 | if err != nil { 120 | t.Fatal(err) 121 | } 122 | actual := builder.GetBuiltObject() 123 | if !equivalence.IsEquivalent(expected, actual) { 124 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 125 | } 126 | 127 | stream := bytes.NewBuffer(document) 128 | err = decoder.Decode(stream, builder) 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | actual = builder.GetBuiltObject() 133 | if !equivalence.IsEquivalent(expected, actual) { 134 | t.Fatalf("Expected %v but got %v", describe.D(expected), describe.D(actual)) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /ce/decoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package ce 22 | 23 | import ( 24 | "bufio" 25 | "fmt" 26 | "io" 27 | 28 | "github.com/kstenerud/go-concise-encoding/cbe" 29 | "github.com/kstenerud/go-concise-encoding/ce/events" 30 | "github.com/kstenerud/go-concise-encoding/configuration" 31 | "github.com/kstenerud/go-concise-encoding/cte" 32 | ) 33 | 34 | // Decoder decodes a byte stream, converting it to events. 35 | // Its operation is similar to a lexer. 36 | type Decoder interface { 37 | // Decode the stream of bytes from reader, sending all events to eventReceiver. 38 | Decode(reader io.Reader, eventReceiver events.DataEventReceiver) error 39 | 40 | // Decode from the specified document, sending all events to eventReceiver. 41 | DecodeDocument(document []byte, eventReceiver events.DataEventReceiver) (err error) 42 | } 43 | 44 | // A universal decoder automatically chooses a decoding method based on the first byte of the document: 45 | // - 0x63 = Decode as CTE 46 | // - 0x81 = Decode as CBE 47 | type UniversalDecoder struct { 48 | config *configuration.Configuration 49 | } 50 | 51 | func (_this *UniversalDecoder) Decode(reader io.Reader, eventReceiver events.DataEventReceiver) error { 52 | bufReader := bufio.NewReader(reader) 53 | firstByte, err := bufReader.Peek(1) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | if decoder, err := chooseDecoder(firstByte[0], _this.config); err == nil { 59 | return decoder.Decode(bufReader, eventReceiver) 60 | } else { 61 | return err 62 | } 63 | } 64 | 65 | func (_this *UniversalDecoder) DecodeDocument(document []byte, eventReceiver events.DataEventReceiver) error { 66 | if decoder, err := chooseDecoder(document[0], _this.config); err == nil { 67 | return decoder.DecodeDocument(document, eventReceiver) 68 | } else { 69 | return err 70 | } 71 | } 72 | 73 | func chooseDecoder(identifier byte, config *configuration.Configuration) (decoder Decoder, err error) { 74 | switch identifier { 75 | case 'c', 'C': 76 | decoder = cte.NewDecoder(config) 77 | case cbe.CBESignatureByte: 78 | decoder = cbe.NewDecoder(config) 79 | default: 80 | err = fmt.Errorf("%02d: Unknown CE identifier", identifier) 81 | } 82 | return 83 | } 84 | -------------------------------------------------------------------------------- /ce/encoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package ce 22 | 23 | import ( 24 | "io" 25 | 26 | "github.com/kstenerud/go-concise-encoding/ce/events" 27 | ) 28 | 29 | // Encoder accepts events and encodes them to a byte stream. 30 | type Encoder interface { 31 | // Prepare the encoder for encoding. All events will be encoded to writer. 32 | // PrepareToEncode MUST be called before using the encoder. 33 | PrepareToEncode(writer io.Writer) 34 | 35 | events.DataEventReceiver 36 | } 37 | -------------------------------------------------------------------------------- /ce/marshaler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package ce 22 | 23 | import ( 24 | "io" 25 | ) 26 | 27 | // Marshaler marshals an object into a stream of encoded bytes. 28 | type Marshaler interface { 29 | // Marshal the given object, writing the encoded stream to writer. 30 | Marshal(object interface{}, writer io.Writer) error 31 | 32 | // Marshal the given object, returning the encoded document. 33 | MarshalToDocument(object interface{}) (document []byte, err error) 34 | } 35 | -------------------------------------------------------------------------------- /ce/unmarshaler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package ce 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | 27 | "github.com/kstenerud/go-concise-encoding/cbe" 28 | "github.com/kstenerud/go-concise-encoding/configuration" 29 | "github.com/kstenerud/go-concise-encoding/cte" 30 | ) 31 | 32 | // Unmarshaler decodes bytes from an input source, then creates, fills out, and 33 | // returns an object that is compatible with the given template. 34 | type Unmarshaler interface { 35 | // Unmarshal an object from the given reader, in a type compatible with template. 36 | // If template is nil, a best-guess type will be returned (likely a slice or map). 37 | // If an error occurs while decoding, err will be non-null, and decoded will contain whatever was successfully decoded thus far. 38 | Unmarshal(reader io.Reader, template interface{}) (decoded interface{}, err error) 39 | 40 | // Unmarshal an object from the given document, in a type compatible with template. 41 | // If template is nil, a best-guess type will be returned (likely a slice or map). 42 | // If an error occurs while decoding, err will be non-null, and decoded will contain whatever was successfully decoded thus far. 43 | UnmarshalFromDocument(document []byte, template interface{}) (decoded interface{}, err error) 44 | } 45 | 46 | func chooseUnmarshaler(identifier byte, config *configuration.Configuration) (unmarshaler Unmarshaler, err error) { 47 | switch identifier { 48 | case 'c': 49 | unmarshaler = cte.NewUnmarshaler(config) 50 | case cbe.CBESignatureByte: 51 | unmarshaler = cbe.NewUnmarshaler(config) 52 | default: 53 | err = fmt.Errorf("%02d: Unknown CE identifier", identifier) 54 | } 55 | return 56 | } 57 | -------------------------------------------------------------------------------- /codegen/.gitignore: -------------------------------------------------------------------------------- 1 | codegen 2 | -------------------------------------------------------------------------------- /codegen/README.md: -------------------------------------------------------------------------------- 1 | Code Generator for go-concise-encoding 2 | ====================================== 3 | 4 | This program performs all code generation for the go-concise-encoding project. 5 | 6 | To build everything, you'll need to extract https://www.unicode.org/Public/UCD/latest/ucdxml/ucd.all.flat.zip 7 | 8 | ### Usage 9 | 10 | * Build all but chars: `go build && ./codegen` 11 | * Build everything: `go build && ./codegen -unicode /path/to/ucd.all.flat.xml` 12 | 13 | Use `--help` for a list of all options. 14 | -------------------------------------------------------------------------------- /codegen/antlr/antlr-4.12.0-complete.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kstenerud/go-concise-encoding/2a7aebf568df6ec3681f38109d0d38e466cb91c8/codegen/antlr/antlr-4.12.0-complete.jar -------------------------------------------------------------------------------- /codegen/antlr/antlr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package antlr 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | "io/ioutil" 27 | "os" 28 | "os/exec" 29 | "path/filepath" 30 | "strings" 31 | ) 32 | 33 | func RunAntlr(antlrPath string, grammarPath string, dstPath string) { 34 | 35 | javaPath, err := exec.LookPath("java") 36 | if err != nil { 37 | panic(err) 38 | } 39 | if err := os.RemoveAll(dstPath); err != nil { 40 | panic(err) 41 | } 42 | if err := os.MkdirAll(dstPath, 0755); err != nil { 43 | panic(err) 44 | } 45 | 46 | antlrPath = filepath.Join(antlrPath, "antlr-4.12.0-complete.jar") 47 | lexerPath, err := findLexerFile(grammarPath) 48 | if err != nil { 49 | panic(err) 50 | } 51 | parserPath, err := findParserFile(grammarPath) 52 | if err != nil { 53 | panic(err) 54 | } 55 | 56 | cmd := exec.Command( 57 | javaPath, 58 | "-cp", antlrPath, 59 | "org.antlr.v4.Tool", 60 | "-o", dstPath, 61 | "-Dlanguage=Go", 62 | lexerPath, parserPath, 63 | ) 64 | stdout := &bytes.Buffer{} 65 | stderr := &bytes.Buffer{} 66 | cmd.Stdout = stdout 67 | cmd.Stderr = stderr 68 | if err := cmd.Run(); err != nil { 69 | panic(fmt.Errorf("failed to run %v: %w\nStdout = [%v]\nStderr = [%v]", cmd.Args, err, stdout.String(), stderr.String())) 70 | } 71 | } 72 | 73 | func findLexerFile(basePath string) (path string, err error) { 74 | files, err := ioutil.ReadDir(basePath) 75 | if err != nil { 76 | return "", err 77 | } 78 | 79 | for _, finfo := range files { 80 | if strings.HasSuffix(finfo.Name(), "Lexer.g4") { 81 | return filepath.Join(basePath, finfo.Name()), nil 82 | } 83 | } 84 | return "", fmt.Errorf("could not find *Lexer.g4 in %v", basePath) 85 | } 86 | 87 | func findParserFile(basePath string) (path string, err error) { 88 | files, err := ioutil.ReadDir(basePath) 89 | if err != nil { 90 | return "", err 91 | } 92 | 93 | for _, finfo := range files { 94 | if strings.HasSuffix(finfo.Name(), "Parser.g4") { 95 | return filepath.Join(basePath, finfo.Name()), nil 96 | } 97 | } 98 | return "", fmt.Errorf("could not find *Parser.g4 in %v", basePath) 99 | } 100 | -------------------------------------------------------------------------------- /codegen/cte/.gitignore: -------------------------------------------------------------------------------- 1 | .antlr 2 | -------------------------------------------------------------------------------- /codegen/cte/README.md: -------------------------------------------------------------------------------- 1 | ANTLR4 Grammar for Concise Text Encoding 2 | ======================================== 3 | 4 | This Antlr4 grammar is used to generate the CTE parser for this library. 5 | 6 | [`codegen`](..) generates the parser code from this grammar. 7 | 8 | [Parser.go](../../cte/parser.go) integrates the generated parser code into the library. 9 | 10 | ### TODO 11 | 12 | - Define the proper character ranges for stringlike (see the bottom of [CTELexer.g4](CTELexer.g4)). 13 | -------------------------------------------------------------------------------- /codegen/cte/antlr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package cte 22 | 23 | import ( 24 | "path/filepath" 25 | 26 | "github.com/kstenerud/go-concise-encoding/codegen/antlr" 27 | ) 28 | 29 | func generateAntlrCode(projectDir string) { 30 | antlr.RunAntlr( 31 | filepath.Join(projectDir, "codegen", "antlr"), 32 | filepath.Join(projectDir, "codegen", "cte"), 33 | filepath.Join(projectDir, "cte", "parser"), 34 | ) 35 | } 36 | -------------------------------------------------------------------------------- /codegen/cte/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package cte 22 | 23 | func GenerateCode(projectDir string) { 24 | generateAntlrCode(projectDir) 25 | } 26 | -------------------------------------------------------------------------------- /codegen/datatypes/datatypes.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package datatypes 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "reflect" 27 | "strings" 28 | ) 29 | 30 | type FlagDataTypeWriter struct { 31 | writer io.Writer 32 | typeName string 33 | highestValue interface{} 34 | isFirst bool 35 | } 36 | 37 | func NewFlagDataTypeWriter(writer io.Writer, typeName string, endMarkerValue interface{}) *FlagDataTypeWriter { 38 | endValue := reflect.ValueOf(endMarkerValue) 39 | highestValue := reflect.New(reflect.TypeOf(endMarkerValue)).Elem() 40 | highestValue.SetUint(endValue.Uint() >> 1) 41 | return &FlagDataTypeWriter{ 42 | writer: writer, 43 | typeName: typeName, 44 | highestValue: highestValue.Interface(), 45 | isFirst: true, 46 | } 47 | } 48 | 49 | func (_this *FlagDataTypeWriter) BeginType() { 50 | endMarkerValue := reflect.ValueOf(_this.highestValue).Uint() << 1 51 | var baseType string 52 | switch { 53 | case endMarkerValue <= 0x100: 54 | baseType = "uint8" 55 | case endMarkerValue <= 0x10000: 56 | baseType = "uint16" 57 | case endMarkerValue <= 0x100000000: 58 | baseType = "uint32" 59 | default: 60 | baseType = "uint64" 61 | } 62 | 63 | if _, err := fmt.Fprintf(_this.writer, "type %v %v\n\nconst (", _this.typeName, baseType); err != nil { 64 | panic(err) 65 | } 66 | } 67 | 68 | func (_this *FlagDataTypeWriter) AddNamed(valueName interface{}) { 69 | if _, err := fmt.Fprintf(_this.writer, "\n\t%v", valueName); err != nil { 70 | panic(err) 71 | } 72 | if _this.isFirst { 73 | _this.isFirst = false 74 | if _, err := fmt.Fprintf(_this.writer, " %v = 1 << iota", _this.typeName); err != nil { 75 | panic(err) 76 | } 77 | } 78 | } 79 | 80 | func (_this *FlagDataTypeWriter) AddCustom(valueName interface{}, value interface{}) { 81 | if reflect.TypeOf(value) == reflect.TypeOf(uint64(0)) { 82 | value = fmt.Sprintf("0x%x", value) 83 | } 84 | if _, err := fmt.Fprintf(_this.writer, "\n\t%v = %v", valueName, value); err != nil { 85 | panic(err) 86 | } 87 | } 88 | 89 | func (_this *FlagDataTypeWriter) EndType() { 90 | if _, err := fmt.Fprintf(_this.writer, "\n)\n"); err != nil { 91 | panic(err) 92 | } 93 | } 94 | 95 | func (_this *FlagDataTypeWriter) BeginStringer() { 96 | typeName := _this.typeName 97 | lowerName := strings.ToLower(typeName) 98 | fmt.Fprintf(_this.writer, ` 99 | func (_this %v) String() string { 100 | asString := "" 101 | if _this == 0 { 102 | asString = %vNames[_this] 103 | } else { 104 | isFirst := true 105 | builder := strings.Builder{} 106 | for i := %v(1); i <= %v; i <<= 1 { 107 | if _this&i != 0 { 108 | if isFirst { 109 | isFirst = false 110 | } else { 111 | builder.WriteString(" | ") 112 | } 113 | builder.WriteString(%vNames[i]) 114 | } 115 | } 116 | asString = builder.String() 117 | } 118 | if asString == "" { 119 | asString = fmt.Sprintf("%%d", _this) 120 | } 121 | return asString 122 | } 123 | 124 | var %vNames = map[%v]string{ 125 | `, typeName, lowerName, typeName, _this.highestValue, lowerName, lowerName, typeName) 126 | } 127 | 128 | func (_this *FlagDataTypeWriter) AddStringer(typeName interface{}) { 129 | fmt.Fprintf(_this.writer, "\n\t%v: \"%v\",", typeName, typeName) 130 | } 131 | 132 | func (_this *FlagDataTypeWriter) EndStringer() { 133 | fmt.Fprintf(_this.writer, "\n}\n") 134 | } 135 | 136 | func FlagToString(valueMap map[interface{}]string, value interface{}) string { 137 | uintValue := reflect.ValueOf(value).Uint() 138 | if uintValue == 0 { 139 | return valueMap[value] 140 | } 141 | 142 | isFirst := true 143 | builder := strings.Builder{} 144 | lookupValue := reflect.New(reflect.TypeOf(value)).Elem() 145 | for i := 0; i < 64; i++ { 146 | testValue := uint64(1) << i 147 | if uintValue&testValue != 0 { 148 | lookupValue.SetUint(testValue) 149 | stringValue, ok := valueMap[lookupValue.Interface()] 150 | if !ok { 151 | continue 152 | } 153 | 154 | if isFirst { 155 | isFirst = false 156 | } else { 157 | builder.WriteString(" | ") 158 | } 159 | builder.WriteString(stringValue) 160 | } 161 | } 162 | return builder.String() 163 | } 164 | -------------------------------------------------------------------------------- /codegen/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kstenerud/go-concise-encoding/codegen 2 | 3 | go 1.14 4 | 5 | replace github.com/kstenerud/go-concise-encoding => ../ 6 | 7 | require ( 8 | github.com/kstenerud/go-compact-float v1.6.1 9 | github.com/kstenerud/go-compact-time v1.8.3 10 | // github.com/kstenerud/go-concise-encoding v0.0.0-20221011041601-2ce2abafcdc6 11 | github.com/kstenerud/go-concise-encoding v0.0.0 12 | ) 13 | -------------------------------------------------------------------------------- /codegen/go.sum: -------------------------------------------------------------------------------- 1 | github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220626175859-9abda183db8e h1:bt6SW1eSSvdmmsG0KqyxYXorcTnFBTX7hfVR1+68+jg= 2 | github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220626175859-9abda183db8e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= 3 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= 4 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= 5 | github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= 6 | github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= 7 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 8 | github.com/kstenerud/go-compact-float v1.6.1 h1:FhRID85bjfOSKrGkHEKgwSZTYPXoMxaiH/L7TsvpikA= 9 | github.com/kstenerud/go-compact-float v1.6.1/go.mod h1:86Q5GkzClM2g0hObQndeX90g5kKMhG7eptDFsVrubbk= 10 | github.com/kstenerud/go-compact-time v1.8.3 h1:VPhN2UwJsJjAHnHbGuqLibNF8e7PXG4nBFpZ3UkjsRg= 11 | github.com/kstenerud/go-compact-time v1.8.3/go.mod h1:wBgxWliL/ON5b1/EaefPqQ44u2UgTtiNiIvkmByYFFc= 12 | github.com/kstenerud/go-describe v1.2.13/go.mod h1:+B7K/fXHhObqwmMhcUT0CPT68fR1SQ9XNmJ5E4ck2RI= 13 | github.com/kstenerud/go-describe v1.2.15 h1:EVa8A9rerri6NGih4Koleg1cadp4wtzSLom802jnMGY= 14 | github.com/kstenerud/go-describe v1.2.15/go.mod h1:+B7K/fXHhObqwmMhcUT0CPT68fR1SQ9XNmJ5E4ck2RI= 15 | github.com/kstenerud/go-duplicates v1.1.1 h1:Z8pdWHv842d/vszSeoWGAswbxx/8SHxz+/DNYQUl4jU= 16 | github.com/kstenerud/go-duplicates v1.1.1/go.mod h1:OoPQ8B7sw/LFlPpLJT6bbCrHdOtaC9yWBF1uZXqOoxw= 17 | github.com/kstenerud/go-equivalence v1.0.4 h1:gbyVM157tVSlYSbCpzi6AZnXLugmetvYdvxUBnrEqmU= 18 | github.com/kstenerud/go-equivalence v1.0.4/go.mod h1:FKUKQCiRiJ8M0rgXy/0jR+yIet5mydc2zyj6XQ1PK60= 19 | github.com/kstenerud/go-uleb128 v1.1.0 h1:QrNuuZVFkNvn3tEEJHDSUZRckNZt1haE23uYPKJ8qAA= 20 | github.com/kstenerud/go-uleb128 v1.1.0/go.mod h1:0DgQsdpP8sW2B3Qd2yx3vO7oaGNqDUZW7h4PoKIQj14= 21 | github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= 22 | github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 23 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 24 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 25 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 26 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 27 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 28 | golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= 29 | golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= 30 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= 31 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 32 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 33 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 34 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 35 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 36 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 37 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 38 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 39 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 40 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 41 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 42 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 43 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 44 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 45 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 46 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 47 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 48 | golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= 49 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 50 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 51 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 52 | -------------------------------------------------------------------------------- /codegen/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Package build generates code for other parts of the library. The lack of 22 | // generics and inheritance makes a number of things tedious and error prone, 23 | // which these generators attempt to deal with. 24 | package main 25 | 26 | import ( 27 | "flag" 28 | "os" 29 | "path/filepath" 30 | 31 | "github.com/kstenerud/go-concise-encoding/codegen/builder" 32 | "github.com/kstenerud/go-concise-encoding/codegen/chars" 33 | "github.com/kstenerud/go-concise-encoding/codegen/cte" 34 | "github.com/kstenerud/go-concise-encoding/codegen/rules" 35 | gentest "github.com/kstenerud/go-concise-encoding/codegen/test" 36 | "github.com/kstenerud/go-concise-encoding/codegen/tests" 37 | ) 38 | 39 | func main() { 40 | projectPath := getProjectPath() 41 | 42 | unicodePath := flag.String("unicode", "", "/path/to/ucd.all.flat.xml. Get it from https://www.unicode.org/Public/UCD/latest/ucdxml/ucd.all.flat.zip") 43 | flag.Parse() 44 | 45 | cte.GenerateCode(projectPath) 46 | builder.GenerateCode(projectPath) 47 | rules.GenerateCode(projectPath) 48 | gentest.GenerateCode(projectPath) 49 | tests.GenerateCode(projectPath) 50 | 51 | if *unicodePath != "" { 52 | chars.GenerateCode(projectPath, *unicodePath) 53 | } 54 | } 55 | 56 | func getExePath() string { 57 | ex, err := os.Executable() 58 | if err != nil { 59 | panic(err) 60 | } 61 | return filepath.Dir(ex) 62 | } 63 | 64 | func getProjectPath() string { 65 | return filepath.Dir(getExePath()) 66 | } 67 | -------------------------------------------------------------------------------- /codegen/test/.gitignore: -------------------------------------------------------------------------------- 1 | .antlr 2 | -------------------------------------------------------------------------------- /codegen/test/antlr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test 22 | 23 | import ( 24 | "path/filepath" 25 | 26 | "github.com/kstenerud/go-concise-encoding/codegen/antlr" 27 | ) 28 | 29 | func generateAntlrCode(projectDir string) { 30 | antlr.RunAntlr( 31 | filepath.Join(projectDir, "codegen", "antlr"), 32 | filepath.Join(projectDir, "codegen", "test"), 33 | filepath.Join(projectDir, "test", "event_parser", "parser"), 34 | ) 35 | } 36 | -------------------------------------------------------------------------------- /codegen/test/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test 22 | 23 | func GenerateCode(projectDir string) { 24 | generateAntlrCode(projectDir) 25 | generateEvents(projectDir) 26 | } 27 | -------------------------------------------------------------------------------- /codegen/tests/golang_generators.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package tests 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "os" 27 | "path" 28 | "strings" 29 | 30 | "github.com/kstenerud/go-concise-encoding/codegen/common" 31 | ) 32 | 33 | var testsImports = []*common.Import{ 34 | {As: "", Import: "fmt"}, 35 | {As: "", Import: "math"}, 36 | {As: "", Import: "math/big"}, 37 | {As: "", Import: "github.com/kstenerud/go-concise-encoding/configuration"}, 38 | {As: "", Import: "github.com/kstenerud/go-concise-encoding/test"}, 39 | {As: "", Import: "github.com/kstenerud/go-concise-encoding/test/test_runner"}, 40 | } 41 | 42 | func generateTestGenerators(basePath string) { 43 | common.GenerateGoFile(basePath, testsImports, func(writer io.Writer) { 44 | generateArrayTestGenerator(basePath, writer) 45 | }) 46 | } 47 | 48 | func generateArrayTestGenerator(basePath string, writer io.Writer) { 49 | template := readFunctionTemplate(basePath, "generateArrayInt32Tests") 50 | writer.Write([]byte(strings.ReplaceAll(template, "32", "8"))) 51 | writer.Write([]byte(strings.ReplaceAll(template, "32", "16"))) 52 | writer.Write([]byte(strings.ReplaceAll(template, "32", "64"))) 53 | 54 | template = readFunctionTemplate(basePath, "generateArrayUint32Tests") 55 | writer.Write([]byte(strings.ReplaceAll(template, "32", "8"))) 56 | writer.Write([]byte(strings.ReplaceAll(template, "32", "16"))) 57 | writer.Write([]byte(strings.ReplaceAll(template, "32", "64"))) 58 | 59 | template = readFunctionTemplate(basePath, "generateArrayFloat32Tests") 60 | writer.Write([]byte(strings.ReplaceAll(strings.ReplaceAll(template, "32", "16"), "float16", "float32"))) 61 | writer.Write([]byte(strings.ReplaceAll(template, "32", "64"))) 62 | } 63 | 64 | func readFunctionTemplate(basePath string, functionName string) string { 65 | var templateData string 66 | path := path.Join(basePath, "templates.go") 67 | if file, err := os.ReadFile(path); err != nil { 68 | panic(fmt.Errorf("could not read function template file %v: %w", path, err)) 69 | } else { 70 | templateData = string(file) 71 | } 72 | 73 | iStart := strings.Index(templateData, fmt.Sprintf("\nfunc %v(", functionName)) 74 | if iStart < 0 { 75 | panic(fmt.Errorf("could not find function %v in template file %v", functionName, path)) 76 | } 77 | 78 | templateData = templateData[iStart+1:] 79 | iEnd := strings.Index(templateData, "\n}\n") 80 | if iEnd < 0 { 81 | panic(fmt.Errorf("could not find end of function %v in template file %v", functionName, path)) 82 | } 83 | 84 | return templateData[:iEnd+3] 85 | } 86 | -------------------------------------------------------------------------------- /codegen/tests/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package tests 22 | 23 | import ( 24 | "path/filepath" 25 | ) 26 | 27 | func GenerateCode(projectDir string) { 28 | generateTestGenerators(filepath.Join(projectDir, "codegen/tests")) 29 | generateTestFiles(projectDir) 30 | } 31 | -------------------------------------------------------------------------------- /configuration/builder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | import ( 24 | "fmt" 25 | "reflect" 26 | ) 27 | 28 | type BuilderConfiguration struct { 29 | // Max base-10 exponent allowed when converting from floating point to big integer. 30 | // As exponents get very large, it takes geometrically more CPU to convert. 31 | FloatToBigIntMaxBase10Exponent int 32 | 33 | // Max base-2 exponent allowed when converting from floating point to big integer. 34 | FloatToBigIntMaxBase2Exponent int 35 | 36 | // Match struct field names in a case insensitive manner 37 | CaseInsensitiveStructFieldNames bool 38 | 39 | // TODO: If true, don't raise an error on a lossy floating point conversion. 40 | AllowLossyFloatConversion bool 41 | 42 | // TODO: If true, don't raise an error on unknown fields 43 | IgnoreUnknownFields bool 44 | 45 | // Specifies which types will be built using custom text/binary build 46 | // functions. You must also set one or both of CustomBinaryBuildFunction 47 | // and CustomTextBuildFunction in order to use this feature. 48 | // Both CBE and CTE will attempt to use either the binary or text version 49 | // depending on the data type (custom binary, custom text) encoded in the 50 | // source document. 51 | CustomBuiltTypes []reflect.Type 52 | 53 | // Build function to use when building from a custom binary source. 54 | CustomBinaryBuildFunction CustomBinaryBuildFunction 55 | 56 | // Build function to use when building from a custom text source. 57 | CustomTextBuildFunction CustomTextBuildFunction 58 | } 59 | 60 | func (_this *BuilderConfiguration) init() {} 61 | 62 | var defaultBuilderConfiguration = BuilderConfiguration{ 63 | FloatToBigIntMaxBase10Exponent: maxBase10Exp, 64 | FloatToBigIntMaxBase2Exponent: maxBase10Exp * 10 / 3, 65 | AllowLossyFloatConversion: true, 66 | IgnoreUnknownFields: true, 67 | CaseInsensitiveStructFieldNames: true, 68 | CustomBinaryBuildFunction: func(customType uint64, src []byte, dst reflect.Value) error { 69 | return fmt.Errorf("no builder has been registered to handle custom binary data") 70 | }, 71 | CustomTextBuildFunction: func(customType uint64, src string, dst reflect.Value) error { 72 | return fmt.Errorf("no builder has been registered to handle custom text data") 73 | }, 74 | CustomBuiltTypes: []reflect.Type{}, 75 | } 76 | 77 | // Fills out a value from custom data. 78 | // See https://github.com/kstenerud/concise-encoding/blob/master/cbe-specification.md#custom-binary 79 | // See https://github.com/kstenerud/concise-encoding/blob/master/cbe-specification.md#custom-text 80 | // See https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#custom-binary 81 | // See https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#custom-text 82 | type CustomBinaryBuildFunction func(customType uint64, src []byte, dst reflect.Value) error 83 | type CustomTextBuildFunction func(customType uint64, src string, dst reflect.Value) error 84 | 85 | const maxBase10Exp = 50 86 | -------------------------------------------------------------------------------- /configuration/configuration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // All configuration that can be used to fine-tune the behavior of various aspects of 22 | // this library. 23 | package configuration 24 | 25 | func New() *Configuration { 26 | config := defaultConfiguration 27 | config.init() 28 | return &config 29 | } 30 | 31 | type Configuration struct { 32 | Rules RuleConfiguration 33 | Iterator IteratorConfiguration 34 | Builder BuilderConfiguration 35 | Decoder DecoderConfiguration 36 | Encoder EncoderConfiguration 37 | Marshal MarshalConfiguration 38 | Debug DebugConfiguration 39 | } 40 | 41 | func (_this *Configuration) init() { 42 | _this.Rules.init() 43 | _this.Iterator.init() 44 | _this.Builder.init() 45 | _this.Decoder.init() 46 | _this.Encoder.init() 47 | _this.Marshal.init() 48 | _this.Debug.init() 49 | } 50 | 51 | var defaultConfiguration = Configuration{ 52 | Rules: defaultRuleConfiguration, 53 | Iterator: defaultIteratorConfiguration, 54 | Builder: defaultBuilderConfiguration, 55 | Decoder: defaultDecoderConfiguration, 56 | Encoder: defaultEncoderConfiguration, 57 | Marshal: defaultMarshalConfiguration, 58 | Debug: defaultDebugConfiguration, 59 | } 60 | -------------------------------------------------------------------------------- /configuration/debug.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | type DebugConfiguration struct { 24 | PassThroughPanics bool 25 | } 26 | 27 | func (_this *DebugConfiguration) init() {} 28 | 29 | var defaultDebugConfiguration = DebugConfiguration{ 30 | PassThroughPanics: false, 31 | } 32 | -------------------------------------------------------------------------------- /configuration/decoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | type DecoderConfiguration struct { 24 | } 25 | 26 | func (_this *DecoderConfiguration) init() {} 27 | 28 | var defaultDecoderConfiguration = DecoderConfiguration{} 29 | 30 | type DataErrorCallback func(object interface{}, err error) 31 | -------------------------------------------------------------------------------- /configuration/encoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | import ( 24 | "fmt" 25 | "math" 26 | ) 27 | 28 | type EncoderConfiguration struct { 29 | CTE CTEEncoderConfiguration 30 | } 31 | 32 | func (_this *EncoderConfiguration) init() { 33 | _this.CTE.init() 34 | } 35 | 36 | var defaultEncoderConfiguration = EncoderConfiguration{ 37 | CTE: defaultCTEEncoderConfiguration, 38 | } 39 | 40 | // ============================================================================ 41 | // CTE Encoder 42 | 43 | type CTEEncoderConfiguration struct { 44 | // Indentation to use when pretty printing 45 | Indent string 46 | 47 | // TODO: Max column before forcing a newline (if possible) 48 | MaxColumn uint 49 | 50 | // TODO: Convert line endings to escapes 51 | EscapeLineEndings bool 52 | 53 | DefaultNumericFormats CTEEncoderDefaultNumericFormats 54 | } 55 | 56 | func (_this *CTEEncoderConfiguration) init() {} 57 | 58 | var defaultCTEEncoderConfiguration = CTEEncoderConfiguration{ 59 | Indent: " ", 60 | MaxColumn: math.MaxUint64, 61 | EscapeLineEndings: true, 62 | DefaultNumericFormats: CTEEncoderDefaultNumericFormats{ 63 | BinaryFloat: CTEEncodingFormatHexadecimal, 64 | Int: CTEEncodingFormatDecimal, 65 | Uint: CTEEncodingFormatDecimal, 66 | Array: CTEEncoderDefaultArrayFormats{ 67 | Int8: CTEEncodingFormatDecimal, 68 | Int16: CTEEncodingFormatDecimal, 69 | Int32: CTEEncodingFormatDecimal, 70 | Int64: CTEEncodingFormatDecimal, 71 | Uint8: CTEEncodingFormatDecimal, 72 | Uint16: CTEEncodingFormatDecimal, 73 | Uint32: CTEEncodingFormatDecimal, 74 | Uint64: CTEEncodingFormatDecimal, 75 | Float16: CTEEncodingFormatHexadecimal, 76 | Float32: CTEEncodingFormatHexadecimal, 77 | Float64: CTEEncodingFormatHexadecimal, 78 | }, 79 | }, 80 | } 81 | 82 | // How to encode binary floats 83 | type FloatEncoding int 84 | 85 | const ( 86 | // Use decimal encoding (1.2e4) 87 | FloatEncodingDecimal = iota 88 | // Use binary encoding (0x1.2p4) 89 | FloatEncodingBinary 90 | ) 91 | 92 | type IntEncoding int 93 | 94 | const ( 95 | IntEncodingDecimal = iota 96 | IntEncodingBinary 97 | IntEncodingOctal 98 | IntEncodingHexadecimal 99 | ) 100 | 101 | type CTEEncoderDefaultNumericFormats struct { 102 | Int CTENumericFormat 103 | Uint CTENumericFormat 104 | BinaryFloat CTENumericFormat 105 | Array CTEEncoderDefaultArrayFormats 106 | } 107 | 108 | type CTEEncoderDefaultArrayFormats struct { 109 | Int8 CTENumericFormat 110 | Int16 CTENumericFormat 111 | Int32 CTENumericFormat 112 | Int64 CTENumericFormat 113 | Uint8 CTENumericFormat 114 | Uint16 CTENumericFormat 115 | Uint32 CTENumericFormat 116 | Uint64 CTENumericFormat 117 | Float16 CTENumericFormat 118 | Float32 CTENumericFormat 119 | Float64 CTENumericFormat 120 | } 121 | 122 | type CTENumericFormat uint8 123 | 124 | const ( 125 | CTEEncodingFormatDecimal CTENumericFormat = 0 126 | CTEEncodingFormatFlagZeroFilled CTENumericFormat = 1 127 | CTEEncodingFormatBinary CTENumericFormat = 2 + iota 128 | CTEEncodingFormatBinaryZeroFilled 129 | CTEEncodingFormatOctal 130 | CTEEncodingFormatOctalZeroFilled 131 | CTEEncodingFormatHexadecimal 132 | CTEEncodingFormatHexadecimalZeroFilled 133 | cteEncodingFormatCount 134 | ) 135 | 136 | var cteEncodingFormatStrings = []string{ 137 | CTEEncodingFormatDecimal: "CTEEncodingFormatDecimal", 138 | CTEEncodingFormatFlagZeroFilled: "CTEEncodingFormatFlagZeroFilled", 139 | CTEEncodingFormatBinary: "CTEEncodingFormatBinary", 140 | CTEEncodingFormatBinaryZeroFilled: "CTEEncodingFormatBinaryZeroFilled", 141 | CTEEncodingFormatOctal: "CTEEncodingFormatOctal", 142 | CTEEncodingFormatOctalZeroFilled: "CTEEncodingFormatOctalZeroFilled", 143 | CTEEncodingFormatHexadecimal: "CTEEncodingFormatHexadecimal", 144 | CTEEncodingFormatHexadecimalZeroFilled: "CTEEncodingFormatHexadecimalZeroFilled", 145 | } 146 | 147 | func (_this CTENumericFormat) String() string { 148 | if _this < cteEncodingFormatCount { 149 | return cteEncodingFormatStrings[_this] 150 | } 151 | return fmt.Sprintf("CTEEncodingFormat(%d)", _this) 152 | } 153 | -------------------------------------------------------------------------------- /configuration/iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | import ( 24 | "reflect" 25 | ) 26 | 27 | type IteratorConfiguration struct { 28 | FieldNameStyle FieldNameStyle 29 | 30 | // If RecursionSupport is true, the iterator will also look for duplicate 31 | // pointers to data, generating marker and reference events rather than 32 | // walking the object again. This is useful for cyclic or recursive data 33 | // structures, but has a performance cost. 34 | RecursionSupport bool 35 | 36 | // What to do by default when an empty field is encountered. 37 | // This can be overridden at the field level using the ce tags 38 | // "omit", "omit_never", "omit_empty", and "omit_zero". 39 | // 40 | // Defaults to OmitFieldEmpty 41 | DefaultFieldOmitBehavior FieldOmitBehavior 42 | 43 | // Every struct type added here will have a record type with the associated 44 | // name generated at the top of the file, and all instances will be generated 45 | // as records instead of maps. 46 | // Only go struct types are allowed. 47 | // Don't map multiple types to the same name unless you're sure you know what you're doing. 48 | RecordTypes map[reflect.Type]string 49 | 50 | // Specifies which types to convert to custom binary data, and how to do it. 51 | // Note: You should only fill out one of these maps, depending on your 52 | // intended encoding (binary or text). The iterator session will consult 53 | // the binary map first and the text map second, choosing the first match. 54 | CustomBinaryConverters map[reflect.Type]ConvertToCustomFunction 55 | 56 | // Specifies which types to convert to custom text data, and how to do it 57 | // Note: You should only fill out one of these maps, depending on your 58 | // intended encoding (binary or text). The iterator session will consult 59 | // the binary map first and the text map second, choosing the first match. 60 | CustomTextConverters map[reflect.Type]ConvertToCustomFunction 61 | } 62 | 63 | func (_this *IteratorConfiguration) init() { 64 | _this.CustomBinaryConverters = make(map[reflect.Type]ConvertToCustomFunction) 65 | _this.CustomTextConverters = make(map[reflect.Type]ConvertToCustomFunction) 66 | _this.RecordTypes = make(map[reflect.Type]string) 67 | } 68 | 69 | var defaultIteratorConfiguration = IteratorConfiguration{ 70 | FieldNameStyle: FieldNameSnakeCase, 71 | RecursionSupport: false, 72 | DefaultFieldOmitBehavior: OmitFieldEmpty, 73 | } 74 | 75 | // Converts a value to custom binary data. 76 | // See https://github.com/kstenerud/concise-encoding/blob/master/cbe-specification.md#custom-binary 77 | // See https://github.com/kstenerud/concise-encoding/blob/master/cbe-specification.md#custom-text 78 | // See https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#custom-binary 79 | // See https://github.com/kstenerud/concise-encoding/blob/master/cte-specification.md#custom-text 80 | type ConvertToCustomFunction func(v reflect.Value) (customType uint64, asBytes []byte, err error) 81 | 82 | type FieldNameStyle int 83 | 84 | const ( 85 | FieldNameCamelCase FieldNameStyle = iota 86 | FieldNameSnakeCase 87 | ) 88 | 89 | type FieldOmitBehavior int 90 | 91 | const ( 92 | OmitFieldChooseDefault FieldOmitBehavior = iota 93 | 94 | OmitFieldNever 95 | 96 | OmitFieldAlways 97 | 98 | // An "empty" field is: 99 | // * A container with no contents 100 | // * An empty array 101 | // * A nil pointer 102 | OmitFieldEmpty 103 | 104 | // A zero field ebcompasses OmitFieldEmpty and also omits 105 | // any golang zero value. 106 | OmitFieldZero 107 | ) 108 | -------------------------------------------------------------------------------- /configuration/marshal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | type MarshalConfiguration struct { 24 | EnforceRules bool // If false, disable all rule checks. 25 | } 26 | 27 | func (_this *MarshalConfiguration) init() {} 28 | 29 | var defaultMarshalConfiguration = MarshalConfiguration{ 30 | EnforceRules: true, 31 | } 32 | -------------------------------------------------------------------------------- /configuration/rules.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package configuration 22 | 23 | // ============================================================================ 24 | // Rules 25 | 26 | type RuleConfiguration struct { 27 | MaxDocumentSizeBytes uint64 28 | MaxArraySizeBytes uint64 29 | MaxIdentifierLength uint64 30 | MaxObjectCount uint64 31 | MaxContainerDepth uint64 32 | MaxIntegerDigitCount uint64 33 | MaxFloatCoefficientDigitCount uint64 34 | MaxFloatExponentDigitCount uint64 35 | MaxYearDigitCount uint64 36 | MaxMarkerCount uint64 37 | MaxLocalReferenceCount uint64 38 | AllowRecursiveLocalReferences bool 39 | AutoCompleteTruncatedDocument bool 40 | } 41 | 42 | func (_this *RuleConfiguration) init() {} 43 | 44 | var defaultRuleConfiguration = RuleConfiguration{ 45 | MaxDocumentSizeBytes: 5 * gigabyte, 46 | MaxArraySizeBytes: 1 * gigabyte, 47 | MaxIdentifierLength: 1000, 48 | MaxObjectCount: 1000000, 49 | MaxContainerDepth: 1000, 50 | MaxIntegerDigitCount: 100, 51 | MaxFloatCoefficientDigitCount: 100, 52 | MaxFloatExponentDigitCount: 5, 53 | MaxYearDigitCount: 11, 54 | MaxMarkerCount: 10000, 55 | MaxLocalReferenceCount: 10000, 56 | AllowRecursiveLocalReferences: false, 57 | AutoCompleteTruncatedDocument: false, 58 | } 59 | 60 | const gigabyte = 1024 * 1024 * 1024 61 | -------------------------------------------------------------------------------- /cte/decoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package cte 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "strings" 27 | 28 | "github.com/kstenerud/go-concise-encoding/ce/events" 29 | "github.com/kstenerud/go-concise-encoding/configuration" 30 | ) 31 | 32 | type Decoder struct { 33 | config *configuration.Configuration 34 | } 35 | 36 | // Create a new CTE decoder, which will read from reader and send data events 37 | // to nextReceiver. 38 | func NewDecoder(config *configuration.Configuration) *Decoder { 39 | _this := &Decoder{} 40 | _this.Init(config) 41 | return _this 42 | } 43 | 44 | // Initialize this decoder, which will read from reader and send data events 45 | // to nextReceiver. 46 | func (_this *Decoder) Init(config *configuration.Configuration) { 47 | _this.config = config 48 | } 49 | 50 | func (_this *Decoder) Decode(reader io.Reader, eventReceiver events.DataEventReceiver) (err error) { 51 | defer func() { 52 | if !_this.config.Debug.PassThroughPanics { 53 | if r := recover(); r != nil { 54 | switch v := r.(type) { 55 | case error: 56 | err = v 57 | default: 58 | err = fmt.Errorf("%v", r) 59 | } 60 | } 61 | } 62 | }() 63 | 64 | buf := new(strings.Builder) 65 | if _, err = io.Copy(buf, reader); err != nil { 66 | return 67 | } 68 | 69 | _this.markBytesRead(buf.Len()) 70 | 71 | return ParseDocument(buf.String(), eventReceiver) 72 | } 73 | 74 | func (_this *Decoder) DecodeDocument(document []byte, eventReceiver events.DataEventReceiver) (err error) { 75 | defer func() { 76 | if !_this.config.Debug.PassThroughPanics { 77 | if r := recover(); r != nil { 78 | switch v := r.(type) { 79 | case error: 80 | err = v 81 | default: 82 | err = fmt.Errorf("%v", r) 83 | } 84 | } 85 | } 86 | }() 87 | 88 | _this.markBytesRead(len(document)) 89 | 90 | return ParseDocument(string(document), eventReceiver) 91 | } 92 | 93 | func (_this *Decoder) markBytesRead(byteCount int) { 94 | if uint64(byteCount) > _this.config.Rules.MaxDocumentSizeBytes { 95 | _this.errorf("exceeded maximum document size of %v", _this.config.Rules.MaxDocumentSizeBytes) 96 | } 97 | } 98 | 99 | func (_this *Decoder) errorf(format string, args ...interface{}) { 100 | // TODO: Diagnostics 101 | panic(fmt.Errorf(format, args...)) 102 | } 103 | -------------------------------------------------------------------------------- /cte/escapes.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package cte 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | 27 | "github.com/kstenerud/go-concise-encoding/internal/chars" 28 | ) 29 | 30 | func getEscapeCount(str string) (escapeCount int) { 31 | for _, ch := range str { 32 | if !chars.IsRuneSafeFor(ch, chars.SafetyString) { 33 | escapeCount++ 34 | } 35 | } 36 | return 37 | } 38 | 39 | func getEscapeCountBytes(str []byte) (escapeCount int) { 40 | for _, ch := range string(str) { 41 | if !chars.IsRuneSafeFor(ch, chars.SafetyString) { 42 | escapeCount++ 43 | } 44 | } 45 | return 46 | } 47 | 48 | // ============================================================================ 49 | 50 | func escapeCharQuoted(ch rune) []byte { 51 | switch ch { 52 | case '\t': 53 | return []byte(`\t`) 54 | case '\r': 55 | return []byte(`\r`) 56 | case '\n': 57 | return []byte(`\n`) 58 | case '"': 59 | return []byte(`\"`) 60 | case '*': 61 | return []byte(`\*`) 62 | case '/': 63 | return []byte(`\/`) 64 | case '\\': 65 | return []byte(`\\`) 66 | } 67 | return unicodeEscape(ch) 68 | } 69 | 70 | func unicodeEscape(ch rune) []byte { 71 | return []byte(fmt.Sprintf("\\[%s]", fmt.Sprintf("%x", ch))) 72 | } 73 | 74 | // Ordered from least common to most common, chosen to not be confused by 75 | // a human with other CTE document structural characters. 76 | var verbatimSentinelAlphabet = []byte("~%*+;=^_23456789ZQXJVKBPYGCFMWULDHSNOIRATE10zqxjvkbpygcfmwuldhsnoirate") 77 | 78 | func generateVerbatimSentinelBytes(str []byte) []byte { 79 | // Try all 1, 2, and 3-character sequences picked from a safe alphabet. 80 | 81 | usedChars := [256]bool{} 82 | for _, ch := range str { 83 | usedChars[ch] = true 84 | } 85 | 86 | var sentinelBuff [3]byte 87 | 88 | for _, ch := range verbatimSentinelAlphabet { 89 | if !usedChars[ch] { 90 | return []byte{ch} 91 | } 92 | } 93 | 94 | for _, ch0 := range verbatimSentinelAlphabet { 95 | for _, ch1 := range verbatimSentinelAlphabet { 96 | sentinelBuff[0] = ch0 97 | sentinelBuff[1] = ch1 98 | sentinel := sentinelBuff[:2] 99 | if !bytes.Contains(str, sentinel) { 100 | return sentinel 101 | } 102 | } 103 | } 104 | 105 | for _, ch0 := range verbatimSentinelAlphabet { 106 | for _, ch1 := range verbatimSentinelAlphabet { 107 | for _, ch2 := range verbatimSentinelAlphabet { 108 | sentinelBuff[0] = ch0 109 | sentinelBuff[1] = ch1 110 | sentinelBuff[2] = ch2 111 | sentinel := sentinelBuff[:3] 112 | if !bytes.Contains(str, sentinel) { 113 | return sentinel 114 | } 115 | } 116 | } 117 | } 118 | 119 | // If we're here, all 450,000 three-character sequences have occurred in 120 | // the string. At this point, we conclude that it's a specially crafted 121 | // attack string, and not naturally occurring. 122 | panic(fmt.Errorf("could not generate verbatim sentinel for malicious string [%v]", str)) 123 | } 124 | -------------------------------------------------------------------------------- /cte/parser/CTELexer.tokens: -------------------------------------------------------------------------------- 1 | VERSION=1 2 | CTE_VERSION=2 3 | WSL=3 4 | COMMENT_LINE=4 5 | COMMENT_BLOCK=5 6 | LIST_BEGIN=6 7 | LIST_END=7 8 | MAP_BEGIN=8 9 | MAP_OR_RECORD_END=9 10 | KV_SEPARATOR=10 11 | NODE_BEGIN=11 12 | EDGE_OR_NODE_END=12 13 | NULL=13 14 | TRUE=14 15 | FALSE=15 16 | PINT_BIN=16 17 | NINT_BIN=17 18 | PINT_DEC=18 19 | NINT_DEC=19 20 | PINT_OCT=20 21 | NINT_OCT=21 22 | PINT_HEX=22 23 | NINT_HEX=23 24 | FLOAT_DEC=24 25 | FLOAT_HEX=25 26 | FLOAT_INF=26 27 | FLOAT_NINF=27 28 | FLOAT_NAN=28 29 | FLOAT_SNAN=29 30 | DATE=30 31 | TIME=31 32 | VALUE_UID=32 33 | STRING_BEGIN=33 34 | RREF_BEGIN=34 35 | MARKER=35 36 | REFERENCE=36 37 | RECORD_TYPE_END=37 38 | RID_BEGIN=38 39 | EDGE_BEGIN=39 40 | RECORD_TYPE_BEGIN=40 41 | RECORD_BEGIN=41 42 | ARRAY_TYPE_I8=42 43 | ARRAY_TYPE_I8B=43 44 | ARRAY_TYPE_I8O=44 45 | ARRAY_TYPE_I8X=45 46 | ARRAY_TYPE_I16=46 47 | ARRAY_TYPE_I16B=47 48 | ARRAY_TYPE_I16O=48 49 | ARRAY_TYPE_I16X=49 50 | ARRAY_TYPE_I32=50 51 | ARRAY_TYPE_I32B=51 52 | ARRAY_TYPE_I32O=52 53 | ARRAY_TYPE_I32X=53 54 | ARRAY_TYPE_I64=54 55 | ARRAY_TYPE_I64B=55 56 | ARRAY_TYPE_I64O=56 57 | ARRAY_TYPE_I64X=57 58 | ARRAY_TYPE_U8=58 59 | ARRAY_TYPE_U8B=59 60 | ARRAY_TYPE_U8O=60 61 | ARRAY_TYPE_U8X=61 62 | ARRAY_TYPE_U16=62 63 | ARRAY_TYPE_U16B=63 64 | ARRAY_TYPE_U16O=64 65 | ARRAY_TYPE_U16X=65 66 | ARRAY_TYPE_U32=66 67 | ARRAY_TYPE_U32B=67 68 | ARRAY_TYPE_U32O=68 69 | ARRAY_TYPE_U32X=69 70 | ARRAY_TYPE_U64=70 71 | ARRAY_TYPE_U64B=71 72 | ARRAY_TYPE_U64O=72 73 | ARRAY_TYPE_U64X=73 74 | ARRAY_TYPE_F16=74 75 | ARRAY_TYPE_F16X=75 76 | ARRAY_TYPE_F32=76 77 | ARRAY_TYPE_F32X=77 78 | ARRAY_TYPE_F64=78 79 | ARRAY_TYPE_F64X=79 80 | ARRAY_TYPE_UID=80 81 | ARRAY_TYPE_BIT=81 82 | ARRAY_TYPE_CUSTOM_BIN=82 83 | ARRAY_TYPE_CUSTOM_TXT=83 84 | ARRAY_TYPE_MEDIA_BIN=84 85 | ARRAY_TYPE_MEDIA_TXT=85 86 | ARRAY_I_ELEM_B=86 87 | ARRAY_I_ELEM_O=87 88 | ARRAY_I_ELEM_H=88 89 | ARRAY_I_ELEM_D=89 90 | ARRAY_I_END=90 91 | ARRAY_I_WSL=91 92 | ARRAY_U_ELEM_B=92 93 | ARRAY_U_ELEM_O=93 94 | ARRAY_U_ELEM_H=94 95 | ARRAY_U_ELEM_D=95 96 | ARRAY_U_END=96 97 | ARRAY_U_WSL=97 98 | ARRAY_F_ELEM_D=98 99 | ARRAY_F_ELEM_H=99 100 | ARRAY_F_NAN=100 101 | ARRAY_F_SNAN=101 102 | ARRAY_F_INF=102 103 | ARRAY_F_NINF=103 104 | ARRAY_F_END=104 105 | ARRAY_F_WSL=105 106 | ARRAY_F_X_ELEM=106 107 | ARRAY_F_X_NAN=107 108 | ARRAY_F_X_SNAN=108 109 | ARRAY_F_X_INF=109 110 | ARRAY_F_X_NINF=110 111 | ARRAY_F_X_END=111 112 | ARRAY_F_X_WSL=112 113 | ARRAY_I_B_ELEM=113 114 | ARRAY_I_B_END=114 115 | ARRAY_I_B_WSL=115 116 | ARRAY_I_O_ELEM=116 117 | ARRAY_I_O_END=117 118 | ARRAY_I_O_WSL=118 119 | ARRAY_I_X_ELEM=119 120 | ARRAY_I_X_END=120 121 | ARRAY_I_X_WSL=121 122 | ARRAY_U_B_ELEM=122 123 | ARRAY_U_B_END=123 124 | ARRAY_U_B_WSL=124 125 | ARRAY_U_O_ELEM=125 126 | ARRAY_U_O_END=126 127 | ARRAY_U_O_WSL=127 128 | ARRAY_U_X_ELEM=128 129 | ARRAY_U_X_END=129 130 | ARRAY_U_X_WSL=130 131 | ARRAY_UID_ELEM=131 132 | ARRAY_UID_END=132 133 | ARRAY_UID_WSL=133 134 | ARRAY_BIT_BITS=134 135 | ARRAY_BIT_END=135 136 | ARRAY_BIT_WSL=136 137 | BYTES_ELEM=137 138 | BYTES_END=138 139 | BYTES_WS=139 140 | STRING_END=140 141 | STRING_ESCAPE=141 142 | STRING_CONTENTS=142 143 | VERBATIM_INIT=143 144 | CODEPOINT_INIT=144 145 | CONTINUATION=145 146 | ESCAPE_CHAR=146 147 | VERBATIM_SENTINEL=147 148 | VERBATIM_SEPARATOR=148 149 | VERBATIM_EMPTY=149 150 | VERBATIM_CONTENTS=150 151 | VERBATIM_END=151 152 | CODEPOINT=152 153 | ']'=7 154 | '{'=8 155 | '}'=9 156 | '='=10 157 | '('=11 158 | ')'=12 159 | '$"'=34 160 | '>'=37 161 | '@"'=38 162 | '@('=39 163 | '\\'=141 164 | '.'=143 165 | -------------------------------------------------------------------------------- /cte/parser/CTEParser.tokens: -------------------------------------------------------------------------------- 1 | VERSION=1 2 | CTE_VERSION=2 3 | WSL=3 4 | COMMENT_LINE=4 5 | COMMENT_BLOCK=5 6 | LIST_BEGIN=6 7 | LIST_END=7 8 | MAP_BEGIN=8 9 | MAP_OR_RECORD_END=9 10 | KV_SEPARATOR=10 11 | NODE_BEGIN=11 12 | EDGE_OR_NODE_END=12 13 | NULL=13 14 | TRUE=14 15 | FALSE=15 16 | PINT_BIN=16 17 | NINT_BIN=17 18 | PINT_DEC=18 19 | NINT_DEC=19 20 | PINT_OCT=20 21 | NINT_OCT=21 22 | PINT_HEX=22 23 | NINT_HEX=23 24 | FLOAT_DEC=24 25 | FLOAT_HEX=25 26 | FLOAT_INF=26 27 | FLOAT_NINF=27 28 | FLOAT_NAN=28 29 | FLOAT_SNAN=29 30 | DATE=30 31 | TIME=31 32 | VALUE_UID=32 33 | STRING_BEGIN=33 34 | RREF_BEGIN=34 35 | MARKER=35 36 | REFERENCE=36 37 | RECORD_TYPE_END=37 38 | RID_BEGIN=38 39 | EDGE_BEGIN=39 40 | RECORD_TYPE_BEGIN=40 41 | RECORD_BEGIN=41 42 | ARRAY_TYPE_I8=42 43 | ARRAY_TYPE_I8B=43 44 | ARRAY_TYPE_I8O=44 45 | ARRAY_TYPE_I8X=45 46 | ARRAY_TYPE_I16=46 47 | ARRAY_TYPE_I16B=47 48 | ARRAY_TYPE_I16O=48 49 | ARRAY_TYPE_I16X=49 50 | ARRAY_TYPE_I32=50 51 | ARRAY_TYPE_I32B=51 52 | ARRAY_TYPE_I32O=52 53 | ARRAY_TYPE_I32X=53 54 | ARRAY_TYPE_I64=54 55 | ARRAY_TYPE_I64B=55 56 | ARRAY_TYPE_I64O=56 57 | ARRAY_TYPE_I64X=57 58 | ARRAY_TYPE_U8=58 59 | ARRAY_TYPE_U8B=59 60 | ARRAY_TYPE_U8O=60 61 | ARRAY_TYPE_U8X=61 62 | ARRAY_TYPE_U16=62 63 | ARRAY_TYPE_U16B=63 64 | ARRAY_TYPE_U16O=64 65 | ARRAY_TYPE_U16X=65 66 | ARRAY_TYPE_U32=66 67 | ARRAY_TYPE_U32B=67 68 | ARRAY_TYPE_U32O=68 69 | ARRAY_TYPE_U32X=69 70 | ARRAY_TYPE_U64=70 71 | ARRAY_TYPE_U64B=71 72 | ARRAY_TYPE_U64O=72 73 | ARRAY_TYPE_U64X=73 74 | ARRAY_TYPE_F16=74 75 | ARRAY_TYPE_F16X=75 76 | ARRAY_TYPE_F32=76 77 | ARRAY_TYPE_F32X=77 78 | ARRAY_TYPE_F64=78 79 | ARRAY_TYPE_F64X=79 80 | ARRAY_TYPE_UID=80 81 | ARRAY_TYPE_BIT=81 82 | ARRAY_TYPE_CUSTOM_BIN=82 83 | ARRAY_TYPE_CUSTOM_TXT=83 84 | ARRAY_TYPE_MEDIA_BIN=84 85 | ARRAY_TYPE_MEDIA_TXT=85 86 | ARRAY_I_ELEM_B=86 87 | ARRAY_I_ELEM_O=87 88 | ARRAY_I_ELEM_H=88 89 | ARRAY_I_ELEM_D=89 90 | ARRAY_I_END=90 91 | ARRAY_I_WSL=91 92 | ARRAY_U_ELEM_B=92 93 | ARRAY_U_ELEM_O=93 94 | ARRAY_U_ELEM_H=94 95 | ARRAY_U_ELEM_D=95 96 | ARRAY_U_END=96 97 | ARRAY_U_WSL=97 98 | ARRAY_F_ELEM_D=98 99 | ARRAY_F_ELEM_H=99 100 | ARRAY_F_NAN=100 101 | ARRAY_F_SNAN=101 102 | ARRAY_F_INF=102 103 | ARRAY_F_NINF=103 104 | ARRAY_F_END=104 105 | ARRAY_F_WSL=105 106 | ARRAY_F_X_ELEM=106 107 | ARRAY_F_X_NAN=107 108 | ARRAY_F_X_SNAN=108 109 | ARRAY_F_X_INF=109 110 | ARRAY_F_X_NINF=110 111 | ARRAY_F_X_END=111 112 | ARRAY_F_X_WSL=112 113 | ARRAY_I_B_ELEM=113 114 | ARRAY_I_B_END=114 115 | ARRAY_I_B_WSL=115 116 | ARRAY_I_O_ELEM=116 117 | ARRAY_I_O_END=117 118 | ARRAY_I_O_WSL=118 119 | ARRAY_I_X_ELEM=119 120 | ARRAY_I_X_END=120 121 | ARRAY_I_X_WSL=121 122 | ARRAY_U_B_ELEM=122 123 | ARRAY_U_B_END=123 124 | ARRAY_U_B_WSL=124 125 | ARRAY_U_O_ELEM=125 126 | ARRAY_U_O_END=126 127 | ARRAY_U_O_WSL=127 128 | ARRAY_U_X_ELEM=128 129 | ARRAY_U_X_END=129 130 | ARRAY_U_X_WSL=130 131 | ARRAY_UID_ELEM=131 132 | ARRAY_UID_END=132 133 | ARRAY_UID_WSL=133 134 | ARRAY_BIT_BITS=134 135 | ARRAY_BIT_END=135 136 | ARRAY_BIT_WSL=136 137 | BYTES_ELEM=137 138 | BYTES_END=138 139 | BYTES_WS=139 140 | STRING_END=140 141 | STRING_ESCAPE=141 142 | STRING_CONTENTS=142 143 | VERBATIM_INIT=143 144 | CODEPOINT_INIT=144 145 | CONTINUATION=145 146 | ESCAPE_CHAR=146 147 | VERBATIM_SENTINEL=147 148 | VERBATIM_SEPARATOR=148 149 | VERBATIM_EMPTY=149 150 | VERBATIM_CONTENTS=150 151 | VERBATIM_END=151 152 | CODEPOINT=152 153 | ']'=7 154 | '{'=8 155 | '}'=9 156 | '='=10 157 | '('=11 158 | ')'=12 159 | '$"'=34 160 | '>'=37 161 | '@"'=38 162 | '@('=39 163 | '\\'=141 164 | '.'=143 165 | -------------------------------------------------------------------------------- /docs.go: -------------------------------------------------------------------------------- 1 | // Package concise_encoding implements a concise encoding codec and marshaler. 2 | // https://concise-encoding.org/ 3 | // 4 | // Most people will only care about the highest level APIs for mashaling in 5 | // package ce. 6 | // 7 | // If all you're interested in is (de)serializing to go objects, the marshaler 8 | // API is sufficient. The codecs provide more control over the process, and 9 | // can handle more data types (such as comments). The event 10 | // handlers, builders and iterators are the lowest level API, providing maximum 11 | // control but the highest complexity. 12 | // 13 | // The primary architecture design is one of filtered message pipelines, 14 | // consisting of data events (which report what kind of data is encountered), 15 | // and builder directives (which direct the parts of a complex data structure 16 | // that is to be built). All software components are designed around this 17 | // principle to promote interchangeability. 18 | // 19 | // 20 | // High Level API (package ce) 21 | // 22 | // * Marshalers: (de)serializes to/from go objects. 23 | // 24 | // 25 | // Medium Level API (package ce) 26 | // 27 | // * Encoder: Accepts data events and generates a CBE or CTE encoded document. 28 | // 29 | // * Decoder: Decodes a CBE or CTE document, generating data events 30 | // 31 | // 32 | // Low Level API (packages builder, events, iterator, rules) 33 | // 34 | // * Iterator: Iterates through an object, generating data events. 35 | // 36 | // * DataEventReceiver: Receives data events and acts upon them. 37 | // 38 | // * Builder: DataEventReceiver that builds objects in response to events. 39 | // 40 | // * Rules: DataEventReceiver that validates events, ensuring their contents 41 | // and order match a valid CBE/CTE document. 42 | package concise_encoding 43 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kstenerud/go-concise-encoding 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 7 | github.com/cockroachdb/apd/v2 v2.0.2 8 | github.com/kstenerud/go-compact-float v1.6.1 9 | github.com/kstenerud/go-compact-time v1.8.3 10 | github.com/kstenerud/go-describe v1.2.15 11 | github.com/kstenerud/go-duplicates v1.1.1 12 | github.com/kstenerud/go-equivalence v1.0.4 13 | github.com/kstenerud/go-uleb128 v1.1.0 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= 2 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= 3 | github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= 4 | github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= 5 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 6 | github.com/kstenerud/go-compact-float v1.6.1 h1:FhRID85bjfOSKrGkHEKgwSZTYPXoMxaiH/L7TsvpikA= 7 | github.com/kstenerud/go-compact-float v1.6.1/go.mod h1:86Q5GkzClM2g0hObQndeX90g5kKMhG7eptDFsVrubbk= 8 | github.com/kstenerud/go-compact-time v1.8.3 h1:VPhN2UwJsJjAHnHbGuqLibNF8e7PXG4nBFpZ3UkjsRg= 9 | github.com/kstenerud/go-compact-time v1.8.3/go.mod h1:wBgxWliL/ON5b1/EaefPqQ44u2UgTtiNiIvkmByYFFc= 10 | github.com/kstenerud/go-describe v1.2.13/go.mod h1:+B7K/fXHhObqwmMhcUT0CPT68fR1SQ9XNmJ5E4ck2RI= 11 | github.com/kstenerud/go-describe v1.2.15 h1:EVa8A9rerri6NGih4Koleg1cadp4wtzSLom802jnMGY= 12 | github.com/kstenerud/go-describe v1.2.15/go.mod h1:+B7K/fXHhObqwmMhcUT0CPT68fR1SQ9XNmJ5E4ck2RI= 13 | github.com/kstenerud/go-duplicates v1.1.1 h1:Z8pdWHv842d/vszSeoWGAswbxx/8SHxz+/DNYQUl4jU= 14 | github.com/kstenerud/go-duplicates v1.1.1/go.mod h1:OoPQ8B7sw/LFlPpLJT6bbCrHdOtaC9yWBF1uZXqOoxw= 15 | github.com/kstenerud/go-equivalence v1.0.4 h1:gbyVM157tVSlYSbCpzi6AZnXLugmetvYdvxUBnrEqmU= 16 | github.com/kstenerud/go-equivalence v1.0.4/go.mod h1:FKUKQCiRiJ8M0rgXy/0jR+yIet5mydc2zyj6XQ1PK60= 17 | github.com/kstenerud/go-uleb128 v1.1.0 h1:QrNuuZVFkNvn3tEEJHDSUZRckNZt1haE23uYPKJ8qAA= 18 | github.com/kstenerud/go-uleb128 v1.1.0/go.mod h1:0DgQsdpP8sW2B3Qd2yx3vO7oaGNqDUZW7h4PoKIQj14= 19 | github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= 20 | github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 21 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 22 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 23 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 24 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 25 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 26 | golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= 27 | golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= 28 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= 29 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 30 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 31 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 32 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 33 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 34 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 35 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 37 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 38 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 39 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 40 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 41 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 42 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 43 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 44 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 45 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 46 | golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= 47 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 48 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 49 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 50 | -------------------------------------------------------------------------------- /internal/arrays/arrays_purego.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | //go:build purego 22 | // +build purego 23 | 24 | package arrays 25 | 26 | func Uint8SliceAsBytes(data []uint8) []byte { return []byte(data) } 27 | 28 | func Int8SliceAsBytes(data []int8) []byte { return int8SliceToBytes(data) } 29 | 30 | func Uint16SliceAsBytes(data []uint16) []byte { return uint16SliceToBytes(data) } 31 | 32 | func Int16SliceAsBytes(data []int16) []byte { return int16SliceToBytes(data) } 33 | 34 | func Float16SliceAsBytes(data []float32) []byte { return float16SliceToBytes(data) } 35 | 36 | func Uint32SliceAsBytes(data []uint32) []byte { return uint32SliceToBytes(data) } 37 | 38 | func Int32SliceAsBytes(data []int32) []byte { return int32SliceToBytes(data) } 39 | 40 | func Float32SliceAsBytes(data []float32) []byte { return float32SliceToBytes(data) } 41 | 42 | func Uint64SliceAsBytes(data []uint64) []byte { return uint64SliceToBytes(data) } 43 | 44 | func Int64SliceAsBytes(data []int64) []byte { return int64SliceToBytes(data) } 45 | 46 | func Float64SliceAsBytes(data []float64) []byte { return float64SliceToBytes(data) } 47 | 48 | func BytesToInt8Slice(data []byte) []int8 { return bytesToInt8Slice(data) } 49 | 50 | func BytesToUint16Slice(data []byte) []uint16 { return bytesToUint16Slice(data) } 51 | 52 | func BytesToInt16Slice(data []byte) []int16 { return bytesToInt16Slice(data) } 53 | 54 | func BytesToFloat16Slice(data []byte) []float32 { return bytesToFloat16Slice(data) } 55 | 56 | func BytesToUint32Slice(data []byte) []uint32 { return bytesToUint32Slice(data) } 57 | 58 | func BytesToInt32Slice(data []byte) []int32 { return bytesToInt32Slice(data) } 59 | 60 | func BytesToFloat32Slice(data []byte) []float32 { return bytesToFloat32Slice(data) } 61 | 62 | func BytesToUint64Slice(data []byte) []uint64 { return bytesToUint64Slice(data) } 63 | 64 | func BytesToInt64Slice(data []byte) []int64 { return bytesToInt64Slice(data) } 65 | 66 | func BytesToFloat64Slice(data []byte) []float64 { return bytesToFloat64Slice(data) } 67 | 68 | func UUIDSliceAsBytes(data [][]byte) []byte { return uuidSliceToBytes(data) } 69 | 70 | func BytesToUUIDSlice(data []byte) [][]byte { return bytesToUUIDSlice(data) } 71 | -------------------------------------------------------------------------------- /internal/chars/chars.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package chars 22 | 23 | import ( 24 | "unicode/utf8" 25 | ) 26 | 27 | const EOFMarker = 0x100 28 | 29 | func CalculateRuneByteCount(startByte byte) int { 30 | return int(runeByteCounts[startByte>>3]) 31 | } 32 | 33 | // Returns the index of the start of the last UTF-8 rune in data, and whether 34 | // it's complete or not. 35 | // 36 | // If data is empty, returns (0, true) 37 | // If there are no rune starts, returns (0, false) 38 | // If there is a rune start, returns (index-of-rune-start, is-a-complete-rune) 39 | func IndexOfLastRuneStart(data []byte) (index int, isCompleteRune bool) { 40 | dataLength := len(data) 41 | if dataLength == 0 { 42 | return 0, true 43 | } 44 | 45 | for index = dataLength - 1; index >= 0; index-- { 46 | runeByteCount := CalculateRuneByteCount(data[index]) 47 | if runeByteCount > 0 { 48 | isCompleteRune = index+runeByteCount == dataLength 49 | return 50 | } 51 | } 52 | return 53 | } 54 | 55 | func (_this Properties) HasProperty(property Properties) bool { 56 | return _this&property != 0 57 | } 58 | 59 | func IsRuneSafeFor(r rune, flags SafetyFlags) bool { 60 | if getBitArrayValue(stringlikeSafe[:], int(r)) { 61 | return true 62 | } 63 | unsafety := stringlikeUnsafe[r] 64 | if unsafety == 0 { 65 | // unsafety 0 actually means "all". when the corresponding stringlikeSafe 66 | // entry indicates "unsafe". This keeps the stringlikeUnsafe map small. 67 | unsafety = SafetyAll 68 | } 69 | return unsafety&flags == 0 70 | } 71 | 72 | func IsIdentifierSafe(str []byte) bool { 73 | if len(str) == 0 { 74 | return false 75 | } 76 | for len(str) > 0 { 77 | r, size := utf8.DecodeRune(str) 78 | if !IsRuneValidIdentifier(r) { 79 | return false 80 | } 81 | str = str[size:] 82 | } 83 | return true 84 | } 85 | 86 | func ByteHasProperty(b byte, property Properties) bool { 87 | return properties[b].HasProperty(property) 88 | } 89 | 90 | type Byte uint8 91 | 92 | func (_this Byte) HasProperty(property Properties) bool { 93 | return properties[_this].HasProperty(property) 94 | } 95 | 96 | type ByteWithEOF uint16 97 | 98 | func (_this ByteWithEOF) HasProperty(property Properties) bool { 99 | return properties[_this].HasProperty(property) 100 | } 101 | 102 | func IsRuneValidIdentifier(r rune) bool { 103 | return getBitArrayValue(identifierSafe[:], int(r)) 104 | } 105 | 106 | func getBitArrayValue(array []byte, index int) bool { 107 | bits := array[index>>3] 108 | return bits&(1<<(index&7)) != 0 109 | } 110 | 111 | var HexCharValues = [256]byte{} 112 | 113 | const InvalidHexChar = byte(0x80) 114 | 115 | func init() { 116 | for i := 0; i < len(HexCharValues); i++ { 117 | HexCharValues[i] = InvalidHexChar 118 | } 119 | for i := '0'; i <= '9'; i++ { 120 | HexCharValues[i] = byte(i - '0') 121 | } 122 | for i := 'A'; i <= 'F'; i++ { 123 | HexCharValues[i] = byte(i - 'A' + 10) 124 | } 125 | for i := 'a'; i <= 'f'; i++ { 126 | HexCharValues[i] = byte(i - 'a' + 10) 127 | } 128 | } 129 | 130 | var HexChars = "0123456789abcdef" 131 | 132 | const DigitBase16 = DigitBase10 | LowerAF | UpperAF 133 | -------------------------------------------------------------------------------- /internal/common/go_tags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | "fmt" 25 | "math" 26 | "reflect" 27 | "strconv" 28 | "strings" 29 | 30 | "github.com/kstenerud/go-concise-encoding/configuration" 31 | ) 32 | 33 | func DecodeGoTags(field reflect.StructField) (tags GoTags) { 34 | requiresValue := func(kv []string, key string) { 35 | if len(kv) != 2 { 36 | panic(fmt.Errorf(`tag key "%s" requires a value`, key)) 37 | } 38 | } 39 | 40 | tags.Name = field.Name 41 | tags.Order = math.MaxInt64 42 | 43 | tagString := strings.TrimSpace(field.Tag.Get("ce")) 44 | if len(tagString) == 0 { 45 | return 46 | } 47 | 48 | for _, entry := range strings.Split(tagString, ",") { 49 | kv := strings.Split(entry, "=") 50 | switch strings.TrimSpace(kv[0]) { 51 | /* TODO: 52 | * - lowercase/origcase 53 | * - omit specific value? 54 | * - recurse/no_recurse? 55 | * - type=f16, f10.x, i2, i8, i10, i16, string, vstring? 56 | */ 57 | case "omit": 58 | tags.OmitBehavior = configuration.OmitFieldAlways 59 | case "omit_empty": 60 | tags.OmitBehavior = configuration.OmitFieldEmpty 61 | case "omit_zero": 62 | tags.OmitBehavior = configuration.OmitFieldZero 63 | case "omit_never": 64 | tags.OmitBehavior = configuration.OmitFieldNever 65 | case "name": 66 | requiresValue(kv, "name") 67 | tags.Name = strings.TrimSpace(kv[1]) 68 | case "order": 69 | order, err := strconv.ParseInt(strings.TrimSpace(kv[1]), 10, 64) 70 | if err != nil { 71 | panic(err) 72 | } 73 | tags.Order = order 74 | default: 75 | panic(fmt.Errorf("%v: Unknown Concise Encoding struct tag field decoding [%v] in field %v", entry, tagString, field.Name)) 76 | } 77 | } 78 | 79 | return 80 | } 81 | 82 | type GoTags struct { 83 | Name string 84 | OmitBehavior configuration.FieldOmitBehavior 85 | Order int64 86 | } 87 | -------------------------------------------------------------------------------- /internal/common/golang_1_12_pre.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | //go:build !go1.12 22 | // +build !go1.12 23 | 24 | package common 25 | 26 | import ( 27 | "reflect" 28 | ) 29 | 30 | type mapIter struct { 31 | mapInstance reflect.Value 32 | keys []reflect.Value 33 | index int 34 | } 35 | 36 | func (_this *mapIter) Key() reflect.Value { 37 | return _this.keys[_this.index] 38 | } 39 | 40 | func (_this *mapIter) Value() reflect.Value { 41 | return _this.mapInstance.MapIndex(_this.Key()) 42 | } 43 | 44 | func (_this *mapIter) Next() bool { 45 | _this.index++ 46 | return _this.index < len(_this.keys) 47 | } 48 | 49 | func MapRange(v reflect.Value) *mapIter { 50 | return &mapIter{ 51 | mapInstance: v, 52 | keys: v.MapKeys(), 53 | index: -1, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /internal/common/golang_1_12_up.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | //go:build go1.12 22 | // +build go1.12 23 | 24 | package common 25 | 26 | import ( 27 | "reflect" 28 | ) 29 | 30 | func MapRange(v reflect.Value) *reflect.MapIter { 31 | return v.MapRange() 32 | } 33 | -------------------------------------------------------------------------------- /iterator/context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package iterator 22 | 23 | import ( 24 | "reflect" 25 | "sort" 26 | 27 | "github.com/kstenerud/go-concise-encoding/ce/events" 28 | "github.com/kstenerud/go-concise-encoding/configuration" 29 | ) 30 | 31 | // Common function signatures 32 | type GetIteratorForType func(reflect.Type) IteratorFunction 33 | type TryAddLocalReference func(reflect.Value) (didGenerateReferenceEvent bool) 34 | 35 | type Context struct { 36 | // Per-session data 37 | GetIteratorForType GetIteratorForType 38 | Configuration *configuration.Configuration 39 | RecordTypeOrder []recordTypeEntry 40 | 41 | // Per-root-iterator data 42 | EventReceiver events.DataEventReceiver 43 | TryAddLocalReference TryAddLocalReference 44 | } 45 | 46 | func (_this *Context) NotifyNil() { 47 | _this.EventReceiver.OnNull() 48 | } 49 | 50 | type recordTypeEntry struct { 51 | Name string 52 | Type reflect.Type 53 | Iterator IteratorFunction 54 | } 55 | 56 | func sessionContext(getIteratorFunc GetIteratorForType, config *configuration.Configuration) Context { 57 | orderedEntries := make([]recordTypeEntry, 0, len(config.Iterator.RecordTypes)) 58 | for rtype, name := range config.Iterator.RecordTypes { 59 | orderedEntries = append(orderedEntries, recordTypeEntry{ 60 | Name: name, 61 | Type: rtype, 62 | Iterator: nil, // Will be set in Session.Init 63 | }) 64 | } 65 | sort.SliceStable(orderedEntries, func(i, j int) bool { 66 | // TODO: case where names are the same 67 | return orderedEntries[i].Name < orderedEntries[j].Name 68 | }) 69 | 70 | return Context{ 71 | GetIteratorForType: getIteratorFunc, 72 | Configuration: config, 73 | RecordTypeOrder: orderedEntries, 74 | } 75 | } 76 | 77 | func iteratorContext(sessionContext *Context, 78 | eventReceiver events.DataEventReceiver, 79 | tryAddLocalReference TryAddLocalReference) Context { 80 | 81 | return Context{ 82 | GetIteratorForType: sessionContext.GetIteratorForType, 83 | Configuration: sessionContext.Configuration, 84 | RecordTypeOrder: sessionContext.RecordTypeOrder, 85 | EventReceiver: eventReceiver, 86 | TryAddLocalReference: tryAddLocalReference, 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /iterator/iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Iterators iterate through go objects, producing data events. 22 | package iterator 23 | 24 | import ( 25 | "reflect" 26 | ) 27 | 28 | // Iterator interface. All iterators follow this signature. 29 | type IteratorFunction func(context *Context, value reflect.Value) 30 | -------------------------------------------------------------------------------- /iterator/iterator_root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package iterator 22 | 23 | import ( 24 | "reflect" 25 | "strconv" 26 | 27 | "github.com/kstenerud/go-concise-encoding/ce/events" 28 | "github.com/kstenerud/go-concise-encoding/configuration" 29 | "github.com/kstenerud/go-concise-encoding/version" 30 | "github.com/kstenerud/go-duplicates" 31 | ) 32 | 33 | // RootObjectIterator acts as a top-level iterator, coordinating iteration 34 | // through an arbitrary object via sub-iterators. 35 | // 36 | // Note: This is a LOW LEVEL API. Error reporting is done via panics. Be sure 37 | // to recover() at an appropriate location when calling this struct's methods 38 | // directly (with the exception of constructors and initializers, which are not 39 | // designed to panic). 40 | type RootObjectIterator struct { 41 | foundReferences map[duplicates.TypedPointer]bool 42 | namedReferences map[duplicates.TypedPointer]uint32 43 | nextMarkerName uint32 44 | context Context 45 | config *configuration.Configuration 46 | referenceIdBuff []byte 47 | } 48 | 49 | // Create a new root object iterator that will send data events to eventReceiver. 50 | func NewRootObjectIterator(context *Context, 51 | eventReceiver events.DataEventReceiver, 52 | config *configuration.Configuration) *RootObjectIterator { 53 | 54 | _this := &RootObjectIterator{} 55 | _this.Init(context, eventReceiver, config) 56 | return _this 57 | } 58 | 59 | // Initialize this iterator to send data events to eventReceiver. 60 | func (_this *RootObjectIterator) Init(context *Context, 61 | eventReceiver events.DataEventReceiver, 62 | config *configuration.Configuration) { 63 | 64 | _this.config = config 65 | _this.context = iteratorContext(context, 66 | eventReceiver, 67 | _this.addLocalReference) 68 | _this.referenceIdBuff = make([]byte, 0, 16) 69 | } 70 | 71 | // Iterates over an object, sending events to the root iterator's 72 | // DataEventReceiver as it visits all elements of the value. 73 | // 74 | // Note: This is a LOW LEVEL API. Error reporting is done via panics. Be sure 75 | // to recover() at an appropriate location when calling this function. 76 | func (_this *RootObjectIterator) Iterate(object interface{}) { 77 | _this.context.EventReceiver.OnBeginDocument() 78 | _this.context.EventReceiver.OnVersion(version.ConciseEncodingVersion) 79 | 80 | if object == nil { 81 | _this.context.NotifyNil() 82 | _this.context.EventReceiver.OnEndDocument() 83 | return 84 | } 85 | 86 | if _this.config.Iterator.RecursionSupport { 87 | _this.foundReferences = duplicates.FindDuplicatePointers(object) 88 | _this.namedReferences = make(map[duplicates.TypedPointer]uint32) 89 | } 90 | 91 | // Generate all record types at the top of the document 92 | for _, entry := range _this.context.RecordTypeOrder { 93 | entry.Iterator(&_this.context, reflect.ValueOf(nil)) 94 | } 95 | 96 | rv := reflect.ValueOf(object) 97 | iterate := _this.context.GetIteratorForType(rv.Type()) 98 | iterate(&_this.context, rv) 99 | _this.context.EventReceiver.OnEndDocument() 100 | } 101 | 102 | // ============================================================================ 103 | // Internal 104 | 105 | func (_this *RootObjectIterator) getNamedLocalReference(ptr duplicates.TypedPointer) (name []byte, exists bool) { 106 | num, exists := _this.namedReferences[ptr] 107 | if !exists { 108 | num = _this.nextMarkerName 109 | _this.namedReferences[ptr] = num 110 | _this.nextMarkerName++ 111 | } 112 | 113 | _this.referenceIdBuff = strconv.AppendInt(_this.referenceIdBuff[:0], int64(num), 10) 114 | return _this.referenceIdBuff, exists 115 | } 116 | 117 | func (_this *RootObjectIterator) addLocalReference(v reflect.Value) (didGenerateReferenceEvent bool) { 118 | if !_this.config.Iterator.RecursionSupport { 119 | return false 120 | } 121 | 122 | ptr := duplicates.TypedPointerOfRV(v) 123 | if !_this.foundReferences[ptr] { 124 | return false 125 | } 126 | 127 | name, exists := _this.getNamedLocalReference(ptr) 128 | if !exists { 129 | _this.context.EventReceiver.OnMarker(name) 130 | return false 131 | } 132 | 133 | _this.context.EventReceiver.OnReferenceLocal(name) 134 | return true 135 | } 136 | -------------------------------------------------------------------------------- /nullevent/null_event_receiver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package nullevent 22 | 23 | import ( 24 | "math/big" 25 | 26 | "github.com/cockroachdb/apd/v2" 27 | compact_float "github.com/kstenerud/go-compact-float" 28 | compact_time "github.com/kstenerud/go-compact-time" 29 | "github.com/kstenerud/go-concise-encoding/ce/events" 30 | ) 31 | 32 | // NullEventReceiver receives events and does nothing with them. 33 | type NullEventReceiver struct{} 34 | 35 | func NewNullEventReceiver() *NullEventReceiver { 36 | return &NullEventReceiver{} 37 | } 38 | func (_this *NullEventReceiver) OnBeginDocument() {} 39 | func (_this *NullEventReceiver) OnVersion(uint64) {} 40 | func (_this *NullEventReceiver) OnComment(bool, []byte) {} 41 | func (_this *NullEventReceiver) OnPadding() {} 42 | func (_this *NullEventReceiver) OnNull() {} 43 | func (_this *NullEventReceiver) OnBoolean(bool) {} 44 | func (_this *NullEventReceiver) OnTrue() {} 45 | func (_this *NullEventReceiver) OnFalse() {} 46 | func (_this *NullEventReceiver) OnPositiveInt(uint64) {} 47 | func (_this *NullEventReceiver) OnNegativeInt(uint64) {} 48 | func (_this *NullEventReceiver) OnInt(int64) {} 49 | func (_this *NullEventReceiver) OnBigInt(*big.Int) {} 50 | func (_this *NullEventReceiver) OnFloat(float64) {} 51 | func (_this *NullEventReceiver) OnBigFloat(*big.Float) {} 52 | func (_this *NullEventReceiver) OnDecimalFloat(compact_float.DFloat) {} 53 | func (_this *NullEventReceiver) OnBigDecimalFloat(*apd.Decimal) {} 54 | func (_this *NullEventReceiver) OnNan(bool) {} 55 | func (_this *NullEventReceiver) OnUID([]byte) {} 56 | func (_this *NullEventReceiver) OnTime(compact_time.Time) {} 57 | func (_this *NullEventReceiver) OnArray(events.ArrayType, uint64, []byte) {} 58 | func (_this *NullEventReceiver) OnStringlikeArray(events.ArrayType, string) {} 59 | func (_this *NullEventReceiver) OnMedia(string, []byte) {} 60 | func (_this *NullEventReceiver) OnCustomBinary(uint64, []byte) {} 61 | func (_this *NullEventReceiver) OnCustomText(uint64, string) {} 62 | func (_this *NullEventReceiver) OnArrayBegin(events.ArrayType) {} 63 | func (_this *NullEventReceiver) OnMediaBegin(string) {} 64 | func (_this *NullEventReceiver) OnCustomBegin(events.ArrayType, uint64) {} 65 | func (_this *NullEventReceiver) OnArrayChunk(uint64, bool) {} 66 | func (_this *NullEventReceiver) OnArrayData([]byte) {} 67 | func (_this *NullEventReceiver) OnList() {} 68 | func (_this *NullEventReceiver) OnMap() {} 69 | func (_this *NullEventReceiver) OnRecordType([]byte) {} 70 | func (_this *NullEventReceiver) OnRecord([]byte) {} 71 | func (_this *NullEventReceiver) OnEdge() {} 72 | func (_this *NullEventReceiver) OnNode() {} 73 | func (_this *NullEventReceiver) OnEndContainer() {} 74 | func (_this *NullEventReceiver) OnMarker([]byte) {} 75 | func (_this *NullEventReceiver) OnReferenceLocal([]byte) {} 76 | func (_this *NullEventReceiver) OnEndDocument() {} 77 | func (_this *NullEventReceiver) OnError() {} 78 | -------------------------------------------------------------------------------- /rules/rules.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Imposes the structural rules that enforce a well-formed concise encoding 22 | // document. 23 | package rules 24 | 25 | import ( 26 | "fmt" 27 | 28 | "github.com/kstenerud/go-concise-encoding/ce/events" 29 | ) 30 | 31 | const maxMarkerIDRuneCount = 50 32 | const maxMarkerIDByteCount = 4 * maxMarkerIDRuneCount // max 4 bytes per rune 33 | 34 | type negint uint64 35 | type rid string 36 | 37 | type EventRule interface { 38 | OnBeginDocument(ctx *Context) 39 | OnEndDocument(ctx *Context) 40 | OnChildContainerEnded(ctx *Context, containerType DataType) 41 | OnVersion(ctx *Context, version uint64) 42 | OnPadding(ctx *Context) 43 | OnComment(ctx *Context) 44 | OnKeyableObject(ctx *Context, objType DataType, key interface{}) 45 | OnNonKeyableObject(ctx *Context, objType DataType) 46 | OnNull(ctx *Context) 47 | OnList(ctx *Context) 48 | OnMap(ctx *Context) 49 | OnRecordType(ctx *Context, identifier []byte) 50 | OnRecord(ctx *Context, identifier []byte) 51 | OnEdge(ctx *Context) 52 | OnNode(ctx *Context) 53 | OnEnd(ctx *Context) 54 | OnMarker(ctx *Context, identifier []byte) 55 | OnReferenceLocal(ctx *Context, identifier []byte) 56 | OnArray(ctx *Context, arrayType events.ArrayType, elementCount uint64, data []uint8) 57 | OnStringlikeArray(ctx *Context, arrayType events.ArrayType, data string) 58 | OnArrayBegin(ctx *Context, arrayType events.ArrayType) 59 | OnArrayChunk(ctx *Context, length uint64, moreChunksFollow bool) 60 | OnArrayData(ctx *Context, data []byte) 61 | } 62 | 63 | var ( 64 | beginDocumentRule BeginDocumentRule 65 | endDocumentRule EndDocumentRule 66 | terminalRule TerminalRule 67 | versionRule VersionRule 68 | topLevelRule TopLevelRule 69 | listRule ListRule 70 | mapKeyRule MapKeyRule 71 | mapValueRule MapValueRule 72 | recordTypeRule RecordTypeRule 73 | recordRule RecordRule 74 | arrayRule ArrayRule 75 | arrayChunkRule ArrayChunkRule 76 | stringRule StringRule 77 | stringChunkRule StringChunkRule 78 | markedObjectKeyableRule MarkedObjectKeyableRule 79 | markedObjectAnyTypeRule MarkedObjectAnyTypeRule 80 | stringBuilderRule StringBuilderRule 81 | stringBuilderChunkRule StringBuilderChunkRule 82 | edgeSourceRule EdgeSourceRule 83 | edgeDescriptionRule EdgeDescriptionRule 84 | edgeDestinationRule EdgeDestinationRule 85 | nodeRule NodeRule 86 | awaitEndRule AwaitEndRule 87 | ) 88 | 89 | var arrayTypeToDataType = []DataType{ 90 | events.ArrayTypeInvalid: DataTypeInvalid, 91 | events.ArrayTypeString: DataTypeString, 92 | events.ArrayTypeResourceID: DataTypeResourceID, 93 | events.ArrayTypeReferenceRemote: DataTypeRemoteReference, 94 | events.ArrayTypeCustomText: DataTypeCustomText, 95 | events.ArrayTypeCustomBinary: DataTypeCustomBinary, 96 | events.ArrayTypeBit: DataTypeArrayBit, 97 | events.ArrayTypeUint8: DataTypeArrayUint8, 98 | events.ArrayTypeUint16: DataTypeArrayUint16, 99 | events.ArrayTypeUint32: DataTypeArrayUint32, 100 | events.ArrayTypeUint64: DataTypeArrayUint64, 101 | events.ArrayTypeInt8: DataTypeArrayInt8, 102 | events.ArrayTypeInt16: DataTypeArrayInt16, 103 | events.ArrayTypeInt32: DataTypeArrayInt32, 104 | events.ArrayTypeInt64: DataTypeArrayInt64, 105 | events.ArrayTypeFloat16: DataTypeArrayFloat16, 106 | events.ArrayTypeFloat32: DataTypeArrayFloat32, 107 | events.ArrayTypeFloat64: DataTypeArrayFloat64, 108 | events.ArrayTypeUID: DataTypeArrayUID, 109 | events.ArrayTypeMedia: DataTypeMedia, 110 | } 111 | 112 | func wrongType(context interface{}, dataType interface{}) { 113 | panic(fmt.Errorf("%v is not allowed while processing %v", dataType, context)) 114 | } 115 | -------------------------------------------------------------------------------- /rules/rules_array.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Imposes the structural rules that enforce a well-formed concise encoding 22 | // document. 23 | package rules 24 | 25 | type ArrayRule struct{} 26 | 27 | func (_this *ArrayRule) String() string { return "Array Rule" } 28 | func (_this *ArrayRule) OnArrayChunk(ctx *Context, length uint64, moreChunksFollow bool) { 29 | if length == 0 { 30 | ctx.tryEndArray(moreChunksFollow, nil) 31 | return 32 | } 33 | 34 | ctx.BeginChunkAnyType(length, moreChunksFollow) 35 | } 36 | func (_this *ArrayRule) OnComment(ctx *Context) { /* Nothing to do */ } 37 | 38 | // ============================================================================= 39 | 40 | type ArrayChunkRule struct{} 41 | 42 | func (_this *ArrayChunkRule) String() string { return "Array Chunk Rule" } 43 | func (_this *ArrayChunkRule) OnArrayData(ctx *Context, data []byte) { 44 | ctx.MarkCompletedChunkByteCount(uint64(len(data))) 45 | if ctx.chunkActualByteCount == ctx.chunkExpectedByteCount { 46 | ctx.EndChunkAnyType() 47 | } 48 | } 49 | 50 | // ============================================================================= 51 | 52 | type StringRule struct{} 53 | 54 | func (_this *StringRule) String() string { return "String Rule" } 55 | func (_this *StringRule) OnArrayChunk(ctx *Context, length uint64, moreChunksFollow bool) { 56 | if length == 0 { 57 | ctx.tryEndArray(moreChunksFollow, nil) 58 | return 59 | } 60 | 61 | ctx.BeginChunkString(length, moreChunksFollow) 62 | } 63 | 64 | // ============================================================================= 65 | 66 | type StringChunkRule struct{} 67 | 68 | func (_this *StringChunkRule) String() string { return "String Chunk Rule" } 69 | func (_this *StringChunkRule) OnArrayData(ctx *Context, data []byte) { 70 | ctx.MarkCompletedChunkByteCount(uint64(len(data))) 71 | firstRuneBytes, nextRunesBytes := ctx.StreamStringData(data) 72 | 73 | ctx.ValidateArrayDataFunc(firstRuneBytes) 74 | ctx.ValidateArrayDataFunc(nextRunesBytes) 75 | 76 | ctx.AddBuiltArrayBytes(firstRuneBytes) 77 | ctx.AddBuiltArrayBytes(nextRunesBytes) 78 | 79 | if ctx.chunkActualByteCount == ctx.chunkExpectedByteCount { 80 | ctx.EndChunkString() 81 | } 82 | } 83 | 84 | // ============================================================================= 85 | 86 | type StringBuilderRule struct{} 87 | 88 | func (_this *StringBuilderRule) String() string { return "String Builder Rule" } 89 | func (_this *StringBuilderRule) OnArrayChunk(ctx *Context, length uint64, moreChunksFollow bool) { 90 | if length == 0 { 91 | ctx.tryEndArray(moreChunksFollow, nil) 92 | return 93 | } 94 | 95 | ctx.BeginChunkStringBuilder(length, moreChunksFollow) 96 | } 97 | 98 | // ============================================================================= 99 | 100 | type StringBuilderChunkRule struct{} 101 | 102 | func (_this *StringBuilderChunkRule) String() string { return "String Builder Chunk Rule" } 103 | func (_this *StringBuilderChunkRule) OnArrayData(ctx *Context, data []byte) { 104 | ctx.MarkCompletedChunkByteCount(uint64(len(data))) 105 | ctx.AddBuiltArrayBytes(data) 106 | if ctx.chunkActualByteCount == ctx.chunkExpectedByteCount { 107 | ctx.EndChunkString() 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /rules/rules_other.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Imposes the structural rules that enforce a well-formed concise encoding 22 | // document. 23 | package rules 24 | 25 | import ( 26 | "fmt" 27 | "math/big" 28 | 29 | "github.com/cockroachdb/apd/v2" 30 | compact_float "github.com/kstenerud/go-compact-float" 31 | "github.com/kstenerud/go-concise-encoding/ce/events" 32 | ) 33 | 34 | type BeginDocumentRule struct{} 35 | 36 | func (_this *BeginDocumentRule) String() string { return "Begin Document Rule" } 37 | func (_this *BeginDocumentRule) OnBeginDocument(ctx *Context) { ctx.ChangeRule(&versionRule) } 38 | 39 | // ============================================================================= 40 | 41 | type EndDocumentRule struct{} 42 | 43 | func (_this *EndDocumentRule) String() string { return "End Document Rule" } 44 | func (_this *EndDocumentRule) OnEndDocument(ctx *Context) { ctx.EndDocument() } 45 | 46 | // ============================================================================= 47 | 48 | type TerminalRule struct{} 49 | 50 | func (_this *TerminalRule) String() string { return "Terminal Rule" } 51 | 52 | // ============================================================================= 53 | 54 | type VersionRule struct{} 55 | 56 | func (_this *VersionRule) String() string { return "Version Rule" } 57 | func (_this *VersionRule) OnVersion(ctx *Context, version uint64) { 58 | if version != ctx.ExpectedVersion { 59 | panic(fmt.Errorf("expected version %v but got version %v", ctx.ExpectedVersion, version)) 60 | } 61 | ctx.ChangeRule(&topLevelRule) 62 | } 63 | 64 | // ============================================================================= 65 | 66 | type TopLevelRule struct{} 67 | 68 | func (_this *TopLevelRule) String() string { return "Top Level Rule" } 69 | func (_this *TopLevelRule) switchEndDocument(ctx *Context) { ctx.ChangeRule(&endDocumentRule) } 70 | func (_this *TopLevelRule) OnKeyableObject(ctx *Context, _ DataType, _ interface{}) { 71 | _this.switchEndDocument(ctx) 72 | } 73 | func (_this *TopLevelRule) OnNonKeyableObject(ctx *Context, _ DataType) { _this.switchEndDocument(ctx) } 74 | func (_this *TopLevelRule) OnChildContainerEnded(ctx *Context, _ DataType) { 75 | _this.switchEndDocument(ctx) 76 | } 77 | func (_this *TopLevelRule) OnNull(ctx *Context) { _this.switchEndDocument(ctx) } 78 | func (_this *TopLevelRule) OnInt(ctx *Context, value int64) { _this.switchEndDocument(ctx) } 79 | func (_this *TopLevelRule) OnPositiveInt(ctx *Context, value uint64) { _this.switchEndDocument(ctx) } 80 | func (_this *TopLevelRule) OnBigInt(ctx *Context, value *big.Int) { _this.switchEndDocument(ctx) } 81 | func (_this *TopLevelRule) OnFloat(ctx *Context, value float64) { _this.switchEndDocument(ctx) } 82 | func (_this *TopLevelRule) OnBigFloat(ctx *Context, value *big.Float) { _this.switchEndDocument(ctx) } 83 | func (_this *TopLevelRule) OnDecimalFloat(ctx *Context, value compact_float.DFloat) { 84 | _this.switchEndDocument(ctx) 85 | } 86 | func (_this *TopLevelRule) OnBigDecimalFloat(ctx *Context, value *apd.Decimal) { 87 | _this.switchEndDocument(ctx) 88 | } 89 | func (_this *TopLevelRule) OnArray(ctx *Context, arrayType events.ArrayType, elementCount uint64, data []uint8) { 90 | ctx.ValidateFullArrayAnyType(arrayType, elementCount, data) 91 | _this.switchEndDocument(ctx) 92 | } 93 | func (_this *TopLevelRule) OnStringlikeArray(ctx *Context, arrayType events.ArrayType, data string) { 94 | ctx.ValidateFullArrayStringlike(arrayType, data) 95 | _this.switchEndDocument(ctx) 96 | } 97 | -------------------------------------------------------------------------------- /test/event.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Test helper code. 22 | package test 23 | 24 | import ( 25 | "bytes" 26 | "fmt" 27 | 28 | "github.com/kstenerud/go-concise-encoding/ce/events" 29 | ) 30 | 31 | type EventInvocation func(receiver events.DataEventReceiver) 32 | 33 | type Event interface { 34 | Name() string 35 | String() string 36 | ArrayElementCount() int 37 | Invoke(events.DataEventReceiver) 38 | IsEquivalentTo(Event) bool 39 | Comparable() string 40 | Value() interface{} 41 | Expand() Events 42 | } 43 | 44 | type Events []Event 45 | 46 | func (_this Events) String() string { 47 | sb := bytes.Buffer{} 48 | for i, v := range _this { 49 | if i > 0 { 50 | sb.WriteByte(' ') 51 | } 52 | sb.WriteByte('"') 53 | sb.WriteString(v.String()) 54 | sb.WriteByte('"') 55 | } 56 | return sb.String() 57 | } 58 | 59 | func (_this Events) AreEquivalentTo(that Events) bool { 60 | return AreEventsEquivalent(_this, that) 61 | } 62 | func (_this Events) Expand() Events { 63 | var expanded Events 64 | for _, event := range _this { 65 | expanded = append(expanded, event.Expand()...) 66 | } 67 | return expanded 68 | } 69 | 70 | type BaseEvent struct { 71 | shortName string 72 | invocation EventInvocation 73 | values []interface{} 74 | comparable string 75 | stringified string 76 | arrayElementCount int 77 | } 78 | 79 | func (_this *BaseEvent) Invoke(receiver events.DataEventReceiver) { _this.invocation(receiver) } 80 | func (_this *BaseEvent) Name() string { return _this.shortName } 81 | func (_this *BaseEvent) String() string { return _this.stringified } 82 | func (_this *BaseEvent) Comparable() string { return _this.comparable } 83 | func (_this *BaseEvent) ArrayElementCount() int { return _this.arrayElementCount } 84 | func (_this *BaseEvent) IsEquivalentTo(that Event) bool { 85 | return _this.Comparable() == that.Comparable() 86 | } 87 | func (_this *BaseEvent) Value() interface{} { 88 | if len(_this.values) == 0 { 89 | return nil 90 | } 91 | return _this.values[len(_this.values)-1] 92 | } 93 | 94 | func ConstructEvent(shortName string, invocation EventInvocation, values ...interface{}) BaseEvent { 95 | return BaseEvent{ 96 | shortName: shortName, 97 | invocation: invocation, 98 | values: values, 99 | arrayElementCount: getArrayElementCount(values...), 100 | comparable: constructComparable(shortName, values...), 101 | stringified: constructStringified(shortName, values...), 102 | } 103 | } 104 | 105 | func constructComparable(shortName string, values ...interface{}) string { 106 | switch len(values) { 107 | case 0: 108 | return shortName 109 | case 1: 110 | return fmt.Sprintf("%v=%v", shortName, toComparable(values[0])) 111 | case 2: 112 | return fmt.Sprintf("%v=%v %v", shortName, toComparable(values[0]), toComparable(values[1])) 113 | default: 114 | panic(fmt.Errorf("expected 0, 1, or 2 values but got %v", len(values))) 115 | } 116 | } 117 | 118 | func constructStringified(shortName string, values ...interface{}) string { 119 | switch len(values) { 120 | case 0: 121 | return shortName 122 | case 1: 123 | return fmt.Sprintf("%v=%v", shortName, toStringified(values[0])) 124 | case 2: 125 | return fmt.Sprintf("%v=%v %v", shortName, toStringified(values[0]), toStringified(values[1])) 126 | default: 127 | panic(fmt.Errorf("expected 0, 1, or 2 values but got %v", len(values))) 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /test/event_parser/parser/CEEventLexer.tokens: -------------------------------------------------------------------------------- 1 | EVENT_AB=1 2 | EVENT_AB_ARGS=2 3 | EVENT_ACL=3 4 | EVENT_ACM=4 5 | EVENT_ADB=5 6 | EVENT_ADF16=6 7 | EVENT_ADF32=7 8 | EVENT_ADF64=8 9 | EVENT_ADI16=9 10 | EVENT_ADI32=10 11 | EVENT_ADI64=11 12 | EVENT_ADI8=12 13 | EVENT_ADT=13 14 | EVENT_ADU16=14 15 | EVENT_ADU16X=15 16 | EVENT_ADU32=16 17 | EVENT_ADU32X=17 18 | EVENT_ADU64=18 19 | EVENT_ADU64X=19 20 | EVENT_ADU8=20 21 | EVENT_ADU8X=21 22 | EVENT_ADU=22 23 | EVENT_AF16=23 24 | EVENT_AF16_ARGS=24 25 | EVENT_AF32=25 26 | EVENT_AF32_ARGS=26 27 | EVENT_AF64=27 28 | EVENT_AF64_ARGS=28 29 | EVENT_AI16=29 30 | EVENT_AI16_ARGS=30 31 | EVENT_AI32=31 32 | EVENT_AI32_ARGS=32 33 | EVENT_AI64=33 34 | EVENT_AI64_ARGS=34 35 | EVENT_AI8=35 36 | EVENT_AI8_ARGS=36 37 | EVENT_AU16=37 38 | EVENT_AU16_ARGS=38 39 | EVENT_AU16X=39 40 | EVENT_AU16X_ARGS=40 41 | EVENT_AU32=41 42 | EVENT_AU32_ARGS=42 43 | EVENT_AU32X=43 44 | EVENT_AU32X_ARGS=44 45 | EVENT_AU64=45 46 | EVENT_AU64_ARGS=46 47 | EVENT_AU64X=47 48 | EVENT_AU64X_ARGS=48 49 | EVENT_AU8=49 50 | EVENT_AU8_ARGS=50 51 | EVENT_AU8X=51 52 | EVENT_AU8X_ARGS=52 53 | EVENT_AU=53 54 | EVENT_AU_ARGS=54 55 | EVENT_B=55 56 | EVENT_BAB=56 57 | EVENT_BAF16=57 58 | EVENT_BAF32=58 59 | EVENT_BAF64=59 60 | EVENT_BAI16=60 61 | EVENT_BAI32=61 62 | EVENT_BAI64=62 63 | EVENT_BAI8=63 64 | EVENT_BAU16=64 65 | EVENT_BAU32=65 66 | EVENT_BAU64=66 67 | EVENT_BAU8=67 68 | EVENT_BAU=68 69 | EVENT_BCB=69 70 | EVENT_BCT=70 71 | EVENT_BMEDIA=71 72 | EVENT_BREFR=72 73 | EVENT_BRID=73 74 | EVENT_BS=74 75 | EVENT_CB=75 76 | EVENT_CM=76 77 | EVENT_CM_ARGS=77 78 | EVENT_CS=78 79 | EVENT_CS_ARGS=79 80 | EVENT_CT=80 81 | EVENT_E=81 82 | EVENT_EDGE=82 83 | EVENT_L=83 84 | EVENT_M=84 85 | EVENT_MARK=85 86 | EVENT_MEDIA=86 87 | EVENT_N=87 88 | EVENT_NODE=88 89 | EVENT_NULL=89 90 | EVENT_PAD=90 91 | EVENT_REFL=91 92 | EVENT_REFR=92 93 | EVENT_REFR_ARGS=93 94 | EVENT_RID=94 95 | EVENT_RID_ARGS=95 96 | EVENT_REC=96 97 | EVENT_RT=97 98 | EVENT_S=98 99 | EVENT_S_ARGS=99 100 | EVENT_T=100 101 | EVENT_UID=101 102 | EVENT_V=102 103 | TRUE=103 104 | FALSE=104 105 | FLOAT_NAN=105 106 | FLOAT_SNAN=106 107 | FLOAT_INF=107 108 | FLOAT_DEC=108 109 | FLOAT_HEX=109 110 | INT_BIN=110 111 | INT_OCT=111 112 | INT_DEC=112 113 | INT_HEX=113 114 | UID=114 115 | VALUE_UINT_BIN=115 116 | VALUE_UINT_OCT=116 117 | VALUE_UINT_DEC=117 118 | VALUE_UINT_HEX=118 119 | MODE_UINT_WS=119 120 | VALUE_UINTX=120 121 | MODE_UINTX_WS=121 122 | VALUE_INT_BIN=122 123 | VALUE_INT_OCT=123 124 | VALUE_INT_DEC=124 125 | VALUE_INT_HEX=125 126 | MODE_INT_WS=126 127 | VALUE_FLOAT_NAN=127 128 | VALUE_FLOAT_SNAN=128 129 | VALUE_FLOAT_INF=129 130 | VALUE_FLOAT_DEC=130 131 | VALUE_FLOAT_HEX=131 132 | MODE_FLOAT_WS=132 133 | VALUE_UID=133 134 | MODE_UID_WS=134 135 | TZ_PINT=135 136 | TZ_NINT=136 137 | TZ_INT=137 138 | TZ_COORD=138 139 | TZ_STRING=139 140 | TIME_ZONE=140 141 | TIME=141 142 | DATE=142 143 | DATETIME=143 144 | MODE_TIME_WS=144 145 | STRING=145 146 | MODE_BYTES_WS=146 147 | BYTE=147 148 | VALUE_BIT=148 149 | MODE_BITS_WS=149 150 | CUSTOM_BINARY_TYPE=150 151 | CUSTOM_TEXT_TYPE=151 152 | CUSTOM_TEXT_SEPARATOR=152 153 | MEDIA_TYPE=153 154 | 'ab'=1 155 | 'ab='=2 156 | 'acl='=3 157 | 'acm='=4 158 | 'adb='=5 159 | 'adf16='=6 160 | 'adf32='=7 161 | 'adf64='=8 162 | 'adi16='=9 163 | 'adi32='=10 164 | 'adi64='=11 165 | 'adi8='=12 166 | 'adt='=13 167 | 'adu16='=14 168 | 'adu16x='=15 169 | 'adu32='=16 170 | 'adu32x='=17 171 | 'adu64='=18 172 | 'adu64x='=19 173 | 'adu8='=20 174 | 'adu8x='=21 175 | 'adu='=22 176 | 'af16'=23 177 | 'af16='=24 178 | 'af32'=25 179 | 'af32='=26 180 | 'af64'=27 181 | 'af64='=28 182 | 'ai16'=29 183 | 'ai16='=30 184 | 'ai32'=31 185 | 'ai32='=32 186 | 'ai64'=33 187 | 'ai64='=34 188 | 'ai8'=35 189 | 'ai8='=36 190 | 'au16'=37 191 | 'au16='=38 192 | 'au16x'=39 193 | 'au16x='=40 194 | 'au32'=41 195 | 'au32='=42 196 | 'au32x'=43 197 | 'au32x='=44 198 | 'au64'=45 199 | 'au64='=46 200 | 'au64x'=47 201 | 'au64x='=48 202 | 'au8'=49 203 | 'au8='=50 204 | 'au8x'=51 205 | 'au8x='=52 206 | 'au'=53 207 | 'au='=54 208 | 'b='=55 209 | 'bab'=56 210 | 'baf16'=57 211 | 'baf32'=58 212 | 'baf64'=59 213 | 'bai16'=60 214 | 'bai32'=61 215 | 'bai64'=62 216 | 'bai8'=63 217 | 'bau16'=64 218 | 'bau32'=65 219 | 'bau64'=66 220 | 'bau8'=67 221 | 'bau'=68 222 | 'bcb='=69 223 | 'bct='=70 224 | 'bmedia='=71 225 | 'brefr'=72 226 | 'brid'=73 227 | 'bs'=74 228 | 'cb='=75 229 | 'cm'=76 230 | 'cm='=77 231 | 'cs'=78 232 | 'cs='=79 233 | 'ct='=80 234 | 'e'=81 235 | 'edge'=82 236 | 'l'=83 237 | 'm'=84 238 | 'mark='=85 239 | 'media='=86 240 | 'n='=87 241 | 'node'=88 242 | 'null'=89 243 | 'pad'=90 244 | 'refl='=91 245 | 'refr'=92 246 | 'refr='=93 247 | 'rid'=94 248 | 'rid='=95 249 | 'rec='=96 250 | 'rt='=97 251 | 's'=98 252 | 's='=99 253 | 't='=100 254 | 'uid='=101 255 | 'v='=102 256 | 'true'=103 257 | 'false'=104 258 | ' '=152 259 | -------------------------------------------------------------------------------- /test/event_parser/parser/CEEventParser.tokens: -------------------------------------------------------------------------------- 1 | EVENT_AB=1 2 | EVENT_AB_ARGS=2 3 | EVENT_ACL=3 4 | EVENT_ACM=4 5 | EVENT_ADB=5 6 | EVENT_ADF16=6 7 | EVENT_ADF32=7 8 | EVENT_ADF64=8 9 | EVENT_ADI16=9 10 | EVENT_ADI32=10 11 | EVENT_ADI64=11 12 | EVENT_ADI8=12 13 | EVENT_ADT=13 14 | EVENT_ADU16=14 15 | EVENT_ADU16X=15 16 | EVENT_ADU32=16 17 | EVENT_ADU32X=17 18 | EVENT_ADU64=18 19 | EVENT_ADU64X=19 20 | EVENT_ADU8=20 21 | EVENT_ADU8X=21 22 | EVENT_ADU=22 23 | EVENT_AF16=23 24 | EVENT_AF16_ARGS=24 25 | EVENT_AF32=25 26 | EVENT_AF32_ARGS=26 27 | EVENT_AF64=27 28 | EVENT_AF64_ARGS=28 29 | EVENT_AI16=29 30 | EVENT_AI16_ARGS=30 31 | EVENT_AI32=31 32 | EVENT_AI32_ARGS=32 33 | EVENT_AI64=33 34 | EVENT_AI64_ARGS=34 35 | EVENT_AI8=35 36 | EVENT_AI8_ARGS=36 37 | EVENT_AU16=37 38 | EVENT_AU16_ARGS=38 39 | EVENT_AU16X=39 40 | EVENT_AU16X_ARGS=40 41 | EVENT_AU32=41 42 | EVENT_AU32_ARGS=42 43 | EVENT_AU32X=43 44 | EVENT_AU32X_ARGS=44 45 | EVENT_AU64=45 46 | EVENT_AU64_ARGS=46 47 | EVENT_AU64X=47 48 | EVENT_AU64X_ARGS=48 49 | EVENT_AU8=49 50 | EVENT_AU8_ARGS=50 51 | EVENT_AU8X=51 52 | EVENT_AU8X_ARGS=52 53 | EVENT_AU=53 54 | EVENT_AU_ARGS=54 55 | EVENT_B=55 56 | EVENT_BAB=56 57 | EVENT_BAF16=57 58 | EVENT_BAF32=58 59 | EVENT_BAF64=59 60 | EVENT_BAI16=60 61 | EVENT_BAI32=61 62 | EVENT_BAI64=62 63 | EVENT_BAI8=63 64 | EVENT_BAU16=64 65 | EVENT_BAU32=65 66 | EVENT_BAU64=66 67 | EVENT_BAU8=67 68 | EVENT_BAU=68 69 | EVENT_BCB=69 70 | EVENT_BCT=70 71 | EVENT_BMEDIA=71 72 | EVENT_BREFR=72 73 | EVENT_BRID=73 74 | EVENT_BS=74 75 | EVENT_CB=75 76 | EVENT_CM=76 77 | EVENT_CM_ARGS=77 78 | EVENT_CS=78 79 | EVENT_CS_ARGS=79 80 | EVENT_CT=80 81 | EVENT_E=81 82 | EVENT_EDGE=82 83 | EVENT_L=83 84 | EVENT_M=84 85 | EVENT_MARK=85 86 | EVENT_MEDIA=86 87 | EVENT_N=87 88 | EVENT_NODE=88 89 | EVENT_NULL=89 90 | EVENT_PAD=90 91 | EVENT_REFL=91 92 | EVENT_REFR=92 93 | EVENT_REFR_ARGS=93 94 | EVENT_RID=94 95 | EVENT_RID_ARGS=95 96 | EVENT_REC=96 97 | EVENT_RT=97 98 | EVENT_S=98 99 | EVENT_S_ARGS=99 100 | EVENT_T=100 101 | EVENT_UID=101 102 | EVENT_V=102 103 | TRUE=103 104 | FALSE=104 105 | FLOAT_NAN=105 106 | FLOAT_SNAN=106 107 | FLOAT_INF=107 108 | FLOAT_DEC=108 109 | FLOAT_HEX=109 110 | INT_BIN=110 111 | INT_OCT=111 112 | INT_DEC=112 113 | INT_HEX=113 114 | UID=114 115 | VALUE_UINT_BIN=115 116 | VALUE_UINT_OCT=116 117 | VALUE_UINT_DEC=117 118 | VALUE_UINT_HEX=118 119 | MODE_UINT_WS=119 120 | VALUE_UINTX=120 121 | MODE_UINTX_WS=121 122 | VALUE_INT_BIN=122 123 | VALUE_INT_OCT=123 124 | VALUE_INT_DEC=124 125 | VALUE_INT_HEX=125 126 | MODE_INT_WS=126 127 | VALUE_FLOAT_NAN=127 128 | VALUE_FLOAT_SNAN=128 129 | VALUE_FLOAT_INF=129 130 | VALUE_FLOAT_DEC=130 131 | VALUE_FLOAT_HEX=131 132 | MODE_FLOAT_WS=132 133 | VALUE_UID=133 134 | MODE_UID_WS=134 135 | TZ_PINT=135 136 | TZ_NINT=136 137 | TZ_INT=137 138 | TZ_COORD=138 139 | TZ_STRING=139 140 | TIME_ZONE=140 141 | TIME=141 142 | DATE=142 143 | DATETIME=143 144 | MODE_TIME_WS=144 145 | STRING=145 146 | MODE_BYTES_WS=146 147 | BYTE=147 148 | VALUE_BIT=148 149 | MODE_BITS_WS=149 150 | CUSTOM_BINARY_TYPE=150 151 | CUSTOM_TEXT_TYPE=151 152 | CUSTOM_TEXT_SEPARATOR=152 153 | MEDIA_TYPE=153 154 | 'ab'=1 155 | 'ab='=2 156 | 'acl='=3 157 | 'acm='=4 158 | 'adb='=5 159 | 'adf16='=6 160 | 'adf32='=7 161 | 'adf64='=8 162 | 'adi16='=9 163 | 'adi32='=10 164 | 'adi64='=11 165 | 'adi8='=12 166 | 'adt='=13 167 | 'adu16='=14 168 | 'adu16x='=15 169 | 'adu32='=16 170 | 'adu32x='=17 171 | 'adu64='=18 172 | 'adu64x='=19 173 | 'adu8='=20 174 | 'adu8x='=21 175 | 'adu='=22 176 | 'af16'=23 177 | 'af16='=24 178 | 'af32'=25 179 | 'af32='=26 180 | 'af64'=27 181 | 'af64='=28 182 | 'ai16'=29 183 | 'ai16='=30 184 | 'ai32'=31 185 | 'ai32='=32 186 | 'ai64'=33 187 | 'ai64='=34 188 | 'ai8'=35 189 | 'ai8='=36 190 | 'au16'=37 191 | 'au16='=38 192 | 'au16x'=39 193 | 'au16x='=40 194 | 'au32'=41 195 | 'au32='=42 196 | 'au32x'=43 197 | 'au32x='=44 198 | 'au64'=45 199 | 'au64='=46 200 | 'au64x'=47 201 | 'au64x='=48 202 | 'au8'=49 203 | 'au8='=50 204 | 'au8x'=51 205 | 'au8x='=52 206 | 'au'=53 207 | 'au='=54 208 | 'b='=55 209 | 'bab'=56 210 | 'baf16'=57 211 | 'baf32'=58 212 | 'baf64'=59 213 | 'bai16'=60 214 | 'bai32'=61 215 | 'bai64'=62 216 | 'bai8'=63 217 | 'bau16'=64 218 | 'bau32'=65 219 | 'bau64'=66 220 | 'bau8'=67 221 | 'bau'=68 222 | 'bcb='=69 223 | 'bct='=70 224 | 'bmedia='=71 225 | 'brefr'=72 226 | 'brid'=73 227 | 'bs'=74 228 | 'cb='=75 229 | 'cm'=76 230 | 'cm='=77 231 | 'cs'=78 232 | 'cs='=79 233 | 'ct='=80 234 | 'e'=81 235 | 'edge'=82 236 | 'l'=83 237 | 'm'=84 238 | 'mark='=85 239 | 'media='=86 240 | 'n='=87 241 | 'node'=88 242 | 'null'=89 243 | 'pad'=90 244 | 'refl='=91 245 | 'refr'=92 246 | 'refr='=93 247 | 'rid'=94 248 | 'rid='=95 249 | 'rec='=96 250 | 'rt='=97 251 | 's'=98 252 | 's='=99 253 | 't='=100 254 | 'uid='=101 255 | 'v='=102 256 | 'true'=103 257 | 'false'=104 258 | ' '=152 259 | -------------------------------------------------------------------------------- /test/test_runner/common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test_runner 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | ) 27 | 28 | func wrapPanic(recovery interface{}, format string, args ...interface{}) { 29 | if recovery != nil { 30 | message := fmt.Sprintf(format, args...) 31 | 32 | switch e := recovery.(type) { 33 | case error: 34 | panic(fmt.Errorf("%v: %w", message, e)) 35 | default: 36 | panic(fmt.Errorf("%v: %v", message, e)) 37 | } 38 | } 39 | } 40 | 41 | func asHex(data []byte) string { 42 | sb := strings.Builder{} 43 | for i, b := range data { 44 | if i > 0 { 45 | sb.WriteByte(' ') 46 | } 47 | sb.WriteString(fmt.Sprintf("%02x", b)) 48 | } 49 | return sb.String() 50 | } 51 | -------------------------------------------------------------------------------- /test/test_runner/test_must_fail.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test_runner 22 | 23 | // Contrived file name to get around the idiotic go "feature" that gives 24 | // special meaning to filenames ending in _test. Every experienced engineer 25 | // knows that you NEVER add extra constraints to existing published standards 26 | // (no matter how "clever" you think you are) because it always bites you in 27 | // the ass eventually, and requires ugly workarounds. Simplicity indeed... 28 | 29 | import ( 30 | "fmt" 31 | ) 32 | 33 | type MustFailTest struct { 34 | BaseTest 35 | } 36 | 37 | func (_this *MustFailTest) PostDecodeInit(ceVersion int, context string, index int) error { 38 | if _this.Skip { 39 | return nil 40 | } 41 | 42 | // A bit hacky, but it makes things easier to have Debug in the base class. 43 | _this.Debug = false 44 | 45 | context = fmt.Sprintf(`%v, "must fail" test #%v`, context, index+1) 46 | if err := _this.BaseTest.PostDecodeInit(ceVersion, context); err != nil { 47 | return err 48 | } 49 | 50 | total := 0 51 | if len(_this.CTE) > 0 { 52 | total++ 53 | } 54 | if len(_this.CBE) > 0 { 55 | total++ 56 | } 57 | if len(_this.Events) > 0 { 58 | total++ 59 | } 60 | 61 | if total != 1 { 62 | return _this.errorf("must have one and only one of: cbe, cte, events") 63 | } 64 | return nil 65 | } 66 | 67 | func (_this *MustFailTest) Run() error { 68 | if _this.Skip { 69 | return nil 70 | } 71 | defer func() { 72 | if r := recover(); r != nil { 73 | switch v := r.(type) { 74 | case error: 75 | panic(fmt.Errorf("%v: %w", _this.context, v)) 76 | default: 77 | panic(fmt.Errorf("%v: %v", _this.context, v)) 78 | } 79 | } 80 | }() 81 | 82 | if len(_this.CTE) > 0 { 83 | if events, err := _this.cteToEvents(_this.CTE); err == nil { 84 | return _this.errorf("expected CTE document [%v] to fail but it produced [%v]", 85 | _this.CTE, events) 86 | } 87 | } else { 88 | if events, err := _this.cbeToEvents(_this.CBE); err == nil { 89 | return _this.errorf("expected CBE document [%v] to fail but it produced [%v]", 90 | asHex(_this.CBE), events) 91 | } 92 | } 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /test/test_runner/test_runner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test_runner 22 | 23 | import ( 24 | "fmt" 25 | "testing" 26 | ) 27 | 28 | func RunTests(t *testing.T, sourceFile string) { 29 | suite, errors := loadTestSuite(sourceFile) 30 | if len(errors) > 0 { 31 | fmt.Printf("❌ %v\n", sourceFile) 32 | reportFailedTestLoad(errors) 33 | t.Fail() 34 | return 35 | } 36 | 37 | errors = suite.Run() 38 | if len(errors) > 0 { 39 | fmt.Printf("❌ %v\n", sourceFile) 40 | reportTestFailures(errors) 41 | t.Fail() 42 | return 43 | } 44 | fmt.Printf("✅ %v\n", sourceFile) 45 | } 46 | 47 | func reportFailedTestLoad(errors []error) { 48 | for _, err := range errors { 49 | fmt.Printf(" ❗ %v\n", err) 50 | } 51 | } 52 | 53 | func reportTestFailures(errors []error) { 54 | for _, err := range errors { 55 | fmt.Printf(" 💣 %v\n", err) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /test/test_runner/test_suite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test_runner 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/kstenerud/go-concise-encoding/ce" 28 | "github.com/kstenerud/go-concise-encoding/configuration" 29 | "github.com/kstenerud/go-concise-encoding/version" 30 | ) 31 | 32 | const TestSuiteVersion = 1 33 | 34 | type TestSuiteType struct { 35 | Identifier string `ce:"order=1"` 36 | Version int `ce:"order=2"` 37 | } 38 | 39 | type TestSuite struct { 40 | Type TestSuiteType `ce:"order=1"` 41 | CEVersion *int `ce:"order=2"` 42 | Tests []*UnitTest `ce:"order=3"` 43 | context string 44 | } 45 | 46 | func (_this *TestSuite) PostDecodeInit(sourceFile string) (errors []error) { 47 | _this.context = sourceFile 48 | 49 | if len(_this.Type.Identifier) == 0 { 50 | return []error{_this.errorf("missing type identifier field")} 51 | } 52 | 53 | if _this.Type.Identifier != "ce-test" { 54 | return []error{_this.errorf("%v: unrecognized type identifier", _this.Type.Identifier)} 55 | } 56 | 57 | if _this.Type.Version != TestSuiteVersion { 58 | return []error{_this.errorf("ce test format version %v runner cannot load from version %v file", TestSuiteVersion, _this.Type.Version)} 59 | } 60 | 61 | if _this.CEVersion == nil { 62 | return []error{_this.errorf("missing ceversion field")} 63 | } 64 | 65 | if *_this.CEVersion != version.ConciseEncodingVersion { 66 | return []error{_this.errorf("This codec is for Concise Encoding version %v and cannot run tests for version %v", 67 | version.ConciseEncodingVersion, _this.CEVersion)} 68 | } 69 | 70 | for index, test := range _this.Tests { 71 | if nextErrors := test.PostDecodeInit(*_this.CEVersion, _this.context, index); nextErrors != nil { 72 | errors = append(errors, nextErrors...) 73 | } 74 | } 75 | return 76 | } 77 | 78 | func (_this *TestSuite) Run() (errors []error) { 79 | for i, test := range _this.Tests { 80 | defer func() { wrapPanic(recover(), "while running test %v in %v", i, _this.context) }() 81 | if nextErrors := test.Run(); nextErrors != nil { 82 | errors = append(errors, nextErrors...) 83 | } 84 | } 85 | return 86 | } 87 | 88 | func (_this *TestSuite) errorf(format string, args ...interface{}) error { 89 | message := fmt.Sprintf(format, args...) 90 | return fmt.Errorf("%v: %v", _this.context, message) 91 | } 92 | 93 | func loadTestSuite(testDescriptorFile string) (suite *TestSuite, errors []error) { 94 | defer func() { wrapPanic(recover(), "while loading test suite from %v", testDescriptorFile) }() 95 | 96 | file, err := os.Open(testDescriptorFile) 97 | if err != nil { 98 | panic(fmt.Errorf("unexpected error opening test suite file %v: %w", testDescriptorFile, err)) 99 | } 100 | 101 | config := configuration.New() 102 | config.Debug.PassThroughPanics = true 103 | loadedTest, err := ce.UnmarshalCTE(file, suite, config) 104 | if err != nil { 105 | panic(fmt.Errorf("malformed unit test: Unexpected CTE decode error in test suite file %v: %w", testDescriptorFile, err)) 106 | } 107 | 108 | suite = loadedTest.(*TestSuite) 109 | errors = suite.PostDecodeInit(testDescriptorFile) 110 | 111 | return 112 | } 113 | -------------------------------------------------------------------------------- /test/test_runner/test_unit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package test_runner 22 | 23 | // Contrived file name to get around the idiotic go "feature" that gives 24 | // special meaning to filenames ending in _test. Every experienced engineer 25 | // knows that you NEVER add extra constraints to existing published standards 26 | // (no matter how "clever" you think you are) because it always bites you in 27 | // the ass eventually, and requires ugly workarounds. Simplicity indeed... 28 | 29 | import ( 30 | "fmt" 31 | ) 32 | 33 | type UnitTest struct { 34 | Name string `ce:"order=1"` 35 | MustSucceed []*MustSucceedTest `ce:"order=2,omit_empty"` 36 | MustFail []*MustFailTest `ce:"order=3,omit_empty"` 37 | } 38 | 39 | func (_this *UnitTest) PostDecodeInit(ceVersion int, context string, testIndex int) (errors []error) { 40 | context = fmt.Sprintf("%v, unit test #%v (%v)", context, testIndex+1, _this.Name) 41 | 42 | for index, test := range _this.MustSucceed { 43 | if err := test.PostDecodeInit(ceVersion, context, index); err != nil { 44 | errors = append(errors, err) 45 | } 46 | } 47 | 48 | for index, test := range _this.MustFail { 49 | if err := test.PostDecodeInit(ceVersion, context, index); err != nil { 50 | errors = append(errors, err) 51 | } 52 | } 53 | 54 | return 55 | } 56 | 57 | func (_this *UnitTest) Run() (errors []error) { 58 | for _, test := range _this.MustSucceed { 59 | if err := test.Run(); err != nil { 60 | errors = append(errors, err) 61 | } 62 | } 63 | 64 | for _, test := range _this.MustFail { 65 | if err := test.Run(); err != nil { 66 | errors = append(errors, err) 67 | } 68 | } 69 | 70 | return 71 | } 72 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | __debug_bin 2 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | Portable Tests 2 | ============== 3 | 4 | This directory contains standard CE compliance tests that can be run against other implementations. 5 | 6 | TODO: Flesh this out more. 7 | 8 | See [template.cte](suites/template.cte) for test templates. 9 | 10 | Implementors: See [event_parser](../test/event_parser) and [test_runner](../test/test_runner) for sample test runner code. 11 | -------------------------------------------------------------------------------- /tests/snippet_check/.gitignore: -------------------------------------------------------------------------------- 1 | snippet_check 2 | -------------------------------------------------------------------------------- /tests/snippet_check/README.md: -------------------------------------------------------------------------------- 1 | Snippet Check 2 | ============= 3 | 4 | This tool checks documents for CTE snippets and validates them, printing out any errors it encounters. This can be useful for verifying that examples in documents are actually valid. 5 | 6 | A snippet is anything within a standard markdown snippet marker using a "cte" tag. For example: 7 | 8 | ```cte 9 | c1 10 | { 11 | // An example CTE snippet 12 | 13 | "a" = [ 14 | 1 15 | 2 16 | 3 17 | ] 18 | } 19 | ``` 20 | 21 | 22 | ### Usage: 23 | 24 | ``` 25 | Usage: ./snippet_check [opts] 26 | -q quiet 27 | -v verbose 28 | ``` 29 | -------------------------------------------------------------------------------- /tests/suites/general/cte-complex.cte: -------------------------------------------------------------------------------- 1 | c0 2 | { 3 | "type" = { 4 | "identifier" = "ce-test" 5 | "version" = 1 6 | } 7 | "ceversion" = 0 8 | "tests" = [ 9 | 10 | // ==================================================================== 11 | 12 | { 13 | "name" = "Complex example" 14 | "mustSucceed" = [ 15 | { 16 | "events" = ["m" "cm= Comments look very C-like, except: /* Nested comments are allowed! */ " 17 | "cm= Notice that there are no commas in maps and lists " 18 | "s=a_list" "l" 19 | "n=1" 20 | "n=2" 21 | "s=a string" 22 | "e" 23 | "s=map" "m" 24 | "n=2" "s=two" 25 | "n=3" "n=3000" 26 | "n=1" "s=one" 27 | "e" 28 | "s=string" "s=A string value" 29 | "s=boolean" "b=true" 30 | "s=regular int" "n=-10000000" 31 | "s=decimal float" "n=-14.125" 32 | "s=uid" "uid=f1ce4567-e89b-12d3-a456-426655440000" 33 | "s=date" "t=2019-07-01" 34 | "s=time" "t=18:04:00.940231541/Europe/Prague" 35 | "s=timestamp" "t=2010-07-15/13:28:15.415942344" 36 | "s=null" "null" 37 | "s=bytes" "au8x=10 ff 38 9a dd 00 4f 4f 91" 38 | "s=url" "rid=https://example.com/" 39 | "s=email" "rid=mailto:me@somewhere.com" 40 | "n=15" "s=Keys don't have to be strings" 41 | "s=marked_object" "mark=tag1" "m" 42 | "s=description" "s=This map will be referenced later using $tag1" 43 | "s=value" "n=-inf" 44 | "s=child_elements" "null" 45 | "s=recursive" "refl=tag1" 46 | "e" 47 | "s=ref1" "refl=tag1" 48 | "s=ref2" "refl=tag1" 49 | "s=outside_ref" "refr=https://somewhere.else.com/path/to/document.cte#some_tag" 50 | "e"] 51 | "cte" = "\.^ 52 | { 53 | /* Comments look very C-like, except: /* Nested comments are allowed! */ */ 54 | /* Notice that there are no commas in maps and lists */ 55 | "a_list" = [ 56 | 1 57 | 2 58 | "a string" 59 | ] 60 | "map" = { 61 | 2 = "two" 62 | 3 = 3000 63 | 1 = "one" 64 | } 65 | "string" = "A string value" 66 | "boolean" = true 67 | "regular int" = -10000000 68 | "decimal float" = -14.125 69 | "uid" = f1ce4567-e89b-12d3-a456-426655440000 70 | "date" = 2019-07-01 71 | "time" = 18:04:00.940231541/Europe/Prague 72 | "timestamp" = 2010-07-15/13:28:15.415942344 73 | "null" = null 74 | "bytes" = @u8[16 255 56 154 221 0 79 79 145] 75 | "url" = @"https://example.com/" 76 | "email" = @"mailto:me@somewhere.com" 77 | 15 = "Keys don't have to be strings" 78 | "marked_object" = &tag1:{ 79 | "description" = "This map will be referenced later using $tag1" 80 | "value" = -inf 81 | "child_elements" = null 82 | "recursive" = $tag1 83 | } 84 | "ref1" = $tag1 85 | "ref2" = $tag1 86 | "outside_ref" = $"https://somewhere.else.com/path/to/document.cte#some_tag" 87 | }^" 88 | } 89 | ] 90 | } 91 | 92 | // ==================================================================== 93 | 94 | ] 95 | } 96 | -------------------------------------------------------------------------------- /tests/suites/general/cte-spacing.cte: -------------------------------------------------------------------------------- 1 | c0 2 | { 3 | "type" = { 4 | "identifier" = "ce-test" 5 | "version" = 1 6 | } 7 | "ceversion" = 0 8 | "tests" = [ 9 | { 10 | "name" = "Spacing" 11 | "mustSucceed" = [ 12 | { 13 | "cte" = "\.# 14 | [ 15 | "a" 16 | /* comment */ 17 | "b" 18 | ]#" 19 | "events" = ["l" "s=a" "cm= comment " "s=b" "e"] 20 | } 21 | { 22 | "from_cte" = "\.# ["a"/* comment */ "b"]#" 23 | "events" = ["l" "s=a" "cm= comment " "s=b" "e"] 24 | } 25 | ] 26 | "mustFail" = [ 27 | { 28 | "rawdocument" = true 29 | "cte"="c1[]" 30 | } 31 | {"cte"="\.# ["a""b"]#"} 32 | {"cte"="\.# ["a"[]]#"} 33 | {"cte"="\.# [[]"a"]#"} 34 | {"cte"="\.# [[][]]#"} 35 | {"cte"="\.# [{}"a"]#"} 36 | {"cte"="\.# [{}{}]#"} 37 | {"cte"="\.# ["a"]#"} 38 | {"cte"="\.# []#"} 39 | {"cte"="\.# [(@"a" @"a" 1)"a"]#"} 40 | {"cte"="\.# [(@"a" @"a" 1)(@"a" @"a" 1)]#"} 41 | 42 | ] 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /tests/suites/general/version.cte: -------------------------------------------------------------------------------- 1 | c0 2 | { 3 | "type" = { 4 | "identifier" = "ce-test" 5 | "version" = 1 6 | } 7 | "ceversion" = 0 8 | "tests" = [ 9 | { 10 | "name" = "This is a test" 11 | "mustSucceed" = [ 12 | { // TODO: Remove when v1 released 13 | "raw_document" = true 14 | "cbe" = @u8x[81 00 01] 15 | "cte" = "c0 16 | 1" 17 | "events" = ["v=0" "n=1"] 18 | } 19 | //{ // TODO: Uncomment when v1 released 20 | // "rawDocument" = true 21 | // "cbe" = @u8x[81 01 01] 22 | // "cte" = "c1 1" 23 | // "events" = ["v=1" "n=1"] 24 | //} 25 | ] 26 | "mustFail" = [ 27 | //{ 28 | // "rawDocument" = true 29 | // "cbe" = @u8x[81 00 00] // TODO: Uncomment when v1 released 30 | //} 31 | { 32 | "rawDocument" = true 33 | "cbe" = @u8x[81 02 00] // v2 not released yet 34 | } 35 | ] 36 | } 37 | { 38 | "name" = "Version specifier" 39 | 40 | "mustSucceed" = [ 41 | {"rawDocument"=true "from_cte"="c0 1" "events"=["v=0" "n=1"]} 42 | {"rawDocument"=true "from_cte"="C0 1" "events"=["v=0" "n=1"]} 43 | {"rawDocument"=true "from_cte"="c0\n1" "events"=["v=0" "n=1"]} 44 | {"rawDocument"=true "from_cte"="C0\n1" "events"=["v=0" "n=1"]} 45 | {"rawDocument"=true "from_cte"="c0\t1" "events"=["v=0" "n=1"]} 46 | {"rawDocument"=true "from_cte"="C0\t1" "events"=["v=0" "n=1"]} 47 | {"rawDocument"=true "from_cte"="c0\r\n1" "events"=["v=0" "n=1"]} 48 | {"rawDocument"=true "from_cte"="C0\r\n1" "events"=["v=0" "n=1"]} 49 | {"rawDocument"=true "from_cte"="\r\n\t c0 1" "events"=["v=0" "n=1"]} 50 | {"rawDocument"=true "from_cte"="\r\n\t C0 1" "events"=["v=0" "n=1"]} 51 | {"rawDocument"=true "from_cte"="c0 \r\n\t 1" "events"=["v=0" "n=1"]} 52 | {"rawDocument"=true "from_cte"="C0 \r\n\t 1" "events"=["v=0" "n=1"]} 53 | ] 54 | "mustFail" = [ 55 | {"cte"="c0{}"} // Missing whitespace 56 | {"cte"="c2 1"} // Future version number 57 | {"cte"="c-1 1"} // Bad version number 58 | {"cte"="ca 1"} // Bad version number 59 | {"cte"="d0 1"} // Bad first char 60 | ] 61 | } 62 | ] 63 | } 64 | -------------------------------------------------------------------------------- /tests/suites/issues/github-issues.cte: -------------------------------------------------------------------------------- 1 | c0 2 | { 3 | "type" = { 4 | "identifier" = "ce-test" 5 | "version" = 1 6 | } 7 | "ceversion" = 0 8 | "tests" = [ 9 | 10 | // ==================================================================== 11 | 12 | { 13 | "name" = "Whitespace before an edge closure" 14 | "issue link" = @"https://github.com/kstenerud/enctool/issues/27" 15 | "mustSucceed" = [ 16 | { 17 | "from_cte" = "\.%%%% 18 | @(@"homer" @"married-to" @"marge" ) 19 | %%%%" 20 | "events" = ["edge" "rid=homer" "rid=married-to" "rid=marge" "e"] 21 | } 22 | ] 23 | } 24 | 25 | // ==================================================================== 26 | 27 | { 28 | "name" = "Date issues" 29 | "issue link" = @"https://github.com/kstenerud/enctool/issues/10" 30 | "mustSucceed" = [ 31 | { 32 | "from_cte" = "\.%%%% 33 | [ 34 | 17:41:03/-13.54/-172.36 35 | 9:04:21 36 | 0:15:00 37 | 0:15:00/-13.54/-172.36 38 | ] 39 | %%%%" 40 | "cbe" = @u8x[9a 7b 19 d2 f8 6d f5 ac bc 7b a8 88 f4 7b 00 1e f0 7b 01 1e f0 6d f5 ac bc 9b] 41 | "events" = [ 42 | "l" 43 | "t=17:41:03/-13.54/-172.36" 44 | "t=9:04:21" 45 | "t=0:15:00" 46 | "t=0:15:00/-13.54/-172.36" 47 | "e" 48 | ] 49 | } 50 | ] 51 | } 52 | 53 | // ==================================================================== 54 | 55 | { 56 | "name" = "Date issues pt 2" 57 | "issue link" = @"https://github.com/kstenerud/enctool/issues/10" 58 | "mustFail" = [ 59 | { 60 | "cte" = "\.%%%% 61 | 0:hello world 62 | %%%%" 63 | } 64 | ] 65 | } 66 | 67 | // ==================================================================== 68 | 69 | { 70 | "name" = "-0" 71 | "issue link" = @"https://github.com/kstenerud/enctool/issues/22" 72 | "mustSucceed" = [ 73 | { 74 | "cte" = "\.%%%% 75 | -0 76 | %%%%" 77 | "cbe" = @u8x[69 00] 78 | "events" = ["n=-0"] 79 | } 80 | ] 81 | } 82 | 83 | // ==================================================================== 84 | 85 | ] 86 | } 87 | -------------------------------------------------------------------------------- /tests/tests_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | package tests 22 | 23 | import ( 24 | "os" 25 | "path/filepath" 26 | "strings" 27 | "testing" 28 | 29 | "github.com/kstenerud/go-concise-encoding/test/test_runner" 30 | ) 31 | 32 | func TestSuites(t *testing.T) { 33 | runTestsInPath(t, "suites") 34 | } 35 | 36 | func runTestsInPath(t *testing.T, testDir string) { 37 | filepath.Walk(testDir, func(path string, info os.FileInfo, err error) error { 38 | if info.IsDir() { 39 | return nil 40 | } 41 | if !strings.HasSuffix(strings.ToLower(info.Name()), ".cte") { 42 | return nil 43 | } 44 | test_runner.RunTests(t, path) 45 | return nil 46 | }) 47 | } 48 | 49 | func TestBugReportTemplates(t *testing.T) { 50 | // Make sure the bug report templates are valid 51 | test_runner.RunTests(t, "../bugreport/templates/incorrectly_allowed.cte") 52 | test_runner.RunTests(t, "../bugreport/templates/incorrectly_rejected.cte") 53 | 54 | // Make sure the default bug report test is valid 55 | test_runner.RunTests(t, "../bugreport/bugreport.cte") 56 | } 57 | -------------------------------------------------------------------------------- /types/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Imposes the structural rules that enforce a well-formed concise encoding 22 | // document. 23 | 24 | // Concise Encoding types that don't exist in the standard library. 25 | // 26 | // Go is not expressive enough, nor is its type system capable enough to support 27 | // these properly, so it's necessary to use interface{} everywhere, which also 28 | // removes all compile-time type protections. 29 | package types 30 | 31 | // 128-bit universal identifier. 32 | type UID [16]byte 33 | 34 | func NewUID(contents []byte) UID { 35 | return UID{ 36 | contents[0], 37 | contents[1], 38 | contents[2], 39 | contents[3], 40 | contents[4], 41 | contents[5], 42 | contents[6], 43 | contents[7], 44 | contents[8], 45 | contents[9], 46 | contents[10], 47 | contents[11], 48 | contents[12], 49 | contents[13], 50 | contents[14], 51 | contents[15], 52 | } 53 | } 54 | 55 | // Media, containing an IANA media type string, and the media data. 56 | type Media struct { 57 | MediaType string 58 | Data []byte 59 | } 60 | 61 | // An edge between two vertices in a graph. 62 | type Edge struct { 63 | Source interface{} 64 | Description interface{} 65 | Destination interface{} 66 | } 67 | 68 | const ( 69 | EdgeFieldIndexSource = iota 70 | EdgeFieldIndexDescription 71 | EdgeFieldIndexDestination 72 | ) 73 | 74 | // A node in a tree. 75 | type Node struct { 76 | Value interface{} 77 | Children []interface{} 78 | } 79 | 80 | const ( 81 | NodeFieldIndexValue = iota 82 | NodeFieldIndexChildren 83 | ) 84 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Karl Stenerud 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to 5 | // deal in the Software without restriction, including without limitation the 6 | // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | // sell copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | // IN THE SOFTWARE. 20 | 21 | // Which version of concise encoding this library adheres to. 22 | package version 23 | 24 | // Version will be 0 until release 25 | const ConciseEncodingVersion = 0 26 | --------------------------------------------------------------------------------