├── testdata ├── input │ ├── null.json │ ├── true.json │ ├── false.json │ ├── simpleString.json │ ├── unicode.json │ ├── arrays.json │ ├── french.json │ ├── structures.json │ ├── values.json │ └── weird.json ├── output │ ├── false.json │ ├── null.json │ ├── true.json │ ├── unicode.json │ ├── arrays.json │ ├── simpleString.json │ ├── structures.json │ ├── values.json │ ├── french.json │ └── weird.json ├── outhex │ ├── null.txt │ ├── true.txt │ ├── false.txt │ ├── unicode.txt │ ├── arrays.txt │ ├── structures.txt │ ├── values.txt │ ├── french.txt │ └── weird.txt └── README.md ├── go.mod ├── .gitignore ├── go.sum ├── es6numfmt.go ├── test ├── verify-canonicalization │ └── verify-canonicalization.go └── verify-numbers │ └── verify-numbers.go ├── jcs_test.go ├── es6numfmt_test.go ├── README.md ├── LICENSE └── jcs.go /testdata/input/null.json: -------------------------------------------------------------------------------- 1 | null -------------------------------------------------------------------------------- /testdata/input/true.json: -------------------------------------------------------------------------------- 1 | true -------------------------------------------------------------------------------- /testdata/input/false.json: -------------------------------------------------------------------------------- 1 | false -------------------------------------------------------------------------------- /testdata/output/false.json: -------------------------------------------------------------------------------- 1 | false -------------------------------------------------------------------------------- /testdata/output/null.json: -------------------------------------------------------------------------------- 1 | null -------------------------------------------------------------------------------- /testdata/output/true.json: -------------------------------------------------------------------------------- 1 | true -------------------------------------------------------------------------------- /testdata/outhex/null.txt: -------------------------------------------------------------------------------- 1 | 6E 75 6C 6C -------------------------------------------------------------------------------- /testdata/outhex/true.txt: -------------------------------------------------------------------------------- 1 | 74 72 75 65 -------------------------------------------------------------------------------- /testdata/outhex/false.txt: -------------------------------------------------------------------------------- 1 | 66 61 6C 73 65 -------------------------------------------------------------------------------- /testdata/output/unicode.json: -------------------------------------------------------------------------------- 1 | {"Unnormalized Unicode":"Å"} -------------------------------------------------------------------------------- /testdata/output/arrays.json: -------------------------------------------------------------------------------- 1 | [56,{"1":[],"10":null,"d":true}] -------------------------------------------------------------------------------- /testdata/output/simpleString.json: -------------------------------------------------------------------------------- 1 | "this is a test string 1235" -------------------------------------------------------------------------------- /testdata/input/simpleString.json: -------------------------------------------------------------------------------- 1 | "this is a test string 1235" -------------------------------------------------------------------------------- /testdata/input/unicode.json: -------------------------------------------------------------------------------- 1 | { 2 | "Unnormalized Unicode":"A\u030a" 3 | } 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gowebpki/jcs 2 | 3 | go 1.15 4 | 5 | require github.com/stretchr/testify v1.7.0 6 | -------------------------------------------------------------------------------- /testdata/input/arrays.json: -------------------------------------------------------------------------------- 1 | [ 2 | 56, 3 | { 4 | "d": true, 5 | "10": null, 6 | "1": [ ] 7 | } 8 | ] 9 | -------------------------------------------------------------------------------- /testdata/outhex/unicode.txt: -------------------------------------------------------------------------------- 1 | 7b 22 55 6e 6e 6f 72 6d 61 6c 69 7a 65 64 20 55 6e 69 63 6f 64 65 22 3a 22 41 cc 8a 22 7d 2 | -------------------------------------------------------------------------------- /testdata/outhex/arrays.txt: -------------------------------------------------------------------------------- 1 | 5b 35 36 2c 7b 22 31 22 3a 5b 5d 2c 22 31 30 22 3a 6e 75 6c 6c 2c 22 64 22 3a 74 72 75 65 7d 5d 2 | -------------------------------------------------------------------------------- /testdata/output/structures.json: -------------------------------------------------------------------------------- 1 | {"":"empty","1":{"\n":56,"f":{"F":5,"f":"hi"}},"10":{},"111":[{"E":"no","e":"yes"}],"A":{},"a":{}} -------------------------------------------------------------------------------- /testdata/output/values.json: -------------------------------------------------------------------------------- 1 | {"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],"string":"€$\u000f\nA'B\"\\\\\"/"} -------------------------------------------------------------------------------- /testdata/output/french.json: -------------------------------------------------------------------------------- 1 | {"peach":"This sorting order","péché":"is wrong according to French","pêche":"but canonicalization MUST","sin":"ignore locale"} -------------------------------------------------------------------------------- /testdata/input/french.json: -------------------------------------------------------------------------------- 1 | { 2 | "peach": "This sorting order", 3 | "péché": "is wrong according to French", 4 | "pêche": "but canonicalization MUST", 5 | "sin": "ignore locale" 6 | } 7 | -------------------------------------------------------------------------------- /testdata/input/structures.json: -------------------------------------------------------------------------------- 1 | { 2 | "1": {"f": {"f": "hi","F": 5} ,"\n": 56.0}, 3 | "10": { }, 4 | "": "empty", 5 | "a": { }, 6 | "111": [ {"e": "yes","E": "no" } ], 7 | "A": { } 8 | } -------------------------------------------------------------------------------- /testdata/input/values.json: -------------------------------------------------------------------------------- 1 | { 2 | "numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], 3 | "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", 4 | "literals": [null, true, false] 5 | } -------------------------------------------------------------------------------- /testdata/output/weird.json: -------------------------------------------------------------------------------- 1 | {"\n":"Newline","\r":"Carriage Return","1":"One","":"Browser Challenge","€":"Control","ö":"Latin Small Letter O With Diaeresis","€":"Euro Sign","😂":"Smiley","דּ":"Hebrew Letter Dalet With Dagesh"} -------------------------------------------------------------------------------- /testdata/outhex/structures.txt: -------------------------------------------------------------------------------- 1 | 7b 22 22 3a 22 65 6d 70 74 79 22 2c 22 31 22 3a 7b 22 5c 6e 22 3a 35 36 2c 22 66 22 3a 7b 22 46 2 | 22 3a 35 2c 22 66 22 3a 22 68 69 22 7d 7d 2c 22 31 30 22 3a 7b 7d 2c 22 31 31 31 22 3a 5b 7b 22 3 | 45 22 3a 22 6e 6f 22 2c 22 65 22 3a 22 79 65 73 22 7d 5d 2c 22 41 22 3a 7b 7d 2c 22 61 22 3a 7b 4 | 7d 7d 5 | -------------------------------------------------------------------------------- /testdata/input/weird.json: -------------------------------------------------------------------------------- 1 | { 2 | "\u20ac": "Euro Sign", 3 | "\r": "Carriage Return", 4 | "\u000a": "Newline", 5 | "1": "One", 6 | "\u0080": "Control\u007f", 7 | "\ud83d\ude02": "Smiley", 8 | "\u00f6": "Latin Small Letter O With Diaeresis", 9 | "\ufb33": "Hebrew Letter Dalet With Dagesh", 10 | "": "Browser Challenge" 11 | } 12 | -------------------------------------------------------------------------------- /testdata/outhex/values.txt: -------------------------------------------------------------------------------- 1 | 7b 22 6c 69 74 65 72 61 6c 73 22 3a 5b 6e 75 6c 6c 2c 74 72 75 65 2c 66 61 6c 73 65 5d 2c 22 6e 2 | 75 6d 62 65 72 73 22 3a 5b 33 33 33 33 33 33 33 33 33 2e 33 33 33 33 33 33 33 2c 31 65 2b 33 30 3 | 2c 34 2e 35 2c 30 2e 30 30 32 2c 31 65 2d 32 37 5d 2c 22 73 74 72 69 6e 67 22 3a 22 e2 82 ac 24 4 | 5c 75 30 30 30 66 5c 6e 41 27 42 5c 22 5c 5c 5c 5c 5c 22 2f 22 7d 5 | -------------------------------------------------------------------------------- /testdata/outhex/french.txt: -------------------------------------------------------------------------------- 1 | 7b 22 70 65 61 63 68 22 3a 22 54 68 69 73 20 73 6f 72 74 69 6e 67 20 6f 72 64 65 72 22 2c 22 70 2 | c3 a9 63 68 c3 a9 22 3a 22 69 73 20 77 72 6f 6e 67 20 61 63 63 6f 72 64 69 6e 67 20 74 6f 20 46 3 | 72 65 6e 63 68 22 2c 22 70 c3 aa 63 68 65 22 3a 22 62 75 74 20 63 61 6e 6f 6e 69 63 61 6c 69 7a 4 | 61 74 69 6f 6e 20 4d 55 53 54 22 2c 22 73 69 6e 22 3a 22 69 67 6e 6f 72 65 20 6c 6f 63 61 6c 65 5 | 22 7d 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Binaries for programs and plugins 7 | *.exe 8 | *.exe~ 9 | *.dll 10 | *.so 11 | *.dylib 12 | 13 | # Test binary, built with `go test -c` 14 | *.test 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | coverageout 19 | coverage.html 20 | goreportcard.db 21 | 22 | # Dependency directories (remove the comment below to include it) 23 | # vendor/ 24 | 25 | # vscode 26 | */.vscode 27 | *.code-workspace 28 | 29 | # sublime text 30 | *.sublime-workspace 31 | *.sublime-project 32 | 33 | # Mac files 34 | .DS_Store 35 | -------------------------------------------------------------------------------- /testdata/outhex/weird.txt: -------------------------------------------------------------------------------- 1 | 7b 22 5c 6e 22 3a 22 4e 65 77 6c 69 6e 65 22 2c 22 5c 72 22 3a 22 43 61 72 72 69 61 67 65 20 52 2 | 65 74 75 72 6e 22 2c 22 31 22 3a 22 4f 6e 65 22 2c 22 3c 2f 73 63 72 69 70 74 3e 22 3a 22 42 72 3 | 6f 77 73 65 72 20 43 68 61 6c 6c 65 6e 67 65 22 2c 22 c2 80 22 3a 22 43 6f 6e 74 72 6f 6c 7f 22 4 | 2c 22 c3 b6 22 3a 22 4c 61 74 69 6e 20 53 6d 61 6c 6c 20 4c 65 74 74 65 72 20 4f 20 57 69 74 68 5 | 20 44 69 61 65 72 65 73 69 73 22 2c 22 e2 82 ac 22 3a 22 45 75 72 6f 20 53 69 67 6e 22 2c 22 f0 6 | 9f 98 82 22 3a 22 53 6d 69 6c 65 79 22 2c 22 ef ac b3 22 3a 22 48 65 62 72 65 77 20 4c 65 74 74 7 | 65 72 20 44 61 6c 65 74 20 57 69 74 68 20 44 61 67 65 73 68 22 7d 8 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 6 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 7 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 8 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 11 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 12 | -------------------------------------------------------------------------------- /testdata/README.md: -------------------------------------------------------------------------------- 1 | ## Test Data 2 | 3 | The [input](input) directory contains files with non-canonicalized data which is 4 | supposed be transformed as specified by the corresponding file in the 5 | [output](output) directory. In the [outhex](outhex) directory the expected 6 | output is expressed in hexadecimal byte notation. 7 | 8 | ## ES6 Numbers 9 | 10 | For testing ES6 number serialization there is a ZIP file on 11 | https://1drv.ms/u/s!AmhUDQ0Od0GTiXeAjaBJFLJlxyg0?e=HFG4Ao 12 | containing about a 100 million of random and edge-case values. The test file consists of lines 13 | ```code 14 | hex-ieee,expected\n 15 | ``` 16 | where `hex-ieee` holds 1-16 ASCII hexadecimal characters representing an IEEE-754 double precision value 17 | while `expected` holds the expected serialized value. Each line is terminated by a single new-line character. 18 | Sample lines: 19 | ```code 20 | 4340000000000001,9007199254740994 21 | 4340000000000002,9007199254740996 22 | 444b1ae4d6e2ef50,1e+21 23 | 3eb0c6f7a0b5ed8d,0.000001 24 | 3eb0c6f7a0b5ed8c,9.999999999999997e-7 25 | 8000000000000000,0 26 | 0,0 27 | ``` 28 | -------------------------------------------------------------------------------- /es6numfmt.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Bret Jordan & Benedikt Thoma, All rights reserved. 2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org). 3 | // 4 | // Use of this source code is governed by an Apache 2.0 license that can be 5 | // found in the LICENSE file in the root of the source tree. 6 | 7 | package jcs 8 | 9 | import ( 10 | "errors" 11 | "math" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | const invalidPattern uint64 = 0x7ff0000000000000 17 | 18 | // NumberToJSON converts numbers in IEEE-754 double precision into the 19 | // format specified for JSON in EcmaScript Version 6 and forward. 20 | // The core application for this is canonicalization per RFC 8785: 21 | func NumberToJSON(ieeeF64 float64) (res string, err error) { 22 | ieeeU64 := math.Float64bits(ieeeF64) 23 | 24 | // Special case: NaN and Infinity are invalid in JSON 25 | if (ieeeU64 & invalidPattern) == invalidPattern { 26 | return "null", errors.New("Invalid JSON number: " + strconv.FormatUint(ieeeU64, 16)) 27 | } 28 | 29 | // Special case: eliminate "-0" as mandated by the ES6-JSON/JCS specifications 30 | if ieeeF64 == 0 { // Right, this line takes both -0 and 0 31 | return "0", nil 32 | } 33 | 34 | // Deal with the sign separately 35 | var sign string = "" 36 | if ieeeF64 < 0 { 37 | ieeeF64 = -ieeeF64 38 | sign = "-" 39 | } 40 | 41 | // ES6 has a unique "g" format 42 | var format byte = 'e' 43 | if ieeeF64 < 1e+21 && ieeeF64 >= 1e-6 { 44 | format = 'f' 45 | } 46 | 47 | // The following should (in "theory") do the trick: 48 | es6Formatted := strconv.FormatFloat(ieeeF64, format, -1, 64) 49 | 50 | // Ryu version 51 | exponent := strings.IndexByte(es6Formatted, 'e') 52 | if exponent > 0 { 53 | // Go outputs "1e+09" which must be rewritten as "1e+9" 54 | if es6Formatted[exponent+2] == '0' { 55 | es6Formatted = es6Formatted[:exponent+2] + es6Formatted[exponent+3:] 56 | } 57 | } 58 | return sign + es6Formatted, nil 59 | } 60 | -------------------------------------------------------------------------------- /test/verify-canonicalization/verify-canonicalization.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org). 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // https://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | // This program verifies the JSON canonicalizer using a test suite 18 | // containing sample data and expected output 19 | 20 | package main 21 | 22 | import ( 23 | "bytes" 24 | "fmt" 25 | "io/ioutil" 26 | "path/filepath" 27 | "runtime" 28 | 29 | "github.com/gowebpki/jcs" 30 | ) 31 | 32 | func check(e error) { 33 | if e != nil { 34 | panic(e) 35 | } 36 | } 37 | 38 | var testdata string 39 | 40 | var failures = 0 41 | 42 | func read(fileName string, directory string) []byte { 43 | data, err := ioutil.ReadFile(filepath.Join(filepath.Join(testdata, directory), fileName)) 44 | check(err) 45 | return data 46 | } 47 | 48 | func verify(fileName string) { 49 | actual, err := jcs.Transform(read(fileName, "input")) 50 | check(err) 51 | recycled, err2 := jcs.Transform(actual) 52 | check(err2) 53 | expected := read(fileName, "output") 54 | utf8InHex := "\nFile: " + fileName 55 | byteCount := 0 56 | next := false 57 | for _, b := range actual { 58 | if byteCount%32 == 0 { 59 | utf8InHex = utf8InHex + "\n" 60 | next = false 61 | } 62 | byteCount++ 63 | if next { 64 | utf8InHex = utf8InHex + " " 65 | } 66 | next = true 67 | utf8InHex = utf8InHex + fmt.Sprintf("%02x", b) 68 | } 69 | fmt.Println(utf8InHex + "\n") 70 | if !bytes.Equal(actual, expected) || !bytes.Equal(actual, recycled) { 71 | failures++ 72 | fmt.Println("THE TEST ABOVE FAILED!") 73 | } 74 | } 75 | 76 | func main() { 77 | _, executable, _, _ := runtime.Caller(0) 78 | testdata = filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(executable))), "jcs/testdata") 79 | fmt.Println(testdata) 80 | files, err := ioutil.ReadDir(filepath.Join(testdata, "input")) 81 | check(err) 82 | for _, file := range files { 83 | verify(file.Name()) 84 | } 85 | if failures == 0 { 86 | fmt.Println("All tests succeeded!") 87 | } else { 88 | fmt.Printf("\n****** ERRORS: %d *******\n", failures) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /test/verify-numbers/verify-numbers.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org). 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // https://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | // This program tests the JSON number serializer using both a few discrete 18 | // values as well as the 100 million value test suite 19 | 20 | package main 21 | 22 | import ( 23 | "bufio" 24 | "fmt" 25 | "math" 26 | "os" 27 | "strconv" 28 | "strings" 29 | 30 | "github.com/gowebpki/jcs" 31 | ) 32 | 33 | func check(e error) { 34 | if e != nil { 35 | panic(e) 36 | } 37 | } 38 | 39 | // Change the file name to suit your environment 40 | const testFile = "c:\\es6\\numbers\\es6testfile100m.txt" 41 | 42 | const invalidNumber = "null" 43 | 44 | var conversionErrors int = 0 45 | 46 | func verify(ieeeHex string, expected string) { 47 | for len(ieeeHex) < 16 { 48 | ieeeHex = "0" + ieeeHex 49 | } 50 | ieeeU64, err := strconv.ParseUint(ieeeHex, 16, 64) 51 | check(err) 52 | ieeeF64 := math.Float64frombits(ieeeU64) 53 | es6Created, err := jcs.NumberToJSON(ieeeF64) 54 | if expected == invalidNumber { 55 | if err == nil { 56 | panic("Missing error") 57 | } 58 | return 59 | } 60 | check(err) 61 | 62 | if es6Created != expected { 63 | conversionErrors++ 64 | fmt.Println("\n" + ieeeHex) 65 | fmt.Println(es6Created) 66 | fmt.Println(expected) 67 | } 68 | esParsed, err := strconv.ParseFloat(expected, 64) 69 | check(err) 70 | if esParsed != ieeeF64 { 71 | panic("Parsing error ieeeHex: " + ieeeHex + " expected: " + expected) 72 | } 73 | } 74 | 75 | func main() { 76 | verify("4340000000000001", "9007199254740994") 77 | verify("4340000000000002", "9007199254740996") 78 | verify("444b1ae4d6e2ef50", "1e+21") 79 | verify("3eb0c6f7a0b5ed8d", "0.000001") 80 | verify("3eb0c6f7a0b5ed8c", "9.999999999999997e-7") 81 | verify("8000000000000000", "0") 82 | verify("7fffffffffffffff", invalidNumber) 83 | verify("7ff0000000000000", invalidNumber) 84 | verify("fff0000000000000", invalidNumber) 85 | 86 | file, err := os.Open(testFile) 87 | check(err) 88 | defer file.Close() 89 | scanner := bufio.NewScanner(file) 90 | var lineCount int = 0 91 | for scanner.Scan() { 92 | lineCount++ 93 | if lineCount%1000000 == 0 { 94 | fmt.Printf("line: %d\n", lineCount) 95 | } 96 | line := scanner.Text() 97 | comma := strings.IndexByte(line, ',') 98 | if comma <= 0 { 99 | panic("Missing comma!") 100 | } 101 | verify(line[:comma], line[comma+1:]) 102 | } 103 | check(scanner.Err()) 104 | if conversionErrors == 0 { 105 | fmt.Printf("\nSuccessful Operation. Lines read: %d\n", lineCount) 106 | } else { 107 | fmt.Printf("\n****** ERRORS: %d *******\n", conversionErrors) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /jcs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Bret Jordan & Benedikt Thoma, All rights reserved. 2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org). 3 | // 4 | // Use of this source code is governed by an Apache 2.0 license that can be 5 | // found in the LICENSE file in the root of the source tree. 6 | 7 | package jcs 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "os" 13 | "path/filepath" 14 | "testing" 15 | 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | const ( 20 | pathTestData = "./testdata" 21 | pathInputRelativeToTestData = "/input" 22 | pathOutputRelativeToTestData = "/output" 23 | ) 24 | 25 | func failedBecause(errormsg string) string { 26 | return fmt.Sprintf("Failed because %s", errormsg) 27 | } 28 | 29 | func errorOccurred(activity string, err error) string { 30 | return failedBecause(fmt.Sprintf("an error occurred while %s: %s\n", activity, err)) 31 | } 32 | 33 | func doesNotMatchExpected(expectedField, expectedValue, actualField, actualValue string) string { 34 | return failedBecause( 35 | fmt.Sprintf( 36 | "%s [%s] does not match expected %s [%s]\n", 37 | actualField, 38 | actualValue, 39 | expectedField, 40 | expectedValue, 41 | ), 42 | ) 43 | } 44 | 45 | func TestTransform(t *testing.T) { 46 | testCases := []struct { 47 | desc string 48 | filename string 49 | }{ 50 | { 51 | desc: "Null", 52 | filename: "null.json", 53 | }, 54 | { 55 | desc: "True", 56 | filename: "true.json", 57 | }, 58 | { 59 | desc: "False", 60 | filename: "false.json", 61 | }, 62 | { 63 | desc: "Arrays", 64 | filename: "arrays.json", 65 | }, 66 | { 67 | desc: "French", 68 | filename: "french.json", 69 | }, 70 | { 71 | desc: "SimpleString", 72 | filename: "simpleString.json", 73 | }, 74 | { 75 | desc: "Structures", 76 | filename: "structures.json", 77 | }, 78 | { 79 | desc: "Unicode", 80 | filename: "unicode.json", 81 | }, 82 | { 83 | desc: "Values", 84 | filename: "values.json", 85 | }, 86 | { 87 | desc: "Weird", 88 | filename: "weird.json", 89 | }, 90 | } 91 | for _, tC := range testCases { 92 | t.Run(tC.desc, func(t *testing.T) { 93 | tC := tC 94 | t.Parallel() 95 | r := require.New(t) 96 | 97 | input, err := os.ReadFile(filepath.Join(pathTestData, 98 | pathInputRelativeToTestData, tC.filename)) 99 | r.NoError(err, errorOccurred("reading test input json", err)) 100 | 101 | output, err := os.ReadFile(filepath.Join(pathTestData, 102 | pathOutputRelativeToTestData, tC.filename)) 103 | r.NoError(err, errorOccurred("reading expected transformed output sample", err)) 104 | 105 | transformed, err := Transform(input) 106 | r.NoError(err, errorOccurred("transforming test input", err)) 107 | 108 | twiceTransformed, err := Transform(input) 109 | r.NoError(err, errorOccurred("transforming transformed input", err)) 110 | 111 | r.True( 112 | bytes.Equal(transformed, output), 113 | doesNotMatchExpected( 114 | "JSON", 115 | string(output), 116 | "transformed JSON", 117 | string(transformed), 118 | ), 119 | ) 120 | r.True( 121 | bytes.Equal(twiceTransformed, transformed), 122 | doesNotMatchExpected( 123 | "transformed JSON", 124 | string(transformed), 125 | "twice transformed JSON", 126 | string(twiceTransformed), 127 | ), 128 | ) 129 | }) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /es6numfmt_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Bret Jordan & Benedikt Thoma, All rights reserved. 2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org). 3 | // 4 | // Use of this source code is governed by an Apache 2.0 license that can be 5 | // found in the LICENSE file in the root of the source tree. 6 | 7 | package jcs 8 | 9 | import ( 10 | "bufio" 11 | "math" 12 | "os" 13 | "strconv" 14 | "strings" 15 | "testing" 16 | 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | const testFile = "/home/test/es6testfile100m.txt" 21 | 22 | const invalidNumber = "null" 23 | 24 | type numberParsingTestCase struct { 25 | desc string 26 | ieeeHex string 27 | expected string 28 | } 29 | 30 | func TestNumberToJSON(t *testing.T) { 31 | r := require.New(t) 32 | 33 | testCases := []numberParsingTestCase{ 34 | { 35 | ieeeHex: "4340000000000001", 36 | expected: "9007199254740994", 37 | }, 38 | { 39 | ieeeHex: "4340000000000002", 40 | expected: "9007199254740996", 41 | }, 42 | { 43 | ieeeHex: "444b1ae4d6e2ef50", 44 | expected: "1e+21", 45 | }, 46 | { 47 | ieeeHex: "3eb0c6f7a0b5ed8d", 48 | expected: "0.000001", 49 | }, 50 | { 51 | ieeeHex: "3eb0c6f7a0b5ed8c", 52 | expected: "9.999999999999997e-7", 53 | }, 54 | { 55 | ieeeHex: "8000000000000000", 56 | expected: "0", 57 | }, 58 | { 59 | ieeeHex: "7fffffffffffffff", 60 | expected: invalidNumber, 61 | }, 62 | { 63 | ieeeHex: "7ff0000000000000", 64 | expected: invalidNumber, 65 | }, 66 | { 67 | ieeeHex: "fff0000000000000", 68 | expected: invalidNumber, 69 | }, 70 | } 71 | 72 | if _, err := os.Stat(testFile); err == nil { 73 | file, err := os.Open(testFile) 74 | r.NoErrorf(err, "Failed at reading test sample file: %s\n", err) 75 | 76 | scanner := bufio.NewScanner(file) 77 | var lineCount int 78 | for scanner.Scan() { 79 | lineCount++ 80 | line := scanner.Text() 81 | lineSl := strings.Split(line, ",") 82 | r.Equalf(len(lineSl), 2, "Failed because a comma is missing in line %v\n", lineCount) 83 | 84 | hex := lineSl[0] 85 | expected := lineSl[1] 86 | t.Run(expected, func(t *testing.T) { 87 | testNumberParsing(t, r, numberParsingTestCase{ 88 | desc: expected, 89 | ieeeHex: hex, 90 | expected: expected, 91 | }) 92 | }) 93 | } 94 | r.NoErrorf(err, "Failed at scanning test file: %s\n", err) 95 | file.Close() 96 | } else if os.IsNotExist(err) { 97 | } else { 98 | r.NoErrorf(err, "Failed at testing if numbers test file exists: %s\n", err) 99 | } 100 | 101 | for _, tC := range testCases { 102 | t.Run(tC.desc, func(t *testing.T) { 103 | testNumberParsing(t, r, tC) 104 | }) 105 | } 106 | } 107 | 108 | func testNumberParsing(t *testing.T, r *require.Assertions, tC numberParsingTestCase) { 109 | for len(tC.ieeeHex) < 16 { 110 | tC.ieeeHex = "0" + tC.ieeeHex 111 | } 112 | ieeeU64, err := strconv.ParseUint(tC.ieeeHex, 16, 64) 113 | r.NoErrorf(err, "Failed at parsing tC.ieeeHex [%s]: %s\n", tC.ieeeHex, err) 114 | 115 | ieeeF64 := math.Float64frombits(ieeeU64) 116 | es6Created, err := NumberToJSON(ieeeF64) 117 | if tC.expected == invalidNumber { 118 | r.Errorf(err, "Failed because parsing number to json did not return an error even though the expected JSON is null\n") 119 | } else { 120 | r.NoErrorf(err, "Failed at converting float to json: %s\n", err) 121 | } 122 | 123 | r.Equalf(tC.expected, es6Created, "Failed because converted number [%s] (tC.ieeeHex [%s]) does not match tC.expected [%s]\n", es6Created, tC.ieeeHex, tC.expected) 124 | } 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![JCS](https://cyberphone.github.io/doc/security/jcs.svg) 2 | 3 | # JSON Canonicalization 4 | 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/gowebpki/jcs)](https://goreportcard.com/report/github.com/gowebpki/jcs) 6 | [![godoc](https://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://pkg.go.dev/github.com/gowebpki/jcs) 7 | [![GitHub license](https://img.shields.io/github/license/gowebpki/jcs.svg?style=flat)](https://github.com/gowebpki/jcs/blob/master/LICENSE) 8 | [![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/gowebpki/jcs.svg?style=flat)](https://github.com/gowebpki/jcs) 9 | 10 | Cryptographic operations like hashing and signing depend on that the target 11 | data does not change during serialization, transport, or parsing. 12 | By applying the rules defined by JCS (JSON Canonicalization Scheme), 13 | data provided in the JSON [[RFC8259](https://tools.ietf.org/html/rfc8259)] 14 | format can be exchanged "as is", while still being subject to secure cryptographic operations. 15 | JCS achieves this by building on the serialization formats for JSON 16 | primitives as defined by ECMAScript [[ES](https://ecma-international.org/ecma-262/)], 17 | constraining JSON data to the I-JSON [[RFC7493](https://tools.ietf.org/html//rfc7493)] subset, 18 | and through a platform independent property sorting scheme. 19 | 20 | Public RFC: https://tools.ietf.org/html/rfc8785 21 | 22 | The JSON Canonicalization Scheme concept in a nutshell: 23 | - Serialization of primitive JSON data types using methods compatible with ECMAScript's `JSON.stringify()` 24 | - Lexicographic sorting of JSON `Object` properties in a *recursive* process 25 | - JSON `Array` data is also subject to canonicalization, *but element order remains untouched* 26 | 27 | ### Original Work 28 | 29 | This code was originally created by Anders Rundgren aka cyberphone and can be found here: 30 | https://github.com/cyberphone/json-canonicalization. This fork and work is done with Anders' 31 | permission and is an attempt to clean up the Golang version. 32 | 33 | 34 | ### Sample Input 35 | ```code 36 | { 37 | "numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], 38 | "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", 39 | "literals": [null, true, false] 40 | } 41 | ``` 42 | ### Expected Output 43 | ```code 44 | {"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],"string":"€$\u000f\nA'B\"\\\\\"/"} 45 | ``` 46 | 47 | Note: for platform interoperable canonicalization, the output must be converted to UTF-8 48 | as well, here shown in hexadecimal notation: 49 | 50 | ```code 51 | 7b 22 6c 69 74 65 72 61 6c 73 22 3a 5b 6e 75 6c 6c 2c 74 72 75 65 2c 66 61 6c 73 65 5d 2c 22 6e 52 | 75 6d 62 65 72 73 22 3a 5b 33 33 33 33 33 33 33 33 33 2e 33 33 33 33 33 33 33 2c 31 65 2b 33 30 53 | 2c 34 2e 35 2c 30 2e 30 30 32 2c 31 65 2d 32 37 5d 2c 22 73 74 72 69 6e 67 22 3a 22 e2 82 ac 24 54 | 5c 75 30 30 30 66 5c 6e 41 27 42 5c 22 5c 5c 5c 5c 5c 22 2f 22 7d 55 | ``` 56 | ### Combining JCS and JWS (RFC7515) 57 | [JWS-JCS](https://github.com/cyberphone/jws-jcs#combining-detached-jws-with-jcs-json-canonicalization-scheme) 58 | 59 | ### On-line Browser JCS Test 60 | https://cyberphone.github.io/doc/security/browser-json-canonicalization.html 61 | 62 | ### ECMAScript Proposal: JSON.canonify() 63 | [JSON.canonify()](https://github.com/cyberphone/json-canonicalization/blob/master/JSON.canonify.md) 64 | 65 | ### Other Canonicalization Efforts 66 | https://tools.ietf.org/html/draft-staykov-hu-json-canonical-form-00 67 | 68 | http://wiki.laptop.org/go/Canonical_JSON 69 | 70 | https://gibson042.github.io/canonicaljson-spec/ 71 | 72 | https://gist.github.com/mikesamuel/20710f94a53e440691f04bf79bc3d756 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /jcs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Bret Jordan & Benedikt Thoma, All rights reserved. 2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org). 3 | // 4 | // Use of this source code is governed by an Apache 2.0 license that can be 5 | // found in the LICENSE file in the root of the source tree. 6 | 7 | // Package jcs transforms UTF-8 JSON data into a canonicalized version according RFC 8785 8 | package jcs 9 | 10 | import ( 11 | "container/list" 12 | "errors" 13 | "fmt" 14 | "strconv" 15 | "strings" 16 | "unicode/utf16" 17 | ) 18 | 19 | type nameValueType struct { 20 | name string 21 | sortKey []uint16 22 | value string 23 | } 24 | 25 | type jcsData struct { 26 | // JSON data MUST be UTF-8 encoded 27 | jsonData []byte 28 | // Current pointer in jsonData 29 | index int 30 | } 31 | 32 | // JSON standard escapes (modulo \u) 33 | var ( 34 | asciiEscapes = []byte{'\\', '"', 'b', 'f', 'n', 'r', 't'} 35 | binaryEscapes = []byte{'\\', '"', '\b', '\f', '\n', '\r', '\t'} 36 | ) 37 | 38 | // JSON literals 39 | var literals = []string{"true", "false", "null"} 40 | 41 | // Transform converts raw JSON data from a []byte array into a canonicalized version according RFC 8785 42 | func Transform(jsonData []byte) ([]byte, error) { 43 | if jsonData == nil { 44 | return nil, errors.New("No JSON data provided") 45 | } 46 | 47 | // Create a JCS Data struct to store the JSON Data and the index. 48 | var jd jcsData 49 | jd.jsonData = jsonData 50 | j := &jd 51 | 52 | transformed, err := j.parseEntry() 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | for j.index < len(j.jsonData) { 58 | if !j.isWhiteSpace(j.jsonData[j.index]) { 59 | return nil, errors.New("Improperly terminated JSON object") 60 | } 61 | j.index++ 62 | } 63 | return []byte(transformed), err 64 | } 65 | 66 | func (j *jcsData) isWhiteSpace(c byte) bool { 67 | return c == 0x20 || c == 0x0a || c == 0x0d || c == 0x09 68 | } 69 | 70 | func (j *jcsData) nextChar() (byte, error) { 71 | if j.index < len(j.jsonData) { 72 | c := j.jsonData[j.index] 73 | if c > 0x7f { 74 | return 0, errors.New("Unexpected non-ASCII character") 75 | } 76 | j.index++ 77 | return c, nil 78 | } 79 | return 0, errors.New("Unexpected EOF reached") 80 | } 81 | 82 | // scan advances index on jsonData to the first non whitespace character and returns it. 83 | func (j *jcsData) scan() (byte, error) { 84 | for { 85 | c, err := j.nextChar() 86 | if err != nil { 87 | return 0, err 88 | } 89 | 90 | if j.isWhiteSpace(c) { 91 | continue 92 | } 93 | 94 | return c, nil 95 | } 96 | } 97 | 98 | func (j *jcsData) scanFor(expected byte) error { 99 | c, err := j.scan() 100 | if err != nil { 101 | return err 102 | } 103 | if c != expected { 104 | return fmt.Errorf("Expected %s but got %s", string(expected), string(c)) 105 | } 106 | return nil 107 | } 108 | 109 | func (j *jcsData) getUEscape() (rune, error) { 110 | start := j.index 111 | for i := 0; i < 4; i++ { 112 | _, err := j.nextChar() 113 | if err != nil { 114 | return 0, err 115 | } 116 | } 117 | 118 | u16, err := strconv.ParseUint(string(j.jsonData[start:j.index]), 16, 64) 119 | if err != nil { 120 | return 0, err 121 | } 122 | return rune(u16), nil 123 | } 124 | 125 | func (j *jcsData) decorateString(rawUTF8 string) string { 126 | var quotedString strings.Builder 127 | quotedString.WriteByte('"') 128 | 129 | CoreLoop: 130 | for _, c := range []byte(rawUTF8) { 131 | // Is this within the JSON standard escapes? 132 | for i, esc := range binaryEscapes { 133 | if esc == c { 134 | quotedString.WriteByte('\\') 135 | quotedString.WriteByte(asciiEscapes[i]) 136 | 137 | continue CoreLoop 138 | } 139 | } 140 | if c < 0x20 { 141 | // Other ASCII control characters must be escaped with \uhhhh 142 | quotedString.WriteString(fmt.Sprintf("\\u%04x", c)) 143 | } else { 144 | quotedString.WriteByte(c) 145 | } 146 | } 147 | quotedString.WriteByte('"') 148 | 149 | return quotedString.String() 150 | } 151 | 152 | // parseEntry is the entrypoint into the parsing control flow 153 | func (j *jcsData) parseEntry() (string, error) { 154 | c, err := j.scan() 155 | if err != nil { 156 | return "", err 157 | } 158 | j.index-- 159 | 160 | switch c { 161 | case '{', '"', '[': 162 | return j.parseElement() 163 | default: 164 | value, err := parseLiteral(string(j.jsonData)) 165 | if err != nil { 166 | return "", err 167 | } 168 | 169 | j.index = len(j.jsonData) 170 | return value, nil 171 | } 172 | } 173 | 174 | func (j *jcsData) parseQuotedString() (string, error) { 175 | var rawString strings.Builder 176 | 177 | CoreLoop: 178 | for { 179 | var c byte 180 | if j.index < len(j.jsonData) { 181 | c = j.jsonData[j.index] 182 | j.index++ 183 | } else { 184 | return "", errors.New("Unexpected EOF reached") 185 | } 186 | 187 | if c == '"' { 188 | break 189 | } 190 | 191 | if c < ' ' { 192 | return "", errors.New("Unterminated string literal") 193 | } else if c == '\\' { 194 | // Escape sequence 195 | c, err := j.nextChar() 196 | if err != nil { 197 | return "", err 198 | } 199 | 200 | if c == 'u' { 201 | // The \u escape 202 | firstUTF16, err := j.getUEscape() 203 | if err != nil { 204 | return "", err 205 | } 206 | 207 | if utf16.IsSurrogate(firstUTF16) { 208 | // If the first UTF-16 code unit has a certain value there must be 209 | // another succeeding UTF-16 code unit as well 210 | backslash, err := j.nextChar() 211 | if err != nil { 212 | return "", err 213 | } 214 | u, err := j.nextChar() 215 | if err != nil { 216 | return "", err 217 | } 218 | 219 | if backslash != '\\' || u != 'u' { 220 | return "", errors.New("Missing surrogate") 221 | } 222 | 223 | // Output the UTF-32 code point as UTF-8 224 | uEscape, err := j.getUEscape() 225 | if err != nil { 226 | return "", err 227 | } 228 | rawString.WriteRune(utf16.DecodeRune(firstUTF16, uEscape)) 229 | 230 | } else { 231 | // Single UTF-16 code identical to UTF-32. Output as UTF-8 232 | rawString.WriteRune(firstUTF16) 233 | } 234 | } else if c == '/' { 235 | // Benign but useless escape 236 | rawString.WriteByte('/') 237 | } else { 238 | // The JSON standard escapes 239 | for i, esc := range asciiEscapes { 240 | if esc == c { 241 | rawString.WriteByte(binaryEscapes[i]) 242 | continue CoreLoop 243 | } 244 | } 245 | return "", fmt.Errorf("Unexpected escape: \\%s", string(c)) 246 | } 247 | } else { 248 | // Just an ordinary ASCII character alternatively a UTF-8 byte 249 | // outside of ASCII. 250 | // Note that properly formatted UTF-8 never clashes with ASCII 251 | // making byte per byte search for ASCII break characters work 252 | // as expected. 253 | rawString.WriteByte(c) 254 | } 255 | } 256 | 257 | return rawString.String(), nil 258 | } 259 | 260 | func (j *jcsData) parseSimpleType() (string, error) { 261 | var token strings.Builder 262 | 263 | j.index-- 264 | 265 | // no condition is needed here. 266 | // if the buffer reaches EOF scan returns an error, or we terminate because the 267 | // json simple type terminates 268 | for { 269 | c, err := j.scan() 270 | if err != nil { 271 | return "", err 272 | } 273 | 274 | if c == ',' || c == ']' || c == '}' { 275 | j.index-- 276 | break 277 | } 278 | 279 | token.WriteByte(c) 280 | } 281 | 282 | if token.Len() == 0 { 283 | return "", errors.New("Missing argument") 284 | } 285 | 286 | return parseLiteral(token.String()) 287 | } 288 | 289 | func parseLiteral(value string) (string, error) { 290 | // Is it a JSON literal? 291 | for _, literal := range literals { 292 | if literal == value { 293 | return literal, nil 294 | } 295 | } 296 | 297 | // Apparently not so we assume that it is a I-JSON number 298 | ieeeF64, err := strconv.ParseFloat(value, 64) 299 | if err != nil { 300 | return "", err 301 | } 302 | 303 | value, err = NumberToJSON(ieeeF64) 304 | if err != nil { 305 | return "", err 306 | } 307 | 308 | return value, nil 309 | } 310 | 311 | func (j *jcsData) parseElement() (string, error) { 312 | c, err := j.scan() 313 | if err != nil { 314 | return "", err 315 | } 316 | 317 | switch c { 318 | case '{': 319 | return j.parseObject() 320 | case '"': 321 | str, err := j.parseQuotedString() 322 | if err != nil { 323 | return "", err 324 | } 325 | return j.decorateString(str), nil 326 | case '[': 327 | return j.parseArray() 328 | default: 329 | return j.parseSimpleType() 330 | } 331 | } 332 | 333 | func (j *jcsData) peek() (byte, error) { 334 | c, err := j.scan() 335 | if err != nil { 336 | return 0, err 337 | } 338 | 339 | j.index-- 340 | return c, nil 341 | } 342 | 343 | func (j *jcsData) parseArray() (string, error) { 344 | var arrayData strings.Builder 345 | var next bool 346 | 347 | arrayData.WriteByte('[') 348 | 349 | for { 350 | c, err := j.peek() 351 | if err != nil { 352 | return "", err 353 | } 354 | 355 | if c == ']' { 356 | j.index++ 357 | break 358 | } 359 | 360 | if next { 361 | err = j.scanFor(',') 362 | if err != nil { 363 | return "", err 364 | } 365 | arrayData.WriteByte(',') 366 | } else { 367 | next = true 368 | } 369 | 370 | element, err := j.parseElement() 371 | if err != nil { 372 | return "", err 373 | } 374 | arrayData.WriteString(element) 375 | } 376 | 377 | arrayData.WriteByte(']') 378 | return arrayData.String(), nil 379 | } 380 | 381 | func (j *jcsData) lexicographicallyPrecedes(sortKey []uint16, e *list.Element) (bool, error) { 382 | // Find the minimum length of the sortKeys 383 | oldSortKey := e.Value.(nameValueType).sortKey 384 | minLength := len(oldSortKey) 385 | if minLength > len(sortKey) { 386 | minLength = len(sortKey) 387 | } 388 | for q := 0; q < minLength; q++ { 389 | diff := int(sortKey[q]) - int(oldSortKey[q]) 390 | if diff < 0 { 391 | // Smaller => Precedes 392 | return true, nil 393 | } else if diff > 0 { 394 | // Bigger => No match 395 | return false, nil 396 | } 397 | // Still equal => Continue 398 | } 399 | // The sortKeys compared equal up to minLength 400 | if len(sortKey) < len(oldSortKey) { 401 | // Shorter => Precedes 402 | return true, nil 403 | } 404 | if len(sortKey) == len(oldSortKey) { 405 | return false, fmt.Errorf("Duplicate key: %s", e.Value.(nameValueType).name) 406 | } 407 | // Longer => No match 408 | return false, nil 409 | } 410 | 411 | func (j *jcsData) parseObject() (string, error) { 412 | nameValueList := list.New() 413 | var next bool = false 414 | CoreLoop: 415 | for { 416 | c, err := j.peek() 417 | if err != nil { 418 | return "", err 419 | } 420 | 421 | if c == '}' { 422 | // advance index because of peeked '}' 423 | j.index++ 424 | break 425 | } 426 | 427 | if next { 428 | err = j.scanFor(',') 429 | if err != nil { 430 | return "", err 431 | } 432 | } 433 | next = true 434 | 435 | err = j.scanFor('"') 436 | if err != nil { 437 | return "", err 438 | } 439 | rawUTF8, err := j.parseQuotedString() 440 | if err != nil { 441 | break 442 | } 443 | // Sort keys on UTF-16 code units 444 | // Since UTF-8 doesn't have endianess this is just a value transformation 445 | // In the Go case the transformation is UTF-8 => UTF-32 => UTF-16 446 | sortKey := utf16.Encode([]rune(rawUTF8)) 447 | err = j.scanFor(':') 448 | if err != nil { 449 | return "", err 450 | } 451 | 452 | element, err := j.parseElement() 453 | if err != nil { 454 | return "", err 455 | } 456 | nameValue := nameValueType{rawUTF8, sortKey, element} 457 | for e := nameValueList.Front(); e != nil; e = e.Next() { 458 | // Check if the key is smaller than a previous key 459 | if precedes, err := j.lexicographicallyPrecedes(sortKey, e); err != nil { 460 | return "", err 461 | } else if precedes { 462 | // Precedes => Insert before and exit sorting 463 | nameValueList.InsertBefore(nameValue, e) 464 | continue CoreLoop 465 | } 466 | // Continue searching for a possibly succeeding sortKey 467 | // (which is straightforward since the list is ordered) 468 | } 469 | // The sortKey is either the first or is succeeding all previous sortKeys 470 | nameValueList.PushBack(nameValue) 471 | } 472 | 473 | // Now everything is sorted so we can properly serialize the object 474 | var objectData strings.Builder 475 | objectData.WriteByte('{') 476 | next = false 477 | for e := nameValueList.Front(); e != nil; e = e.Next() { 478 | if next { 479 | objectData.WriteByte(',') 480 | } 481 | next = true 482 | nameValue := e.Value.(nameValueType) 483 | objectData.WriteString(j.decorateString(nameValue.name)) 484 | objectData.WriteByte(':') 485 | objectData.WriteString(nameValue.value) 486 | } 487 | objectData.WriteByte('}') 488 | return objectData.String(), nil 489 | } 490 | --------------------------------------------------------------------------------