(
159 | new UTF8Encoding(false, true).GetString(Base64UrlConverter.Decode(jwsHeaderB64U)),
160 | new JsonSerializerSettings
161 | {
162 | MissingMemberHandling = MissingMemberHandling.Error, // Reject undeclared properties
163 | });
164 |
165 | // Verify correctness of container
166 | if (!jwsHeader.Algorithm.Equals(ALGORITHM) || !jwsHeader.KeyId.Equals(KEY_ID))
167 | {
168 | throw new CryptographicException("Unexpected JWS header arguments: " +
169 | JsonConvert.SerializeObject(jwsHeader));
170 | }
171 |
172 | // Fetch signature value
173 | byte[] signatureValue = Base64UrlConverter.Decode(jwsString.Substring(jwsString.LastIndexOf('.') + 1));
174 |
175 | // Data to be signed
176 | byte[] dataToBeSigned = new UTF8Encoding(false, true).GetBytes((jwsHeaderB64U + "." + payloadB64U));
177 |
178 | // Verify signature value
179 | return signatureValue.SequenceEqual(HmacObject(dataToBeSigned));
180 | }
181 | }
182 | }
183 |
--------------------------------------------------------------------------------
/dotnet/json.net.sign/UTCStrictDateConverter.cs:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | using System;
19 | using System.Globalization;
20 | using System.Text.RegularExpressions;
21 |
22 | using Newtonsoft.Json;
23 | using Newtonsoft.Json.Converters;
24 |
25 | // Annotation argument for DateTime encoding/decoding
26 |
27 | namespace json.net.signaturesupport
28 | {
29 | public class UTCStrictDateConverter : DateTimeConverterBase
30 | {
31 | static Regex UTC_DATE_NO_FRACTION_PATTERN = new Regex("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$");
32 |
33 | public override bool CanConvert(Type objectType)
34 | {
35 | return true;
36 | }
37 |
38 | public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
39 | {
40 | DateTime date = ((DateTime)value).ToUniversalTime();
41 | writer.WriteValue(date.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture));
42 | }
43 |
44 | public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
45 | {
46 | string value = (string)reader.Value;
47 | if (UTC_DATE_NO_FRACTION_PATTERN.IsMatch(value))
48 | {
49 | return DateTime.Parse(value);
50 | }
51 | throw new ArgumentException("Invalid DateTime format: " + value);
52 | }
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/dotnet/json.net.sign/json.net.sign.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exe
5 | netcoreapp3.1
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/dotnet/jsoncanonicalizer/README.md:
--------------------------------------------------------------------------------
1 | ## JCS (RFC 8785) compatible canonicalizer for .NET
2 |
3 | The project **verify-canonicalization** contains the actual test program.
4 |
--------------------------------------------------------------------------------
/dotnet/jsoncanonicalizer/jsoncanonicalizer.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | netstandard2.0
5 | true
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/dotnet/verify-canonicalization/ArrayUtil.cs:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | using System.IO;
19 |
20 | namespace VerifyJsonCanonicalizer
21 | {
22 | public static class ArrayUtil
23 | {
24 | public static byte[] ReadFile(string fileName)
25 | {
26 | using (FileStream fsSource = new FileStream(fileName, FileMode.Open, FileAccess.Read))
27 | {
28 | // Read the source file into a byte array.
29 | byte[] bytes = new byte[fsSource.Length];
30 | int numBytesToRead = (int)fsSource.Length;
31 | int numBytesRead = 0;
32 | while (numBytesToRead > 0)
33 | {
34 | // Read may return anything from 0 to numBytesToRead.
35 | int n = fsSource.Read(bytes, numBytesRead, numBytesToRead);
36 |
37 | // Break when the end of the file is reached.
38 | if (n == 0)
39 | break;
40 |
41 | numBytesRead += n;
42 | numBytesToRead -= n;
43 | }
44 | // Write the byte array to memory.
45 | using (MemoryStream result = new MemoryStream())
46 | {
47 | result.Write(bytes, 0, bytes.Length);
48 | return result.ToArray();
49 | }
50 | }
51 | }
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/dotnet/verify-canonicalization/Program.cs:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | using System;
19 | using System.IO;
20 | using System.Linq;
21 | using System.Text;
22 |
23 | using Org.Webpki.JsonCanonicalizer;
24 |
25 | // Test program for verifying the JsonCanonicalizer
26 |
27 | namespace VerifyJsonCanonicalizer
28 | {
29 | class Program
30 | {
31 | static string testData;
32 |
33 | static int failures = 0;
34 |
35 | static void PerformOneTest(string inputFilePath)
36 | {
37 | string fileName = Path.GetFileName(inputFilePath);
38 | byte[] actual = new JsonCanonicalizer(ArrayUtil.ReadFile(inputFilePath)).GetEncodedUTF8();
39 | byte[] recycled = new JsonCanonicalizer(actual).GetEncodedUTF8();
40 | byte[] expected = ArrayUtil.ReadFile(Path.Combine(Path.Combine(testData, "output"), fileName));
41 | StringBuilder utf8InHex = new StringBuilder("\nFile: ");
42 | utf8InHex.Append(fileName);
43 | int byteCount = 0;
44 | bool next = false;
45 | foreach (byte b in actual)
46 | {
47 | if (byteCount++ % 32 == 0)
48 | {
49 | utf8InHex.Append('\n');
50 | next = false;
51 | }
52 | if (next)
53 | {
54 | utf8InHex.Append(' ');
55 | }
56 | next = true;
57 | utf8InHex.Append(((int)b).ToString("x02"));
58 | }
59 | Console.WriteLine(utf8InHex.Append('\n').ToString());
60 | if (!actual.SequenceEqual(expected) || !actual.SequenceEqual(recycled))
61 | {
62 | failures++;
63 | Console.WriteLine("THE TEST ABOVE FAILED!");
64 | }
65 | }
66 |
67 | static void Main(string[] args)
68 | {
69 | Console.OutputEncoding = System.Text.Encoding.Unicode;
70 | if (args.Length == 0)
71 | {
72 | // This code is based on the directory structure of the repository
73 | int q = 6;
74 | string path = new Uri(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().CodeBase)).LocalPath;
75 | while (--q > 0)
76 | {
77 | int i = path.LastIndexOf(Path.DirectorySeparatorChar);
78 | if (i <= 0)
79 | {
80 | throw new Exception("Strange file path");
81 | }
82 | path = path.Substring(0, i);
83 | }
84 | testData = Path.Combine(path, "testdata");
85 | }
86 | else
87 | {
88 | // Alternatively you may give the full path to the testdata folder
89 | testData = args[0];
90 | }
91 | foreach (string file in Directory.GetFiles(Path.Combine(testData, "input")))
92 | {
93 | PerformOneTest(file);
94 | }
95 | if (failures == 0)
96 | {
97 | Console.WriteLine("All tests succeeded!\n");
98 | }
99 | else
100 | {
101 | Console.WriteLine("\n****** ERRORS: " + failures + " *******\n");
102 | }
103 | }
104 | }
105 | }
106 |
--------------------------------------------------------------------------------
/dotnet/verify-canonicalization/verify-canonicalization.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exe
5 | netcoreapp3.1
6 | verify_canonicalization
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/dotnet/verify-numbers/Program.cs:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | using System;
19 | using System.IO;
20 |
21 | using Org.Webpki.Es6NumberSerialization;
22 |
23 | namespace verify_numbers
24 | {
25 | // Test program for a .NET ES6 Number Serializer
26 | class Program
27 | {
28 | static int conversionErrors = 0;
29 |
30 | const string INVALID_NUMBER = "null";
31 |
32 | static void Verify(string ieeeHex, string expected)
33 | {
34 | while (ieeeHex.Length < 16)
35 | {
36 | ieeeHex = '0' + ieeeHex;
37 | }
38 | double ieeeF64 = BitConverter.Int64BitsToDouble((long)Convert.ToUInt64(ieeeHex, 16));
39 | try
40 | {
41 | String es6Created = NumberToJson.SerializeNumber(ieeeF64);
42 | if (!es6Created.Equals(expected))
43 | {
44 | conversionErrors++;
45 | Console.WriteLine("ES6={0,-24:S} C#={1,-24:S} Original=" + ieeeHex, expected, es6Created);
46 | }
47 | else
48 | {
49 | if (ieeeF64 != double.Parse(expected, System.Globalization.CultureInfo.InvariantCulture))
50 | {
51 | Console.WriteLine("ES6={0,-24:S} C#={1,-24:S} Original=" + ieeeHex, expected, es6Created);
52 | }
53 | }
54 | }
55 | catch (ArgumentException)
56 | {
57 | if (!expected.Equals(INVALID_NUMBER))
58 | {
59 | conversionErrors++;
60 | Console.WriteLine("ES6={0,-24:S} Original=" + ieeeHex, expected);
61 | }
62 | }
63 | }
64 |
65 | static void Main(string[] args)
66 | {
67 | Verify("4340000000000001", "9007199254740994");
68 | Verify("4340000000000002", "9007199254740996");
69 | Verify("444b1ae4d6e2ef50", "1e+21");
70 | Verify("3eb0c6f7a0b5ed8d", "0.000001");
71 | Verify("3eb0c6f7a0b5ed8c", "9.999999999999997e-7");
72 | Verify("8000000000000000", "0");
73 | Verify("7fffffffffffffff", INVALID_NUMBER);
74 | Verify("7ff0000000000000", INVALID_NUMBER);
75 | Verify("fff0000000000000", INVALID_NUMBER);
76 | using (StreamReader sr = new StreamReader("c:\\es6\\numbers\\es6testfile100m.txt"))
77 | {
78 | string line;
79 | // Read test lines from the file until EOF is reached
80 | long lineCount = 0;
81 | while ((line = sr.ReadLine()) != null)
82 | {
83 | // Each line contains
84 | // Hexadecimal,Number\n
85 | // where Hexadecimal is the IEEE-754 double precision
86 | // equivalent of an optimal (ES6 compliant) Number
87 | int comma = line.IndexOf(',');
88 | Verify(line.Substring(0, comma), line.Substring(comma + 1));
89 | if (++lineCount % 1000000 == 0)
90 | {
91 | Console.WriteLine("Line: " + lineCount);
92 | }
93 | }
94 | if (conversionErrors == 0)
95 | {
96 | Console.WriteLine("\nSuccessful Operation. Lines read: " + lineCount);
97 | }
98 | else
99 | {
100 | Console.WriteLine("\nNumber of failures: " + conversionErrors);
101 | }
102 | }
103 | }
104 | }
105 | }
106 |
--------------------------------------------------------------------------------
/dotnet/verify-numbers/verify-numbers.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exe
5 | netcoreapp3.1
6 | verify_numbers
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/dotnet/version.json:
--------------------------------------------------------------------------------
1 | {
2 | "version": "1.0",
3 | "publicReleaseRefSpec": [
4 | "^refs/heads/rel$"
5 | ],
6 | }
--------------------------------------------------------------------------------
/go/README.md:
--------------------------------------------------------------------------------
1 | ## JSON Canonicalizer for Go
2 |
3 | The [jsoncanonicalizer](src/webpki.org/jsoncanonicalizer)
4 | folder contains the source code for a
5 | JCS (RFC 8785) compliant JSON canonicalizer written in Go.
6 |
7 | ### Building and testing
8 |
9 | - Set GOPATH to this directory.
10 |
11 | - For running `verify-numbers.go` you need to download a 3Gb+ file with test
12 | data described in the root directory [testdata](../testdata). This file can be stored in
13 | any directory and requires updating the file path in `verify-numbers.go`.
14 |
15 | - Perform the commands:
16 | ```code
17 | $ cd test
18 | $ go build webpki.org/jsoncanonicalizer
19 | $ go run verify-canonicalization.go
20 | $ go run verify-numbers.go
21 | ```
22 |
23 |
24 | ### Using the JSON canonicalizer
25 |
26 | ```go
27 | import "webpki.org/jsoncanonicalizer"
28 |
29 | func Transform(jsonData []byte) (result []byte, e error)
30 | ```
31 | Note that both the input and the result is assumed to be in UTF-8 format.
32 |
33 | ### Constraints
34 | The JSON canonicalizer only accepts a JSON _Object_ or _Array_ as the top level data type.
35 |
--------------------------------------------------------------------------------
/go/src/webpki.org/jsoncanonicalizer/es6numfmt.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // https://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | // This package converts numbers in IEEE-754 double precision into the
18 | // format specified for JSON in EcmaScript Version 6 and forward.
19 | // The core application for this is canonicalization:
20 | // https://tools.ietf.org/html/draft-rundgren-json-canonicalization-scheme-02
21 |
22 | package jsoncanonicalizer
23 |
24 | import (
25 | "errors"
26 | "math"
27 | "strconv"
28 | "strings"
29 | )
30 |
31 | const invalidPattern uint64 = 0x7ff0000000000000
32 |
33 | func NumberToJSON(ieeeF64 float64) (res string, err error) {
34 | ieeeU64 := math.Float64bits(ieeeF64)
35 |
36 | // Special case: NaN and Infinity are invalid in JSON
37 | if (ieeeU64 & invalidPattern) == invalidPattern {
38 | return "null", errors.New("Invalid JSON number: " + strconv.FormatUint(ieeeU64, 16))
39 | }
40 |
41 | // Special case: eliminate "-0" as mandated by the ES6-JSON/JCS specifications
42 | if ieeeF64 == 0 { // Right, this line takes both -0 and 0
43 | return "0", nil
44 | }
45 |
46 | // Deal with the sign separately
47 | var sign string = ""
48 | if ieeeF64 < 0 {
49 | ieeeF64 =-ieeeF64
50 | sign = "-"
51 | }
52 |
53 | // ES6 has a unique "g" format
54 | var format byte = 'e'
55 | if ieeeF64 < 1e+21 && ieeeF64 >= 1e-6 {
56 | format = 'f'
57 | }
58 |
59 | // The following should do the trick:
60 | es6Formatted := strconv.FormatFloat(ieeeF64, format, -1, 64)
61 |
62 | // Minor cleanup
63 | exponent := strings.IndexByte(es6Formatted, 'e')
64 | if exponent > 0 {
65 | // Go outputs "1e+09" which must be rewritten as "1e+9"
66 | if es6Formatted[exponent + 2] == '0' {
67 | es6Formatted = es6Formatted[:exponent + 2] + es6Formatted[exponent + 3:]
68 | }
69 | }
70 | return sign + es6Formatted, nil
71 | }
72 |
--------------------------------------------------------------------------------
/go/test/verify-canonicalization.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // https://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | // This program verifies the JSON canonicalizer using a test suite
18 | // containing sample data and expected output
19 |
20 | package main
21 |
22 | import (
23 | "fmt"
24 | "io/ioutil"
25 | "runtime"
26 | "path/filepath"
27 | "bytes"
28 | "webpki.org/jsoncanonicalizer"
29 | )
30 |
31 | func check(e error) {
32 | if e != nil {
33 | panic(e)
34 | }
35 | }
36 |
37 | var testdata string
38 |
39 | var failures = 0
40 |
41 | func read(fileName string, directory string) []byte {
42 | data, err := ioutil.ReadFile(filepath.Join(filepath.Join(testdata, directory), fileName))
43 | check(err)
44 | return data
45 | }
46 |
47 | func verify(fileName string) {
48 | actual, err := jsoncanonicalizer.Transform(read(fileName, "input"))
49 | check(err)
50 | recycled, err2 := jsoncanonicalizer.Transform(actual)
51 | check(err2)
52 | expected := read(fileName, "output")
53 | var utf8InHex = "\nFile: " + fileName
54 | var byteCount = 0
55 | var next = false
56 | for _, b := range actual {
57 | if byteCount % 32 == 0 {
58 | utf8InHex = utf8InHex + "\n"
59 | next = false
60 | }
61 | byteCount++
62 | if next {
63 | utf8InHex = utf8InHex + " "
64 | }
65 | next = true
66 | utf8InHex = utf8InHex + fmt.Sprintf("%02x", b)
67 | }
68 | fmt.Println(utf8InHex + "\n")
69 | if !bytes.Equal(actual, expected) || !bytes.Equal(actual, recycled) {
70 | failures++
71 | fmt.Println("THE TEST ABOVE FAILED!");
72 | }
73 | }
74 |
75 | func main() {
76 | _, executable, _, _ := runtime.Caller(0)
77 | testdata = filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(executable))), "testdata")
78 | fmt.Println(testdata)
79 | files, err := ioutil.ReadDir(filepath.Join(testdata, "input"))
80 | check(err)
81 | for _, file := range files {
82 | verify(file.Name())
83 | }
84 | if failures == 0 {
85 | fmt.Println("All tests succeeded!\n")
86 | } else {
87 | fmt.Printf("\n****** ERRORS: %d *******\n", failures)
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/go/test/verify-numbers.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2006-2019 WebPKI.org (http://webpki.org).
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // https://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | // This program tests the JSON number serializer using both a few discrete
18 | // values as well as the 100 million value test suite
19 |
20 | package main
21 |
22 | import (
23 | "bufio"
24 | "strings"
25 | "strconv"
26 | "math"
27 | "fmt"
28 | "os"
29 | "webpki.org/jsoncanonicalizer"
30 | )
31 |
32 | func check(e error) {
33 | if e != nil {
34 | panic(e)
35 | }
36 | }
37 |
38 | // Change the file name to suit your environment
39 | const testFile = "c:\\es6\\numbers\\es6testfile100m.txt"
40 |
41 | const invalidNumber = "null"
42 |
43 | var conversionErrors int = 0
44 |
45 | func verify(ieeeHex string, expected string) {
46 | for len(ieeeHex) < 16 {
47 | ieeeHex = "0" + ieeeHex
48 | }
49 | ieeeU64, err := strconv.ParseUint(ieeeHex, 16, 64)
50 | check(err)
51 | ieeeF64 := math.Float64frombits(ieeeU64)
52 | es6Created, err := jsoncanonicalizer.NumberToJSON(ieeeF64)
53 | if expected == invalidNumber {
54 | if err == nil {
55 | panic("Missing error")
56 | }
57 | return
58 | } else {
59 | check(err);
60 | }
61 | if es6Created != expected {
62 | conversionErrors++
63 | fmt.Println("\n" + ieeeHex)
64 | fmt.Println(es6Created)
65 | fmt.Println(expected)
66 | }
67 | esParsed, err := strconv.ParseFloat(expected, 64)
68 | check(err)
69 | if esParsed != ieeeF64 {
70 | panic("Parsing error ieeeHex: " + ieeeHex + " expected: " + expected)
71 | }
72 | }
73 |
74 | func main() {
75 | verify("4340000000000001", "9007199254740994")
76 | verify("4340000000000002", "9007199254740996")
77 | verify("444b1ae4d6e2ef50", "1e+21")
78 | verify("3eb0c6f7a0b5ed8d", "0.000001")
79 | verify("3eb0c6f7a0b5ed8c", "9.999999999999997e-7")
80 | verify("8000000000000000", "0")
81 | verify("7fffffffffffffff", invalidNumber)
82 | verify("7ff0000000000000", invalidNumber)
83 | verify("fff0000000000000", invalidNumber)
84 |
85 | file, err := os.Open(testFile)
86 | check(err)
87 | defer file.Close()
88 | scanner := bufio.NewScanner(file)
89 | var lineCount int = 0
90 | for scanner.Scan() {
91 | lineCount++
92 | if lineCount % 1000000 == 0 {
93 | fmt.Printf("line: %d\n", lineCount)
94 | }
95 | line := scanner.Text()
96 | comma := strings.IndexByte(line, ',')
97 | if comma <= 0 {
98 | panic("Missing comma!")
99 | }
100 | verify(line[:comma], line[comma + 1:])
101 | }
102 | check(scanner.Err())
103 | if conversionErrors == 0 {
104 | fmt.Printf("\nSuccessful Operation. Lines read: %d\n", lineCount)
105 | } else {
106 | fmt.Printf("\n****** ERRORS: %d *******\n", conversionErrors)
107 | }
108 | }
109 |
110 |
--------------------------------------------------------------------------------
/java/.gitignore:
--------------------------------------------------------------------------------
1 | .tmp/
2 | bin/
3 |
4 |
--------------------------------------------------------------------------------
/java/canonicalizer/.classpath:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/java/canonicalizer/.project:
--------------------------------------------------------------------------------
1 |
2 |
3 | canonicalizer
4 |
5 |
6 |
7 |
8 |
9 | org.eclipse.jdt.core.javabuilder
10 |
11 |
12 |
13 |
14 |
15 | org.eclipse.jdt.core.javanature
16 |
17 |
18 |
19 | 1723925813691
20 |
21 | 30
22 |
23 | org.eclipse.core.resources.regexFilterMatcher
24 | node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/java/canonicalizer/README.md:
--------------------------------------------------------------------------------
1 | # JSON Canonicalizer for Java
2 | This library implements the canonicalization scheme described in
3 | the [core](https://github.com/cyberphone/json-canonicalization#json-canonicalization) of this repository.
4 |
5 | A compiled JAR is available in the [dist](dist) sub directory while the source is available in the [src](src) sub directory.
6 |
7 | ### Using the JSON canonicalizer
8 |
9 | ```java
10 | import org.webpki.jcs.JsonCanonicalizer;
11 |
12 | JsonCanonicalizer jsonCanonicalizer = new JsonCanonicalizer(jsonString);
13 | String result = jsonCanonicalizer.getEncodedString();
14 |
15 | ```
16 | The `JsonCanonicalizer()` may also be invoked with a `byte[]` array holding JSON data in UTF-8 format.
17 |
18 | In addition to `getEncodedString()` there is a method `getEncodedUTF8()` returning canonicalized data as
19 | a `byte[]` array.
20 |
21 | ### Constraints
22 | The JSON canonicalizer only accepts a JSON _Object_ or _Array_ as the top level data type.
23 |
24 | ### ES6 Number Formatting
25 | For formatting the JSON Number data type in an ES6 compliant way, there is a static utility method
26 | (which is also used internally by the JSON canonicalizer):
27 | ```java
28 | public static String NumberToJSON.serializeNumber(double value) throws IOException;
29 | ```
30 |
--------------------------------------------------------------------------------
/java/canonicalizer/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
27 |
28 |
29 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/java/canonicalizer/dist/json-canonicalizer.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cyberphone/json-canonicalization/19d51d7fe467d4706a3ff08adf8a748f29fc21e0/java/canonicalizer/dist/json-canonicalizer.jar
--------------------------------------------------------------------------------
/java/canonicalizer/src/org/webpki/jcs/NumberToJSON.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Ulf Adams.
3 | *
4 | * Modifications for ECMAScript / RFC 8785 by Anders Rundgren
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * https://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 | package org.webpki.jcs;
20 |
21 | import java.io.IOException;
22 |
23 | /**
24 | * JSON as specified by ECMAScript
25 | */
26 | public final class NumberToJSON {
27 | /**
28 | * Formats a number according to ECMAScript.
29 | *
30 | * This code is emulating 7.1.12.1 of the EcmaScript V6 specification.
31 | *
32 | *
33 | * @param value Value to be formatted
34 | * @return String representation
35 | */
36 | public static String serializeNumber(double value) throws IOException {
37 | // First, handle the JSON cases.
38 | if (value == 0.0) {
39 | return "0";
40 | }
41 | if (Double.isNaN(value) || Double.isInfinite(value)) {
42 | throw new IOException("NaN/Infinity not allowed in JSON");
43 | }
44 | return DoubleCoreSerializer.serialize(value, true);
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/java/deprecated/org/webpki/jcs/NumberCachedPowers.java:
--------------------------------------------------------------------------------
1 | // Copyright 2010 the V8 project authors. All rights reserved.
2 | // Redistribution and use in source and binary forms, with or without
3 | // modification, are permitted provided that the following conditions are
4 | // met:
5 | //
6 | // * Redistributions of source code must retain the above copyright
7 | // notice, this list of conditions and the following disclaimer.
8 | // * Redistributions in binary form must reproduce the above
9 | // copyright notice, this list of conditions and the following
10 | // disclaimer in the documentation and/or other materials provided
11 | // with the distribution.
12 | // * Neither the name of Google Inc. nor the names of its
13 | // contributors may be used to endorse or promote products derived
14 | // from this software without specific prior written permission.
15 | //
16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 |
28 | // Ported to Java from Mozilla's version of V8-dtoa by Hannes Wallnoefer.
29 | // The original revision was 67d1049b0bf9 from the mozilla-central tree.
30 |
31 | package org.webpki.jcs;
32 |
33 | class NumberCachedPowers {
34 |
35 |
36 | static final double kD_1_LOG2_10 = 0.30102999566398114; // 1 / lg(10)
37 |
38 | static class CachedPower {
39 | long significand;
40 | short binaryExponent;
41 | short decimalExponent;
42 |
43 | CachedPower(long significand, short binaryExponent, short decimalExponent) {
44 | this.significand = significand;
45 | this.binaryExponent = binaryExponent;
46 | this.decimalExponent = decimalExponent;
47 | }
48 | }
49 |
50 |
51 | static int getCachedPower(int e, int alpha, int gamma, NumberDiyFp c_mk) {
52 | int kQ = NumberDiyFp.kSignificandSize;
53 | double k = Math.ceil((alpha - e + kQ - 1) * kD_1_LOG2_10);
54 | int index = (GRISU_CACHE_OFFSET + (int) k - 1) / CACHED_POWERS_SPACING + 1;
55 | CachedPower cachedPower = CACHED_POWERS[index];
56 |
57 | c_mk.setF(cachedPower.significand);
58 | c_mk.setE(cachedPower.binaryExponent);
59 | assert ((alpha <= c_mk.e() + e) && (c_mk.e() + e <= gamma));
60 | return cachedPower.decimalExponent;
61 | }
62 |
63 | // Code below is converted from GRISU_CACHE_NAME(8) in file "powers-ten.h"
64 | // Regexp to convert this from original C++ source:
65 | // \{GRISU_UINT64_C\((\w+), (\w+)\), (\-?\d+), (\-?\d+)\}
66 |
67 | // interval between entries of the powers cache below
68 | static final int CACHED_POWERS_SPACING = 8;
69 |
70 | static final CachedPower[] CACHED_POWERS = {
71 | new CachedPower(0xe61acf033d1a45dfL, (short) -1087, (short) -308),
72 | new CachedPower(0xab70fe17c79ac6caL, (short) -1060, (short) -300),
73 | new CachedPower(0xff77b1fcbebcdc4fL, (short) -1034, (short) -292),
74 | new CachedPower(0xbe5691ef416bd60cL, (short) -1007, (short) -284),
75 | new CachedPower(0x8dd01fad907ffc3cL, (short) -980, (short) -276),
76 | new CachedPower(0xd3515c2831559a83L, (short) -954, (short) -268),
77 | new CachedPower(0x9d71ac8fada6c9b5L, (short) -927, (short) -260),
78 | new CachedPower(0xea9c227723ee8bcbL, (short) -901, (short) -252),
79 | new CachedPower(0xaecc49914078536dL, (short) -874, (short) -244),
80 | new CachedPower(0x823c12795db6ce57L, (short) -847, (short) -236),
81 | new CachedPower(0xc21094364dfb5637L, (short) -821, (short) -228),
82 | new CachedPower(0x9096ea6f3848984fL, (short) -794, (short) -220),
83 | new CachedPower(0xd77485cb25823ac7L, (short) -768, (short) -212),
84 | new CachedPower(0xa086cfcd97bf97f4L, (short) -741, (short) -204),
85 | new CachedPower(0xef340a98172aace5L, (short) -715, (short) -196),
86 | new CachedPower(0xb23867fb2a35b28eL, (short) -688, (short) -188),
87 | new CachedPower(0x84c8d4dfd2c63f3bL, (short) -661, (short) -180),
88 | new CachedPower(0xc5dd44271ad3cdbaL, (short) -635, (short) -172),
89 | new CachedPower(0x936b9fcebb25c996L, (short) -608, (short) -164),
90 | new CachedPower(0xdbac6c247d62a584L, (short) -582, (short) -156),
91 | new CachedPower(0xa3ab66580d5fdaf6L, (short) -555, (short) -148),
92 | new CachedPower(0xf3e2f893dec3f126L, (short) -529, (short) -140),
93 | new CachedPower(0xb5b5ada8aaff80b8L, (short) -502, (short) -132),
94 | new CachedPower(0x87625f056c7c4a8bL, (short) -475, (short) -124),
95 | new CachedPower(0xc9bcff6034c13053L, (short) -449, (short) -116),
96 | new CachedPower(0x964e858c91ba2655L, (short) -422, (short) -108),
97 | new CachedPower(0xdff9772470297ebdL, (short) -396, (short) -100),
98 | new CachedPower(0xa6dfbd9fb8e5b88fL, (short) -369, (short) -92),
99 | new CachedPower(0xf8a95fcf88747d94L, (short) -343, (short) -84),
100 | new CachedPower(0xb94470938fa89bcfL, (short) -316, (short) -76),
101 | new CachedPower(0x8a08f0f8bf0f156bL, (short) -289, (short) -68),
102 | new CachedPower(0xcdb02555653131b6L, (short) -263, (short) -60),
103 | new CachedPower(0x993fe2c6d07b7facL, (short) -236, (short) -52),
104 | new CachedPower(0xe45c10c42a2b3b06L, (short) -210, (short) -44),
105 | new CachedPower(0xaa242499697392d3L, (short) -183, (short) -36),
106 | new CachedPower(0xfd87b5f28300ca0eL, (short) -157, (short) -28),
107 | new CachedPower(0xbce5086492111aebL, (short) -130, (short) -20),
108 | new CachedPower(0x8cbccc096f5088ccL, (short) -103, (short) -12),
109 | new CachedPower(0xd1b71758e219652cL, (short) -77, (short) -4),
110 | new CachedPower(0x9c40000000000000L, (short) -50, (short) 4),
111 | new CachedPower(0xe8d4a51000000000L, (short) -24, (short) 12),
112 | new CachedPower(0xad78ebc5ac620000L, (short) 3, (short) 20),
113 | new CachedPower(0x813f3978f8940984L, (short) 30, (short) 28),
114 | new CachedPower(0xc097ce7bc90715b3L, (short) 56, (short) 36),
115 | new CachedPower(0x8f7e32ce7bea5c70L, (short) 83, (short) 44),
116 | new CachedPower(0xd5d238a4abe98068L, (short) 109, (short) 52),
117 | new CachedPower(0x9f4f2726179a2245L, (short) 136, (short) 60),
118 | new CachedPower(0xed63a231d4c4fb27L, (short) 162, (short) 68),
119 | new CachedPower(0xb0de65388cc8ada8L, (short) 189, (short) 76),
120 | new CachedPower(0x83c7088e1aab65dbL, (short) 216, (short) 84),
121 | new CachedPower(0xc45d1df942711d9aL, (short) 242, (short) 92),
122 | new CachedPower(0x924d692ca61be758L, (short) 269, (short) 100),
123 | new CachedPower(0xda01ee641a708deaL, (short) 295, (short) 108),
124 | new CachedPower(0xa26da3999aef774aL, (short) 322, (short) 116),
125 | new CachedPower(0xf209787bb47d6b85L, (short) 348, (short) 124),
126 | new CachedPower(0xb454e4a179dd1877L, (short) 375, (short) 132),
127 | new CachedPower(0x865b86925b9bc5c2L, (short) 402, (short) 140),
128 | new CachedPower(0xc83553c5c8965d3dL, (short) 428, (short) 148),
129 | new CachedPower(0x952ab45cfa97a0b3L, (short) 455, (short) 156),
130 | new CachedPower(0xde469fbd99a05fe3L, (short) 481, (short) 164),
131 | new CachedPower(0xa59bc234db398c25L, (short) 508, (short) 172),
132 | new CachedPower(0xf6c69a72a3989f5cL, (short) 534, (short) 180),
133 | new CachedPower(0xb7dcbf5354e9beceL, (short) 561, (short) 188),
134 | new CachedPower(0x88fcf317f22241e2L, (short) 588, (short) 196),
135 | new CachedPower(0xcc20ce9bd35c78a5L, (short) 614, (short) 204),
136 | new CachedPower(0x98165af37b2153dfL, (short) 641, (short) 212),
137 | new CachedPower(0xe2a0b5dc971f303aL, (short) 667, (short) 220),
138 | new CachedPower(0xa8d9d1535ce3b396L, (short) 694, (short) 228),
139 | new CachedPower(0xfb9b7cd9a4a7443cL, (short) 720, (short) 236),
140 | new CachedPower(0xbb764c4ca7a44410L, (short) 747, (short) 244),
141 | new CachedPower(0x8bab8eefb6409c1aL, (short) 774, (short) 252),
142 | new CachedPower(0xd01fef10a657842cL, (short) 800, (short) 260),
143 | new CachedPower(0x9b10a4e5e9913129L, (short) 827, (short) 268),
144 | new CachedPower(0xe7109bfba19c0c9dL, (short) 853, (short) 276),
145 | new CachedPower(0xac2820d9623bf429L, (short) 880, (short) 284),
146 | new CachedPower(0x80444b5e7aa7cf85L, (short) 907, (short) 292),
147 | new CachedPower(0xbf21e44003acdd2dL, (short) 933, (short) 300),
148 | new CachedPower(0x8e679c2f5e44ff8fL, (short) 960, (short) 308),
149 | new CachedPower(0xd433179d9c8cb841L, (short) 986, (short) 316),
150 | new CachedPower(0x9e19db92b4e31ba9L, (short) 1013, (short) 324),
151 | new CachedPower(0xeb96bf6ebadf77d9L, (short) 1039, (short) 332),
152 | new CachedPower(0xaf87023b9bf0ee6bL, (short) 1066, (short) 340)
153 | };
154 |
155 | static final int GRISU_CACHE_MAX_DISTANCE = 27;
156 | // nb elements (8): 82
157 |
158 | static final int GRISU_CACHE_OFFSET = 308;
159 |
160 |
161 | }
162 |
--------------------------------------------------------------------------------
/java/deprecated/org/webpki/jcs/NumberDiyFp.java:
--------------------------------------------------------------------------------
1 | // Copyright 2010 the V8 project authors. All rights reserved.
2 | // Redistribution and use in source and binary forms, with or without
3 | // modification, are permitted provided that the following conditions are
4 | // met:
5 | //
6 | // * Redistributions of source code must retain the above copyright
7 | // notice, this list of conditions and the following disclaimer.
8 | // * Redistributions in binary form must reproduce the above
9 | // copyright notice, this list of conditions and the following
10 | // disclaimer in the documentation and/or other materials provided
11 | // with the distribution.
12 | // * Neither the name of Google Inc. nor the names of its
13 | // contributors may be used to endorse or promote products derived
14 | // from this software without specific prior written permission.
15 | //
16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 |
28 | // Ported to Java from Mozilla's version of V8-dtoa by Hannes Wallnoefer.
29 | // The original revision was 67d1049b0bf9 from the mozilla-central tree.
30 |
31 | package org.webpki.jcs;
32 |
33 | // This "Do It Yourself Floating Point" class implements a floating-point number
34 | // with a uint64 significand and an int exponent. Normalized DiyFp numbers will
35 | // have the most significant bit of the significand set.
36 | // Multiplication and Subtraction do not normalize their results.
37 | // DiyFp are not designed to contain special doubles (NaN and Infinity).
38 | class NumberDiyFp {
39 |
40 | private long f;
41 | private int e;
42 |
43 | static final int kSignificandSize = 64;
44 | static final long kUint64MSB = 0x8000000000000000L;
45 |
46 |
47 | NumberDiyFp() {
48 | this.f = 0;
49 | this.e = 0;
50 | }
51 |
52 | NumberDiyFp(long f, int e) {
53 | this.f = f;
54 | this.e = e;
55 | }
56 |
57 | private static boolean uint64_gte(long a, long b) {
58 | // greater-or-equal for unsigned int64 in java-style...
59 | return (a == b) || ((a > b) ^ (a < 0) ^ (b < 0));
60 | }
61 |
62 | // this = this - other.
63 | // The exponents of both numbers must be the same and the significand of this
64 | // must be bigger than the significand of other.
65 | // The result will not be normalized.
66 | void subtract(NumberDiyFp other) {
67 | assert (e == other.e);
68 | assert uint64_gte(f, other.f);
69 | f -= other.f;
70 | }
71 |
72 | // Returns a - b.
73 | // The exponents of both numbers must be the same and this must be bigger
74 | // than other. The result will not be normalized.
75 | static NumberDiyFp minus(NumberDiyFp a, NumberDiyFp b) {
76 | NumberDiyFp result = new NumberDiyFp(a.f, a.e);
77 | result.subtract(b);
78 | return result;
79 | }
80 |
81 |
82 | // this = this * other.
83 | void multiply(NumberDiyFp other) {
84 | // Simply "emulates" a 128 bit multiplication.
85 | // However: the resulting number only contains 64 bits. The least
86 | // significant 64 bits are only used for rounding the most significant 64
87 | // bits.
88 | final long kM32 = 0xFFFFFFFFL;
89 | long a = f >>> 32;
90 | long b = f & kM32;
91 | long c = other.f >>> 32;
92 | long d = other.f & kM32;
93 | long ac = a * c;
94 | long bc = b * c;
95 | long ad = a * d;
96 | long bd = b * d;
97 | long tmp = (bd >>> 32) + (ad & kM32) + (bc & kM32);
98 | // By adding 1U << 31 to tmp we round the final result.
99 | // Halfway cases will be round up.
100 | tmp += 1L << 31;
101 | long result_f = ac + (ad >>> 32) + (bc >>> 32) + (tmp >>> 32);
102 | e += other.e + 64;
103 | f = result_f;
104 | }
105 |
106 | // returns a * b;
107 | static NumberDiyFp times(NumberDiyFp a, NumberDiyFp b) {
108 | NumberDiyFp result = new NumberDiyFp(a.f, a.e);
109 | result.multiply(b);
110 | return result;
111 | }
112 |
113 | void normalize() {
114 | assert (f != 0);
115 | long f = this.f;
116 | int e = this.e;
117 |
118 | // This method is mainly called for normalizing boundaries. In general
119 | // boundaries need to be shifted by 10 bits. We thus optimize for this case.
120 | final long k10MSBits = 0xFFC00000L << 32;
121 | while ((f & k10MSBits) == 0) {
122 | f <<= 10;
123 | e -= 10;
124 | }
125 | while ((f & kUint64MSB) == 0) {
126 | f <<= 1;
127 | e--;
128 | }
129 | this.f = f;
130 | this.e = e;
131 | }
132 |
133 | static NumberDiyFp normalize(NumberDiyFp a) {
134 | NumberDiyFp result = new NumberDiyFp(a.f, a.e);
135 | result.normalize();
136 | return result;
137 | }
138 |
139 | long f() {
140 | return f;
141 | }
142 |
143 | int e() {
144 | return e;
145 | }
146 |
147 | void setF(long new_value) {
148 | f = new_value;
149 | }
150 |
151 | void setE(int new_value) {
152 | e = new_value;
153 | }
154 |
155 | @Override
156 | public String toString() {
157 | return "[DiyFp f:" + f + ", e:" + e + "]";
158 | }
159 |
160 | }
161 |
--------------------------------------------------------------------------------
/java/deprecated/org/webpki/jcs/NumberDoubleHelper.java:
--------------------------------------------------------------------------------
1 | // Copyright 2010 the V8 project authors. All rights reserved.
2 | // Redistribution and use in source and binary forms, with or without
3 | // modification, are permitted provided that the following conditions are
4 | // met:
5 | //
6 | // * Redistributions of source code must retain the above copyright
7 | // notice, this list of conditions and the following disclaimer.
8 | // * Redistributions in binary form must reproduce the above
9 | // copyright notice, this list of conditions and the following
10 | // disclaimer in the documentation and/or other materials provided
11 | // with the distribution.
12 | // * Neither the name of Google Inc. nor the names of its
13 | // contributors may be used to endorse or promote products derived
14 | // from this software without specific prior written permission.
15 | //
16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 |
28 | // Ported to Java from Mozilla's version of V8-dtoa by Hannes Wallnoefer.
29 | // The original revision was 67d1049b0bf9 from the mozilla-central tree.
30 |
31 | package org.webpki.jcs;
32 |
33 | // Helper functions for doubles.
34 | class NumberDoubleHelper {
35 |
36 | static final long kSignMask = 0x8000000000000000L;
37 | static final long kExponentMask = 0x7FF0000000000000L;
38 | static final long kSignificandMask = 0x000FFFFFFFFFFFFFL;
39 | static final long kHiddenBit = 0x0010000000000000L;
40 |
41 | static NumberDiyFp asDiyFp(long d64) {
42 | assert (!isSpecial(d64));
43 | return new NumberDiyFp(significand(d64), exponent(d64));
44 | }
45 |
46 | // this->Significand() must not be 0.
47 | static NumberDiyFp asNormalizedDiyFp(long d64) {
48 | long f = significand(d64);
49 | int e = exponent(d64);
50 |
51 | assert (f != 0);
52 |
53 | // The current double could be a denormal.
54 | while ((f & kHiddenBit) == 0) {
55 | f <<= 1;
56 | e--;
57 | }
58 | // Do the final shifts in one go. Don't forget the hidden bit (the '-1').
59 | f <<= NumberDiyFp.kSignificandSize - kSignificandSize - 1;
60 | e -= NumberDiyFp.kSignificandSize - kSignificandSize - 1;
61 | return new NumberDiyFp(f, e);
62 | }
63 |
64 | static int exponent(long d64) {
65 | if (isDenormal(d64)) return kDenormalExponent;
66 |
67 | int biased_e = (int) (((d64 & kExponentMask) >>> kSignificandSize) & 0xffffffffL);
68 | return biased_e - kExponentBias;
69 | }
70 |
71 | static long significand(long d64) {
72 | long significand = d64 & kSignificandMask;
73 | if (!isDenormal(d64)) {
74 | return significand + kHiddenBit;
75 | } else {
76 | return significand;
77 | }
78 | }
79 |
80 | // Returns true if the double is a denormal.
81 | static boolean isDenormal(long d64) {
82 | return (d64 & kExponentMask) == 0L;
83 | }
84 |
85 | // We consider denormals not to be special.
86 | // Hence only Infinity and NaN are special.
87 | static boolean isSpecial(long d64) {
88 | return (d64 & kExponentMask) == kExponentMask;
89 | }
90 |
91 | static boolean isNan(long d64) {
92 | return ((d64 & kExponentMask) == kExponentMask) &&
93 | ((d64 & kSignificandMask) != 0L);
94 | }
95 |
96 |
97 | static boolean isInfinite(long d64) {
98 | return ((d64 & kExponentMask) == kExponentMask) &&
99 | ((d64 & kSignificandMask) == 0L);
100 | }
101 |
102 |
103 | static int sign(long d64) {
104 | return (d64 & kSignMask) == 0L ? 1 : -1;
105 | }
106 |
107 |
108 | // Returns the two boundaries of first argument.
109 | // The bigger boundary (m_plus) is normalized. The lower boundary has the same
110 | // exponent as m_plus.
111 | static void normalizedBoundaries(long d64, NumberDiyFp m_minus, NumberDiyFp m_plus) {
112 | NumberDiyFp v = asDiyFp(d64);
113 | boolean significand_is_zero = (v.f() == kHiddenBit);
114 | m_plus.setF((v.f() << 1) + 1);
115 | m_plus.setE(v.e() - 1);
116 | m_plus.normalize();
117 | if (significand_is_zero && v.e() != kDenormalExponent) {
118 | // The boundary is closer. Think of v = 1000e10 and v- = 9999e9.
119 | // Then the boundary (== (v - v-)/2) is not just at a distance of 1e9 but
120 | // at a distance of 1e8.
121 | // The only exception is for the smallest normal: the largest denormal is
122 | // at the same distance as its successor.
123 | // Note: denormals have the same exponent as the smallest normals.
124 | m_minus.setF((v.f() << 2) - 1);
125 | m_minus.setE(v.e() - 2);
126 | } else {
127 | m_minus.setF((v.f() << 1) - 1);
128 | m_minus.setE(v.e() - 1);
129 | }
130 | m_minus.setF(m_minus.f() << (m_minus.e() - m_plus.e()));
131 | m_minus.setE(m_plus.e());
132 | }
133 |
134 | private static final int kSignificandSize = 52; // Excludes the hidden bit.
135 | private static final int kExponentBias = 0x3FF + kSignificandSize;
136 | private static final int kDenormalExponent = -kExponentBias + 1;
137 |
138 | }
139 |
--------------------------------------------------------------------------------
/java/deprecated/org/webpki/jcs/NumberFastDtoaBuilder.java:
--------------------------------------------------------------------------------
1 | /* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 | *
3 | * This Source Code Form is subject to the terms of the Mozilla Public
4 | * License, v. 2.0. If a copy of the MPL was not distributed with this
5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 |
7 | package org.webpki.jcs;
8 |
9 | import java.util.Arrays;
10 |
11 | class NumberFastDtoaBuilder {
12 |
13 | // allocate buffer for generated digits + extra notation + padding zeroes
14 | final char[] chars = new char[NumberFastDtoa.kFastDtoaMaximalLength + 8];
15 | int end = 0;
16 | int point;
17 | boolean formatted = false;
18 |
19 | void append(char c) {
20 | chars[end++] = c;
21 | }
22 |
23 | void decreaseLast() {
24 | chars[end - 1]--;
25 | }
26 |
27 | public void reset() {
28 | end = 0;
29 | formatted = false;
30 | }
31 |
32 | @Override
33 | public String toString() {
34 | return "[chars:" + new String(chars, 0, end) + ", point:" + point + "]";
35 | }
36 |
37 | public String format() {
38 | if (!formatted) {
39 | // check for minus sign
40 | int firstDigit = chars[0] == '-' ? 1 : 0;
41 | int decPoint = point - firstDigit;
42 | if (decPoint < -5 || decPoint > 21) {
43 | toExponentialFormat(firstDigit, decPoint);
44 | } else {
45 | toFixedFormat(firstDigit, decPoint);
46 | }
47 | formatted = true;
48 | }
49 | return new String(chars, 0, end);
50 |
51 | }
52 |
53 | private void toFixedFormat(int firstDigit, int decPoint) {
54 | if (point < end) {
55 | // insert decimal point
56 | if (decPoint > 0) {
57 | // >= 1, split decimals and insert point
58 | System.arraycopy(chars, point, chars, point + 1, end - point);
59 | chars[point] = '.';
60 | end++;
61 | } else {
62 | // < 1,
63 | int target = firstDigit + 2 - decPoint;
64 | System.arraycopy(chars, firstDigit, chars, target, end - firstDigit);
65 | chars[firstDigit] = '0';
66 | chars[firstDigit + 1] = '.';
67 | if (decPoint < 0) {
68 | Arrays.fill(chars, firstDigit + 2, target, '0');
69 | }
70 | end += 2 - decPoint;
71 | }
72 | } else if (point > end) {
73 | // large integer, add trailing zeroes
74 | Arrays.fill(chars, end, point, '0');
75 | end += point - end;
76 | }
77 | }
78 |
79 | private void toExponentialFormat(int firstDigit, int decPoint) {
80 | if (end - firstDigit > 1) {
81 | // insert decimal point if more than one digit was produced
82 | int dot = firstDigit + 1;
83 | System.arraycopy(chars, dot, chars, dot + 1, end - dot);
84 | chars[dot] = '.';
85 | end++;
86 | }
87 | chars[end++] = 'e';
88 | char sign = '+';
89 | int exp = decPoint - 1;
90 | if (exp < 0) {
91 | sign = '-';
92 | exp = -exp;
93 | }
94 | chars[end++] = sign;
95 |
96 | int charPos = exp > 99 ? end + 2 : exp > 9 ? end + 1 : end;
97 | end = charPos + 1;
98 |
99 | // code below is needed because Integer.getChars() is not public
100 | for (; ; ) {
101 | int r = exp % 10;
102 | chars[charPos--] = digits[r];
103 | exp = exp / 10;
104 | if (exp == 0) break;
105 | }
106 | }
107 |
108 | final static char[] digits = {
109 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
110 | };
111 | }
112 |
--------------------------------------------------------------------------------
/java/deprecated/org/webpki/jcs/NumberToJSON.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 | package org.webpki.jcs;
18 |
19 | import java.io.IOException;
20 |
21 | /**
22 | * Number serialization support.
23 | */
24 | public class NumberToJSON {
25 |
26 | private NumberToJSON() {}
27 |
28 | /**
29 | * Formats a number according to ES6.
30 | * This code is emulating 7.1.12.1 of the EcmaScript V6 specification.
31 | * @param value Value to be formatted
32 | * @return String representation
33 | * @throws IOException
34 | */
35 | public static String serializeNumber(double value) throws IOException {
36 | // 1. Check for JSON compatibility.
37 | if (Double.isNaN(value) || Double.isInfinite(value)) {
38 | throw new IOException("NaN/Infinity are not permitted in JSON");
39 | }
40 |
41 | // 2.Deal with zero separately. Note that this test takes "-0.0" as well
42 | if (value == 0.0) {
43 | return "0";
44 | }
45 |
46 | // 3. Call the DtoA algorithm crunchers
47 | // V8 FastDtoa can't convert all numbers, so try it first but
48 | // fall back to old DToA in case it fails
49 | String result = NumberFastDtoa.numberToString(value);
50 | if (result != null) {
51 | return result;
52 | }
53 | StringBuilder buffer = new StringBuilder();
54 | NumberDToA.JS_dtostr(buffer, NumberDToA.DTOSTR_STANDARD, 0, value);
55 | return buffer.toString();
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/java/miscellaneous/README.md:
--------------------------------------------------------------------------------
1 | ### Java Test Programs
2 |
3 | This directory contains a number of test programs for the canonicalizer stored in the sibling directory.
4 |
5 | Note that `verify-numbers` require download of a 3Gb+ file with testdata.
6 | See root directory `testdata` for details.
7 | This file can be stored in any suitable place.
8 | After that the file `webpki.properties` must be updated with the actual path.
9 |
10 | ```code
11 | $ ant verify-canonicalization
12 | $ ant verify-numbers
13 | ```
14 |
--------------------------------------------------------------------------------
/java/miscellaneous/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/AddEdgeCases.java:
--------------------------------------------------------------------------------
1 | import java.io.FileReader;
2 | import java.io.FileWriter;
3 | import java.io.BufferedReader;
4 |
5 | import java.util.LinkedHashMap;
6 |
7 | import org.webpki.jcs.NumberToJSON;
8 |
9 | public class AddEdgeCases {
10 |
11 | static final int TURNS = 10000000;
12 |
13 | static LinkedHashMap ieeeValues = new LinkedHashMap();
14 |
15 | static void add(long exponent, long mantissa) throws Exception {
16 | long value = (exponent << 52) + mantissa;
17 | String ieeeHex = Long.toHexString(value);
18 | Double ieeeDouble = Double.longBitsToDouble(value);
19 | if (ieeeDouble == 0 || ieeeDouble.isNaN() || ieeeDouble.isInfinite()) {
20 | System.out.println("Dropped value: " + ieeeHex);
21 | return;
22 | }
23 | String es6Number = NumberToJSON.serializeNumber(ieeeDouble);
24 | System.out.println("New " + ieeeHex + " " + es6Number);
25 | ieeeValues.put(ieeeHex, es6Number);
26 | }
27 |
28 | public static void main(String[] args) throws Exception {
29 | // Add something
30 | /*
31 | ieeeValues.put("3eb0c6f7a0b5ed8d", "0.000001");
32 | add(5, 70l);
33 | */
34 | BufferedReader in = new BufferedReader(new FileReader(args[0]));
35 | int q = 0;
36 | long total = 0;
37 | while (true) {
38 | String s = in.readLine();
39 | if (s == null) {
40 | in.close();
41 | System.out.println("\nTest was successful");
42 | if (ieeeValues.isEmpty()) {
43 | System.out.println("\nThere were no new values to add");
44 | return;
45 | }
46 | FileWriter writer = new FileWriter(args[0], true);
47 | for (String ieeeHex : ieeeValues.keySet()) {
48 | writer.write(ieeeHex + "," + ieeeValues.get(ieeeHex) + "\n");
49 | }
50 | writer.close();
51 | return;
52 | }
53 | String hex = s.substring(0, s.indexOf(','));
54 | String text = s.substring(s.indexOf(',') + 1);
55 | if (ieeeValues.containsKey(hex)) {
56 | System.out.println("Duplicate: " + hex);
57 | ieeeValues.remove(hex);
58 | }
59 | while (hex.length() < 16) {
60 | hex = '0' + hex;
61 | }
62 | double d = Double.longBitsToDouble(Long.parseUnsignedLong(hex,16));
63 | String res = NumberToJSON.serializeNumber(d);
64 | if (!res.equals(text)) {
65 | System.out.println("FAIL res=" + res + " d=" + d);
66 | return;
67 | }
68 | total++;
69 | if (q++ == TURNS) {
70 | System.out.println("TURN:" + total + " " + text + " d=" + d);
71 | q = 0;
72 | }
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/BrowserCodeGenerator.java:
--------------------------------------------------------------------------------
1 | import java.io.File;
2 |
3 | import org.webpki.util.ArrayUtil;
4 | import org.webpki.util.DebugFormatter;
5 |
6 | public class BrowserCodeGenerator {
7 |
8 | static StringBuilder html = new StringBuilder(
9 | "JSON Canonicalization\n" +
10 | "\n" +
11 | "\n" +
12 | "\n" +
23 | "\n" +
180 | "![](\"https://cyberphone.github.io/doc/security/jcs.svg\" "\\"Project")
" +
183 | "JSON Canonicalization Test
\n" +
184 | "\n")
185 | .append(table)
186 | .append("");
187 | ArrayUtil.writeFile(args[1], html.toString().getBytes("utf-8"));
188 | }
189 | }
190 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/CanonicalizerTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | import java.io.File;
19 |
20 | import org.webpki.util.ArrayUtil;
21 | import org.webpki.util.DebugFormatter;
22 |
23 | import org.webpki.jcs.JsonCanonicalizer;
24 |
25 | public class CanonicalizerTest {
26 |
27 | static String inputDirectory;
28 | static String outputDirectory;
29 | static String outhexDirectory;
30 |
31 | static int failures = 0;
32 |
33 | static void performOneTest(String fileName) throws Exception {
34 | byte[] rawInput = ArrayUtil.readFile(inputDirectory + File.separator + fileName);
35 | byte[] expected = ArrayUtil.readFile(outputDirectory + File.separator + fileName);
36 | byte[] outhex = DebugFormatter.getByteArrayFromHex(
37 | new String(ArrayUtil.readFile(outhexDirectory + File.separator
38 | + fileName.substring(0, fileName.indexOf('.')) + ".txt"), "utf-8")
39 | .replace(" ","").replace("\n","").replace("\r",""));
40 | byte[] actual = new JsonCanonicalizer(rawInput).getEncodedUTF8();
41 | byte[] recycled = new JsonCanonicalizer(actual).getEncodedUTF8();
42 | StringBuilder utf8InHex = new StringBuilder("\nFile: ");
43 | utf8InHex.append(fileName);
44 | int byteCount = 0;
45 | boolean next = false;
46 | for (byte b : actual) {
47 | if (byteCount++ % 32 == 0) {
48 | utf8InHex.append('\n');
49 | next = false;
50 | }
51 | if (next) {
52 | utf8InHex.append(" ");
53 | }
54 | next = true;
55 | utf8InHex.append(DebugFormatter.getHexString(new byte[]{b}));
56 | }
57 | System.out.println(utf8InHex.append("\n").toString());
58 | if (!ArrayUtil.compare(expected, actual) ||
59 | !ArrayUtil.compare(recycled, actual) ||
60 | !ArrayUtil.compare(actual, outhex)) {
61 | failures++;
62 | System.out.println("THE TEST ABOVE FAILED!");
63 | }
64 | }
65 |
66 | public static void main(String[] args) throws Exception {
67 | inputDirectory = args[0] + File.separator + "input";
68 | outputDirectory = args[0] + File.separator + "output";
69 | outhexDirectory = args[0] + File.separator + "outhex";
70 | File[] files = new File(inputDirectory).listFiles();
71 | for (File f : files) {
72 | performOneTest(f.getName());
73 | }
74 | if (failures == 0) {
75 | System.out.println("All tests succeeded!\n");
76 | } else {
77 | System.out.println("\n****** ERRORS: " + failures + " *******\n");
78 | }
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/ES6InterActive.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | import java.util.Scanner;
19 |
20 | import org.webpki.jcs.NumberToJSON;
21 |
22 | public class ES6InterActive {
23 |
24 | public static void main(String[] args) throws Exception {
25 | Scanner input = new Scanner(System.in);
26 | while (true) {
27 | System.out.println("\nEnter xhhhhhhhhh or floating point number: ");
28 | String line = input.next();
29 | String inputFp = "N/A";
30 | double d;
31 | if (line.startsWith("x")) {
32 | String hex = line.substring(1);
33 | while (hex.length() < 16) {
34 | hex = '0' + hex;
35 | }
36 | d = Double.longBitsToDouble(Long.parseUnsignedLong(hex,16));
37 | } else {
38 | inputFp = line;
39 | d = Double.valueOf(inputFp);
40 | }
41 | long ieee = Double.doubleToRawLongBits(d);
42 | String ieeeHex = Long.toHexString(ieee);
43 | while (ieeeHex.length() < 16) {
44 | ieeeHex = '0' + ieeeHex;
45 | }
46 | String ieeeBin = Long.toBinaryString(ieee);
47 | while (ieeeBin.length() < 64) {
48 | ieeeBin = '0' + ieeeBin;
49 | }
50 | ieeeBin = ieeeBin.substring(0,1) + ' ' +
51 | ieeeBin.substring(1,12) + ' ' +
52 | ieeeBin.substring(12);
53 | String outputFp = NumberToJSON.serializeNumber(d);
54 | System.out.println("\nInput floating point: " + inputFp);
55 | System.out.println("Output floating point: " + outputFp);
56 | System.out.println("Hex value: " + ieeeHex);
57 | System.out.println("Binary value: " + ieeeBin);
58 | }
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/ES6NumberTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | import java.io.FileReader;
19 | import java.io.BufferedReader;
20 | import java.io.IOException;
21 |
22 | import org.webpki.jcs.NumberToJSON;
23 |
24 | public class ES6NumberTest {
25 |
26 | static final int TURNS = 1000000;
27 |
28 | static final String INVALID_NUMBER = "null";
29 |
30 | static int conversionErrors;
31 |
32 | static void verify(String ieeeHex, String expected) {
33 | while (ieeeHex.length() < 16) {
34 | ieeeHex = '0' + ieeeHex;
35 | }
36 | double ieeeF64 = Double.longBitsToDouble(Long.parseUnsignedLong(ieeeHex, 16));
37 | try {
38 | String es6Created = NumberToJSON.serializeNumber(ieeeF64);
39 | if (!es6Created.equals(expected)) {
40 | conversionErrors++;
41 | System.out.println("ieeeHex=" + ieeeHex +
42 | "\ncreated=" + es6Created +
43 | "\nexpected=" + expected + "\n");
44 | }
45 | } catch (IOException e) {
46 | if (!expected.equals(INVALID_NUMBER)) {
47 | conversionErrors++;
48 | System.out.println("ieeeHex=" + ieeeHex);
49 | }
50 | }
51 | if (!expected.equals(INVALID_NUMBER)) {
52 | if (ieeeF64 != Double.valueOf(expected)) {
53 | throw new RuntimeException("Parser error: " + expected);
54 | }
55 | }
56 | }
57 |
58 | public static void main(String[] args) throws IOException {
59 | verify("4340000000000001", "9007199254740994");
60 | verify("4340000000000002", "9007199254740996");
61 | verify("444b1ae4d6e2ef50", "1e+21");
62 | verify("3eb0c6f7a0b5ed8d", "0.000001");
63 | verify("3eb0c6f7a0b5ed8c", "9.999999999999997e-7");
64 | verify("8000000000000000", "0");
65 | verify("7fffffffffffffff", INVALID_NUMBER);
66 | verify("7ff0000000000000", INVALID_NUMBER);
67 | verify("fff0000000000000", INVALID_NUMBER);
68 | BufferedReader in = new BufferedReader(new FileReader(args[0]));
69 | long lineCount = 0;
70 | while (true) {
71 | String line = in.readLine();
72 | if (line == null) {
73 | if (conversionErrors == 0) {
74 | System.out.println("\nSuccessful Operation. Lines read: " + lineCount);
75 | } else {
76 | System.out.println("\nFailures: " + conversionErrors);
77 | }
78 | return;
79 | }
80 | int comma = line.indexOf(',');
81 | verify(line.substring(0, comma), line.substring(comma + 1));
82 | if (++lineCount % TURNS == 0) {
83 | System.out.println("Line: " + lineCount);
84 | }
85 | }
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/F32InterActive.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | import java.util.Scanner;
19 |
20 | import org.webpki.jcs.NumberToJSON;
21 |
22 | public class F32InterActive {
23 |
24 | public static void main(String[] args) throws Exception {
25 | Scanner input = new Scanner(System.in);
26 | while (true) {
27 | System.out.println("\nEnter xhhhhhhhhh or floating point 32-bit number: ");
28 | String line = input.next();
29 | String inputFp = "N/A";
30 | float f;
31 | if (line.startsWith("x")) {
32 | String hex = line.substring(1);
33 | while (hex.length() < 8) {
34 | hex = '0' + hex;
35 | }
36 | f = Float.intBitsToFloat(Integer.parseUnsignedInt(hex,16));
37 | } else {
38 | inputFp = line;
39 | f = Float.valueOf(inputFp);
40 | }
41 | double d = f;
42 | long ieee64 = Double.doubleToRawLongBits(d);
43 | String ieeeHex64 = Long.toHexString(ieee64);
44 | while (ieeeHex64.length() < 16) {
45 | ieeeHex64 = '0' + ieeeHex64;
46 | }
47 | String ieeeBin64 = Long.toBinaryString(ieee64);
48 | while (ieeeBin64.length() < 64) {
49 | ieeeBin64 = '0' + ieeeBin64;
50 | }
51 | ieeeBin64 = ieeeBin64.substring(0,1) + ' ' +
52 | ieeeBin64.substring(1,12) + ' ' +
53 | ieeeBin64.substring(12);
54 | long ieee32 = Float.floatToRawIntBits(f) & 0xffffffffL;
55 | String ieeeHex32 = Long.toHexString(ieee32);
56 | while (ieeeHex32.length() < 8) {
57 | ieeeHex32 = '0' + ieeeHex32;
58 | }
59 | String ieeeBin32 = Long.toBinaryString(ieee32);
60 | while (ieeeBin32.length() < 32) {
61 | ieeeBin32 = '0' + ieeeBin32;
62 | }
63 | ieeeBin32 = ieeeBin32.substring(0,1) + ' ' +
64 | ieeeBin32.substring(1,9) + ' ' +
65 | ieeeBin32.substring(9);
66 | String outputFp = NumberToJSON.serializeNumber(d);
67 | System.out.println("\nInput floating point: " + inputFp);
68 | System.out.println("Output floating point: " + outputFp);
69 | System.out.println("Hex 64 value: " + ieeeHex64);
70 | System.out.println("Binary 64 value: " + ieeeBin64);
71 | System.out.println("Hex 32 value: " + ieeeHex32);
72 | System.out.println("Binary 32 value: " + ieeeBin32);
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/SortingSchemes.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | import java.io.IOException;
19 |
20 | import java.util.TreeMap;
21 | import java.util.LinkedHashMap;
22 |
23 | import org.webpki.jcs.JsonCanonicalizer;
24 |
25 | public class SortingSchemes {
26 |
27 | static LinkedHashMap elements = new LinkedHashMap();
28 | static LinkedHashMap rawKeys = new LinkedHashMap();
29 |
30 | static {
31 | elements.put("\\u20ac", "Euro Sign");
32 | elements.put("\\r", "Carriage Return");
33 | elements.put("\\ufb33", "Hebrew Letter Dalet With Dagesh");
34 | elements.put("1", "One");
35 | elements.put("\\u0080", "Control");
36 | elements.put("\\ud83d\\ude00", "Emoji: Grinning Face");
37 | elements.put("\\u00f6", "Latin Small Letter O With Diaeresis");
38 | }
39 |
40 | static char getHexChar(char c) throws IOException {
41 | switch (c) {
42 | case '0':
43 | case '1':
44 | case '2':
45 | case '3':
46 | case '4':
47 | case '5':
48 | case '6':
49 | case '7':
50 | case '8':
51 | case '9':
52 | return (char) (c - '0');
53 |
54 | case 'a':
55 | case 'b':
56 | case 'c':
57 | case 'd':
58 | case 'e':
59 | case 'f':
60 | return (char) (c - 'a' + 10);
61 |
62 | case 'A':
63 | case 'B':
64 | case 'C':
65 | case 'D':
66 | case 'E':
67 | case 'F':
68 | return (char) (c - 'A' + 10);
69 | }
70 | throw new IOException("Bad hex in \\u escape: " + c);
71 | }
72 |
73 | static String parseString(String property) throws IOException {
74 | StringBuilder result = new StringBuilder();
75 | int index = 0;
76 | while (index < property.length()) {
77 | char c = property.charAt(index++);
78 | if (c == '\\') {
79 | switch (c = property.charAt(index++)) {
80 | case '"':
81 | case '\\':
82 | case '/':
83 | break;
84 |
85 | case 'b':
86 | c = '\b';
87 | break;
88 |
89 | case 'f':
90 | c = '\f';
91 | break;
92 |
93 | case 'n':
94 | c = '\n';
95 | break;
96 |
97 | case 'r':
98 | c = '\r';
99 | break;
100 |
101 | case 't':
102 | c = '\t';
103 | break;
104 |
105 | case 'u':
106 | c = 0;
107 | for (int i = 0; i < 4; i++) {
108 | c = (char) ((c << 4) + getHexChar(property.charAt(index++)));
109 | }
110 | break;
111 |
112 | default:
113 | throw new IOException("Unsupported escape:" + c);
114 | }
115 | }
116 | result.append(c);
117 | }
118 | return result.toString();
119 | }
120 |
121 | public static void main(String[] args) throws Exception {
122 | StringBuilder json = new StringBuilder("{\n");
123 | boolean next = false;
124 | for (String key : elements.keySet()) {
125 | rawKeys.put(key, parseString(key));
126 | if (next) {
127 | json.append(",\n");
128 | }
129 | next = true;
130 | json.append(" \"")
131 | .append(key)
132 | .append("\": \"")
133 | .append(elements.get(key))
134 | .append('"');
135 | }
136 | json.append("\n}");
137 | System.out.println(json.toString());
138 | String canon = new JsonCanonicalizer(json.toString()).getEncodedString();
139 | int i = 0;
140 | while ((i = canon.indexOf(":\"", i)) > 0) {
141 | i += 2;
142 | int j = canon.indexOf('"', i);
143 | System.out.println(canon.substring(i, j));
144 | }
145 | json = new StringBuilder("{\n");
146 | next = false;
147 | for (String key : elements.keySet()) {
148 | if (next) {
149 | json.append(",\n");
150 | }
151 | byte[] utf8 = rawKeys.get(key).getBytes("utf-8");
152 | String utf8Key = "";
153 | for (byte b : utf8) {
154 | utf8Key += "\\u00";
155 | String hex = Integer.toUnsignedString(b & 0xff, 16);
156 | if (hex.length() == 1) {
157 | hex = "0" + hex;
158 | }
159 | utf8Key += hex;
160 | }
161 | next = true;
162 | json.append(" \"")
163 | .append(utf8Key)
164 | .append("\": \"")
165 | .append(elements.get(key))
166 | .append('"');
167 | }
168 | json.append("\n}");
169 | System.out.println(json.toString());
170 | canon = new JsonCanonicalizer(json.toString()).getEncodedString();
171 | i = 0;
172 | while ((i = canon.indexOf(":\"", i)) > 0) {
173 | i += 2;
174 | int j = canon.indexOf('"', i);
175 | System.out.println(canon.substring(i, j));
176 | }
177 | }
178 | }
179 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/Unicode2UTF16.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 |
18 | import java.util.Scanner;
19 |
20 | public class Unicode2UTF16 {
21 |
22 | public static void main(String[] args) throws Exception {
23 | Scanner input = new Scanner(System.in);
24 | while (true) {
25 | System.out.println("\nEnter uncode code point in hex: ");
26 | String line = input.next();
27 | char[] utf16 = Character.toChars(Integer.parseInt(line, 16));
28 | for (char c : utf16) {
29 | String hex = Integer.toHexString(c);
30 | while (hex.length() < 4) {
31 | hex = "0" + hex;
32 | }
33 | System.out.print(" " + hex);
34 | }
35 | System.out.println();
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/java/miscellaneous/src/org/webpki/util/DebugFormatter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2006-2018 WebPKI.org (http://webpki.org).
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | */
17 | package org.webpki.util;
18 |
19 |
20 | import java.io.IOException;
21 |
22 | public class DebugFormatter {
23 |
24 | private DebugFormatter() {}
25 |
26 | private StringBuilder res = new StringBuilder(1000);
27 |
28 | private void put(char c) {
29 | res.append(c);
30 | }
31 |
32 | private void hex(int i) {
33 | if (i < 10) {
34 | put((char) (i + '0'));
35 | } else {
36 | put((char) (i + 'a' - 10));
37 | }
38 | }
39 |
40 | private void twohex(int i) {
41 | i &= 0xFF;
42 | hex(i / 16);
43 | hex(i % 16);
44 | }
45 |
46 | private void addrhex(int i) {
47 | if (i > 65535) {
48 | twohex(i / 65536);
49 | i %= 65536;
50 | }
51 | twohex(i / 256);
52 | twohex(i % 256);
53 | }
54 |
55 | private String toHexDebugData(byte[] indata, int bytesPerLine) {
56 | int index = 0;
57 | int i = 0;
58 | if (indata.length == 0) {
59 | return "No data";
60 | }
61 | boolean onlyData = false;
62 | if (bytesPerLine < 0) {
63 | bytesPerLine = -bytesPerLine;
64 | onlyData = true;
65 | }
66 | while (index < indata.length) {
67 | if (index > 0) {
68 | put('\n');
69 | }
70 | addrhex(index);
71 | put(':');
72 | int q = indata.length - index;
73 | if (q > bytesPerLine) {
74 | q = bytesPerLine;
75 | }
76 | for (i = 0; i < q; i++) {
77 | put(' ');
78 | twohex(indata[index + i]);
79 | }
80 | if (onlyData) {
81 | index += q;
82 | continue;
83 | }
84 | while (i++ <= bytesPerLine) {
85 | put(' ');
86 | put(' ');
87 | put(' ');
88 | }
89 | put('\'');
90 | for (i = 0; i < q; i++) {
91 | int c = (int) indata[index++];
92 | if (c < 32 || c >= 127) {
93 | put('.');
94 | } else {
95 | put((char) c);
96 | }
97 | }
98 | put('\'');
99 | while (i++ < bytesPerLine) {
100 | put(' ');
101 | }
102 | }
103 | return res.toString();
104 | }
105 |
106 | private String toHexString(byte[] indata) {
107 | int i = 0;
108 | while (i < indata.length) {
109 | twohex(indata[i++]);
110 | }
111 | return res.toString();
112 | }
113 |
114 | public static String getHexDebugData(byte[] binaryBlob, int bytesPerLine) {
115 | return new DebugFormatter().toHexDebugData(binaryBlob, bytesPerLine);
116 | }
117 |
118 | public static String getHexDebugData(byte[] binaryBlob) {
119 | return getHexDebugData(binaryBlob, 16);
120 | }
121 |
122 | public static String getHexString(byte[] binaryBlob) {
123 | return new DebugFormatter().toHexString(binaryBlob);
124 | }
125 |
126 | public static int toHex(char c) throws IOException {
127 | if (c >= '0') {
128 | if (c <= '9') return c - '0';
129 | if (c >= 'a') {
130 | if (c <= 'f') return c - ('a' - 10);
131 | }
132 | if (c >= 'A') {
133 | if (c <= 'F') return c - ('A' - 10);
134 | }
135 | }
136 | throw new IOException("Bad hexchar: " + c);
137 | }
138 |
139 | public static byte[] getByteArrayFromHex(String hex) throws IOException {
140 | int l = hex.length();
141 | int bl;
142 | if (l == 0 || l % 2 != 0) throw new IOException("Bad hexstring: " + hex);
143 | byte[] data = new byte[bl = l / 2];
144 | while (--bl >= 0) {
145 | data[bl] = (byte) (toHex(hex.charAt(--l)) + (toHex(hex.charAt(--l)) << 4));
146 | }
147 | return data;
148 | }
149 |
150 |
151 | /*##################################################################*/
152 | /* */
153 | /* Method: main */
154 | /* */
155 | /* Description: This is a command-line interface for testing only */
156 | /* */
157 | /*##################################################################*/
158 |
159 | public static void main(String[] args) throws IOException {
160 | if (args.length == 3 && args[0].equals("tobin")) {
161 | ArrayUtil.writeFile(args[2], DebugFormatter.getByteArrayFromHex(new String(ArrayUtil.readFile(args[1]), "UTF-8")));
162 | System.exit(0);
163 | }
164 | if (args.length != 2 || !(args[0].equals("hex") || args[0].equals("dump"))) {
165 | System.out.println("Usage: DebugFormatter hex|dump bininputfile \n" +
166 | " tobin inputfileinhex outputfilebin\n");
167 | System.exit(0);
168 | }
169 | byte[] data = ArrayUtil.readFile(args[1]);
170 | System.out.print(args[0].equals("dump") ? DebugFormatter.getHexDebugData(data) : DebugFormatter.getHexString(data));
171 | }
172 |
173 | }
174 |
--------------------------------------------------------------------------------
/java/miscellaneous/webpki.properties:
--------------------------------------------------------------------------------
1 | es6test.file=c:/es6/numbers/es6testfile100m.txt
2 |
3 |
--------------------------------------------------------------------------------
/node-es6/README.md:
--------------------------------------------------------------------------------
1 | ### Node.js canonicalizer + test programs
2 |
3 | For running `verify-numbers.js` you need to download a 3Gb+ file with test
4 | data described in the root directory `testdata`. This file can be stored in
5 | any directory and requires updating the file path in `verify-numbers.js`
6 |
7 |
8 | ```code
9 | $ node verify-canonicalization.js
10 | $ node verify-numbers.js
11 | ```
12 |
--------------------------------------------------------------------------------
/node-es6/canonicalize.js:
--------------------------------------------------------------------------------
1 | // ES6 based JSON canonicalizer
2 | 'use strict';
3 | var canonicalize = function(object) {
4 |
5 | var buffer = '';
6 | serialize(object);
7 | return buffer;
8 |
9 | function serialize(object) {
10 | if (object === null || typeof object !== 'object' ||
11 | object.toJSON != null) {
12 | /////////////////////////////////////////////////
13 | // Primitive type or toJSON - Use ES6/JSON //
14 | /////////////////////////////////////////////////
15 | buffer += JSON.stringify(object);
16 |
17 | } else if (Array.isArray(object)) {
18 | /////////////////////////////////////////////////
19 | // Array - Maintain element order //
20 | /////////////////////////////////////////////////
21 | buffer += '[';
22 | let next = false;
23 | object.forEach((element) => {
24 | if (next) {
25 | buffer += ',';
26 | }
27 | next = true;
28 | /////////////////////////////////////////
29 | // Array element - Recursive expansion //
30 | /////////////////////////////////////////
31 | serialize(element);
32 | });
33 | buffer += ']';
34 |
35 | } else {
36 | /////////////////////////////////////////////////
37 | // Object - Sort properties before serializing //
38 | /////////////////////////////////////////////////
39 | buffer += '{';
40 | let next = false;
41 | Object.keys(object).sort().forEach((property) => {
42 | if (next) {
43 | buffer += ',';
44 | }
45 | next = true;
46 | ///////////////////////////////////////////////
47 | // Property names are strings - Use ES6/JSON //
48 | ///////////////////////////////////////////////
49 | buffer += JSON.stringify(property);
50 | buffer += ':';
51 | //////////////////////////////////////////
52 | // Property value - Recursive expansion //
53 | //////////////////////////////////////////
54 | serialize(object[property]);
55 | });
56 | buffer += '}';
57 | }
58 | }
59 | };
60 |
61 | module.exports = canonicalize;
62 |
--------------------------------------------------------------------------------
/node-es6/verify-canonicalization.js:
--------------------------------------------------------------------------------
1 | // JavaScript source code for testing the JSON canonicalizer
2 |
3 | 'use strict';
4 | const Fs = require('fs');
5 | const canonicalize = require('./canonicalize.js');
6 |
7 | const pathSeparator = __dirname.indexOf('\\') >= 0 ? '\\' : '/';
8 | const rootPath = __dirname.substring(0, __dirname.lastIndexOf(pathSeparator))
9 |
10 | const inputData = rootPath + '/testdata/input';
11 | const outputData = rootPath + '/testdata/output';
12 |
13 | function readFile(path) {
14 | return Fs.readFileSync(path);
15 | }
16 |
17 | var failures = 0;
18 |
19 | Fs.readdirSync(inputData).forEach((fileName) => {
20 | var expected = readFile(outputData + '/' + fileName);
21 | var actual = Buffer.from(canonicalize(JSON.parse(readFile(inputData + '/' + fileName))));
22 | var next = false;
23 | var byteCount = 0;
24 | var utf8 = '\nFile: ' + fileName;
25 | for (let i = 0; i < actual.length; i++) {
26 | if (byteCount++ % 32 == 0) {
27 | utf8 += '\n';
28 | next = false;
29 | }
30 | if (next) {
31 | utf8 += ' ';
32 | }
33 | next = true;
34 | utf8 += actual[i].toString(16);
35 | }
36 | console.log(utf8 + '\n');
37 | if (expected.compare(actual)) {
38 | failures++;
39 | console.log('THE TEST ABOVE FAILED!');
40 | }
41 | });
42 |
43 | if (failures == 0) {
44 | console.log('All tests succeeded!');
45 | } else {
46 | console.log('\n****** ERRORS: ' + failures + ' *******\n');
47 | }
48 |
--------------------------------------------------------------------------------
/node-es6/verify-numbers.js:
--------------------------------------------------------------------------------
1 | // JavaScript source code for testing number parsing and serialization
2 |
3 | 'use strict';
4 | const Fs = require('fs');
5 |
6 | var conversionErrors = 0;
7 |
8 | const INVALID_NUMBER = "null";
9 |
10 | function verify(ieeeHex, expected) {
11 | while (ieeeHex.length < 16) {
12 | ieeeHex = '0' + ieeeHex;
13 | }
14 | var ieeeDouble = Buffer.from(ieeeHex, 'hex').readDoubleBE();
15 | var es6Created = JSON.stringify(ieeeDouble);
16 | if (es6Created == INVALID_NUMBER && expected == INVALID_NUMBER) {
17 | return;
18 | } else if (es6Created == expected && Number(expected) == ieeeDouble) {
19 | return;
20 | }
21 | conversionErrors++;
22 | console.log("Hex=" + ieeeHex + " Expected=" + expected + " Created=" + es6Created);
23 | }
24 |
25 | verify("4340000000000001", "9007199254740994");
26 | verify("4340000000000002", "9007199254740996");
27 | verify("444b1ae4d6e2ef50", "1e+21");
28 | verify("3eb0c6f7a0b5ed8d", "0.000001");
29 | verify("3eb0c6f7a0b5ed8c", "9.999999999999997e-7");
30 | verify("8000000000000000", "0");
31 | verify("7fffffffffffffff", INVALID_NUMBER);
32 | verify("7ff0000000000000", INVALID_NUMBER);
33 | verify("fff0000000000000", INVALID_NUMBER);
34 |
35 | // Change the file name below to fit your environment
36 | var file = Fs.openSync("c:\\es6\\numbers\\es6testfile100m.txt", "r");
37 | var lineCount = 0;
38 | var fileBuffer = Buffer.alloc(1024);
39 | var line = "";
40 | var length = 0;
41 | while (length = Fs.readSync(file, fileBuffer, 0, 1024, null)) {
42 | for (let q = 0; q < length; q++) {
43 | var c = fileBuffer[q];
44 | if (c == 0x0a) {
45 | if (++lineCount % 1000000 == 0) {
46 | console.log("Line: " + lineCount);
47 | }
48 | let comma = line.indexOf(',');
49 | verify(line.substring(0, comma), line.substring(comma + 1));
50 | line = "";
51 | } else {
52 | line += String.fromCharCode(c);
53 | }
54 | }
55 | }
56 | Fs.closeSync(file);
57 | if (conversionErrors) {
58 | console.log("\n****** ERRORS: " + conversionErrors + " *******");
59 | } else {
60 | console.log("\nSuccessful Operation. Lines read: " + lineCount);
61 | }
62 |
--------------------------------------------------------------------------------
/python3/.gitignore:
--------------------------------------------------------------------------------
1 | **/__pycache__
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/python3/README.md:
--------------------------------------------------------------------------------
1 | ## JSON Canonicalizer for Python
2 |
3 | The [src/org/webpki/json](src/org/webpki/json)
4 | folder contains the source code for a
5 | JCS (RFC 8785) compliant canonicalizer written in Python.
6 |
7 | ### Building and testing
8 |
9 | - Set PYTHONPATH to the `src` directory.
10 |
11 | - For running `verify-numbers.py` you need to download a 3Gb+ file with test
12 | data described in the root directory [testdata](../testdata). This file can be stored in
13 | any directory and requires updating the file path in `verify-numbers.py`.
14 |
15 | - Perform the commands:
16 | ```code
17 | $ cd test
18 | $ python verify-canonicalization.py
19 | $ python verify-numbers.py
20 | ```
21 |
22 |
23 | ### Using the JSON canonicalizer
24 |
25 | ```python
26 | from org.webpki.json.Canonicalize import canonicalize
27 |
28 | data = canonicalize({"tag":4})
29 | ```
30 | Note that the input is Python data structures while result is an UTF-8 formatted byte array.
31 |
32 | If you rather need a free-standing canonicalizer you can achive that by using standard Python tools:
33 | ```python
34 | from org.webpki.json.Canonicalize import canonicalize
35 | from json import loads
36 |
37 | data = canonicalize(loads('{"tag":4}'))
38 | ```
39 |
--------------------------------------------------------------------------------
/python3/src/org/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/python3/src/org/webpki/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/python3/src/org/webpki/json/LICENSE.PSF:
--------------------------------------------------------------------------------
1 | PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
2 | --------------------------------------------
3 |
4 | 1. This LICENSE AGREEMENT is between the Python Software Foundation
5 | ("PSF"), and the Individual or Organization ("Licensee") accessing and
6 | otherwise using this software ("Python") in source or binary form and
7 | its associated documentation.
8 |
9 | 2. Subject to the terms and conditions of this License Agreement, PSF hereby
10 | grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
11 | analyze, test, perform and/or display publicly, prepare derivative works,
12 | distribute, and otherwise use Python alone or in any derivative version,
13 | provided, however, that PSF's License Agreement and PSF's notice of copyright,
14 | i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
15 | 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
16 | Rights Reserved" are retained in Python alone or in any derivative version
17 | prepared by Licensee.
18 |
19 | 3. In the event Licensee prepares a derivative work that is based on
20 | or incorporates Python or any part thereof, and wants to make
21 | the derivative work available to others as provided herein, then
22 | Licensee hereby agrees to include in any such work a brief summary of
23 | the changes made to Python.
24 |
25 | 4. PSF is making Python available to Licensee on an "AS IS"
26 | basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
27 | IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
28 | DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
29 | FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
30 | INFRINGE ANY THIRD PARTY RIGHTS.
31 |
32 | 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
33 | FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
34 | A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
35 | OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
36 |
37 | 6. This License Agreement will automatically terminate upon a material
38 | breach of its terms and conditions.
39 |
40 | 7. Nothing in this License Agreement shall be deemed to create any
41 | relationship of agency, partnership, or joint venture between PSF and
42 | Licensee. This License Agreement does not grant permission to use PSF
43 | trademarks or trade name in a trademark sense to endorse or promote
44 | products or services of Licensee, or any third party.
45 |
46 | 8. By copying, installing or otherwise using Python, Licensee
47 | agrees to be bound by the terms and conditions of this License
48 | Agreement.
49 |
--------------------------------------------------------------------------------
/python3/src/org/webpki/json/NumberToJson.py:
--------------------------------------------------------------------------------
1 | ##############################################################################
2 | # #
3 | # Copyright 2006-2019 WebPKI.org (http://webpki.org). #
4 | # #
5 | # Licensed under the Apache License, Version 2.0 (the "License"); #
6 | # you may not use this file except in compliance with the License. #
7 | # You may obtain a copy of the License at #
8 | # #
9 | # https://www.apache.org/licenses/LICENSE-2.0 #
10 | # #
11 | # Unless required by applicable law or agreed to in writing, software #
12 | # distributed under the License is distributed on an "AS IS" BASIS, #
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
14 | # See the License for the specific language governing permissions and #
15 | # limitations under the License. #
16 | # #
17 | ##############################################################################
18 |
19 | ##################################################################
20 | # Convert a Python double/float into an ES6/V8 compatible string #
21 | ##################################################################
22 |
23 | def convert2Es6Format(value):
24 | """Convert double/float to str using the native Python formatter"""
25 |
26 | fvalue = float(value)
27 |
28 | # Zero is a special case. The following line takes "-0" case as well
29 | if fvalue == 0:
30 | return '0'
31 |
32 | # The rest of the algorithm works on the textual representation only
33 | pyDouble = str(fvalue)
34 |
35 | # The following line catches the "inf" and "nan" values returned by str(fvalue)
36 | if 'n' in pyDouble:
37 | raise ValueError("Invalid JSON number: " + pyDouble)
38 |
39 | # Save sign separately, it doesn't have any role in the algorithm
40 | pySign = ''
41 | if pyDouble[0] == '-':
42 | pySign = '-'
43 | pyDouble = pyDouble[1:]
44 |
45 | # Now we should only have valid non-zero values
46 | pyExpStr = ''
47 | pyExpVal = 0
48 | q = pyDouble.find('e')
49 | if q > 0:
50 | # Grab the exponent and remove it from the number
51 | pyExpStr = pyDouble[q:]
52 | if pyExpStr[2:3] == '0':
53 | # Suppress leading zero on exponents
54 | pyExpStr = pyExpStr[:2] + pyExpStr[3:]
55 | pyDouble = pyDouble[0:q]
56 | pyExpVal = int(pyExpStr[1:])
57 |
58 | # Split number in pyFirst + pyDot + pyLast
59 | pyFirst = pyDouble
60 | pyDot = ''
61 | pyLast = ''
62 | q = pyDouble.find('.')
63 | if q > 0:
64 | pyDot = '.'
65 | pyFirst = pyDouble[:q]
66 | pyLast = pyDouble[q + 1:]
67 |
68 | # Now the string is split into: pySign + pyFirst + pyDot + pyLast + pyExpStr
69 | if pyLast == '0':
70 | # Always remove trailing .0
71 | pyDot = ''
72 | pyLast = ''
73 |
74 | if pyExpVal > 0 and pyExpVal < 21:
75 | # Integers are shown as is with up to 21 digits
76 | pyFirst += pyLast
77 | pyLast = ''
78 | pyDot = ''
79 | pyExpStr = ''
80 | q = pyExpVal - len(pyFirst)
81 | while q >= 0:
82 | q -= 1;
83 | pyFirst += '0'
84 | elif pyExpVal < 0 and pyExpVal > -7:
85 | # Small numbers are shown as 0.etc with e-6 as lower limit
86 | pyLast = pyFirst + pyLast
87 | pyFirst = '0'
88 | pyDot = '.'
89 | pyExpStr = ''
90 | q = pyExpVal
91 | while q < -1:
92 | q += 1;
93 | pyLast = '0' + pyLast
94 |
95 | # The resulting sub-strings are concatenated
96 | return pySign + pyFirst + pyDot + pyLast + pyExpStr
97 |
--------------------------------------------------------------------------------
/python3/src/org/webpki/json/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/python3/test/verify-canonicalization.py:
--------------------------------------------------------------------------------
1 | ##############################################################################
2 | # #
3 | # Copyright 2006-2019 WebPKI.org (http://webpki.org). #
4 | # #
5 | # Licensed under the Apache License, Version 2.0 (the "License"); #
6 | # you may not use this file except in compliance with the License. #
7 | # You may obtain a copy of the License at #
8 | # #
9 | # https://www.apache.org/licenses/LICENSE-2.0 #
10 | # #
11 | # Unless required by applicable law or agreed to in writing, software #
12 | # distributed under the License is distributed on an "AS IS" BASIS, #
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
14 | # See the License for the specific language governing permissions and #
15 | # limitations under the License. #
16 | # #
17 | ##############################################################################
18 |
19 | # Test program for JCS
20 |
21 | import sys
22 | import codecs
23 | import os
24 |
25 | from json import loads
26 |
27 | from org.webpki.json.Canonicalize import canonicalize
28 |
29 | # Our test program
30 | if not len(sys.argv) in (1,2):
31 | print ('[JSON-in-file]')
32 | sys.exit(1)
33 |
34 | failures = 0
35 |
36 | def readFile(name):
37 | return codecs.open(name, "r", "utf-8").read()
38 |
39 | def oneTurn(fileName):
40 | global failures
41 | jsonData = readFile(os.path.join(inputPath,fileName))
42 | actual = canonicalize(loads(jsonData))
43 | recycled = canonicalize(loads(actual.decode("utf-8","strict")))
44 | expected = readFile(os.path.join(outputPath,fileName)).encode()
45 | if actual == expected and actual == recycled:
46 | result = "\n"
47 | else:
48 | failures += 1
49 | result = "\n\nTHE TEST ABOVE FAILED!"
50 | byteCount = 0
51 | next = False
52 | utf8InHex = "\nFile: " + fileName
53 | for c in actual:
54 | if byteCount % 32 == 0:
55 | utf8InHex += '\n'
56 | next = False
57 | byteCount += 1
58 | if next:
59 | utf8InHex += ' '
60 | next = True
61 | utf8InHex += '{0:02x}'.format(c)
62 | print(utf8InHex + result)
63 |
64 | testData = os.path.join(os.path.split(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0])[0],'testdata')
65 | inputPath = os.path.join(testData, 'input')
66 | outputPath = os.path.join(testData, 'output')
67 | if len(sys.argv) == 1:
68 | for fileName in os.listdir(inputPath):
69 | oneTurn(fileName)
70 | else:
71 | oneTurn(sys.argv[1])
72 | if failures == 0:
73 | print("All tests succeeded!");
74 | else:
75 | print("\n****** ERRORS: " + str(failures) + " *******");
76 |
--------------------------------------------------------------------------------
/python3/test/verify-numbers.py:
--------------------------------------------------------------------------------
1 | ##############################################################################
2 | # #
3 | # Copyright 2006-2019 WebPKI.org (http://webpki.org). #
4 | # #
5 | # Licensed under the Apache License, Version 2.0 (the "License"); #
6 | # you may not use this file except in compliance with the License. #
7 | # You may obtain a copy of the License at #
8 | # #
9 | # https://www.apache.org/licenses/LICENSE-2.0 #
10 | # #
11 | # Unless required by applicable law or agreed to in writing, software #
12 | # distributed under the License is distributed on an "AS IS" BASIS, #
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
14 | # See the License for the specific language governing permissions and #
15 | # limitations under the License. #
16 | # #
17 | ##############################################################################
18 |
19 | # Test program for ES6-JSON/JCS number serialization
20 |
21 | import binascii
22 | import struct
23 |
24 | from org.webpki.json.NumberToJson import convert2Es6Format
25 |
26 | INVALID_NUMBER = 'null'
27 |
28 | #
29 | # Test program using a 100 million value file formatted as follows:
30 | # value in ieee hex representation (1-16 digits) + ',' + correct ES6 format + '\n'
31 | #
32 | def verify(ieeeHex, expected):
33 | while len(ieeeHex) < 16:
34 | ieeeHex = '0' + ieeeHex
35 | value = struct.unpack('>d',binascii.a2b_hex(ieeeHex))[0]
36 | try:
37 | pyFormat = convert2Es6Format(value)
38 | except ValueError:
39 | if expected == INVALID_NUMBER:
40 | return
41 | if pyFormat == expected and value == float(pyFormat) and repr(value) == str(value):
42 | return
43 | print('IEEE: ' + ieeeHex + '\nPython: ' + pyFormat + '\nExpected: ' + expected)
44 | exit(0)
45 |
46 | verify('4340000000000001', '9007199254740994')
47 | verify('4340000000000002', '9007199254740996')
48 | verify('444b1ae4d6e2ef50', '1e+21')
49 | verify('3eb0c6f7a0b5ed8d', '0.000001')
50 | verify('3eb0c6f7a0b5ed8c', '9.999999999999997e-7')
51 | verify('8000000000000000', '0')
52 | verify('7fffffffffffffff', INVALID_NUMBER)
53 | verify('7ff0000000000000', INVALID_NUMBER)
54 | verify('fff0000000000000', INVALID_NUMBER)
55 | # Change the file path below according to your installation
56 | file = open('c:\\es6\\numbers\\es6testfile100m.txt','r')
57 | lineCount = 0;
58 | while True:
59 | line = file.readline();
60 | if not line:
61 | print('\nSuccessful Operation. Lines read: ' + str(lineCount))
62 | exit(0)
63 | lineCount = lineCount + 1;
64 | i = line.find(',')
65 | if i <= 0 or i >= len(line) - 1:
66 | print('Bad line: ' + line)
67 | exit(0)
68 | verify(line[:i], line[i + 1:len(line) - 1])
69 | if lineCount % 1000000 == 0:
70 | print('Line: ' + str(lineCount))
71 |
--------------------------------------------------------------------------------
/testdata/README.md:
--------------------------------------------------------------------------------
1 | ## Test Data
2 |
3 | The [input](input) directory contains files with non-canonicalized data which is
4 | supposed be transformed as specified by the corresponding file in the
5 | [output](output) directory. In the [outhex](outhex) directory the expected
6 | output is expressed in hexadecimal byte notation.
7 |
8 | ## ES6 Numbers
9 |
10 | For testing ES6 number serialization there is a GZIP file hosted at
11 | https://github.com/cyberphone/json-canonicalization/releases/download/es6testfile/es6testfile100m.txt.gz
12 | containing a test vector of 100 million of random and edge-case values.
13 | The test file consists of lines
14 | ```code
15 | hex-ieee,expected\n
16 | ```
17 | where `hex-ieee` holds 1-16 ASCII hexadecimal characters
18 | representing an IEEE-754 double precision value
19 | while `expected` holds the expected serialized value.
20 | Each line is terminated by a single new-line character.
21 |
22 | Sample lines:
23 | ```code
24 | 4340000000000001,9007199254740994
25 | 4340000000000002,9007199254740996
26 | 444b1ae4d6e2ef50,1e+21
27 | 3eb0c6f7a0b5ed8d,0.000001
28 | 3eb0c6f7a0b5ed8c,9.999999999999997e-7
29 | 8000000000000000,0
30 | 0,0
31 | ```
32 |
33 | The sequence of IEEE-754 double precision values is deterministically generated
34 | according to the following algorithm:
35 | ```js
36 | const crypto = require("crypto")
37 |
38 | staticU64s = new BigUint64Array([
39 | 0x0000000000000000n, 0x8000000000000000n, 0x0000000000000001n, 0x8000000000000001n,
40 | 0xc46696695dbd1cc3n, 0xc43211ede4974a35n, 0xc3fce97ca0f21056n, 0xc3c7213080c1a6acn,
41 | 0xc39280f39a348556n, 0xc35d9b1f5d20d557n, 0xc327af4c4a80aaacn, 0xc2f2f2a36ecd5556n,
42 | 0xc2be51057e155558n, 0xc28840d131aaaaacn, 0xc253670dc1555557n, 0xc21f0b4935555557n,
43 | 0xc1e8d5d42aaaaaacn, 0xc1b3de4355555556n, 0xc17fca0555555556n, 0xc1496e6aaaaaaaabn,
44 | 0xc114585555555555n, 0xc0e046aaaaaaaaabn, 0xc0aa0aaaaaaaaaaan, 0xc074d55555555555n,
45 | 0xc040aaaaaaaaaaabn, 0xc00aaaaaaaaaaaabn, 0xbfd5555555555555n, 0xbfa1111111111111n,
46 | 0xbf6b4e81b4e81b4fn, 0xbf35d867c3ece2a5n, 0xbf0179ec9cbd821en, 0xbecbf647612f3696n,
47 | 0xbe965e9f80f29212n, 0xbe61e54c672874dbn, 0xbe2ca213d840baf8n, 0xbdf6e80fe033c8c6n,
48 | 0xbdc2533fe68fd3d2n, 0xbd8d51ffd74c861cn, 0xbd5774ccac3d3817n, 0xbd22c3d6f030f9acn,
49 | 0xbcee0624b3818f79n, 0xbcb804ea293472c7n, 0xbc833721ba905bd3n, 0xbc4ebe9c5db3c61en,
50 | 0xbc18987d17c304e5n, 0xbbe3ad30dfcf371dn, 0xbbaf7b816618582fn, 0xbb792f9ab81379bfn,
51 | 0xbb442615600f9499n, 0xbb101e77800c76e1n, 0xbad9ca58cce0be35n, 0xbaa4a1e0a3e6fe90n,
52 | 0xba708180831f320dn, 0xba3a68cd9e985016n, 0x446696695dbd1cc3n, 0x443211ede4974a35n,
53 | 0x43fce97ca0f21056n, 0x43c7213080c1a6acn, 0x439280f39a348556n, 0x435d9b1f5d20d557n,
54 | 0x4327af4c4a80aaacn, 0x42f2f2a36ecd5556n, 0x42be51057e155558n, 0x428840d131aaaaacn,
55 | 0x4253670dc1555557n, 0x421f0b4935555557n, 0x41e8d5d42aaaaaacn, 0x41b3de4355555556n,
56 | 0x417fca0555555556n, 0x41496e6aaaaaaaabn, 0x4114585555555555n, 0x40e046aaaaaaaaabn,
57 | 0x40aa0aaaaaaaaaaan, 0x4074d55555555555n, 0x4040aaaaaaaaaaabn, 0x400aaaaaaaaaaaabn,
58 | 0x3fd5555555555555n, 0x3fa1111111111111n, 0x3f6b4e81b4e81b4fn, 0x3f35d867c3ece2a5n,
59 | 0x3f0179ec9cbd821en, 0x3ecbf647612f3696n, 0x3e965e9f80f29212n, 0x3e61e54c672874dbn,
60 | 0x3e2ca213d840baf8n, 0x3df6e80fe033c8c6n, 0x3dc2533fe68fd3d2n, 0x3d8d51ffd74c861cn,
61 | 0x3d5774ccac3d3817n, 0x3d22c3d6f030f9acn, 0x3cee0624b3818f79n, 0x3cb804ea293472c7n,
62 | 0x3c833721ba905bd3n, 0x3c4ebe9c5db3c61en, 0x3c18987d17c304e5n, 0x3be3ad30dfcf371dn,
63 | 0x3baf7b816618582fn, 0x3b792f9ab81379bfn, 0x3b442615600f9499n, 0x3b101e77800c76e1n,
64 | 0x3ad9ca58cce0be35n, 0x3aa4a1e0a3e6fe90n, 0x3a708180831f320dn, 0x3a3a68cd9e985016n,
65 | 0x4024000000000000n, 0x4014000000000000n, 0x3fe0000000000000n, 0x3fa999999999999an,
66 | 0x3f747ae147ae147bn, 0x3f40624dd2f1a9fcn, 0x3f0a36e2eb1c432dn, 0x3ed4f8b588e368f1n,
67 | 0x3ea0c6f7a0b5ed8dn, 0x3e6ad7f29abcaf48n, 0x3e35798ee2308c3an, 0x3ed539223589fa95n,
68 | 0x3ed4ff26cd5a7781n, 0x3ed4f95a762283ffn, 0x3ed4f8c60703520cn, 0x3ed4f8b72f19cd0dn,
69 | 0x3ed4f8b5b31c0c8dn, 0x3ed4f8b58d1c461an, 0x3ed4f8b5894f7f0en, 0x3ed4f8b588ee37f3n,
70 | 0x3ed4f8b588e47da4n, 0x3ed4f8b588e3849cn, 0x3ed4f8b588e36bb5n, 0x3ed4f8b588e36937n,
71 | 0x3ed4f8b588e368f8n, 0x3ed4f8b588e368f1n, 0x3ff0000000000000n, 0xbff0000000000000n,
72 | 0xbfeffffffffffffan, 0xbfeffffffffffffbn, 0x3feffffffffffffan, 0x3feffffffffffffbn,
73 | 0x3feffffffffffffcn, 0x3feffffffffffffen, 0xbfefffffffffffffn, 0xbfefffffffffffffn,
74 | 0x3fefffffffffffffn, 0x3fefffffffffffffn, 0x3fd3333333333332n, 0x3fd3333333333333n,
75 | 0x3fd3333333333334n, 0x0010000000000000n, 0x000ffffffffffffdn, 0x000fffffffffffffn,
76 | 0x7fefffffffffffffn, 0xffefffffffffffffn, 0x4340000000000000n, 0xc340000000000000n,
77 | 0x4430000000000000n, 0x44b52d02c7e14af5n, 0x44b52d02c7e14af6n, 0x44b52d02c7e14af7n,
78 | 0x444b1ae4d6e2ef4en, 0x444b1ae4d6e2ef4fn, 0x444b1ae4d6e2ef50n, 0x3eb0c6f7a0b5ed8cn,
79 | 0x3eb0c6f7a0b5ed8dn, 0x41b3de4355555553n, 0x41b3de4355555554n, 0x41b3de4355555555n,
80 | 0x41b3de4355555556n, 0x41b3de4355555557n, 0xbecbf647612f3696n, 0x43143ff3c1cb0959n,
81 | ])
82 | staticF64s = new Float64Array(staticU64s.buffer)
83 | serialU64s = new BigUint64Array(2000)
84 | for (i = 0; i < 2000; i++) {
85 | serialU64s[i] = 0x0010000000000000n + BigInt(i)
86 | }
87 | serialF64s = new Float64Array(serialU64s.buffer)
88 |
89 | var state = {
90 | idx: 0,
91 | data: new Float64Array(),
92 | block: new ArrayBuffer(32),
93 | }
94 | while (true) {
95 | // Generate the next IEEE-754 value in the sequence.
96 | var f = 0.0;
97 | if (state.idx < staticF64s.length) {
98 | f = staticF64s[state.idx]
99 | } else if (state.idx < staticF64s.length + serialF64s.length) {
100 | f = serialF64s[state.idx - staticF64s.length]
101 | } else {
102 | while (f == 0.0 || !isFinite(f)) {
103 | if (state.data.length == 0) {
104 | state.block = crypto.createHash("sha256").update(new Buffer(state.block)).digest().buffer
105 | state.data = new Float64Array(state.block)
106 | }
107 | f = state.data[0]
108 | state.data = state.data.slice(1)
109 | }
110 | }
111 | state.idx++
112 |
113 | // Print the raw IEEE-754 value as hexadecimal.
114 | var u64 = new BigUint64Array(1)
115 | var f64 = new Float64Array(u64.buffer)
116 | f64[0] = f
117 | console.log(u64[0].toString(16))
118 | }
119 | ```
120 | Deterministic generation of the test inputs allows an implementation to verify
121 | correctness of ES6 number formatting without requiring any network bandwidth by
122 | generating the test file locally and computing its hash.
123 |
124 | The following table records the expected hashes:
125 | | SHA-256 checksum | Number of lines | Size in bytes |
126 | | ---------------------------------------------------------------- | --------------- | ------------- |
127 | | be18b62b6f69cdab33a7e0dae0d9cfa869fda80ddc712221570f9f40a5878687 | 1000 | 37967 |
128 | | b9f7a8e75ef22a835685a52ccba7f7d6bdc99e34b010992cbc5864cd12be6892 | 10000 | 399022 |
129 | | 22776e6d4b49fa294a0d0f349268e5c28808fe7e0cb2bcbe28f63894e494d4c7 | 100000 | 4031728 |
130 | | 49415fee2c56c77864931bd3624faad425c3c577d6d74e89a83bc725506dad16 | 1000000 | 40357417 |
131 | | b9f8a44a91d46813b21b9602e72f112613c91408db0b8341fb94603d9db135e0 | 10000000 | 403630048 |
132 | | 0f7dda6b0837dde083c5d6b896f7d62340c8a2415b0c7121d83145e08a755272 | 100000000 | 4036326174 |
133 |
134 | The entries in the table are determined apart from GZIP compression.
135 | The `SHA-256 checksum` is a hash of the generated output file for the
136 | first `Number of lines` in the sequence.
137 |
138 | Both `numgen.go` and `numgen.js` can generate `es6testfile100m.txt`.
139 |
--------------------------------------------------------------------------------
/testdata/input/arrays.json:
--------------------------------------------------------------------------------
1 | [
2 | 56,
3 | {
4 | "d": true,
5 | "10": null,
6 | "1": [ ]
7 | }
8 | ]
9 |
--------------------------------------------------------------------------------
/testdata/input/french.json:
--------------------------------------------------------------------------------
1 | {
2 | "peach": "This sorting order",
3 | "péché": "is wrong according to French",
4 | "pêche": "but canonicalization MUST",
5 | "sin": "ignore locale"
6 | }
7 |
--------------------------------------------------------------------------------
/testdata/input/structures.json:
--------------------------------------------------------------------------------
1 | {
2 | "1": {"f": {"f": "hi","F": 5} ,"\n": 56.0},
3 | "10": { },
4 | "": "empty",
5 | "a": { },
6 | "111": [ {"e": "yes","E": "no" } ],
7 | "A": { }
8 | }
--------------------------------------------------------------------------------
/testdata/input/unicode.json:
--------------------------------------------------------------------------------
1 | {
2 | "Unnormalized Unicode":"A\u030a"
3 | }
4 |
--------------------------------------------------------------------------------
/testdata/input/values.json:
--------------------------------------------------------------------------------
1 | {
2 | "numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001],
3 | "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/",
4 | "literals": [null, true, false]
5 | }
--------------------------------------------------------------------------------
/testdata/input/weird.json:
--------------------------------------------------------------------------------
1 | {
2 | "\u20ac": "Euro Sign",
3 | "\r": "Carriage Return",
4 | "\u000a": "Newline",
5 | "1": "One",
6 | "\u0080": "Control\u007f",
7 | "\ud83d\ude02": "Smiley",
8 | "\u00f6": "Latin Small Letter O With Diaeresis",
9 | "\ufb33": "Hebrew Letter Dalet With Dagesh",
10 | "": "Browser Challenge"
11 | }
12 |
--------------------------------------------------------------------------------
/testdata/numgen.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2006-2021 WebPKI.org (http://webpki.org).
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // https://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | // numgen generates "es6testfile100m.txt.gz".
18 | //
19 | // Run the program with:
20 | // go run numgen.go
21 | //
22 | // Go 1.13 or latter is required due to a rounding bug in strconv.FormatFloat.
23 | // See https://golang.org/issue/29491.
24 | package main
25 |
26 | import (
27 | "compress/gzip"
28 | "crypto/sha256"
29 | "encoding/binary"
30 | "fmt"
31 | "io"
32 | "log"
33 | "math"
34 | "os"
35 | "strconv"
36 | "time"
37 | )
38 |
39 | const (
40 | outputFile = "es6testfile100m.txt.gz"
41 | numLines = 1e8
42 | )
43 |
44 | // generator returns a function that generates the next float64 to format.
45 | func generator() func() float64 {
46 | static := [...]uint64{
47 | 0x0000000000000000, 0x8000000000000000, 0x0000000000000001, 0x8000000000000001,
48 | 0xc46696695dbd1cc3, 0xc43211ede4974a35, 0xc3fce97ca0f21056, 0xc3c7213080c1a6ac,
49 | 0xc39280f39a348556, 0xc35d9b1f5d20d557, 0xc327af4c4a80aaac, 0xc2f2f2a36ecd5556,
50 | 0xc2be51057e155558, 0xc28840d131aaaaac, 0xc253670dc1555557, 0xc21f0b4935555557,
51 | 0xc1e8d5d42aaaaaac, 0xc1b3de4355555556, 0xc17fca0555555556, 0xc1496e6aaaaaaaab,
52 | 0xc114585555555555, 0xc0e046aaaaaaaaab, 0xc0aa0aaaaaaaaaaa, 0xc074d55555555555,
53 | 0xc040aaaaaaaaaaab, 0xc00aaaaaaaaaaaab, 0xbfd5555555555555, 0xbfa1111111111111,
54 | 0xbf6b4e81b4e81b4f, 0xbf35d867c3ece2a5, 0xbf0179ec9cbd821e, 0xbecbf647612f3696,
55 | 0xbe965e9f80f29212, 0xbe61e54c672874db, 0xbe2ca213d840baf8, 0xbdf6e80fe033c8c6,
56 | 0xbdc2533fe68fd3d2, 0xbd8d51ffd74c861c, 0xbd5774ccac3d3817, 0xbd22c3d6f030f9ac,
57 | 0xbcee0624b3818f79, 0xbcb804ea293472c7, 0xbc833721ba905bd3, 0xbc4ebe9c5db3c61e,
58 | 0xbc18987d17c304e5, 0xbbe3ad30dfcf371d, 0xbbaf7b816618582f, 0xbb792f9ab81379bf,
59 | 0xbb442615600f9499, 0xbb101e77800c76e1, 0xbad9ca58cce0be35, 0xbaa4a1e0a3e6fe90,
60 | 0xba708180831f320d, 0xba3a68cd9e985016, 0x446696695dbd1cc3, 0x443211ede4974a35,
61 | 0x43fce97ca0f21056, 0x43c7213080c1a6ac, 0x439280f39a348556, 0x435d9b1f5d20d557,
62 | 0x4327af4c4a80aaac, 0x42f2f2a36ecd5556, 0x42be51057e155558, 0x428840d131aaaaac,
63 | 0x4253670dc1555557, 0x421f0b4935555557, 0x41e8d5d42aaaaaac, 0x41b3de4355555556,
64 | 0x417fca0555555556, 0x41496e6aaaaaaaab, 0x4114585555555555, 0x40e046aaaaaaaaab,
65 | 0x40aa0aaaaaaaaaaa, 0x4074d55555555555, 0x4040aaaaaaaaaaab, 0x400aaaaaaaaaaaab,
66 | 0x3fd5555555555555, 0x3fa1111111111111, 0x3f6b4e81b4e81b4f, 0x3f35d867c3ece2a5,
67 | 0x3f0179ec9cbd821e, 0x3ecbf647612f3696, 0x3e965e9f80f29212, 0x3e61e54c672874db,
68 | 0x3e2ca213d840baf8, 0x3df6e80fe033c8c6, 0x3dc2533fe68fd3d2, 0x3d8d51ffd74c861c,
69 | 0x3d5774ccac3d3817, 0x3d22c3d6f030f9ac, 0x3cee0624b3818f79, 0x3cb804ea293472c7,
70 | 0x3c833721ba905bd3, 0x3c4ebe9c5db3c61e, 0x3c18987d17c304e5, 0x3be3ad30dfcf371d,
71 | 0x3baf7b816618582f, 0x3b792f9ab81379bf, 0x3b442615600f9499, 0x3b101e77800c76e1,
72 | 0x3ad9ca58cce0be35, 0x3aa4a1e0a3e6fe90, 0x3a708180831f320d, 0x3a3a68cd9e985016,
73 | 0x4024000000000000, 0x4014000000000000, 0x3fe0000000000000, 0x3fa999999999999a,
74 | 0x3f747ae147ae147b, 0x3f40624dd2f1a9fc, 0x3f0a36e2eb1c432d, 0x3ed4f8b588e368f1,
75 | 0x3ea0c6f7a0b5ed8d, 0x3e6ad7f29abcaf48, 0x3e35798ee2308c3a, 0x3ed539223589fa95,
76 | 0x3ed4ff26cd5a7781, 0x3ed4f95a762283ff, 0x3ed4f8c60703520c, 0x3ed4f8b72f19cd0d,
77 | 0x3ed4f8b5b31c0c8d, 0x3ed4f8b58d1c461a, 0x3ed4f8b5894f7f0e, 0x3ed4f8b588ee37f3,
78 | 0x3ed4f8b588e47da4, 0x3ed4f8b588e3849c, 0x3ed4f8b588e36bb5, 0x3ed4f8b588e36937,
79 | 0x3ed4f8b588e368f8, 0x3ed4f8b588e368f1, 0x3ff0000000000000, 0xbff0000000000000,
80 | 0xbfeffffffffffffa, 0xbfeffffffffffffb, 0x3feffffffffffffa, 0x3feffffffffffffb,
81 | 0x3feffffffffffffc, 0x3feffffffffffffe, 0xbfefffffffffffff, 0xbfefffffffffffff,
82 | 0x3fefffffffffffff, 0x3fefffffffffffff, 0x3fd3333333333332, 0x3fd3333333333333,
83 | 0x3fd3333333333334, 0x0010000000000000, 0x000ffffffffffffd, 0x000fffffffffffff,
84 | 0x7fefffffffffffff, 0xffefffffffffffff, 0x4340000000000000, 0xc340000000000000,
85 | 0x4430000000000000, 0x44b52d02c7e14af5, 0x44b52d02c7e14af6, 0x44b52d02c7e14af7,
86 | 0x444b1ae4d6e2ef4e, 0x444b1ae4d6e2ef4f, 0x444b1ae4d6e2ef50, 0x3eb0c6f7a0b5ed8c,
87 | 0x3eb0c6f7a0b5ed8d, 0x41b3de4355555553, 0x41b3de4355555554, 0x41b3de4355555555,
88 | 0x41b3de4355555556, 0x41b3de4355555557, 0xbecbf647612f3696, 0x43143ff3c1cb0959,
89 | }
90 | var state struct {
91 | idx int
92 | data []byte
93 | block [sha256.Size]byte
94 | }
95 | return func() float64 {
96 | const numSerial = 2000
97 | var f float64
98 | switch {
99 | case state.idx < len(static):
100 | f = math.Float64frombits(static[state.idx])
101 | case state.idx < len(static)+numSerial:
102 | f = math.Float64frombits(0x0010000000000000 + uint64(state.idx-len(static)))
103 | default:
104 | for f == 0 || math.IsNaN(f) || math.IsInf(f, 0) {
105 | if len(state.data) == 0 {
106 | state.block = sha256.Sum256(state.block[:])
107 | state.data = state.block[:]
108 | }
109 | f = math.Float64frombits(binary.LittleEndian.Uint64(state.data))
110 | state.data = state.data[8:]
111 | }
112 | }
113 | state.idx++
114 | return f
115 | }
116 | }
117 |
118 | // appendNumber implements canonical number formatting per RFC 8785, section 3.2.2.3.
119 | func appendNumber(b []byte, f float64) []byte {
120 | if f == 0 {
121 | f = 0 // normalize -0 as 0
122 | }
123 | fmt := byte('f')
124 | if abs := math.Abs(f); abs != 0 && abs < 1e-6 || abs >= 1e21 {
125 | fmt = 'e'
126 | }
127 | b = strconv.AppendFloat(b, f, fmt, -1, 64)
128 | if fmt == 'e' {
129 | n := len(b)
130 | if n >= 4 && b[n-4] == 'e' && b[n-3] == '-' && b[n-2] == '0' {
131 | b[n-2] = b[n-1]
132 | b = b[:n-1]
133 | }
134 | }
135 | return b
136 | }
137 |
138 | // Generate a compressed es6testfile100m.txt.gz file.
139 | func main() {
140 | // Create a new output file.
141 | f, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0664)
142 | if err != nil {
143 | log.Fatal(err)
144 | }
145 | defer f.Close()
146 |
147 | // Compress the output.
148 | z, err := gzip.NewWriterLevel(f, gzip.BestCompression)
149 | if err != nil {
150 | log.Fatal(err)
151 | }
152 | defer z.Close()
153 |
154 | // Hash the file (before compression is applied).
155 | h := sha256.New()
156 | w := io.MultiWriter(h, z)
157 |
158 | // Format a large quantity of floating-point numbers.
159 | var b []byte
160 | var size int64
161 | type record struct {
162 | hash []byte
163 | lines int
164 | size int64
165 | }
166 | var records []record
167 | next := generator()
168 | recordAt := 1000
169 | start := time.Now()
170 | lastPrint := start
171 | for n := 1; n <= numLines; n++ {
172 | f := next()
173 | b = strconv.AppendUint(b[:0], math.Float64bits(f), 16)
174 | b = append(b, ',')
175 | b = appendNumber(b, f)
176 | b = append(b, '\n')
177 | if _, err := w.Write(b); err != nil {
178 | log.Fatal(err)
179 | }
180 | size += int64(len(b))
181 |
182 | if n == recordAt {
183 | records = append(records, record{h.Sum(nil), n, size})
184 | recordAt *= 10
185 | }
186 |
187 | if now := time.Now(); now.Sub(lastPrint) > time.Second || n == numLines {
188 | log.Printf("%0.3f%%", 100.0*float64(n)/float64(numLines))
189 | lastPrint = now
190 | }
191 | }
192 | end := time.Now()
193 | log.Printf("finished in %v", end.Sub(start))
194 |
195 | // Print records.
196 | for _, r := range records {
197 | fmt.Printf("hash:%x lines:%d size:%d\n", r.hash, r.lines, r.size)
198 | }
199 | }
200 |
--------------------------------------------------------------------------------
/testdata/numgen.js:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2006-2021 WebPKI.org (http://webpki.org).
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // https://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 | //
16 |
17 | // This node.js program generates "es6testfile100m.txt".
18 | //
19 | // Run the program with:
20 | // node numgen.js
21 |
22 | const crypto = require("crypto")
23 | const fs = require('fs')
24 |
25 | const outputFile = "es6testfile100m.txt"
26 | const numLines = 1e8
27 |
28 | staticU64s = new BigUint64Array([
29 | 0x0000000000000000n, 0x8000000000000000n, 0x0000000000000001n, 0x8000000000000001n,
30 | 0xc46696695dbd1cc3n, 0xc43211ede4974a35n, 0xc3fce97ca0f21056n, 0xc3c7213080c1a6acn,
31 | 0xc39280f39a348556n, 0xc35d9b1f5d20d557n, 0xc327af4c4a80aaacn, 0xc2f2f2a36ecd5556n,
32 | 0xc2be51057e155558n, 0xc28840d131aaaaacn, 0xc253670dc1555557n, 0xc21f0b4935555557n,
33 | 0xc1e8d5d42aaaaaacn, 0xc1b3de4355555556n, 0xc17fca0555555556n, 0xc1496e6aaaaaaaabn,
34 | 0xc114585555555555n, 0xc0e046aaaaaaaaabn, 0xc0aa0aaaaaaaaaaan, 0xc074d55555555555n,
35 | 0xc040aaaaaaaaaaabn, 0xc00aaaaaaaaaaaabn, 0xbfd5555555555555n, 0xbfa1111111111111n,
36 | 0xbf6b4e81b4e81b4fn, 0xbf35d867c3ece2a5n, 0xbf0179ec9cbd821en, 0xbecbf647612f3696n,
37 | 0xbe965e9f80f29212n, 0xbe61e54c672874dbn, 0xbe2ca213d840baf8n, 0xbdf6e80fe033c8c6n,
38 | 0xbdc2533fe68fd3d2n, 0xbd8d51ffd74c861cn, 0xbd5774ccac3d3817n, 0xbd22c3d6f030f9acn,
39 | 0xbcee0624b3818f79n, 0xbcb804ea293472c7n, 0xbc833721ba905bd3n, 0xbc4ebe9c5db3c61en,
40 | 0xbc18987d17c304e5n, 0xbbe3ad30dfcf371dn, 0xbbaf7b816618582fn, 0xbb792f9ab81379bfn,
41 | 0xbb442615600f9499n, 0xbb101e77800c76e1n, 0xbad9ca58cce0be35n, 0xbaa4a1e0a3e6fe90n,
42 | 0xba708180831f320dn, 0xba3a68cd9e985016n, 0x446696695dbd1cc3n, 0x443211ede4974a35n,
43 | 0x43fce97ca0f21056n, 0x43c7213080c1a6acn, 0x439280f39a348556n, 0x435d9b1f5d20d557n,
44 | 0x4327af4c4a80aaacn, 0x42f2f2a36ecd5556n, 0x42be51057e155558n, 0x428840d131aaaaacn,
45 | 0x4253670dc1555557n, 0x421f0b4935555557n, 0x41e8d5d42aaaaaacn, 0x41b3de4355555556n,
46 | 0x417fca0555555556n, 0x41496e6aaaaaaaabn, 0x4114585555555555n, 0x40e046aaaaaaaaabn,
47 | 0x40aa0aaaaaaaaaaan, 0x4074d55555555555n, 0x4040aaaaaaaaaaabn, 0x400aaaaaaaaaaaabn,
48 | 0x3fd5555555555555n, 0x3fa1111111111111n, 0x3f6b4e81b4e81b4fn, 0x3f35d867c3ece2a5n,
49 | 0x3f0179ec9cbd821en, 0x3ecbf647612f3696n, 0x3e965e9f80f29212n, 0x3e61e54c672874dbn,
50 | 0x3e2ca213d840baf8n, 0x3df6e80fe033c8c6n, 0x3dc2533fe68fd3d2n, 0x3d8d51ffd74c861cn,
51 | 0x3d5774ccac3d3817n, 0x3d22c3d6f030f9acn, 0x3cee0624b3818f79n, 0x3cb804ea293472c7n,
52 | 0x3c833721ba905bd3n, 0x3c4ebe9c5db3c61en, 0x3c18987d17c304e5n, 0x3be3ad30dfcf371dn,
53 | 0x3baf7b816618582fn, 0x3b792f9ab81379bfn, 0x3b442615600f9499n, 0x3b101e77800c76e1n,
54 | 0x3ad9ca58cce0be35n, 0x3aa4a1e0a3e6fe90n, 0x3a708180831f320dn, 0x3a3a68cd9e985016n,
55 | 0x4024000000000000n, 0x4014000000000000n, 0x3fe0000000000000n, 0x3fa999999999999an,
56 | 0x3f747ae147ae147bn, 0x3f40624dd2f1a9fcn, 0x3f0a36e2eb1c432dn, 0x3ed4f8b588e368f1n,
57 | 0x3ea0c6f7a0b5ed8dn, 0x3e6ad7f29abcaf48n, 0x3e35798ee2308c3an, 0x3ed539223589fa95n,
58 | 0x3ed4ff26cd5a7781n, 0x3ed4f95a762283ffn, 0x3ed4f8c60703520cn, 0x3ed4f8b72f19cd0dn,
59 | 0x3ed4f8b5b31c0c8dn, 0x3ed4f8b58d1c461an, 0x3ed4f8b5894f7f0en, 0x3ed4f8b588ee37f3n,
60 | 0x3ed4f8b588e47da4n, 0x3ed4f8b588e3849cn, 0x3ed4f8b588e36bb5n, 0x3ed4f8b588e36937n,
61 | 0x3ed4f8b588e368f8n, 0x3ed4f8b588e368f1n, 0x3ff0000000000000n, 0xbff0000000000000n,
62 | 0xbfeffffffffffffan, 0xbfeffffffffffffbn, 0x3feffffffffffffan, 0x3feffffffffffffbn,
63 | 0x3feffffffffffffcn, 0x3feffffffffffffen, 0xbfefffffffffffffn, 0xbfefffffffffffffn,
64 | 0x3fefffffffffffffn, 0x3fefffffffffffffn, 0x3fd3333333333332n, 0x3fd3333333333333n,
65 | 0x3fd3333333333334n, 0x0010000000000000n, 0x000ffffffffffffdn, 0x000fffffffffffffn,
66 | 0x7fefffffffffffffn, 0xffefffffffffffffn, 0x4340000000000000n, 0xc340000000000000n,
67 | 0x4430000000000000n, 0x44b52d02c7e14af5n, 0x44b52d02c7e14af6n, 0x44b52d02c7e14af7n,
68 | 0x444b1ae4d6e2ef4en, 0x444b1ae4d6e2ef4fn, 0x444b1ae4d6e2ef50n, 0x3eb0c6f7a0b5ed8cn,
69 | 0x3eb0c6f7a0b5ed8dn, 0x41b3de4355555553n, 0x41b3de4355555554n, 0x41b3de4355555555n,
70 | 0x41b3de4355555556n, 0x41b3de4355555557n, 0xbecbf647612f3696n, 0x43143ff3c1cb0959n,
71 | ])
72 | staticF64s = new Float64Array(staticU64s.buffer)
73 | serialU64s = new BigUint64Array(2000)
74 | for (i = 0; i < 2000; i++) {
75 | serialU64s[i] = 0x0010000000000000n + BigInt(i)
76 | }
77 | serialF64s = new Float64Array(serialU64s.buffer)
78 |
79 | var state = {
80 | idx: 0,
81 | data: new Float64Array(),
82 | block: new ArrayBuffer(32),
83 | }
84 | function next() {
85 | var f = 0.0;
86 | if (state.idx < staticF64s.length) {
87 | f = staticF64s[state.idx]
88 | } else if (state.idx < staticF64s.length + serialF64s.length) {
89 | f = serialF64s[state.idx - staticF64s.length]
90 | } else {
91 | while (f == 0.0 || !isFinite(f)) {
92 | if (state.data.length == 0) {
93 | state.block = crypto.createHash("sha256").update(new Buffer(state.block)).digest().buffer
94 | state.data = new Float64Array(state.block)
95 | }
96 | f = state.data[0]
97 | state.data = state.data.slice(1)
98 | }
99 | }
100 | state.idx++
101 | return f
102 | }
103 |
104 | // TODO: Emit the file as GZIP compressed.
105 | // TODO: Fix a buffering issue where Node.js tries to buffer the entire output.
106 | var f = fs.createWriteStream(outputFile)
107 | var u64 = new BigUint64Array(1)
108 | var f64 = new Float64Array(u64.buffer)
109 | var hash = crypto.createHash("sha256")
110 | for (i = 0; i < numLines; i++) {
111 | f64[0] = next()
112 | line = u64[0].toString(16) + "," + f64[0].toString()+"\n"
113 | f.write(line)
114 | hash.update(line)
115 | }
116 | f.close()
117 | console.log(hash.digest("hex"))
--------------------------------------------------------------------------------
/testdata/outhex/arrays.txt:
--------------------------------------------------------------------------------
1 | 5b 35 36 2c 7b 22 31 22 3a 5b 5d 2c 22 31 30 22 3a 6e 75 6c 6c 2c 22 64 22 3a 74 72 75 65 7d 5d
2 |
--------------------------------------------------------------------------------
/testdata/outhex/french.txt:
--------------------------------------------------------------------------------
1 | 7b 22 70 65 61 63 68 22 3a 22 54 68 69 73 20 73 6f 72 74 69 6e 67 20 6f 72 64 65 72 22 2c 22 70
2 | c3 a9 63 68 c3 a9 22 3a 22 69 73 20 77 72 6f 6e 67 20 61 63 63 6f 72 64 69 6e 67 20 74 6f 20 46
3 | 72 65 6e 63 68 22 2c 22 70 c3 aa 63 68 65 22 3a 22 62 75 74 20 63 61 6e 6f 6e 69 63 61 6c 69 7a
4 | 61 74 69 6f 6e 20 4d 55 53 54 22 2c 22 73 69 6e 22 3a 22 69 67 6e 6f 72 65 20 6c 6f 63 61 6c 65
5 | 22 7d
6 |
--------------------------------------------------------------------------------
/testdata/outhex/structures.txt:
--------------------------------------------------------------------------------
1 | 7b 22 22 3a 22 65 6d 70 74 79 22 2c 22 31 22 3a 7b 22 5c 6e 22 3a 35 36 2c 22 66 22 3a 7b 22 46
2 | 22 3a 35 2c 22 66 22 3a 22 68 69 22 7d 7d 2c 22 31 30 22 3a 7b 7d 2c 22 31 31 31 22 3a 5b 7b 22
3 | 45 22 3a 22 6e 6f 22 2c 22 65 22 3a 22 79 65 73 22 7d 5d 2c 22 41 22 3a 7b 7d 2c 22 61 22 3a 7b
4 | 7d 7d
5 |
--------------------------------------------------------------------------------
/testdata/outhex/unicode.txt:
--------------------------------------------------------------------------------
1 | 7b 22 55 6e 6e 6f 72 6d 61 6c 69 7a 65 64 20 55 6e 69 63 6f 64 65 22 3a 22 41 cc 8a 22 7d
2 |
--------------------------------------------------------------------------------
/testdata/outhex/values.txt:
--------------------------------------------------------------------------------
1 | 7b 22 6c 69 74 65 72 61 6c 73 22 3a 5b 6e 75 6c 6c 2c 74 72 75 65 2c 66 61 6c 73 65 5d 2c 22 6e
2 | 75 6d 62 65 72 73 22 3a 5b 33 33 33 33 33 33 33 33 33 2e 33 33 33 33 33 33 33 2c 31 65 2b 33 30
3 | 2c 34 2e 35 2c 30 2e 30 30 32 2c 31 65 2d 32 37 5d 2c 22 73 74 72 69 6e 67 22 3a 22 e2 82 ac 24
4 | 5c 75 30 30 30 66 5c 6e 41 27 42 5c 22 5c 5c 5c 5c 5c 22 2f 22 7d
5 |
--------------------------------------------------------------------------------
/testdata/outhex/weird.txt:
--------------------------------------------------------------------------------
1 | 7b 22 5c 6e 22 3a 22 4e 65 77 6c 69 6e 65 22 2c 22 5c 72 22 3a 22 43 61 72 72 69 61 67 65 20 52
2 | 65 74 75 72 6e 22 2c 22 31 22 3a 22 4f 6e 65 22 2c 22 3c 2f 73 63 72 69 70 74 3e 22 3a 22 42 72
3 | 6f 77 73 65 72 20 43 68 61 6c 6c 65 6e 67 65 22 2c 22 c2 80 22 3a 22 43 6f 6e 74 72 6f 6c 7f 22
4 | 2c 22 c3 b6 22 3a 22 4c 61 74 69 6e 20 53 6d 61 6c 6c 20 4c 65 74 74 65 72 20 4f 20 57 69 74 68
5 | 20 44 69 61 65 72 65 73 69 73 22 2c 22 e2 82 ac 22 3a 22 45 75 72 6f 20 53 69 67 6e 22 2c 22 f0
6 | 9f 98 82 22 3a 22 53 6d 69 6c 65 79 22 2c 22 ef ac b3 22 3a 22 48 65 62 72 65 77 20 4c 65 74 74
7 | 65 72 20 44 61 6c 65 74 20 57 69 74 68 20 44 61 67 65 73 68 22 7d
8 |
--------------------------------------------------------------------------------
/testdata/output/arrays.json:
--------------------------------------------------------------------------------
1 | [56,{"1":[],"10":null,"d":true}]
--------------------------------------------------------------------------------
/testdata/output/french.json:
--------------------------------------------------------------------------------
1 | {"peach":"This sorting order","péché":"is wrong according to French","pêche":"but canonicalization MUST","sin":"ignore locale"}
--------------------------------------------------------------------------------
/testdata/output/structures.json:
--------------------------------------------------------------------------------
1 | {"":"empty","1":{"\n":56,"f":{"F":5,"f":"hi"}},"10":{},"111":[{"E":"no","e":"yes"}],"A":{},"a":{}}
--------------------------------------------------------------------------------
/testdata/output/unicode.json:
--------------------------------------------------------------------------------
1 | {"Unnormalized Unicode":"Å"}
--------------------------------------------------------------------------------
/testdata/output/values.json:
--------------------------------------------------------------------------------
1 | {"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],"string":"€$\u000f\nA'B\"\\\\\"/"}
--------------------------------------------------------------------------------
/testdata/output/weird.json:
--------------------------------------------------------------------------------
1 | {"\n":"Newline","\r":"Carriage Return","1":"One","":"Browser Challenge","":"Control","ö":"Latin Small Letter O With Diaeresis","€":"Euro Sign","😂":"Smiley","דּ":"Hebrew Letter Dalet With Dagesh"}
--------------------------------------------------------------------------------