├── go.sum ├── api.png ├── internal ├── jsontest │ ├── testdata │ │ ├── citm_catalog.json.zst │ │ ├── golang_source.json.zst │ │ ├── synthea_fhir.json.zst │ │ ├── canada_geometry.json.zst │ │ ├── string_escaped.json.zst │ │ ├── string_unicode.json.zst │ │ └── twitter_status.json.zst │ └── testcase.go ├── internal.go ├── zstd │ ├── window.go │ ├── bits.go │ ├── xxhash.go │ └── huff.go ├── jsonwire │ ├── wire_test.go │ └── wire.go ├── jsonflags │ ├── flags_test.go │ └── flags.go └── jsonopts │ ├── options.go │ └── options_test.go ├── AUTHORS ├── CONTRIBUTORS ├── go.mod ├── fuzz_test.go ├── .github └── workflows │ └── ci.yml ├── LICENSE ├── v1 ├── example_text_marshaling_test.go ├── example_marshaling_test.go ├── fuzz_test.go ├── scanner.go ├── tagkey_test.go ├── indent.go ├── inject.go ├── example_test.go ├── scanner_test.go └── stream.go ├── fold.go ├── jsontext ├── quote.go ├── export.go ├── example_test.go ├── doc.go ├── pools.go ├── errors.go ├── token_test.go ├── fuzz_test.go └── value_test.go ├── migrate.sh ├── intern.go ├── example_orderedobject_test.go ├── fold_test.go ├── intern_test.go ├── inline_test.go ├── errors_test.go ├── arshal_inlined.go ├── alias_gen.go └── arshal_any.go /go.sum: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/api.png -------------------------------------------------------------------------------- /internal/jsontest/testdata/citm_catalog.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/citm_catalog.json.zst -------------------------------------------------------------------------------- /internal/jsontest/testdata/golang_source.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/golang_source.json.zst -------------------------------------------------------------------------------- /internal/jsontest/testdata/synthea_fhir.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/synthea_fhir.json.zst -------------------------------------------------------------------------------- /internal/jsontest/testdata/canada_geometry.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/canada_geometry.json.zst -------------------------------------------------------------------------------- /internal/jsontest/testdata/string_escaped.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/string_escaped.json.zst -------------------------------------------------------------------------------- /internal/jsontest/testdata/string_unicode.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/string_unicode.json.zst -------------------------------------------------------------------------------- /internal/jsontest/testdata/twitter_status.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-json-experiment/json/HEAD/internal/jsontest/testdata/twitter_status.json.zst -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This source code refers to The Go Authors for copyright purposes. 2 | # The master list of authors is in the main Go distribution, 3 | # visible at https://tip.golang.org/AUTHORS. 4 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This source code was written by the Go contributors. 2 | # The master list of contributors is in the main Go distribution, 3 | # visible at https://tip.golang.org/CONTRIBUTORS. 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // WARNING: This package is an experimental implementation in order to explore 2 | // possible API designs for a future proposal of a v2 "encoding/json" package. 3 | // This package will regularly experience breaking changes. 4 | module github.com/go-json-experiment/json 5 | 6 | go 1.25 7 | -------------------------------------------------------------------------------- /internal/jsontest/testcase.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package jsontest 8 | 9 | import ( 10 | "fmt" 11 | "path" 12 | "runtime" 13 | ) 14 | 15 | // TODO(https://go.dev/issue/52751): Replace with native testing support. 16 | 17 | // CaseName is a case name annotated with a file and line. 18 | type CaseName struct { 19 | Name string 20 | Where CasePos 21 | } 22 | 23 | // Name annotates a case name with the file and line of the caller. 24 | func Name(s string) (c CaseName) { 25 | c.Name = s 26 | runtime.Callers(2, c.Where.pc[:]) 27 | return c 28 | } 29 | 30 | // CasePos represents a file and line number. 31 | type CasePos struct{ pc [1]uintptr } 32 | 33 | func (pos CasePos) String() string { 34 | frames := runtime.CallersFrames(pos.pc[:]) 35 | frame, _ := frames.Next() 36 | return fmt.Sprintf("%s:%d", path.Base(frame.File), frame.Line) 37 | } 38 | -------------------------------------------------------------------------------- /fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "bytes" 11 | "testing" 12 | ) 13 | 14 | func FuzzEqualFold(f *testing.F) { 15 | for _, tt := range equalFoldTestdata { 16 | f.Add([]byte(tt.in1), []byte(tt.in2)) 17 | } 18 | 19 | equalFoldSimple := func(x, y []byte) bool { 20 | strip := func(b []byte) []byte { 21 | return bytes.Map(func(r rune) rune { 22 | if r == '_' || r == '-' { 23 | return -1 // ignore underscores and dashes 24 | } 25 | return r 26 | }, b) 27 | } 28 | return bytes.EqualFold(strip(x), strip(y)) 29 | } 30 | 31 | f.Fuzz(func(t *testing.T, s1, s2 []byte) { 32 | // Compare the optimized and simplified implementations. 33 | got := equalFold(s1, s2) 34 | want := equalFoldSimple(s1, s2) 35 | if got != want { 36 | t.Errorf("equalFold(%q, %q) = %v, want %v", s1, s2, got, want) 37 | } 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | jobs: 4 | # Test with a specific version of Go since some tests rely on behavior that 5 | # may drift with versions (e.g., inlining behavior, gofmt behavior, etc.). 6 | test-latest: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Install Go 10 | uses: actions/setup-go@v5 11 | with: 12 | go-version: 1.25.x 13 | - name: Checkout code 14 | uses: actions/checkout@v4 15 | - name: Test 16 | run: go test ./... 17 | - name: Test386 18 | run: go test ./... 19 | env: {GOARCH: "386"} 20 | - name: BuildJSONv2 21 | run: go build -tags=goexperiment.jsonv2 ./... 22 | - name: TestInline 23 | run: go test -v -run=TestInline 24 | env: {TEST_INLINE: true} 25 | - name: Format 26 | run: diff -u <(echo -n) <(gofmt -s -d .) 27 | # Test on a large matrix of Go versions and operating systems. 28 | test-all: 29 | strategy: 30 | matrix: 31 | go-version: [1.25.x] 32 | os: [ubuntu-latest, macos-latest, windows-latest] 33 | runs-on: ${{ matrix.os }} 34 | steps: 35 | - name: Install Go 36 | uses: actions/setup-go@v5 37 | with: 38 | go-version: ${{ matrix.go-version }} 39 | - name: Checkout code 40 | uses: actions/checkout@v4 41 | - name: Test 42 | run: go test ./... 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /v1/example_text_marshaling_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json_test 8 | 9 | import ( 10 | "fmt" 11 | "log" 12 | "strings" 13 | 14 | "github.com/go-json-experiment/json/v1" 15 | ) 16 | 17 | type Size int 18 | 19 | const ( 20 | Unrecognized Size = iota 21 | Small 22 | Large 23 | ) 24 | 25 | func (s *Size) UnmarshalText(text []byte) error { 26 | switch strings.ToLower(string(text)) { 27 | default: 28 | *s = Unrecognized 29 | case "small": 30 | *s = Small 31 | case "large": 32 | *s = Large 33 | } 34 | return nil 35 | } 36 | 37 | func (s Size) MarshalText() ([]byte, error) { 38 | var name string 39 | switch s { 40 | default: 41 | name = "unrecognized" 42 | case Small: 43 | name = "small" 44 | case Large: 45 | name = "large" 46 | } 47 | return []byte(name), nil 48 | } 49 | 50 | func Example_textMarshalJSON() { 51 | blob := `["small","regular","large","unrecognized","small","normal","small","large"]` 52 | var inventory []Size 53 | if err := json.Unmarshal([]byte(blob), &inventory); err != nil { 54 | log.Fatal(err) 55 | } 56 | 57 | counts := make(map[Size]int) 58 | for _, size := range inventory { 59 | counts[size] += 1 60 | } 61 | 62 | fmt.Printf("Inventory Counts:\n* Small: %d\n* Large: %d\n* Unrecognized: %d\n", 63 | counts[Small], counts[Large], counts[Unrecognized]) 64 | 65 | // Output: 66 | // Inventory Counts: 67 | // * Small: 3 68 | // * Large: 2 69 | // * Unrecognized: 3 70 | } 71 | -------------------------------------------------------------------------------- /v1/example_marshaling_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json_test 8 | 9 | import ( 10 | "fmt" 11 | "log" 12 | "strings" 13 | 14 | "github.com/go-json-experiment/json/v1" 15 | ) 16 | 17 | type Animal int 18 | 19 | const ( 20 | Unknown Animal = iota 21 | Gopher 22 | Zebra 23 | ) 24 | 25 | func (a *Animal) UnmarshalJSON(b []byte) error { 26 | var s string 27 | if err := json.Unmarshal(b, &s); err != nil { 28 | return err 29 | } 30 | switch strings.ToLower(s) { 31 | default: 32 | *a = Unknown 33 | case "gopher": 34 | *a = Gopher 35 | case "zebra": 36 | *a = Zebra 37 | } 38 | 39 | return nil 40 | } 41 | 42 | func (a Animal) MarshalJSON() ([]byte, error) { 43 | var s string 44 | switch a { 45 | default: 46 | s = "unknown" 47 | case Gopher: 48 | s = "gopher" 49 | case Zebra: 50 | s = "zebra" 51 | } 52 | 53 | return json.Marshal(s) 54 | } 55 | 56 | func Example_customMarshalJSON() { 57 | blob := `["gopher","armadillo","zebra","unknown","gopher","bee","gopher","zebra"]` 58 | var zoo []Animal 59 | if err := json.Unmarshal([]byte(blob), &zoo); err != nil { 60 | log.Fatal(err) 61 | } 62 | 63 | census := make(map[Animal]int) 64 | for _, animal := range zoo { 65 | census[animal] += 1 66 | } 67 | 68 | fmt.Printf("Zoo Census:\n* Gophers: %d\n* Zebras: %d\n* Unknown: %d\n", 69 | census[Gopher], census[Zebra], census[Unknown]) 70 | 71 | // Output: 72 | // Zoo Census: 73 | // * Gophers: 3 74 | // * Zebras: 2 75 | // * Unknown: 3 76 | } 77 | -------------------------------------------------------------------------------- /v1/fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "bytes" 11 | "io" 12 | "testing" 13 | ) 14 | 15 | func FuzzUnmarshalJSON(f *testing.F) { 16 | f.Add([]byte(`{ 17 | "object": { 18 | "slice": [ 19 | 1, 20 | 2.0, 21 | "3", 22 | [4], 23 | {5: {}} 24 | ] 25 | }, 26 | "slice": [[]], 27 | "string": ":)", 28 | "int": 1e5, 29 | "float": 3e-9" 30 | }`)) 31 | 32 | f.Fuzz(func(t *testing.T, b []byte) { 33 | for _, typ := range []func() any{ 34 | func() any { return new(any) }, 35 | func() any { return new(map[string]any) }, 36 | func() any { return new([]any) }, 37 | } { 38 | i := typ() 39 | if err := Unmarshal(b, i); err != nil { 40 | return 41 | } 42 | 43 | encoded, err := Marshal(i) 44 | if err != nil { 45 | t.Fatalf("failed to marshal: %s", err) 46 | } 47 | 48 | if err := Unmarshal(encoded, i); err != nil { 49 | t.Fatalf("failed to roundtrip: %s", err) 50 | } 51 | } 52 | }) 53 | } 54 | 55 | func FuzzDecoderToken(f *testing.F) { 56 | f.Add([]byte(`{ 57 | "object": { 58 | "slice": [ 59 | 1, 60 | 2.0, 61 | "3", 62 | [4], 63 | {5: {}} 64 | ] 65 | }, 66 | "slice": [[]], 67 | "string": ":)", 68 | "int": 1e5, 69 | "float": 3e-9" 70 | }`)) 71 | 72 | f.Fuzz(func(t *testing.T, b []byte) { 73 | r := bytes.NewReader(b) 74 | d := NewDecoder(r) 75 | for { 76 | _, err := d.Token() 77 | if err != nil { 78 | if err == io.EOF { 79 | break 80 | } 81 | return 82 | } 83 | } 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /fold.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "unicode" 11 | "unicode/utf8" 12 | ) 13 | 14 | // foldName returns a folded string such that foldName(x) == foldName(y) 15 | // is similar to strings.EqualFold(x, y), but ignores underscore and dashes. 16 | // This allows foldName to match common naming conventions. 17 | func foldName(in []byte) []byte { 18 | // This is inlinable to take advantage of "function outlining". 19 | // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ 20 | var arr [32]byte // large enough for most JSON names 21 | return appendFoldedName(arr[:0], in) 22 | } 23 | func appendFoldedName(out, in []byte) []byte { 24 | for i := 0; i < len(in); { 25 | // Handle single-byte ASCII. 26 | if c := in[i]; c < utf8.RuneSelf { 27 | if c != '_' && c != '-' { 28 | if 'a' <= c && c <= 'z' { 29 | c -= 'a' - 'A' 30 | } 31 | out = append(out, c) 32 | } 33 | i++ 34 | continue 35 | } 36 | // Handle multi-byte Unicode. 37 | r, n := utf8.DecodeRune(in[i:]) 38 | out = utf8.AppendRune(out, foldRune(r)) 39 | i += n 40 | } 41 | return out 42 | } 43 | 44 | // foldRune is a variation on unicode.SimpleFold that returns the same rune 45 | // for all runes in the same fold set. 46 | // 47 | // Invariant: 48 | // 49 | // foldRune(x) == foldRune(y) ⇔ strings.EqualFold(string(x), string(y)) 50 | func foldRune(r rune) rune { 51 | for { 52 | r2 := unicode.SimpleFold(r) 53 | if r2 <= r { 54 | return r2 // smallest character in the fold set 55 | } 56 | r = r2 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /jsontext/quote.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package jsontext 8 | 9 | import ( 10 | "github.com/go-json-experiment/json/internal/jsonflags" 11 | "github.com/go-json-experiment/json/internal/jsonwire" 12 | ) 13 | 14 | // AppendQuote appends a double-quoted JSON string literal representing src 15 | // to dst and returns the extended buffer. 16 | // It uses the minimal string representation per RFC 8785, section 3.2.2.2. 17 | // Invalid UTF-8 bytes are replaced with the Unicode replacement character 18 | // and an error is returned at the end indicating the presence of invalid UTF-8. 19 | // The dst must not overlap with the src. 20 | func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) { 21 | dst, err := jsonwire.AppendQuote(dst, src, &jsonflags.Flags{}) 22 | if err != nil { 23 | err = &SyntacticError{Err: err} 24 | } 25 | return dst, err 26 | } 27 | 28 | // AppendUnquote appends the decoded interpretation of src as a 29 | // double-quoted JSON string literal to dst and returns the extended buffer. 30 | // The input src must be a JSON string without any surrounding whitespace. 31 | // Invalid UTF-8 bytes are replaced with the Unicode replacement character 32 | // and an error is returned at the end indicating the presence of invalid UTF-8. 33 | // Any trailing bytes after the JSON string literal results in an error. 34 | // The dst must not overlap with the src. 35 | func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) { 36 | dst, err := jsonwire.AppendUnquote(dst, src) 37 | if err != nil { 38 | err = &SyntacticError{Err: err} 39 | } 40 | return dst, err 41 | } 42 | -------------------------------------------------------------------------------- /internal/internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package internal 8 | 9 | import "errors" 10 | 11 | // NotForPublicUse is a marker type that an API is for internal use only. 12 | // It does not perfectly prevent usage of that API, but helps to restrict usage. 13 | // Anything with this marker is not covered by the Go compatibility agreement. 14 | type NotForPublicUse struct{} 15 | 16 | // AllowInternalUse is passed from "json" to "jsontext" to authenticate 17 | // that the caller can have access to internal functionality. 18 | var AllowInternalUse NotForPublicUse 19 | 20 | // Sentinel error values internally shared between jsonv1 and jsonv2. 21 | var ( 22 | ErrCycle = errors.New("encountered a cycle") 23 | ErrNonNilReference = errors.New("value must be passed as a non-nil pointer reference") 24 | ErrNilInterface = errors.New("cannot derive concrete type for nil interface with finite type set") 25 | ) 26 | 27 | var ( 28 | // TransformMarshalError converts a v2 error into a v1 error. 29 | // It is called only at the top-level of a Marshal function. 30 | TransformMarshalError func(any, error) error 31 | // NewMarshalerError constructs a jsonv1.MarshalerError. 32 | // It is called after a user-defined Marshal method/function fails. 33 | NewMarshalerError func(any, error, string) error 34 | // TransformUnmarshalError converts a v2 error into a v1 error. 35 | // It is called only at the top-level of a Unmarshal function. 36 | TransformUnmarshalError func(any, error) error 37 | 38 | // NewRawNumber returns new(jsonv1.Number). 39 | NewRawNumber func() any 40 | // RawNumberOf returns jsonv1.Number(b). 41 | RawNumberOf func(b []byte) any 42 | ) 43 | -------------------------------------------------------------------------------- /internal/zstd/window.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zstd 6 | 7 | // window stores up to size bytes of data. 8 | // It is implemented as a circular buffer: 9 | // sequential save calls append to the data slice until 10 | // its length reaches configured size and after that, 11 | // save calls overwrite previously saved data at off 12 | // and update off such that it always points at 13 | // the byte stored before others. 14 | type window struct { 15 | size int 16 | data []byte 17 | off int 18 | } 19 | 20 | // reset clears stored data and configures window size. 21 | func (w *window) reset(size int) { 22 | b := w.data[:0] 23 | if cap(b) < size { 24 | b = make([]byte, 0, size) 25 | } 26 | w.data = b 27 | w.off = 0 28 | w.size = size 29 | } 30 | 31 | // len returns the number of stored bytes. 32 | func (w *window) len() uint32 { 33 | return uint32(len(w.data)) 34 | } 35 | 36 | // save stores up to size last bytes from the buf. 37 | func (w *window) save(buf []byte) { 38 | if w.size == 0 { 39 | return 40 | } 41 | if len(buf) == 0 { 42 | return 43 | } 44 | 45 | if len(buf) >= w.size { 46 | from := len(buf) - w.size 47 | w.data = append(w.data[:0], buf[from:]...) 48 | w.off = 0 49 | return 50 | } 51 | 52 | // Update off to point to the oldest remaining byte. 53 | free := w.size - len(w.data) 54 | if free == 0 { 55 | n := copy(w.data[w.off:], buf) 56 | if n == len(buf) { 57 | w.off += n 58 | } else { 59 | w.off = copy(w.data, buf[n:]) 60 | } 61 | } else { 62 | if free >= len(buf) { 63 | w.data = append(w.data, buf...) 64 | } else { 65 | w.data = append(w.data, buf[:free]...) 66 | w.off = copy(w.data, buf[free:]) 67 | } 68 | } 69 | } 70 | 71 | // appendTo appends stored bytes between from and to indices to the buf. 72 | // Index from must be less or equal to index to and to must be less or equal to w.len(). 73 | func (w *window) appendTo(buf []byte, from, to uint32) []byte { 74 | dataLen := uint32(len(w.data)) 75 | from += uint32(w.off) 76 | to += uint32(w.off) 77 | 78 | wrap := false 79 | if from > dataLen { 80 | from -= dataLen 81 | wrap = !wrap 82 | } 83 | if to > dataLen { 84 | to -= dataLen 85 | wrap = !wrap 86 | } 87 | 88 | if wrap { 89 | buf = append(buf, w.data[from:]...) 90 | return append(buf, w.data[:to]...) 91 | } else { 92 | return append(buf, w.data[from:to]...) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /jsontext/export.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package jsontext 8 | 9 | import ( 10 | "io" 11 | 12 | "github.com/go-json-experiment/json/internal" 13 | ) 14 | 15 | // Internal is for internal use only. 16 | // This is exempt from the Go compatibility agreement. 17 | var Internal exporter 18 | 19 | type exporter struct{} 20 | 21 | // Export exposes internal functionality from "jsontext" to "json". 22 | // This cannot be dynamically called by other packages since 23 | // they cannot obtain a reference to the internal.AllowInternalUse value. 24 | func (exporter) Export(p *internal.NotForPublicUse) export { 25 | if p != &internal.AllowInternalUse { 26 | panic("unauthorized call to Export") 27 | } 28 | return export{} 29 | } 30 | 31 | // The export type exposes functionality to packages with visibility to 32 | // the internal.AllowInternalUse variable. The "json" package uses this 33 | // to modify low-level state in the Encoder and Decoder types. 34 | // It mutates the state directly instead of calling ReadToken or WriteToken 35 | // since this is more performant. The public APIs need to track state to ensure 36 | // that users are constructing a valid JSON value, but the "json" implementation 37 | // guarantees that it emits valid JSON by the structure of the code itself. 38 | type export struct{} 39 | 40 | // Encoder returns a pointer to the underlying encoderState. 41 | func (export) Encoder(e *Encoder) *encoderState { return &e.s } 42 | 43 | // Decoder returns a pointer to the underlying decoderState. 44 | func (export) Decoder(d *Decoder) *decoderState { return &d.s } 45 | 46 | func (export) GetBufferedEncoder(o ...Options) *Encoder { 47 | return getBufferedEncoder(o...) 48 | } 49 | func (export) PutBufferedEncoder(e *Encoder) { 50 | putBufferedEncoder(e) 51 | } 52 | 53 | func (export) GetStreamingEncoder(w io.Writer, o ...Options) *Encoder { 54 | return getStreamingEncoder(w, o...) 55 | } 56 | func (export) PutStreamingEncoder(e *Encoder) { 57 | putStreamingEncoder(e) 58 | } 59 | 60 | func (export) GetBufferedDecoder(b []byte, o ...Options) *Decoder { 61 | return getBufferedDecoder(b, o...) 62 | } 63 | func (export) PutBufferedDecoder(d *Decoder) { 64 | putBufferedDecoder(d) 65 | } 66 | 67 | func (export) GetStreamingDecoder(r io.Reader, o ...Options) *Decoder { 68 | return getStreamingDecoder(r, o...) 69 | } 70 | func (export) PutStreamingDecoder(d *Decoder) { 71 | putStreamingDecoder(d) 72 | } 73 | 74 | func (export) IsIOError(err error) bool { 75 | _, ok := err.(*ioError) 76 | return ok 77 | } 78 | -------------------------------------------------------------------------------- /migrate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | GOROOT=${1:-../go} 4 | JSONROOT="." 5 | 6 | cp $JSONROOT/alias_gen.go $JSONROOT/alias_gen.go.bak 7 | rm -r $JSONROOT/*.go $JSONROOT/internal $JSONROOT/jsontext $JSONROOT/v1 8 | mv $JSONROOT/alias_gen.go.bak $JSONROOT/alias_gen.go 9 | cp -r $GOROOT/src/encoding/json/v2/*.go $JSONROOT/ 10 | cp -r $GOROOT/src/encoding/json/internal/ $JSONROOT/internal/ 11 | cp -r $GOROOT/src/encoding/json/jsontext/ $JSONROOT/jsontext/ 12 | mkdir $JSONROOT/v1 13 | for X in $GOROOT/src/encoding/json/v2_*.go; do 14 | cp $X $JSONROOT/v1/$(basename $X | sed "s/v2_//") 15 | done 16 | cd $JSONROOT 17 | for X in $(git ls-files --cached --others --exclude-standard | grep ".*[.]go$"); do 18 | if [ ! -e "$X" ]; then 19 | continue 20 | fi 21 | sed -i 's/go:build goexperiment.jsonv2$/go:build !goexperiment.jsonv2 || !go1.25/' $X 22 | sed -i 's|"encoding/json/v2"|"github.com/go-json-experiment/json"|' $X 23 | sed -i 's|"encoding/json/internal"|"github.com/go-json-experiment/json/internal"|' $X 24 | sed -i 's|"encoding/json/internal/jsonflags"|"github.com/go-json-experiment/json/internal/jsonflags"|' $X 25 | sed -i 's|"encoding/json/internal/jsonopts"|"github.com/go-json-experiment/json/internal/jsonopts"|' $X 26 | sed -i 's|"encoding/json/internal/jsontest"|"github.com/go-json-experiment/json/internal/jsontest"|' $X 27 | sed -i 's|"encoding/json/internal/jsonwire"|"github.com/go-json-experiment/json/internal/jsonwire"|' $X 28 | sed -i 's|"encoding/json/jsontext"|"github.com/go-json-experiment/json/jsontext"|' $X 29 | sed -i 's|"encoding/json"|"github.com/go-json-experiment/json/v1"|' $X 30 | sed -i 's|"internal/zstd"|"github.com/go-json-experiment/json/internal/zstd"|' $X 31 | goimports -w $X 32 | done 33 | sed -i 's/v2[.]struct/json.struct/' $JSONROOT/errors_test.go 34 | sed -i 's|jsonv1 "github.com/go-json-experiment/json/v1"|jsonv1 "encoding/json"|' $JSONROOT/bench_test.go 35 | 36 | # TODO(go1.26): Remove this rewrite once errors.AsType is in the standard library. 37 | sed -i 's/_, ok := errors\.AsType\[\*SyntacticError\](err)/ok := errors.As(err, new(*SyntacticError))/g' $JSONROOT/jsontext/*.go 38 | sed -i 's/serr, ok := errors\.AsType\[\*json.SemanticError\](err)/var serr *json.SemanticError; ok := errors.As(err, \&serr)/g' $JSONROOT/example_test.go 39 | gofmt -w $JSONROOT/example_test.go 40 | 41 | # Remove documentation that only makes sense within the stdlib. 42 | sed -i '/This package .* is experimental/,+4d' $JSONROOT/doc.go 43 | sed -i '/This package .* is experimental/,+4d' $JSONROOT/jsontext/doc.go 44 | 45 | git checkout internal/zstd # we still need local copy of zstd for testing 46 | 47 | go run alias_gen.go "encoding/json" $JSONROOT/v1 48 | go run alias_gen.go "encoding/json/v2" $JSONROOT 49 | go run alias_gen.go "encoding/json/jsontext" $JSONROOT/jsontext 50 | -------------------------------------------------------------------------------- /intern.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "encoding/binary" 11 | "math/bits" 12 | ) 13 | 14 | // stringCache is a cache for strings converted from a []byte. 15 | type stringCache = [256]string // 256*unsafe.Sizeof(string("")) => 4KiB 16 | 17 | // makeString returns the string form of b. 18 | // It returns a pre-allocated string from c if present, otherwise 19 | // it allocates a new string, inserts it into the cache, and returns it. 20 | func makeString(c *stringCache, b []byte) string { 21 | const ( 22 | minCachedLen = 2 // single byte strings are already interned by the runtime 23 | maxCachedLen = 256 // large enough for UUIDs, IPv6 addresses, SHA-256 checksums, etc. 24 | ) 25 | if c == nil || len(b) < minCachedLen || len(b) > maxCachedLen { 26 | return string(b) 27 | } 28 | 29 | // Compute a hash from the fixed-width prefix and suffix of the string. 30 | // This ensures hashing a string is a constant time operation. 31 | var h uint32 32 | switch { 33 | case len(b) >= 8: 34 | lo := binary.LittleEndian.Uint64(b[:8]) 35 | hi := binary.LittleEndian.Uint64(b[len(b)-8:]) 36 | h = hash64(uint32(lo), uint32(lo>>32)) ^ hash64(uint32(hi), uint32(hi>>32)) 37 | case len(b) >= 4: 38 | lo := binary.LittleEndian.Uint32(b[:4]) 39 | hi := binary.LittleEndian.Uint32(b[len(b)-4:]) 40 | h = hash64(lo, hi) 41 | case len(b) >= 2: 42 | lo := binary.LittleEndian.Uint16(b[:2]) 43 | hi := binary.LittleEndian.Uint16(b[len(b)-2:]) 44 | h = hash64(uint32(lo), uint32(hi)) 45 | } 46 | 47 | // Check the cache for the string. 48 | i := h % uint32(len(*c)) 49 | if s := (*c)[i]; s == string(b) { 50 | return s 51 | } 52 | s := string(b) 53 | (*c)[i] = s 54 | return s 55 | } 56 | 57 | // hash64 returns the hash of two uint32s as a single uint32. 58 | func hash64(lo, hi uint32) uint32 { 59 | // If avalanche=true, this is identical to XXH32 hash on a 8B string: 60 | // var b [8]byte 61 | // binary.LittleEndian.PutUint32(b[:4], lo) 62 | // binary.LittleEndian.PutUint32(b[4:], hi) 63 | // return xxhash.Sum32(b[:]) 64 | const ( 65 | prime1 = 0x9e3779b1 66 | prime2 = 0x85ebca77 67 | prime3 = 0xc2b2ae3d 68 | prime4 = 0x27d4eb2f 69 | prime5 = 0x165667b1 70 | ) 71 | h := prime5 + uint32(8) 72 | h += lo * prime3 73 | h = bits.RotateLeft32(h, 17) * prime4 74 | h += hi * prime3 75 | h = bits.RotateLeft32(h, 17) * prime4 76 | // Skip final mix (avalanche) step of XXH32 for performance reasons. 77 | // Empirical testing shows that the improvements in unbiased distribution 78 | // does not outweigh the extra cost in computational complexity. 79 | const avalanche = false 80 | if avalanche { 81 | h ^= h >> 15 82 | h *= prime2 83 | h ^= h >> 13 84 | h *= prime3 85 | h ^= h >> 16 86 | } 87 | return h 88 | } 89 | -------------------------------------------------------------------------------- /v1/scanner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2010 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "errors" 11 | "io" 12 | "strings" 13 | 14 | "github.com/go-json-experiment/json/internal" 15 | "github.com/go-json-experiment/json/internal/jsonflags" 16 | "github.com/go-json-experiment/json/jsontext" 17 | ) 18 | 19 | // export exposes internal functionality of the "jsontext" package. 20 | var export = jsontext.Internal.Export(&internal.AllowInternalUse) 21 | 22 | // Valid reports whether data is a valid JSON encoding. 23 | func Valid(data []byte) bool { 24 | return checkValid(data) == nil 25 | } 26 | 27 | func checkValid(data []byte) error { 28 | d := export.GetBufferedDecoder(data) 29 | defer export.PutBufferedDecoder(d) 30 | xd := export.Decoder(d) 31 | xd.Struct.Flags.Set(jsonflags.AllowDuplicateNames | jsonflags.AllowInvalidUTF8 | 1) 32 | if _, err := d.ReadValue(); err != nil { 33 | if err == io.EOF { 34 | offset := d.InputOffset() + int64(len(d.UnreadBuffer())) 35 | err = &jsontext.SyntacticError{ByteOffset: offset, Err: io.ErrUnexpectedEOF} 36 | } 37 | return transformSyntacticError(err) 38 | } 39 | if err := xd.CheckEOF(); err != nil { 40 | return transformSyntacticError(err) 41 | } 42 | return nil 43 | } 44 | 45 | // A SyntaxError is a description of a JSON syntax error. 46 | // [Unmarshal] will return a SyntaxError if the JSON can't be parsed. 47 | type SyntaxError struct { 48 | msg string // description of error 49 | Offset int64 // error occurred after reading Offset bytes 50 | } 51 | 52 | func (e *SyntaxError) Error() string { return e.msg } 53 | 54 | var errUnexpectedEnd = errors.New("unexpected end of JSON input") 55 | 56 | func transformSyntacticError(err error) error { 57 | switch serr, ok := err.(*jsontext.SyntacticError); { 58 | case serr != nil: 59 | if serr.Err == io.ErrUnexpectedEOF { 60 | serr.Err = errUnexpectedEnd 61 | } 62 | msg := serr.Err.Error() 63 | if i := strings.Index(msg, " (expecting"); i >= 0 && !strings.Contains(msg, " in literal") { 64 | msg = msg[:i] 65 | } 66 | return &SyntaxError{Offset: serr.ByteOffset, msg: syntaxErrorReplacer.Replace(msg)} 67 | case ok: 68 | return (*SyntaxError)(nil) 69 | case export.IsIOError(err): 70 | return errors.Unwrap(err) // v1 historically did not wrap IO errors 71 | default: 72 | return err 73 | } 74 | } 75 | 76 | // syntaxErrorReplacer replaces certain string literals in the v2 error 77 | // to better match the historical string rendering of syntax errors. 78 | // In particular, v2 uses the terminology "object name" to match RFC 8259, 79 | // while v1 uses "object key", which is not a term found in JSON literature. 80 | var syntaxErrorReplacer = strings.NewReplacer( 81 | "object name", "object key", 82 | "at start of value", "looking for beginning of value", 83 | "at start of string", "looking for beginning of object key string", 84 | "after object value", "after object key:value pair", 85 | "in number", "in numeric literal", 86 | ) 87 | -------------------------------------------------------------------------------- /internal/jsonwire/wire_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package jsonwire 8 | 9 | import ( 10 | "cmp" 11 | "slices" 12 | "testing" 13 | "unicode/utf16" 14 | "unicode/utf8" 15 | ) 16 | 17 | func TestQuoteRune(t *testing.T) { 18 | tests := []struct{ in, want string }{ 19 | {"x", `'x'`}, 20 | {"\n", `'\n'`}, 21 | {"'", `'\''`}, 22 | {"\xff", `'\xff'`}, 23 | {"💩", `'💩'`}, 24 | {"💩"[:1], `'\xf0'`}, 25 | {"\uffff", `'\uffff'`}, 26 | {"\U00101234", `'\U00101234'`}, 27 | } 28 | for _, tt := range tests { 29 | got := QuoteRune([]byte(tt.in)) 30 | if got != tt.want { 31 | t.Errorf("quoteRune(%q) = %s, want %s", tt.in, got, tt.want) 32 | } 33 | } 34 | } 35 | 36 | var compareUTF16Testdata = []string{"", "\r", "1", "f\xfe", "f\xfe\xff", "f\xff", "\u0080", "\u00f6", "\u20ac", "\U0001f600", "\ufb33"} 37 | 38 | func TestCompareUTF16(t *testing.T) { 39 | for i, si := range compareUTF16Testdata { 40 | for j, sj := range compareUTF16Testdata { 41 | got := CompareUTF16([]byte(si), []byte(sj)) 42 | want := cmp.Compare(i, j) 43 | if got != want { 44 | t.Errorf("CompareUTF16(%q, %q) = %v, want %v", si, sj, got, want) 45 | } 46 | } 47 | } 48 | } 49 | 50 | func FuzzCompareUTF16(f *testing.F) { 51 | for _, td1 := range compareUTF16Testdata { 52 | for _, td2 := range compareUTF16Testdata { 53 | f.Add([]byte(td1), []byte(td2)) 54 | } 55 | } 56 | 57 | // CompareUTF16Simple is identical to CompareUTF16, 58 | // but relies on naively converting a string to a []uint16 codepoints. 59 | // It is easy to verify as correct, but is slow. 60 | CompareUTF16Simple := func(x, y []byte) int { 61 | ux := utf16.Encode([]rune(string(x))) 62 | uy := utf16.Encode([]rune(string(y))) 63 | return slices.Compare(ux, uy) 64 | } 65 | 66 | f.Fuzz(func(t *testing.T, s1, s2 []byte) { 67 | // Compare the optimized and simplified implementations. 68 | got := CompareUTF16(s1, s2) 69 | want := CompareUTF16Simple(s1, s2) 70 | if got != want && utf8.Valid(s1) && utf8.Valid(s2) { 71 | t.Errorf("CompareUTF16(%q, %q) = %v, want %v", s1, s2, got, want) 72 | } 73 | }) 74 | } 75 | 76 | func TestTruncatePointer(t *testing.T) { 77 | tests := []struct{ in, want string }{ 78 | {"hello", "hello"}, 79 | {"/a/b/c", "/a/b/c"}, 80 | {"/a/b/c/d/e/f/g", "/a/b/…/f/g"}, 81 | {"supercalifragilisticexpialidocious", "super…cious"}, 82 | {"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/supe…/…cious"}, 83 | {"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/supe…/…/…cious"}, 84 | {"/a/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/a/…/…cious"}, 85 | {"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious/b", "/supe…/…/b"}, 86 | {"/fizz/buzz/bazz", "/fizz/…/bazz"}, 87 | {"/fizz/buzz/bazz/razz", "/fizz/…/razz"}, 88 | {"/////////////////////////////", "/////…/////"}, 89 | {"/🎄❤️✨/🎁✅😊/🎅🔥⭐", "/🎄…/…/…⭐"}, 90 | } 91 | for _, tt := range tests { 92 | got := TruncatePointer(tt.in, 10) 93 | if got != tt.want { 94 | t.Errorf("TruncatePointer(%q) = %q, want %q", tt.in, got, tt.want) 95 | } 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /internal/jsonflags/flags_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package jsonflags 8 | 9 | import "testing" 10 | 11 | func TestFlags(t *testing.T) { 12 | type Check struct{ want Flags } 13 | type Join struct{ in Flags } 14 | type Set struct{ in Bools } 15 | type Clear struct{ in Bools } 16 | type Get struct { 17 | in Bools 18 | want bool 19 | wantOk bool 20 | } 21 | 22 | calls := []any{ 23 | Get{in: AllowDuplicateNames, want: false, wantOk: false}, 24 | Set{in: AllowDuplicateNames | 0}, 25 | Get{in: AllowDuplicateNames, want: false, wantOk: true}, 26 | Set{in: AllowDuplicateNames | 1}, 27 | Get{in: AllowDuplicateNames, want: true, wantOk: true}, 28 | Check{want: Flags{Presence: uint64(AllowDuplicateNames), Values: uint64(AllowDuplicateNames)}}, 29 | Get{in: AllowInvalidUTF8, want: false, wantOk: false}, 30 | Set{in: AllowInvalidUTF8 | 1}, 31 | Get{in: AllowInvalidUTF8, want: true, wantOk: true}, 32 | Set{in: AllowInvalidUTF8 | 0}, 33 | Get{in: AllowInvalidUTF8, want: false, wantOk: true}, 34 | Get{in: AllowDuplicateNames, want: true, wantOk: true}, 35 | Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}}, 36 | Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 0}, 37 | Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(0)}}, 38 | Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 0}, 39 | Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(0)}}, 40 | Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 1}, 41 | Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames | AllowInvalidUTF8)}}, 42 | Join{in: Flags{Presence: 0, Values: 0}}, 43 | Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames | AllowInvalidUTF8)}}, 44 | Join{in: Flags{Presence: uint64(Multiline | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}}, 45 | Check{want: Flags{Presence: uint64(Multiline | AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}}, 46 | Clear{in: AllowDuplicateNames | AllowInvalidUTF8}, 47 | Check{want: Flags{Presence: uint64(Multiline), Values: uint64(0)}}, 48 | Set{in: AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics | 1}, 49 | Set{in: Multiline | StringifyNumbers | 0}, 50 | Check{want: Flags{Presence: uint64(AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics | Multiline | StringifyNumbers), Values: uint64(AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics)}}, 51 | Clear{in: ^AllCoderFlags}, 52 | Check{want: Flags{Presence: uint64(AllowInvalidUTF8 | Multiline), Values: uint64(AllowInvalidUTF8)}}, 53 | } 54 | var fs Flags 55 | for i, call := range calls { 56 | switch call := call.(type) { 57 | case Join: 58 | fs.Join(call.in) 59 | case Set: 60 | fs.Set(call.in) 61 | case Clear: 62 | fs.Clear(call.in) 63 | case Get: 64 | got := fs.Get(call.in) 65 | gotOk := fs.Has(call.in) 66 | if got != call.want || gotOk != call.wantOk { 67 | t.Fatalf("%d: GetOk = (%v, %v), want (%v, %v)", i, got, gotOk, call.want, call.wantOk) 68 | } 69 | case Check: 70 | if fs != call.want { 71 | t.Fatalf("%d: got %x, want %x", i, fs, call.want) 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /v1/tagkey_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import "testing" 10 | 11 | type basicLatin2xTag struct { 12 | V string `json:"$%-/"` 13 | } 14 | 15 | type basicLatin3xTag struct { 16 | V string `json:"0123456789"` 17 | } 18 | 19 | type basicLatin4xTag struct { 20 | V string `json:"ABCDEFGHIJKLMO"` 21 | } 22 | 23 | type basicLatin5xTag struct { 24 | V string `json:"PQRSTUVWXYZ_"` 25 | } 26 | 27 | type basicLatin6xTag struct { 28 | V string `json:"abcdefghijklmno"` 29 | } 30 | 31 | type basicLatin7xTag struct { 32 | V string `json:"pqrstuvwxyz"` 33 | } 34 | 35 | type miscPlaneTag struct { 36 | V string `json:"色は匂へど"` 37 | } 38 | 39 | type percentSlashTag struct { 40 | V string `json:"text/html%"` // https://golang.org/issue/2718 41 | } 42 | 43 | type punctuationTag struct { 44 | V string `json:"!#$%&()*+-./:;<=>?@[]^_{|}~ "` // https://golang.org/issue/3546 45 | } 46 | 47 | type dashTag struct { 48 | V string `json:"-,"` 49 | } 50 | 51 | type emptyTag struct { 52 | W string 53 | } 54 | 55 | type misnamedTag struct { 56 | X string `jsom:"Misnamed"` 57 | } 58 | 59 | type badFormatTag struct { 60 | Y string `:"BadFormat"` 61 | } 62 | 63 | type badCodeTag struct { 64 | Z string `json:" !\"#&'()*+,."` 65 | } 66 | 67 | type spaceTag struct { 68 | Q string `json:"With space"` 69 | } 70 | 71 | type unicodeTag struct { 72 | W string `json:"Ελλάδα"` 73 | } 74 | 75 | func TestStructTagObjectKey(t *testing.T) { 76 | tests := []struct { 77 | CaseName 78 | raw any 79 | value string 80 | key string 81 | }{ 82 | {Name(""), basicLatin2xTag{"2x"}, "2x", "$%-/"}, 83 | {Name(""), basicLatin3xTag{"3x"}, "3x", "0123456789"}, 84 | {Name(""), basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"}, 85 | {Name(""), basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"}, 86 | {Name(""), basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, 87 | {Name(""), basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, 88 | {Name(""), miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, 89 | {Name(""), dashTag{"foo"}, "foo", "-"}, 90 | {Name(""), emptyTag{"Pour Moi"}, "Pour Moi", "W"}, 91 | {Name(""), misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, 92 | {Name(""), badFormatTag{"Orfevre"}, "Orfevre", "Y"}, 93 | {Name(""), badCodeTag{"Reliable Man"}, "Reliable Man", "Z"}, 94 | {Name(""), percentSlashTag{"brut"}, "brut", "text/html%"}, 95 | {Name(""), punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:;<=>?@[]^_{|}~ "}, 96 | {Name(""), spaceTag{"Perreddu"}, "Perreddu", "With space"}, 97 | {Name(""), unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"}, 98 | } 99 | for _, tt := range tests { 100 | t.Run(tt.Name, func(t *testing.T) { 101 | b, err := Marshal(tt.raw) 102 | if err != nil { 103 | t.Fatalf("%s: Marshal error: %v", tt.Where, err) 104 | } 105 | var f any 106 | err = Unmarshal(b, &f) 107 | if err != nil { 108 | t.Fatalf("%s: Unmarshal error: %v", tt.Where, err) 109 | } 110 | for k, v := range f.(map[string]any) { 111 | if k == tt.key { 112 | if s, ok := v.(string); !ok || s != tt.value { 113 | t.Fatalf("%s: Unmarshal(%#q) value:\n\tgot: %q\n\twant: %q", tt.Where, b, s, tt.value) 114 | } 115 | } else { 116 | t.Fatalf("%s: Unmarshal(%#q): unexpected key: %q", tt.Where, b, k) 117 | } 118 | } 119 | }) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /example_orderedobject_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json_test 8 | 9 | import ( 10 | "fmt" 11 | "log" 12 | "reflect" 13 | 14 | "github.com/go-json-experiment/json" 15 | "github.com/go-json-experiment/json/jsontext" 16 | ) 17 | 18 | // OrderedObject is an ordered sequence of name/value members in a JSON object. 19 | // 20 | // RFC 8259 defines an object as an "unordered collection". 21 | // JSON implementations need not make "ordering of object members visible" 22 | // to applications nor will they agree on the semantic meaning of an object if 23 | // "the names within an object are not unique". For maximum compatibility, 24 | // applications should avoid relying on ordering or duplicity of object names. 25 | type OrderedObject[V any] []ObjectMember[V] 26 | 27 | // ObjectMember is a JSON object member. 28 | type ObjectMember[V any] struct { 29 | Name string 30 | Value V 31 | } 32 | 33 | // MarshalJSONTo encodes obj as a JSON object into enc. 34 | func (obj *OrderedObject[V]) MarshalJSONTo(enc *jsontext.Encoder) error { 35 | if err := enc.WriteToken(jsontext.BeginObject); err != nil { 36 | return err 37 | } 38 | for i := range *obj { 39 | member := &(*obj)[i] 40 | if err := json.MarshalEncode(enc, &member.Name); err != nil { 41 | return err 42 | } 43 | if err := json.MarshalEncode(enc, &member.Value); err != nil { 44 | return err 45 | } 46 | } 47 | if err := enc.WriteToken(jsontext.EndObject); err != nil { 48 | return err 49 | } 50 | return nil 51 | } 52 | 53 | // UnmarshalJSONFrom decodes a JSON object from dec into obj. 54 | func (obj *OrderedObject[V]) UnmarshalJSONFrom(dec *jsontext.Decoder) error { 55 | if k := dec.PeekKind(); k != '{' { 56 | // The [json] package automatically populates relevant fields 57 | // in a [json.SemanticError] to provide additional context. 58 | return &json.SemanticError{JSONKind: k} 59 | } 60 | if _, err := dec.ReadToken(); err != nil { 61 | return err 62 | } 63 | for dec.PeekKind() != '}' { 64 | *obj = append(*obj, ObjectMember[V]{}) 65 | member := &(*obj)[len(*obj)-1] 66 | if err := json.UnmarshalDecode(dec, &member.Name); err != nil { 67 | return err 68 | } 69 | if err := json.UnmarshalDecode(dec, &member.Value); err != nil { 70 | return err 71 | } 72 | } 73 | if _, err := dec.ReadToken(); err != nil { 74 | return err 75 | } 76 | return nil 77 | } 78 | 79 | // The exact order of JSON object can be preserved through the use of a 80 | // specialized type that implements [MarshalerTo] and [UnmarshalerFrom]. 81 | func Example_orderedObject() { 82 | // Round-trip marshal and unmarshal an ordered object. 83 | // We expect the order and duplicity of JSON object members to be preserved. 84 | // Specify jsontext.AllowDuplicateNames since this object contains "fizz" twice. 85 | want := OrderedObject[string]{ 86 | {"fizz", "buzz"}, 87 | {"hello", "world"}, 88 | {"fizz", "wuzz"}, 89 | } 90 | b, err := json.Marshal(&want, jsontext.AllowDuplicateNames(true)) 91 | if err != nil { 92 | log.Fatal(err) 93 | } 94 | var got OrderedObject[string] 95 | err = json.Unmarshal(b, &got, jsontext.AllowDuplicateNames(true)) 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | 100 | // Sanity check. 101 | if !reflect.DeepEqual(got, want) { 102 | log.Fatalf("roundtrip mismatch: got %v, want %v", got, want) 103 | } 104 | 105 | // Print the serialized JSON object. 106 | (*jsontext.Value)(&b).Indent() // indent for readability 107 | fmt.Println(string(b)) 108 | 109 | // Output: 110 | // { 111 | // "fizz": "buzz", 112 | // "hello": "world", 113 | // "fizz": "wuzz" 114 | // } 115 | } 116 | -------------------------------------------------------------------------------- /internal/zstd/bits.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zstd 6 | 7 | import ( 8 | "math/bits" 9 | ) 10 | 11 | // block is the data for a single compressed block. 12 | // The data starts immediately after the 3 byte block header, 13 | // and is Block_Size bytes long. 14 | type block []byte 15 | 16 | // bitReader reads a bit stream going forward. 17 | type bitReader struct { 18 | r *Reader // for error reporting 19 | data block // the bits to read 20 | off uint32 // current offset into data 21 | bits uint32 // bits ready to be returned 22 | cnt uint32 // number of valid bits in the bits field 23 | } 24 | 25 | // makeBitReader makes a bit reader starting at off. 26 | func (r *Reader) makeBitReader(data block, off int) bitReader { 27 | return bitReader{ 28 | r: r, 29 | data: data, 30 | off: uint32(off), 31 | } 32 | } 33 | 34 | // moreBits is called to read more bits. 35 | // This ensures that at least 16 bits are available. 36 | func (br *bitReader) moreBits() error { 37 | for br.cnt < 16 { 38 | if br.off >= uint32(len(br.data)) { 39 | return br.r.makeEOFError(int(br.off)) 40 | } 41 | c := br.data[br.off] 42 | br.off++ 43 | br.bits |= uint32(c) << br.cnt 44 | br.cnt += 8 45 | } 46 | return nil 47 | } 48 | 49 | // val is called to fetch a value of b bits. 50 | func (br *bitReader) val(b uint8) uint32 { 51 | r := br.bits & ((1 << b) - 1) 52 | br.bits >>= b 53 | br.cnt -= uint32(b) 54 | return r 55 | } 56 | 57 | // backup steps back to the last byte we used. 58 | func (br *bitReader) backup() { 59 | for br.cnt >= 8 { 60 | br.off-- 61 | br.cnt -= 8 62 | } 63 | } 64 | 65 | // makeError returns an error at the current offset wrapping a string. 66 | func (br *bitReader) makeError(msg string) error { 67 | return br.r.makeError(int(br.off), msg) 68 | } 69 | 70 | // reverseBitReader reads a bit stream in reverse. 71 | type reverseBitReader struct { 72 | r *Reader // for error reporting 73 | data block // the bits to read 74 | off uint32 // current offset into data 75 | start uint32 // start in data; we read backward to start 76 | bits uint32 // bits ready to be returned 77 | cnt uint32 // number of valid bits in bits field 78 | } 79 | 80 | // makeReverseBitReader makes a reverseBitReader reading backward 81 | // from off to start. The bitstream starts with a 1 bit in the last 82 | // byte, at off. 83 | func (r *Reader) makeReverseBitReader(data block, off, start int) (reverseBitReader, error) { 84 | streamStart := data[off] 85 | if streamStart == 0 { 86 | return reverseBitReader{}, r.makeError(off, "zero byte at reverse bit stream start") 87 | } 88 | rbr := reverseBitReader{ 89 | r: r, 90 | data: data, 91 | off: uint32(off), 92 | start: uint32(start), 93 | bits: uint32(streamStart), 94 | cnt: uint32(7 - bits.LeadingZeros8(streamStart)), 95 | } 96 | return rbr, nil 97 | } 98 | 99 | // val is called to fetch a value of b bits. 100 | func (rbr *reverseBitReader) val(b uint8) (uint32, error) { 101 | if !rbr.fetch(b) { 102 | return 0, rbr.r.makeEOFError(int(rbr.off)) 103 | } 104 | 105 | rbr.cnt -= uint32(b) 106 | v := (rbr.bits >> rbr.cnt) & ((1 << b) - 1) 107 | return v, nil 108 | } 109 | 110 | // fetch is called to ensure that at least b bits are available. 111 | // It reports false if this can't be done, 112 | // in which case only rbr.cnt bits are available. 113 | func (rbr *reverseBitReader) fetch(b uint8) bool { 114 | for rbr.cnt < uint32(b) { 115 | if rbr.off <= rbr.start { 116 | return false 117 | } 118 | rbr.off-- 119 | c := rbr.data[rbr.off] 120 | rbr.bits <<= 8 121 | rbr.bits |= uint32(c) 122 | rbr.cnt += 8 123 | } 124 | return true 125 | } 126 | 127 | // makeError returns an error at the current offset wrapping a string. 128 | func (rbr *reverseBitReader) makeError(msg string) error { 129 | return rbr.r.makeError(int(rbr.off), msg) 130 | } 131 | -------------------------------------------------------------------------------- /internal/zstd/xxhash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zstd 6 | 7 | import ( 8 | "encoding/binary" 9 | "math/bits" 10 | ) 11 | 12 | const ( 13 | xxhPrime64c1 = 0x9e3779b185ebca87 14 | xxhPrime64c2 = 0xc2b2ae3d27d4eb4f 15 | xxhPrime64c3 = 0x165667b19e3779f9 16 | xxhPrime64c4 = 0x85ebca77c2b2ae63 17 | xxhPrime64c5 = 0x27d4eb2f165667c5 18 | ) 19 | 20 | // xxhash64 is the state of a xxHash-64 checksum. 21 | type xxhash64 struct { 22 | len uint64 // total length hashed 23 | v [4]uint64 // accumulators 24 | buf [32]byte // buffer 25 | cnt int // number of bytes in buffer 26 | } 27 | 28 | // reset discards the current state and prepares to compute a new hash. 29 | // We assume a seed of 0 since that is what zstd uses. 30 | func (xh *xxhash64) reset() { 31 | xh.len = 0 32 | 33 | // Separate addition for awkward constant overflow. 34 | xh.v[0] = xxhPrime64c1 35 | xh.v[0] += xxhPrime64c2 36 | 37 | xh.v[1] = xxhPrime64c2 38 | xh.v[2] = 0 39 | 40 | // Separate negation for awkward constant overflow. 41 | xh.v[3] = xxhPrime64c1 42 | xh.v[3] = -xh.v[3] 43 | 44 | for i := range xh.buf { 45 | xh.buf[i] = 0 46 | } 47 | xh.cnt = 0 48 | } 49 | 50 | // update adds a buffer to the has. 51 | func (xh *xxhash64) update(b []byte) { 52 | xh.len += uint64(len(b)) 53 | 54 | if xh.cnt+len(b) < len(xh.buf) { 55 | copy(xh.buf[xh.cnt:], b) 56 | xh.cnt += len(b) 57 | return 58 | } 59 | 60 | if xh.cnt > 0 { 61 | n := copy(xh.buf[xh.cnt:], b) 62 | b = b[n:] 63 | xh.v[0] = xh.round(xh.v[0], binary.LittleEndian.Uint64(xh.buf[:])) 64 | xh.v[1] = xh.round(xh.v[1], binary.LittleEndian.Uint64(xh.buf[8:])) 65 | xh.v[2] = xh.round(xh.v[2], binary.LittleEndian.Uint64(xh.buf[16:])) 66 | xh.v[3] = xh.round(xh.v[3], binary.LittleEndian.Uint64(xh.buf[24:])) 67 | xh.cnt = 0 68 | } 69 | 70 | for len(b) >= 32 { 71 | xh.v[0] = xh.round(xh.v[0], binary.LittleEndian.Uint64(b)) 72 | xh.v[1] = xh.round(xh.v[1], binary.LittleEndian.Uint64(b[8:])) 73 | xh.v[2] = xh.round(xh.v[2], binary.LittleEndian.Uint64(b[16:])) 74 | xh.v[3] = xh.round(xh.v[3], binary.LittleEndian.Uint64(b[24:])) 75 | b = b[32:] 76 | } 77 | 78 | if len(b) > 0 { 79 | copy(xh.buf[:], b) 80 | xh.cnt = len(b) 81 | } 82 | } 83 | 84 | // digest returns the final hash value. 85 | func (xh *xxhash64) digest() uint64 { 86 | var h64 uint64 87 | if xh.len < 32 { 88 | h64 = xh.v[2] + xxhPrime64c5 89 | } else { 90 | h64 = bits.RotateLeft64(xh.v[0], 1) + 91 | bits.RotateLeft64(xh.v[1], 7) + 92 | bits.RotateLeft64(xh.v[2], 12) + 93 | bits.RotateLeft64(xh.v[3], 18) 94 | h64 = xh.mergeRound(h64, xh.v[0]) 95 | h64 = xh.mergeRound(h64, xh.v[1]) 96 | h64 = xh.mergeRound(h64, xh.v[2]) 97 | h64 = xh.mergeRound(h64, xh.v[3]) 98 | } 99 | 100 | h64 += xh.len 101 | 102 | len := xh.len 103 | len &= 31 104 | buf := xh.buf[:] 105 | for len >= 8 { 106 | k1 := xh.round(0, binary.LittleEndian.Uint64(buf)) 107 | buf = buf[8:] 108 | h64 ^= k1 109 | h64 = bits.RotateLeft64(h64, 27)*xxhPrime64c1 + xxhPrime64c4 110 | len -= 8 111 | } 112 | if len >= 4 { 113 | h64 ^= uint64(binary.LittleEndian.Uint32(buf)) * xxhPrime64c1 114 | buf = buf[4:] 115 | h64 = bits.RotateLeft64(h64, 23)*xxhPrime64c2 + xxhPrime64c3 116 | len -= 4 117 | } 118 | for len > 0 { 119 | h64 ^= uint64(buf[0]) * xxhPrime64c5 120 | buf = buf[1:] 121 | h64 = bits.RotateLeft64(h64, 11) * xxhPrime64c1 122 | len-- 123 | } 124 | 125 | h64 ^= h64 >> 33 126 | h64 *= xxhPrime64c2 127 | h64 ^= h64 >> 29 128 | h64 *= xxhPrime64c3 129 | h64 ^= h64 >> 32 130 | 131 | return h64 132 | } 133 | 134 | // round updates a value. 135 | func (xh *xxhash64) round(v, n uint64) uint64 { 136 | v += n * xxhPrime64c2 137 | v = bits.RotateLeft64(v, 31) 138 | v *= xxhPrime64c1 139 | return v 140 | } 141 | 142 | // mergeRound updates a value in the final round. 143 | func (xh *xxhash64) mergeRound(v, n uint64) uint64 { 144 | n = xh.round(0, n) 145 | v ^= n 146 | v = v*xxhPrime64c1 + xxhPrime64c4 147 | return v 148 | } 149 | -------------------------------------------------------------------------------- /fold_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "fmt" 11 | "reflect" 12 | "testing" 13 | "unicode" 14 | ) 15 | 16 | var equalFoldTestdata = []struct { 17 | in1, in2 string 18 | want bool 19 | }{ 20 | {"", "", true}, 21 | {"abc", "abc", true}, 22 | {"ABcd", "ABcd", true}, 23 | {"123abc", "123ABC", true}, 24 | {"_1_2_-_3__--a-_-b-c-", "123ABC", true}, 25 | {"αβδ", "ΑΒΔ", true}, 26 | {"abc", "xyz", false}, 27 | {"abc", "XYZ", false}, 28 | {"abcdefghijk", "abcdefghijX", false}, 29 | {"abcdefghijk", "abcdefghij\u212A", true}, 30 | {"abcdefghijK", "abcdefghij\u212A", true}, 31 | {"abcdefghijkz", "abcdefghij\u212Ay", false}, 32 | {"abcdefghijKz", "abcdefghij\u212Ay", false}, 33 | {"1", "2", false}, 34 | {"utf-8", "US-ASCII", false}, 35 | {"hello, world!", "hello, world!", true}, 36 | {"hello, world!", "Hello, World!", true}, 37 | {"hello, world!", "HELLO, WORLD!", true}, 38 | {"hello, world!", "jello, world!", false}, 39 | {"γειά, κόσμε!", "γειά, κόσμε!", true}, 40 | {"γειά, κόσμε!", "Γειά, Κόσμε!", true}, 41 | {"γειά, κόσμε!", "ΓΕΙΆ, ΚΌΣΜΕ!", true}, 42 | {"γειά, κόσμε!", "ΛΕΙΆ, ΚΌΣΜΕ!", false}, 43 | {"AESKey", "aesKey", true}, 44 | {"γειά, κόσμε!", "Γ\xce_\xb5ιά, Κόσμε!", false}, 45 | {"aeskey", "AESKEY", true}, 46 | {"AESKEY", "aes_key", true}, 47 | {"aes_key", "AES_KEY", true}, 48 | {"AES_KEY", "aes-key", true}, 49 | {"aes-key", "AES-KEY", true}, 50 | {"AES-KEY", "aesKey", true}, 51 | {"aesKey", "AesKey", true}, 52 | {"AesKey", "AESKey", true}, 53 | {"AESKey", "aeskey", true}, 54 | {"DESKey", "aeskey", false}, 55 | {"AES Key", "aeskey", false}, 56 | {"aes﹏key", "aeskey", false}, // Unicode underscore not handled 57 | {"aes〰key", "aeskey", false}, // Unicode dash not handled 58 | } 59 | 60 | func TestEqualFold(t *testing.T) { 61 | for _, tt := range equalFoldTestdata { 62 | got := equalFold([]byte(tt.in1), []byte(tt.in2)) 63 | if got != tt.want { 64 | t.Errorf("equalFold(%q, %q) = %v, want %v", tt.in1, tt.in2, got, tt.want) 65 | } 66 | } 67 | } 68 | 69 | func equalFold(x, y []byte) bool { 70 | return string(foldName(x)) == string(foldName(y)) 71 | } 72 | 73 | func TestFoldRune(t *testing.T) { 74 | if testing.Short() { 75 | t.Skip() 76 | } 77 | 78 | var foldSet []rune 79 | for r := range rune(unicode.MaxRune + 1) { 80 | // Derive all runes that are all part of the same fold set. 81 | foldSet = foldSet[:0] 82 | for r0 := r; r != r0 || len(foldSet) == 0; r = unicode.SimpleFold(r) { 83 | foldSet = append(foldSet, r) 84 | } 85 | 86 | // Normalized form of each rune in a foldset must be the same and 87 | // also be within the set itself. 88 | var withinSet bool 89 | rr0 := foldRune(foldSet[0]) 90 | for _, r := range foldSet { 91 | withinSet = withinSet || rr0 == r 92 | rr := foldRune(r) 93 | if rr0 != rr { 94 | t.Errorf("foldRune(%q) = %q, want %q", r, rr, rr0) 95 | } 96 | } 97 | if !withinSet { 98 | t.Errorf("foldRune(%q) = %q not in fold set %q", foldSet[0], rr0, string(foldSet)) 99 | } 100 | } 101 | } 102 | 103 | // TestBenchmarkUnmarshalUnknown unmarshals an unknown field into a struct with 104 | // varying number of fields. Since the unknown field does not directly match 105 | // any known field by name, it must fall back on case-insensitive matching. 106 | func TestBenchmarkUnmarshalUnknown(t *testing.T) { 107 | in := []byte(`{"NameUnknown":null}`) 108 | for _, n := range []int{1, 2, 5, 10, 20, 50, 100} { 109 | unmarshal := Unmarshal 110 | 111 | var fields []reflect.StructField 112 | for i := range n { 113 | fields = append(fields, reflect.StructField{ 114 | Name: fmt.Sprintf("Name%d", i), 115 | Type: T[int](), 116 | Tag: `json:",case:ignore"`, 117 | }) 118 | } 119 | out := reflect.New(reflect.StructOf(fields)).Interface() 120 | 121 | t.Run(fmt.Sprintf("N%d", n), func(t *testing.T) { 122 | if err := unmarshal(in, out); err != nil { 123 | t.Fatalf("Unmarshal error: %v", err) 124 | } 125 | }) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /jsontext/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package jsontext_test 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "io" 13 | "log" 14 | "strings" 15 | 16 | "github.com/go-json-experiment/json" 17 | "github.com/go-json-experiment/json/jsontext" 18 | ) 19 | 20 | // This example demonstrates the use of the [Encoder] and [Decoder] to 21 | // parse and modify JSON without unmarshaling it into a concrete Go type. 22 | func Example_stringReplace() { 23 | // Example input with non-idiomatic use of "Golang" instead of "Go". 24 | const input = `{ 25 | "title": "Golang version 1 is released", 26 | "author": "Andrew Gerrand", 27 | "date": "2012-03-28", 28 | "text": "Today marks a major milestone in the development of the Golang programming language.", 29 | "otherArticles": [ 30 | "Twelve Years of Golang", 31 | "The Laws of Reflection", 32 | "Learn Golang from your browser" 33 | ] 34 | }` 35 | 36 | // Using a Decoder and Encoder, we can parse through every token, 37 | // check and modify the token if necessary, and 38 | // write the token to the output. 39 | var replacements []jsontext.Pointer 40 | in := strings.NewReader(input) 41 | dec := jsontext.NewDecoder(in) 42 | out := new(bytes.Buffer) 43 | enc := jsontext.NewEncoder(out, jsontext.Multiline(true)) // expand for readability 44 | for { 45 | // Read a token from the input. 46 | tok, err := dec.ReadToken() 47 | if err != nil { 48 | if err == io.EOF { 49 | break 50 | } 51 | log.Fatal(err) 52 | } 53 | 54 | // Check whether the token contains the string "Golang" and 55 | // replace each occurrence with "Go" instead. 56 | if tok.Kind() == '"' && strings.Contains(tok.String(), "Golang") { 57 | replacements = append(replacements, dec.StackPointer()) 58 | tok = jsontext.String(strings.ReplaceAll(tok.String(), "Golang", "Go")) 59 | } 60 | 61 | // Write the (possibly modified) token to the output. 62 | if err := enc.WriteToken(tok); err != nil { 63 | log.Fatal(err) 64 | } 65 | } 66 | 67 | // Print the list of replacements and the adjusted JSON output. 68 | if len(replacements) > 0 { 69 | fmt.Println(`Replaced "Golang" with "Go" in:`) 70 | for _, where := range replacements { 71 | fmt.Println("\t" + where) 72 | } 73 | fmt.Println() 74 | } 75 | fmt.Println("Result:", out.String()) 76 | 77 | // Output: 78 | // Replaced "Golang" with "Go" in: 79 | // /title 80 | // /text 81 | // /otherArticles/0 82 | // /otherArticles/2 83 | // 84 | // Result: { 85 | // "title": "Go version 1 is released", 86 | // "author": "Andrew Gerrand", 87 | // "date": "2012-03-28", 88 | // "text": "Today marks a major milestone in the development of the Go programming language.", 89 | // "otherArticles": [ 90 | // "Twelve Years of Go", 91 | // "The Laws of Reflection", 92 | // "Learn Go from your browser" 93 | // ] 94 | // } 95 | } 96 | 97 | // Directly embedding JSON within HTML requires special handling for safety. 98 | // Escape certain runes to prevent JSON directly treated as HTML 99 | // from being able to perform `, 112 | } 113 | 114 | b, err := json.Marshal(&page, 115 | // Escape certain runes within a JSON string so that 116 | // JSON will be safe to directly embed inside HTML. 117 | jsontext.EscapeForHTML(true), 118 | jsontext.EscapeForJS(true), 119 | jsontext.Multiline(true)) // expand for readability 120 | if err != nil { 121 | log.Fatal(err) 122 | } 123 | fmt.Println(string(b)) 124 | 125 | // Output: 126 | // { 127 | // "Title": "Example Embedded Javascript", 128 | // "Body": "\u003cscript\u003e console.log(\"Hello, world!\"); \u003c/script\u003e" 129 | // } 130 | } 131 | -------------------------------------------------------------------------------- /intern_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "io" 13 | "testing" 14 | 15 | "github.com/go-json-experiment/json/internal/jsontest" 16 | "github.com/go-json-experiment/json/jsontext" 17 | ) 18 | 19 | func TestIntern(t *testing.T) { 20 | var sc stringCache 21 | const alphabet = "abcdefghijklmnopqrstuvwxyz" 22 | for i := range len(alphabet) + 1 { 23 | want := alphabet[i:] 24 | if got := makeString(&sc, []byte(want)); got != want { 25 | t.Fatalf("make = %v, want %v", got, want) 26 | } 27 | } 28 | for i := range 1000 { 29 | want := fmt.Sprintf("test%b", i) 30 | if got := makeString(&sc, []byte(want)); got != want { 31 | t.Fatalf("make = %v, want %v", got, want) 32 | } 33 | } 34 | } 35 | 36 | var sink string 37 | 38 | func BenchmarkIntern(b *testing.B) { 39 | datasetStrings := func(name string) (out [][]byte) { 40 | var data []byte 41 | for _, ts := range jsontest.Data { 42 | if ts.Name == name { 43 | data = ts.Data() 44 | } 45 | } 46 | dec := jsontext.NewDecoder(bytes.NewReader(data)) 47 | for { 48 | k, n := dec.StackIndex(dec.StackDepth()) 49 | isObjectName := k == '{' && n%2 == 0 50 | tok, err := dec.ReadToken() 51 | if err != nil { 52 | if err == io.EOF { 53 | break 54 | } 55 | b.Fatalf("ReadToken error: %v", err) 56 | } 57 | if tok.Kind() == '"' && !isObjectName { 58 | out = append(out, []byte(tok.String())) 59 | } 60 | } 61 | return out 62 | } 63 | 64 | tests := []struct { 65 | label string 66 | data [][]byte 67 | }{ 68 | // Best is the best case scenario where every string is the same. 69 | {"Best", func() (out [][]byte) { 70 | for range 1000 { 71 | out = append(out, []byte("hello, world!")) 72 | } 73 | return out 74 | }()}, 75 | 76 | // Repeat is a sequence of the same set of names repeated. 77 | // This commonly occurs when unmarshaling a JSON array of JSON objects, 78 | // where the set of all names is usually small. 79 | {"Repeat", func() (out [][]byte) { 80 | for range 100 { 81 | for _, s := range []string{"first_name", "last_name", "age", "address", "street_address", "city", "state", "postal_code", "phone_numbers", "gender"} { 82 | out = append(out, []byte(s)) 83 | } 84 | } 85 | return out 86 | }()}, 87 | 88 | // Synthea is all string values encountered in the Synthea FHIR dataset. 89 | {"Synthea", datasetStrings("SyntheaFhir")}, 90 | 91 | // Twitter is all string values encountered in the Twitter dataset. 92 | {"Twitter", datasetStrings("TwitterStatus")}, 93 | 94 | // Worst is the worst case scenario where every string is different 95 | // resulting in wasted time looking up a string that will never match. 96 | {"Worst", func() (out [][]byte) { 97 | for i := range 1000 { 98 | out = append(out, []byte(fmt.Sprintf("%016x", i))) 99 | } 100 | return out 101 | }()}, 102 | } 103 | 104 | for _, tt := range tests { 105 | b.Run(tt.label, func(b *testing.B) { 106 | // Alloc simply heap allocates each string. 107 | // This provides an upper bound on the number of allocations. 108 | b.Run("Alloc", func(b *testing.B) { 109 | b.ReportAllocs() 110 | for range b.N { 111 | for _, b := range tt.data { 112 | sink = string(b) 113 | } 114 | } 115 | }) 116 | // Cache interns strings using stringCache. 117 | // We want to optimize for having a faster runtime than Alloc, 118 | // and also keeping the number of allocations closer to GoMap. 119 | b.Run("Cache", func(b *testing.B) { 120 | b.ReportAllocs() 121 | for range b.N { 122 | var sc stringCache 123 | for _, b := range tt.data { 124 | sink = makeString(&sc, b) 125 | } 126 | } 127 | }) 128 | // GoMap interns all strings in a simple Go map. 129 | // This provides a lower bound on the number of allocations. 130 | b.Run("GoMap", func(b *testing.B) { 131 | b.ReportAllocs() 132 | for range b.N { 133 | m := make(map[string]string) 134 | for _, b := range tt.data { 135 | s, ok := m[string(b)] 136 | if !ok { 137 | s = string(b) 138 | m[s] = s 139 | } 140 | sink = s 141 | } 142 | } 143 | }) 144 | }) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /inline_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "os" 11 | "os/exec" 12 | "strings" 13 | "testing" 14 | ) 15 | 16 | // Whether a function is inlinable is dependent on the Go compiler version 17 | // and also relies on the presence of the Go toolchain itself being installed. 18 | // This test is disabled by default and explicitly enabled with an 19 | // environment variable that is specified in our integration tests, 20 | // which have fine control over exactly which Go version is being tested. 21 | var testInline = os.Getenv("TEST_INLINE") != "" 22 | 23 | func TestInline(t *testing.T) { 24 | if !testInline { 25 | t.SkipNow() 26 | } 27 | 28 | pkgs := map[string]map[string]bool{ 29 | ".": { 30 | "hash64": true, 31 | "foldName": true, // thin wrapper over appendFoldedName 32 | }, 33 | "./internal/jsonwire": { 34 | "ConsumeWhitespace": true, 35 | "ConsumeNull": true, 36 | "ConsumeFalse": true, 37 | "ConsumeTrue": true, 38 | "ConsumeSimpleString": true, 39 | "ConsumeString": true, // thin wrapper over consumeStringResumable 40 | "ConsumeSimpleNumber": true, 41 | "ConsumeNumber": true, // thin wrapper over consumeNumberResumable 42 | "UnquoteMayCopy": true, // thin wrapper over unescapeString 43 | "HasSuffixByte": true, 44 | "TrimSuffixByte": true, 45 | "TrimSuffixString": true, 46 | "TrimSuffixWhitespace": true, 47 | }, 48 | "./jsontext": { 49 | "encoderState.NeedFlush": true, 50 | "Decoder.ReadToken": true, // thin wrapper over decoderState.ReadToken 51 | "Decoder.ReadValue": true, // thin wrapper over decoderState.ReadValue 52 | "Encoder.WriteToken": true, // thin wrapper over encoderState.WriteToken 53 | "Encoder.WriteValue": true, // thin wrapper over encoderState.WriteValue 54 | "decodeBuffer.needMore": true, 55 | "stateMachine.appendLiteral": true, 56 | "stateMachine.appendNumber": true, 57 | "stateMachine.appendString": true, 58 | "stateMachine.Depth": true, 59 | "stateMachine.reset": true, 60 | "stateMachine.MayAppendDelim": true, 61 | "stateMachine.needDelim": true, 62 | "stateMachine.popArray": true, 63 | "stateMachine.popObject": true, 64 | "stateMachine.pushArray": true, 65 | "stateMachine.pushObject": true, 66 | "stateEntry.Increment": true, 67 | "stateEntry.decrement": true, 68 | "stateEntry.isArray": true, 69 | "stateEntry.isObject": true, 70 | "stateEntry.Length": true, 71 | "stateEntry.needImplicitColon": true, 72 | "stateEntry.needImplicitComma": true, 73 | "stateEntry.NeedObjectName": true, 74 | "stateEntry.needObjectValue": true, 75 | "objectNameStack.reset": true, 76 | "objectNameStack.length": true, 77 | "objectNameStack.getUnquoted": true, 78 | "objectNameStack.push": true, 79 | "objectNameStack.ReplaceLastQuotedOffset": true, 80 | "objectNameStack.replaceLastUnquotedName": true, 81 | "objectNameStack.pop": true, 82 | "objectNameStack.ensureCopiedBuffer": true, 83 | "objectNamespace.insertQuoted": true, // thin wrapper over objectNamespace.insert 84 | "objectNamespace.InsertUnquoted": true, // thin wrapper over objectNamespace.insert 85 | "Token.String": true, // thin wrapper over Token.string 86 | }, 87 | } 88 | 89 | for pkg, fncs := range pkgs { 90 | cmd := exec.Command("go", "build", "-gcflags=-m", pkg) 91 | b, err := cmd.CombinedOutput() 92 | if err != nil { 93 | t.Fatalf("exec.Command error: %v\n\n%s", err, b) 94 | } 95 | for _, line := range strings.Split(string(b), "\n") { 96 | const phrase = ": can inline " 97 | if i := strings.Index(line, phrase); i >= 0 { 98 | fnc := line[i+len(phrase):] 99 | fnc = strings.ReplaceAll(fnc, "(", "") 100 | fnc = strings.ReplaceAll(fnc, "*", "") 101 | fnc = strings.ReplaceAll(fnc, ")", "") 102 | delete(fncs, fnc) 103 | } 104 | } 105 | for fnc := range fncs { 106 | t.Errorf("%v is not inlinable, expected it to be", fnc) 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /v1/indent.go: -------------------------------------------------------------------------------- 1 | // Copyright 2010 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !goexperiment.jsonv2 || !go1.25 6 | 7 | package json 8 | 9 | import ( 10 | "bytes" 11 | "strings" 12 | 13 | "github.com/go-json-experiment/json/jsontext" 14 | ) 15 | 16 | // HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 17 | // characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 18 | // so that the JSON will be safe to embed inside HTML