├── jsoniter_adapter.go ├── jsoniter_bool_test.go ├── jsoniter_float_test.go ├── jsoniter_large_file_test.go ├── LICENSE ├── jsoniter_customize_test.go ├── jsoniter_null_test.go ├── jsoniter_io_test.go ├── jsoniter_int_test.go ├── jsoniter_nested_test.go ├── jsoniter_object_test.go ├── jsoniter_string_test.go ├── jsoniter_array_test.go ├── jsoniter_skip_test.go ├── jsoniter_find_end_test.go ├── README.md ├── jsoniter_reflect_test.go ├── jsoniter_reflect.go └── jsoniter.go /jsoniter_adapter.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import "io" 4 | 5 | // adapt to json/encoding api 6 | 7 | func Unmarshal(data []byte, v interface{}) error { 8 | iter := ParseBytes(data) 9 | iter.Read(v) 10 | if iter.Error == io.EOF { 11 | return nil 12 | } 13 | return iter.Error 14 | } 15 | -------------------------------------------------------------------------------- /jsoniter_bool_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import "testing" 4 | 5 | func Test_true(t *testing.T) { 6 | iter := ParseString(`true`) 7 | if iter.ReadBool() != true { 8 | t.FailNow() 9 | } 10 | } 11 | 12 | func Test_false(t *testing.T) { 13 | iter := ParseString(`false`) 14 | if iter.ReadBool() != false { 15 | t.FailNow() 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /jsoniter_float_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "encoding/json" 6 | ) 7 | 8 | func Test_float64_0(t *testing.T) { 9 | iter := ParseString(`0`) 10 | val := iter.ReadFloat64() 11 | if val != 0 { 12 | t.Fatal(val) 13 | } 14 | } 15 | 16 | func Test_float64_1_dot_1(t *testing.T) { 17 | iter := ParseString(`1.1`) 18 | val := iter.ReadFloat64() 19 | if val != 1.1 { 20 | t.Fatal(val) 21 | } 22 | } 23 | 24 | func Test_float32_1_dot_1_comma(t *testing.T) { 25 | iter := ParseString(`1.1,`) 26 | val := iter.ReadFloat32() 27 | if val != 1.1 { 28 | t.Fatal(val) 29 | } 30 | } 31 | 32 | func Benchmark_jsoniter_float(b *testing.B) { 33 | for n := 0; n < b.N; n++ { 34 | iter := ParseString(`1.1`) 35 | iter.ReadFloat64() 36 | } 37 | } 38 | 39 | func Benchmark_json_float(b *testing.B) { 40 | for n := 0; n < b.N; n++ { 41 | result := float64(0) 42 | json.Unmarshal([]byte(`1.1`), &result) 43 | } 44 | } -------------------------------------------------------------------------------- /jsoniter_large_file_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "os" 6 | "encoding/json" 7 | "io/ioutil" 8 | ) 9 | 10 | func Test_large_file(t *testing.T) { 11 | file, err := os.Open("/tmp/large-file.json") 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | iter := Parse(file, 4096) 16 | count := 0 17 | for iter.ReadArray() { 18 | iter.Skip() 19 | count++ 20 | } 21 | if count != 11351 { 22 | t.Fatal(count) 23 | } 24 | } 25 | 26 | 27 | func Benchmark_jsoniter_large_file(b *testing.B) { 28 | b.ReportAllocs() 29 | for n := 0; n < b.N; n++ { 30 | file, _ := os.Open("/tmp/large-file.json") 31 | iter := Parse(file, 4096) 32 | count := 0 33 | for iter.ReadArray() { 34 | iter.Skip() 35 | count++ 36 | } 37 | file.Close() 38 | } 39 | } 40 | 41 | func Benchmark_json_large_file(b *testing.B) { 42 | b.ReportAllocs() 43 | for n := 0; n < b.N; n++ { 44 | file, _ := os.Open("/tmp/large-file.json") 45 | bytes, _ := ioutil.ReadAll(file) 46 | file.Close() 47 | result := []struct{}{} 48 | json.Unmarshal(bytes, &result) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 json-iterator 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /jsoniter_customize_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | "unsafe" 7 | "strconv" 8 | ) 9 | 10 | func Test_customize_type_decoder(t *testing.T) { 11 | RegisterTypeDecoder("time.Time", func(ptr unsafe.Pointer, iter *Iterator) { 12 | t, err := time.ParseInLocation("2006-01-02 15:04:05", iter.ReadString(), time.UTC) 13 | if err != nil { 14 | iter.Error = err 15 | return 16 | } 17 | *((*time.Time)(ptr)) = t 18 | }) 19 | defer ClearDecoders() 20 | val := time.Time{} 21 | err := Unmarshal([]byte(`"2016-12-05 08:43:28"`), &val) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | year, month, day := val.Date() 26 | if year != 2016 || month != 12 || day != 5 { 27 | t.Fatal(val) 28 | } 29 | } 30 | 31 | type Tom struct { 32 | field1 string 33 | } 34 | 35 | func Test_customize_field_decoder(t *testing.T) { 36 | RegisterFieldDecoder("jsoniter.Tom", "field1", func(ptr unsafe.Pointer, iter *Iterator) { 37 | *((*string)(ptr)) = strconv.Itoa(iter.ReadInt()) 38 | }) 39 | defer ClearDecoders() 40 | tom := Tom{} 41 | err := Unmarshal([]byte(`{"field1": 100}`), &tom) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | } -------------------------------------------------------------------------------- /jsoniter_null_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_null(t *testing.T) { 8 | iter := ParseString(`null`) 9 | if iter.ReadNull() != true { 10 | t.FailNow() 11 | } 12 | } 13 | 14 | func Test_null_object(t *testing.T) { 15 | iter := ParseString(`[null,"a"]`) 16 | iter.ReadArray() 17 | if iter.ReadObject() != "" { 18 | t.FailNow() 19 | } 20 | iter.ReadArray() 21 | if iter.ReadString() != "a" { 22 | t.FailNow() 23 | } 24 | } 25 | 26 | func Test_null_array(t *testing.T) { 27 | iter := ParseString(`[null,"a"]`) 28 | iter.ReadArray() 29 | if iter.ReadArray() != false { 30 | t.FailNow() 31 | } 32 | iter.ReadArray() 33 | if iter.ReadString() != "a" { 34 | t.FailNow() 35 | } 36 | } 37 | 38 | func Test_null_string(t *testing.T) { 39 | iter := ParseString(`[null,"a"]`) 40 | iter.ReadArray() 41 | if iter.ReadString() != "" { 42 | t.FailNow() 43 | } 44 | iter.ReadArray() 45 | if iter.ReadString() != "a" { 46 | t.FailNow() 47 | } 48 | } 49 | 50 | func Test_null_skip(t *testing.T) { 51 | iter := ParseString(`[null,"a"]`) 52 | iter.ReadArray() 53 | iter.Skip() 54 | iter.ReadArray() 55 | if iter.ReadString() != "a" { 56 | t.FailNow() 57 | } 58 | } -------------------------------------------------------------------------------- /jsoniter_io_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "bytes" 6 | "io" 7 | ) 8 | 9 | func Test_read_by_one(t *testing.T) { 10 | iter := Parse(bytes.NewBufferString("abc"), 1) 11 | b := iter.readByte() 12 | if iter.Error != nil { 13 | t.Fatal(iter.Error) 14 | } 15 | if b != 'a' { 16 | t.Fatal(b) 17 | } 18 | iter.unreadByte() 19 | if iter.Error != nil { 20 | t.Fatal(iter.Error) 21 | } 22 | iter.unreadByte() 23 | if iter.Error == nil { 24 | t.FailNow() 25 | } 26 | iter.Error = nil 27 | b = iter.readByte() 28 | if iter.Error != nil { 29 | t.Fatal(iter.Error) 30 | } 31 | if b != 'a' { 32 | t.Fatal(b) 33 | } 34 | } 35 | 36 | func Test_read_by_two(t *testing.T) { 37 | iter := Parse(bytes.NewBufferString("abc"), 2) 38 | b := iter.readByte() 39 | if iter.Error != nil { 40 | t.Fatal(iter.Error) 41 | } 42 | if b != 'a' { 43 | t.Fatal(b) 44 | } 45 | b = iter.readByte() 46 | if iter.Error != nil { 47 | t.Fatal(iter.Error) 48 | } 49 | if b != 'b' { 50 | t.Fatal(b) 51 | } 52 | iter.unreadByte() 53 | if iter.Error != nil { 54 | t.Fatal(iter.Error) 55 | } 56 | iter.unreadByte() 57 | if iter.Error != nil { 58 | t.Fatal(iter.Error) 59 | } 60 | b = iter.readByte() 61 | if iter.Error != nil { 62 | t.Fatal(iter.Error) 63 | } 64 | if b != 'a' { 65 | t.Fatal(b) 66 | } 67 | } 68 | 69 | func Test_read_until_eof(t *testing.T) { 70 | iter := Parse(bytes.NewBufferString("abc"), 2) 71 | iter.readByte() 72 | iter.readByte() 73 | b := iter.readByte() 74 | if iter.Error != nil { 75 | t.Fatal(iter.Error) 76 | } 77 | if b != 'c' { 78 | t.Fatal(b) 79 | } 80 | iter.readByte() 81 | if iter.Error != io.EOF { 82 | t.Fatal(iter.Error) 83 | } 84 | } -------------------------------------------------------------------------------- /jsoniter_int_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "bytes" 6 | "encoding/json" 7 | ) 8 | 9 | func Test_uint64_0(t *testing.T) { 10 | iter := Parse(bytes.NewBufferString("0"), 4096) 11 | val := iter.ReadUint64() 12 | if iter.Error != nil { 13 | t.Fatal(iter.Error) 14 | } 15 | if val != 0 { 16 | t.Fatal(val) 17 | } 18 | } 19 | 20 | func Test_uint64_1(t *testing.T) { 21 | iter := Parse(bytes.NewBufferString("1"), 4096) 22 | val := iter.ReadUint64() 23 | if val != 1 { 24 | t.Fatal(val) 25 | } 26 | } 27 | 28 | func Test_uint64_100(t *testing.T) { 29 | iter := Parse(bytes.NewBufferString("100"), 4096) 30 | val := iter.ReadUint64() 31 | if val != 100 { 32 | t.Fatal(val) 33 | } 34 | } 35 | 36 | func Test_uint64_100_comma(t *testing.T) { 37 | iter := Parse(bytes.NewBufferString("100,"), 4096) 38 | val := iter.ReadUint64() 39 | if iter.Error != nil { 40 | t.Fatal(iter.Error) 41 | } 42 | if val != 100 { 43 | t.Fatal(val) 44 | } 45 | } 46 | 47 | func Test_uint64_invalid(t *testing.T) { 48 | iter := Parse(bytes.NewBufferString(","), 4096) 49 | iter.ReadUint64() 50 | if iter.Error == nil { 51 | t.FailNow() 52 | } 53 | } 54 | 55 | func Test_int64_100(t *testing.T) { 56 | iter := Parse(bytes.NewBufferString("100"), 4096) 57 | val := iter.ReadInt64() 58 | if val != 100 { 59 | t.Fatal(val) 60 | } 61 | } 62 | 63 | func Test_int64_minus_100(t *testing.T) { 64 | iter := Parse(bytes.NewBufferString("-100"), 4096) 65 | val := iter.ReadInt64() 66 | if val != -100 { 67 | t.Fatal(val) 68 | } 69 | } 70 | 71 | func Benchmark_jsoniter_int(b *testing.B) { 72 | for n := 0; n < b.N; n++ { 73 | iter := ParseString(`-100`) 74 | iter.ReadInt64() 75 | } 76 | } 77 | 78 | func Benchmark_json_int(b *testing.B) { 79 | for n := 0; n < b.N; n++ { 80 | result := int64(0) 81 | json.Unmarshal([]byte(`-100`), &result) 82 | } 83 | } -------------------------------------------------------------------------------- /jsoniter_nested_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "reflect" 6 | "encoding/json" 7 | ) 8 | 9 | type Level1 struct { 10 | Hello []Level2 11 | } 12 | 13 | type Level2 struct { 14 | World string 15 | } 16 | 17 | func Test_nested(t *testing.T) { 18 | iter := ParseString(`{"hello": [{"world": "value1"}, {"world": "value2"}]}`) 19 | l1 := Level1{} 20 | for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() { 21 | switch l1Field { 22 | case "hello": 23 | l2Array := []Level2{} 24 | for iter.ReadArray() { 25 | l2 := Level2{} 26 | for l2Field := iter.ReadObject(); l2Field != ""; l2Field = iter.ReadObject() { 27 | switch l2Field { 28 | case "world": 29 | l2.World = iter.ReadString() 30 | default: 31 | iter.ReportError("bind l2", "unexpected field: " + l2Field) 32 | } 33 | } 34 | l2Array = append(l2Array, l2) 35 | } 36 | l1.Hello = l2Array 37 | default: 38 | iter.ReportError("bind l1", "unexpected field: " + l1Field) 39 | } 40 | } 41 | if !reflect.DeepEqual(l1, Level1{ 42 | Hello: []Level2{ 43 | {World: "value1"}, 44 | {World: "value2"}, 45 | }, 46 | }) { 47 | t.Fatal(l1) 48 | } 49 | } 50 | 51 | func Benchmark_jsoniter_nested(b *testing.B) { 52 | for n := 0; n < b.N; n++ { 53 | iter := ParseString(`{"hello": [{"world": "value1"}, {"world": "value2"}]}`) 54 | l1 := Level1{} 55 | for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() { 56 | switch l1Field { 57 | case "hello": 58 | l1.Hello = readLevel1Hello(iter) 59 | default: 60 | iter.Skip() 61 | } 62 | } 63 | } 64 | } 65 | 66 | func readLevel1Hello(iter *Iterator) []Level2 { 67 | l2Array := make([]Level2, 0, 2) 68 | for iter.ReadArray() { 69 | l2 := Level2{} 70 | for l2Field := iter.ReadObject(); l2Field != ""; l2Field = iter.ReadObject() { 71 | switch l2Field { 72 | case "world": 73 | l2.World = iter.ReadString() 74 | default: 75 | iter.Skip() 76 | } 77 | } 78 | l2Array = append(l2Array, l2) 79 | } 80 | return l2Array 81 | } 82 | 83 | func Benchmark_json_nested(b *testing.B) { 84 | for n := 0; n < b.N; n++ { 85 | l1 := Level1{} 86 | json.Unmarshal([]byte(`{"hello": [{"world": "value1"}, {"world": "value2"}]}`), &l1) 87 | } 88 | } -------------------------------------------------------------------------------- /jsoniter_object_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "encoding/json" 6 | "fmt" 7 | ) 8 | 9 | func Test_empty_object(t *testing.T) { 10 | iter := ParseString(`{}`) 11 | field := iter.ReadObject() 12 | if field != "" { 13 | t.Fatal(field) 14 | } 15 | } 16 | 17 | func Test_one_field(t *testing.T) { 18 | iter := ParseString(`{"a": "b"}`) 19 | field := iter.ReadObject() 20 | if field != "a" { 21 | fmt.Println(iter.Error) 22 | t.Fatal(field) 23 | } 24 | value := iter.ReadString() 25 | if value != "b" { 26 | t.Fatal(field) 27 | } 28 | field = iter.ReadObject() 29 | if field != "" { 30 | t.Fatal(field) 31 | } 32 | } 33 | 34 | func Test_two_field(t *testing.T) { 35 | iter := ParseString(`{ "a": "b" , "c": "d" }`) 36 | field := iter.ReadObject() 37 | if field != "a" { 38 | t.Fatal(field) 39 | } 40 | value := iter.ReadString() 41 | if value != "b" { 42 | t.Fatal(field) 43 | } 44 | field = iter.ReadObject() 45 | if field != "c" { 46 | t.Fatal(field) 47 | } 48 | value = iter.ReadString() 49 | if value != "d" { 50 | t.Fatal(field) 51 | } 52 | field = iter.ReadObject() 53 | if field != "" { 54 | t.Fatal(field) 55 | } 56 | iter = ParseString(`{"field1": "1", "field2": 2}`) 57 | for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { 58 | switch field { 59 | case "field1": 60 | iter.ReadString() 61 | case "field2": 62 | iter.ReadInt64() 63 | default: 64 | iter.ReportError("bind object", "unexpected field") 65 | } 66 | } 67 | } 68 | 69 | func Test_cb(t *testing.T) { 70 | iter := ParseString(`{ "a": "b" , "c": "d" }`) 71 | count := 0 72 | iter.ReadObjectCB(func(field string) { 73 | iter.Skip() 74 | count += 1 75 | }) 76 | if count != 2 { 77 | t.Fatal(count) 78 | } 79 | } 80 | 81 | type TestObj struct { 82 | Field1 string 83 | Field2 uint64 84 | } 85 | 86 | func Benchmark_jsoniter_object(b *testing.B) { 87 | for n := 0; n < b.N; n++ { 88 | iter := ParseString(`{"field1": "1", "field2": 2}`) 89 | obj := TestObj{} 90 | for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { 91 | switch field { 92 | case "field1": 93 | obj.Field1 = iter.ReadString() 94 | case "field2": 95 | obj.Field2 = iter.ReadUint64() 96 | default: 97 | iter.ReportError("bind object", "unexpected field") 98 | } 99 | } 100 | } 101 | } 102 | 103 | func Benchmark_jsoniter_cb(b *testing.B) { 104 | for n := 0; n < b.N; n++ { 105 | iter := ParseString(`{"field1": "1", "field2": 2}`) 106 | obj := TestObj{} 107 | iter.ReadObjectCB(func(field string) { 108 | switch field { 109 | case "field1": 110 | obj.Field1 = iter.ReadString() 111 | case "field2": 112 | obj.Field2 = iter.ReadUint64() 113 | default: 114 | iter.ReportError("bind object", "unexpected field") 115 | } 116 | }) 117 | } 118 | } 119 | 120 | func Benchmark_json_object(b *testing.B) { 121 | for n := 0; n < b.N; n++ { 122 | result := TestObj{} 123 | json.Unmarshal([]byte(`{"field1": "1", "field2": 2}`), &result) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /jsoniter_string_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "bytes" 6 | "encoding/json" 7 | ) 8 | 9 | func Test_string_empty(t *testing.T) { 10 | iter := Parse(bytes.NewBufferString(`""`), 4096) 11 | val := iter.ReadString() 12 | if iter.Error != nil { 13 | t.Fatal(iter.Error) 14 | } 15 | if val != "" { 16 | t.Fatal(val) 17 | } 18 | } 19 | 20 | func Test_string_hello(t *testing.T) { 21 | iter := Parse(bytes.NewBufferString(`"hello"`), 4096) 22 | val := iter.ReadString() 23 | if iter.Error != nil { 24 | t.Fatal(iter.Error) 25 | } 26 | if val != "hello" { 27 | t.Fatal(val) 28 | } 29 | } 30 | 31 | func Test_string_escape_quote(t *testing.T) { 32 | iter := Parse(bytes.NewBufferString(`"hel\"lo"`), 4096) 33 | val := iter.ReadString() 34 | if iter.Error != nil { 35 | t.Fatal(iter.Error) 36 | } 37 | if val != `hel"lo` { 38 | t.Fatal(val) 39 | } 40 | } 41 | 42 | func Test_string_escape_newline(t *testing.T) { 43 | iter := Parse(bytes.NewBufferString(`"hel\nlo"`), 4096) 44 | val := iter.ReadString() 45 | if iter.Error != nil { 46 | t.Fatal(iter.Error) 47 | } 48 | if val != "hel\nlo" { 49 | t.Fatal(val) 50 | } 51 | } 52 | 53 | func Test_string_escape_unicode(t *testing.T) { 54 | iter := Parse(bytes.NewBufferString(`"\u4e2d\u6587"`), 4096) 55 | val := iter.ReadString() 56 | if iter.Error != nil { 57 | t.Fatal(iter.Error) 58 | } 59 | if val != "中文" { 60 | t.Fatal(val) 61 | } 62 | } 63 | 64 | func Test_string_escape_unicode_with_surrogate(t *testing.T) { 65 | iter := Parse(bytes.NewBufferString(`"\ud83d\udc4a"`), 4096) 66 | val := iter.ReadString() 67 | if iter.Error != nil { 68 | t.Fatal(iter.Error) 69 | } 70 | if val != "\xf0\x9f\x91\x8a" { 71 | t.Fatal(val) 72 | } 73 | } 74 | 75 | func Test_string_as_bytes(t *testing.T) { 76 | iter := Parse(bytes.NewBufferString(`"hello""world"`), 4096) 77 | val := string(iter.ReadStringAsBytes()) 78 | if val != "hello" { 79 | t.Fatal(val) 80 | } 81 | val = string(iter.ReadStringAsBytes()) 82 | if val != "world" { 83 | t.Fatal(val) 84 | } 85 | } 86 | 87 | func Benchmark_jsoniter_unicode(b *testing.B) { 88 | for n := 0; n < b.N; n++ { 89 | iter := ParseString(`"\ud83d\udc4a"`) 90 | iter.ReadString() 91 | } 92 | } 93 | 94 | func Benchmark_jsoniter_ascii(b *testing.B) { 95 | iter := ParseString(`"hello, world!"`) 96 | b.ResetTimer() 97 | for n := 0; n < b.N; n++ { 98 | iter.Reuse(iter.buf) 99 | iter.ReadString() 100 | } 101 | } 102 | 103 | func Benchmark_jsoniter_string_as_bytes(b *testing.B) { 104 | iter := ParseString(`"hello, world!"`) 105 | b.ResetTimer() 106 | for n := 0; n < b.N; n++ { 107 | iter.Reuse(iter.buf) 108 | iter.ReadStringAsBytes() 109 | } 110 | } 111 | 112 | func Benchmark_json_unicode(b *testing.B) { 113 | for n := 0; n < b.N; n++ { 114 | result := "" 115 | json.Unmarshal([]byte(`"\ud83d\udc4a"`), &result) 116 | } 117 | } 118 | 119 | func Benchmark_json_ascii(b *testing.B) { 120 | for n := 0; n < b.N; n++ { 121 | result := "" 122 | json.Unmarshal([]byte(`"hello"`), &result) 123 | } 124 | } -------------------------------------------------------------------------------- /jsoniter_array_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "encoding/json" 6 | ) 7 | 8 | func Test_empty_array(t *testing.T) { 9 | iter := ParseString(`[]`) 10 | cont := iter.ReadArray() 11 | if cont != false { 12 | t.FailNow() 13 | } 14 | } 15 | 16 | func Test_one_element(t *testing.T) { 17 | iter := ParseString(`[1]`) 18 | cont := iter.ReadArray() 19 | if cont != true { 20 | t.FailNow() 21 | } 22 | if iter.ReadInt64() != 1 { 23 | t.FailNow() 24 | } 25 | cont = iter.ReadArray() 26 | if cont != false { 27 | t.FailNow() 28 | } 29 | } 30 | 31 | func Test_two_elements(t *testing.T) { 32 | iter := ParseString(`[1,2]`) 33 | cont := iter.ReadArray() 34 | if cont != true { 35 | t.FailNow() 36 | } 37 | if iter.ReadInt64() != 1 { 38 | t.FailNow() 39 | } 40 | cont = iter.ReadArray() 41 | if cont != true { 42 | t.FailNow() 43 | } 44 | if iter.ReadInt64() != 2 { 45 | t.FailNow() 46 | } 47 | cont = iter.ReadArray() 48 | if cont != false { 49 | t.FailNow() 50 | } 51 | } 52 | 53 | func Test_two_elements_cb(t *testing.T) { 54 | iter := ParseString(`[1,2]`) 55 | total := int64(0) 56 | iter.ReadArrayCB(func() { 57 | total += iter.ReadInt64() 58 | }) 59 | if total != 3 { 60 | t.Fatal(total) 61 | } 62 | } 63 | 64 | func Test_invalid_array(t *testing.T) { 65 | iter := ParseString(`[`) 66 | iter.ReadArray() 67 | if iter.Error == nil { 68 | t.FailNow() 69 | } 70 | } 71 | 72 | func Test_whitespace_in_head(t *testing.T) { 73 | iter := ParseString(` [1]`) 74 | cont := iter.ReadArray() 75 | if cont != true { 76 | t.FailNow() 77 | } 78 | if iter.ReadUint64() != 1 { 79 | t.FailNow() 80 | } 81 | } 82 | 83 | func Test_whitespace_after_array_start(t *testing.T) { 84 | iter := ParseString(`[ 1]`) 85 | cont := iter.ReadArray() 86 | if cont != true { 87 | t.FailNow() 88 | } 89 | if iter.ReadUint64() != 1 { 90 | t.FailNow() 91 | } 92 | } 93 | 94 | func Test_whitespace_before_array_end(t *testing.T) { 95 | iter := ParseString(`[1 ]`) 96 | cont := iter.ReadArray() 97 | if cont != true { 98 | t.FailNow() 99 | } 100 | if iter.ReadUint64() != 1 { 101 | t.FailNow() 102 | } 103 | cont = iter.ReadArray() 104 | if cont != false { 105 | t.FailNow() 106 | } 107 | } 108 | 109 | func Test_whitespace_before_comma(t *testing.T) { 110 | iter := ParseString(`[1 ,2]`) 111 | cont := iter.ReadArray() 112 | if cont != true { 113 | t.FailNow() 114 | } 115 | if iter.ReadUint64() != 1 { 116 | t.FailNow() 117 | } 118 | cont = iter.ReadArray() 119 | if cont != true { 120 | t.FailNow() 121 | } 122 | if iter.ReadUint64() != 2 { 123 | t.FailNow() 124 | } 125 | cont = iter.ReadArray() 126 | if cont != false { 127 | t.FailNow() 128 | } 129 | } 130 | 131 | func Benchmark_jsoniter_array(b *testing.B) { 132 | b.ReportAllocs() 133 | input := []byte(`[1,2,3,4,5,6,7,8,9]`) 134 | iter := ParseBytes(input) 135 | b.ResetTimer() 136 | for n := 0; n < b.N; n++ { 137 | iter.Reuse(input) 138 | for iter.ReadArray() { 139 | iter.ReadUint64() 140 | } 141 | } 142 | } 143 | 144 | func Benchmark_jsoniter_array_cb(b *testing.B) { 145 | b.ReportAllocs() 146 | input := []byte(`[1,2,3,4,5,6,7,8,9]`) 147 | iter := ParseBytes(input) 148 | b.ResetTimer() 149 | for n := 0; n < b.N; n++ { 150 | iter.Reuse(input) 151 | iter.ReadArrayCB(func() { 152 | iter.ReadUint64() 153 | }) 154 | } 155 | } 156 | 157 | func Benchmark_json_array(b *testing.B) { 158 | for n := 0; n < b.N; n++ { 159 | result := []interface{}{} 160 | json.Unmarshal([]byte(`[1,2,3]`), &result) 161 | } 162 | } -------------------------------------------------------------------------------- /jsoniter_skip_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "encoding/json" 6 | ) 7 | 8 | 9 | func Test_skip_number(t *testing.T) { 10 | iter := ParseString(`[-0.12, "b"]`) 11 | iter.ReadArray() 12 | iter.Skip() 13 | iter.ReadArray() 14 | if iter.ReadString() != "b" { 15 | t.FailNow() 16 | } 17 | } 18 | 19 | func Test_skip_null(t *testing.T) { 20 | iter := ParseString(`[null , "b"]`) 21 | iter.ReadArray() 22 | iter.Skip() 23 | iter.ReadArray() 24 | if iter.ReadString() != "b" { 25 | t.FailNow() 26 | } 27 | } 28 | 29 | func Test_skip_true(t *testing.T) { 30 | iter := ParseString(`[true , "b"]`) 31 | iter.ReadArray() 32 | iter.Skip() 33 | iter.ReadArray() 34 | if iter.ReadString() != "b" { 35 | t.FailNow() 36 | } 37 | } 38 | 39 | func Test_skip_false(t *testing.T) { 40 | iter := ParseString(`[false , "b"]`) 41 | iter.ReadArray() 42 | iter.Skip() 43 | iter.ReadArray() 44 | if iter.ReadString() != "b" { 45 | t.FailNow() 46 | } 47 | } 48 | 49 | func Test_skip_array(t *testing.T) { 50 | iter := ParseString(`[[1, [2, [3], 4]], "b"]`) 51 | iter.ReadArray() 52 | iter.Skip() 53 | iter.ReadArray() 54 | if iter.ReadString() != "b" { 55 | t.FailNow() 56 | } 57 | } 58 | 59 | func Test_skip_empty_array(t *testing.T) { 60 | iter := ParseString(`[ [ ], "b"]`) 61 | iter.ReadArray() 62 | iter.Skip() 63 | iter.ReadArray() 64 | if iter.ReadString() != "b" { 65 | t.FailNow() 66 | } 67 | } 68 | 69 | func Test_skip_nested(t *testing.T) { 70 | iter := ParseString(`[ {"a" : [{"b": "c"}], "d": 102 }, "b"]`) 71 | iter.ReadArray() 72 | iter.Skip() 73 | iter.ReadArray() 74 | if iter.ReadString() != "b" { 75 | t.FailNow() 76 | } 77 | } 78 | 79 | type TestResp struct { 80 | Code uint64 81 | } 82 | 83 | func Benchmark_jsoniter_skip(b *testing.B) { 84 | input := []byte(` 85 | { 86 | "_shards":{ 87 | "total" : 5, 88 | "successful" : 5, 89 | "failed" : 0 90 | }, 91 | "hits":{ 92 | "total" : 1, 93 | "hits" : [ 94 | { 95 | "_index" : "twitter", 96 | "_type" : "tweet", 97 | "_id" : "1", 98 | "_source" : { 99 | "user" : "kimchy", 100 | "postDate" : "2009-11-15T14:12:12", 101 | "message" : "trying out Elasticsearch" 102 | } 103 | } 104 | ] 105 | }, 106 | "code": 200 107 | }`) 108 | for n := 0; n < b.N; n++ { 109 | result := TestResp{} 110 | iter := ParseBytes(input) 111 | for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { 112 | switch field { 113 | case "code": 114 | result.Code = iter.ReadUint64() 115 | default: 116 | iter.Skip() 117 | } 118 | } 119 | } 120 | } 121 | 122 | func Benchmark_json_skip(b *testing.B) { 123 | input := []byte(` 124 | { 125 | "_shards":{ 126 | "total" : 5, 127 | "successful" : 5, 128 | "failed" : 0 129 | }, 130 | "hits":{ 131 | "total" : 1, 132 | "hits" : [ 133 | { 134 | "_index" : "twitter", 135 | "_type" : "tweet", 136 | "_id" : "1", 137 | "_source" : { 138 | "user" : "kimchy", 139 | "postDate" : "2009-11-15T14:12:12", 140 | "message" : "trying out Elasticsearch" 141 | } 142 | } 143 | ] 144 | }, 145 | "code": 200 146 | }`) 147 | for n := 0; n < b.N; n++ { 148 | result := TestResp{} 149 | json.Unmarshal(input, &result) 150 | } 151 | } -------------------------------------------------------------------------------- /jsoniter_find_end_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "io" 6 | ) 7 | 8 | func Test_string_end(t *testing.T) { 9 | end, escaped := ParseString(`abc"`).findStringEnd() 10 | if end != 4 { 11 | t.Fatal(end) 12 | } 13 | if escaped != false { 14 | t.Fatal(escaped) 15 | } 16 | end, escaped = ParseString(`abc\\"`).findStringEnd() 17 | if end != 6 { 18 | t.Fatal(end) 19 | } 20 | if escaped != true { 21 | t.Fatal(escaped) 22 | } 23 | end, escaped = ParseString(`abc\\\\"`).findStringEnd() 24 | if end != 8 { 25 | t.Fatal(end) 26 | } 27 | if escaped != true { 28 | t.Fatal(escaped) 29 | } 30 | end, escaped = ParseString(`abc\"`).findStringEnd() 31 | if end != -1 { 32 | t.Fatal(end) 33 | } 34 | if escaped != false { 35 | t.Fatal(escaped) 36 | } 37 | end, escaped = ParseString(`abc\`).findStringEnd() 38 | if end != -1 { 39 | t.Fatal(end) 40 | } 41 | if escaped != true { 42 | t.Fatal(escaped) 43 | } 44 | end, escaped = ParseString(`abc\\`).findStringEnd() 45 | if end != -1 { 46 | t.Fatal(end) 47 | } 48 | if escaped != false { 49 | t.Fatal(escaped) 50 | } 51 | end, escaped = ParseString(`\\`).findStringEnd() 52 | if end != -1 { 53 | t.Fatal(end) 54 | } 55 | if escaped != false { 56 | t.Fatal(escaped) 57 | } 58 | end, escaped = ParseString(`\`).findStringEnd() 59 | if end != -1 { 60 | t.Fatal(end) 61 | } 62 | if escaped != true { 63 | t.Fatal(escaped) 64 | } 65 | } 66 | 67 | type StagedReader struct { 68 | r1 string 69 | r2 string 70 | r3 string 71 | r int 72 | } 73 | 74 | func (reader *StagedReader) Read(p []byte) (n int, err error) { 75 | reader.r++ 76 | switch reader.r { 77 | case 1: 78 | copy(p, []byte(reader.r1)) 79 | return len(reader.r1), nil 80 | case 2: 81 | copy(p, []byte(reader.r2)) 82 | return len(reader.r2), nil 83 | case 3: 84 | copy(p, []byte(reader.r3)) 85 | return len(reader.r3), nil 86 | default: 87 | return 0, io.EOF 88 | } 89 | } 90 | 91 | func Test_skip_string(t *testing.T) { 92 | iter := ParseString(`"abc`) 93 | iter.skipString() 94 | if iter.head != 1 { 95 | t.Fatal(iter.head) 96 | } 97 | iter = ParseString(`\""abc`) 98 | iter.skipString() 99 | if iter.head != 3 { 100 | t.Fatal(iter.head) 101 | } 102 | reader := &StagedReader{ 103 | r1: `abc`, 104 | r2: `"`, 105 | } 106 | iter = Parse(reader, 4096) 107 | iter.skipString() 108 | if iter.head != 1 { 109 | t.Fatal(iter.head) 110 | } 111 | reader = &StagedReader{ 112 | r1: `abc`, 113 | r2: `1"`, 114 | } 115 | iter = Parse(reader, 4096) 116 | iter.skipString() 117 | if iter.head != 2 { 118 | t.Fatal(iter.head) 119 | } 120 | reader = &StagedReader{ 121 | r1: `abc\`, 122 | r2: `"`, 123 | } 124 | iter = Parse(reader, 4096) 125 | iter.skipString() 126 | if iter.Error != io.EOF { 127 | t.Fatal(iter.Error) 128 | } 129 | reader = &StagedReader{ 130 | r1: `abc\`, 131 | r2: `""`, 132 | } 133 | iter = Parse(reader, 4096) 134 | iter.skipString() 135 | if iter.head != 2 { 136 | t.Fatal(iter.head) 137 | } 138 | } 139 | 140 | func Test_skip_object(t *testing.T) { 141 | iter := ParseString(`}`) 142 | iter.skipObject() 143 | if iter.head != 1 { 144 | t.Fatal(iter.head) 145 | } 146 | iter = ParseString(`a}`) 147 | iter.skipObject() 148 | if iter.head != 2 { 149 | t.Fatal(iter.head) 150 | } 151 | iter = ParseString(`{}}a`) 152 | iter.skipObject() 153 | if iter.head != 3 { 154 | t.Fatal(iter.head) 155 | } 156 | reader := &StagedReader{ 157 | r1: `{`, 158 | r2: `}}a`, 159 | } 160 | iter = Parse(reader, 4096) 161 | iter.skipObject() 162 | if iter.head != 2 { 163 | t.Fatal(iter.head) 164 | } 165 | iter = ParseString(`"}"}a`) 166 | iter.skipObject() 167 | if iter.head != 4 { 168 | t.Fatal(iter.head) 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # json iterator (jsoniter) 2 | 3 | faster than DOM, more usable than SAX/StAX 4 | 5 | # Why json iterator? 6 | 7 | ## 1. It is faster 8 | 9 | jsoniter can work as drop in replacement for json.Unmarshal, reflection-api is not only supported, but recommended. 10 | 11 | for performance numbers, see https://github.com/json-iterator/go-benchmark 12 | 13 | The reflection-api is very fast, on the same scale of hand written ones. 14 | 15 | ## 2. io.Reader as input 16 | 17 | jsoniter does not read the whole json into memory, it parse the document in a streaming way. 18 | There are too many json parser only take []byte as input, this one does not require so. 19 | 20 | ## 3. Pull style api 21 | 22 | jsoniter can be used like drop-in replacement of json.Unmarshal, for example 23 | 24 | ``` 25 | type StructOfTag struct { 26 | field1 string `json:"field-1"` 27 | field2 string `json:"-"` 28 | field3 int `json:",string"` 29 | } 30 | 31 | struct_ := StructOfTag{} 32 | jsoniter.Unmarshal(`{"field-1": "hello", "field2": "", "field3": "100"}`, &struct_) 33 | ``` 34 | 35 | But it allows you to go down one level lower, to control the parsing process using pull style api (like StAX, if you 36 | know what I mean). Here is just a demo of what you can do 37 | 38 | ``` 39 | iter := jsoniter.ParseString(`[1,2,3]`) 40 | for iter.ReadArray() { 41 | iter.ReadUint64() 42 | } 43 | ``` 44 | 45 | ## 4. Customization 46 | 47 | Of course, you can use the low level pull api to do anything you like. But most of the time, 48 | reflection based api is fast enough. How to control the parsing process when we are using the reflection api? 49 | json.Unmarshaller is not flexible enough. Jsoniter provides much better customizability. 50 | 51 | ``` 52 | func Test_customize_type_decoder(t *testing.T) { 53 | RegisterTypeDecoder("time.Time", func(ptr unsafe.Pointer, iter *Iterator) { 54 | t, err := time.ParseInLocation("2006-01-02 15:04:05", iter.ReadString(), time.UTC) 55 | if err != nil { 56 | iter.Error = err 57 | return 58 | } 59 | *((*time.Time)(ptr)) = t 60 | }) 61 | defer ClearDecoders() 62 | val := time.Time{} 63 | err := Unmarshal([]byte(`"2016-12-05 08:43:28"`), &val) 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | year, month, day := val.Date() 68 | if year != 2016 || month != 12 || day != 5 { 69 | t.Fatal(val) 70 | } 71 | } 72 | ``` 73 | 74 | there is no way to add json.Unmarshaller to time.Time as the type is not defined by you (type alias time.Time is not fun to use). 75 | Using jsoniter, we can. 76 | 77 | ``` 78 | type Tom struct { 79 | field1 string 80 | } 81 | 82 | func Test_customize_field_decoder(t *testing.T) { 83 | RegisterFieldDecoder("jsoniter.Tom", "field1", func(ptr unsafe.Pointer, iter *Iterator) { 84 | *((*string)(ptr)) = strconv.Itoa(iter.ReadInt()) 85 | }) 86 | defer ClearDecoders() 87 | tom := Tom{} 88 | err := Unmarshal([]byte(`{"field1": 100}`), &tom) 89 | if err != nil { 90 | t.Fatal(err) 91 | } 92 | } 93 | ``` 94 | 95 | It is very common the input json has certain fields massed up. We want string, but it is int, etc. The old way is to 96 | define a struct of exact type like the json. Then we convert from one struct to a new struct. It is just too much work. 97 | Using jsoniter you can tweak the field conversion. 98 | 99 | ## 5. Minimum work to parse, use whatever fits the job 100 | 101 | I invented this wheel because I find it is tedious to parse json which does not match the object model you want to use. 102 | Parse to `map[string]interface{}` is not only ugly but also slow. Parse to struct is not flexible enough to fix 103 | some field type mismatch or structure mismatch. 104 | 105 | If use low level tokenizer/lexer to work at the token level, it is too much work, not to mention there is very few parser 106 | out there allow you to work on this level. 107 | 108 | jsoniter pull-api is designed to be easy to use, so that you can map your data structure directly to parsing code. 109 | It is still tedious I am not going to lie to you, but easier than pure tokenizer. 110 | The real power is, you can mix the pull-api with reflection-api. 111 | For example: 112 | 113 | ``` 114 | \\ given [1, {"a": "b"}] 115 | iter.ReadArray() 116 | iter.ReadInt() 117 | iter.ReadArray() 118 | iter.Read(&struct_) // reflection-api 119 | ``` 120 | 121 | Also by using type or field callback, we can switch from reflection-api back to pull-api. The seamless mix of both styles 122 | enabled a unique new way to parse our data. 123 | 124 | My advice is always use the reflection-api first. Unless you find pull-api can do a better job in certain area. 125 | 126 | # Why not json iterator? 127 | 128 | jsoniter does not plan to support `map[string]interface{}`, period. 129 | -------------------------------------------------------------------------------- /jsoniter_reflect_test.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "testing" 5 | "fmt" 6 | "encoding/json" 7 | ) 8 | 9 | func Test_reflect_str(t *testing.T) { 10 | iter := ParseString(`"hello"`) 11 | str := "" 12 | iter.Read(&str) 13 | if str != "hello" { 14 | fmt.Println(iter.Error) 15 | t.Fatal(str) 16 | } 17 | } 18 | 19 | func Test_reflect_ptr_str(t *testing.T) { 20 | iter := ParseString(`"hello"`) 21 | var str *string 22 | iter.Read(&str) 23 | if *str != "hello" { 24 | t.Fatal(str) 25 | } 26 | } 27 | 28 | func Test_reflect_int(t *testing.T) { 29 | iter := ParseString(`123`) 30 | val := int(0) 31 | iter.Read(&val) 32 | if val != 123 { 33 | t.Fatal(val) 34 | } 35 | } 36 | 37 | func Test_reflect_int8(t *testing.T) { 38 | iter := ParseString(`123`) 39 | val := int8(0) 40 | iter.Read(&val) 41 | if val != 123 { 42 | t.Fatal(val) 43 | } 44 | } 45 | 46 | func Test_reflect_int16(t *testing.T) { 47 | iter := ParseString(`123`) 48 | val := int16(0) 49 | iter.Read(&val) 50 | if val != 123 { 51 | t.Fatal(val) 52 | } 53 | } 54 | 55 | func Test_reflect_int32(t *testing.T) { 56 | iter := ParseString(`123`) 57 | val := int32(0) 58 | iter.Read(&val) 59 | if val != 123 { 60 | t.Fatal(val) 61 | } 62 | } 63 | 64 | func Test_reflect_int64(t *testing.T) { 65 | iter := ParseString(`123`) 66 | val := int64(0) 67 | iter.Read(&val) 68 | if val != 123 { 69 | t.Fatal(val) 70 | } 71 | } 72 | 73 | func Test_reflect_uint(t *testing.T) { 74 | iter := ParseString(`123`) 75 | val := uint(0) 76 | iter.Read(&val) 77 | if val != 123 { 78 | t.Fatal(val) 79 | } 80 | } 81 | 82 | func Test_reflect_uint8(t *testing.T) { 83 | iter := ParseString(`123`) 84 | val := uint8(0) 85 | iter.Read(&val) 86 | if val != 123 { 87 | t.Fatal(val) 88 | } 89 | } 90 | 91 | func Test_reflect_uint16(t *testing.T) { 92 | iter := ParseString(`123`) 93 | val := uint16(0) 94 | iter.Read(&val) 95 | if val != 123 { 96 | t.Fatal(val) 97 | } 98 | } 99 | 100 | func Test_reflect_uint32(t *testing.T) { 101 | iter := ParseString(`123`) 102 | val := uint32(0) 103 | iter.Read(&val) 104 | if val != 123 { 105 | t.Fatal(val) 106 | } 107 | } 108 | 109 | func Test_reflect_uint64(t *testing.T) { 110 | iter := ParseString(`123`) 111 | val := uint64(0) 112 | iter.Read(&val) 113 | if val != 123 { 114 | t.Fatal(val) 115 | } 116 | } 117 | 118 | func Test_reflect_byte(t *testing.T) { 119 | iter := ParseString(`123`) 120 | val := byte(0) 121 | iter.Read(&val) 122 | if val != 123 { 123 | t.Fatal(val) 124 | } 125 | } 126 | 127 | func Test_reflect_float32(t *testing.T) { 128 | iter := ParseString(`1.23`) 129 | val := float32(0) 130 | iter.Read(&val) 131 | if val != 1.23 { 132 | fmt.Println(iter.Error) 133 | t.Fatal(val) 134 | } 135 | } 136 | 137 | func Test_reflect_float64(t *testing.T) { 138 | iter := ParseString(`1.23`) 139 | val := float64(0) 140 | iter.Read(&val) 141 | if val != 1.23 { 142 | fmt.Println(iter.Error) 143 | t.Fatal(val) 144 | } 145 | } 146 | 147 | func Test_reflect_bool(t *testing.T) { 148 | iter := ParseString(`true`) 149 | val := false 150 | iter.Read(&val) 151 | if val != true { 152 | fmt.Println(iter.Error) 153 | t.Fatal(val) 154 | } 155 | } 156 | 157 | type StructOfString struct { 158 | field1 string 159 | field2 string 160 | } 161 | 162 | func Test_reflect_struct_string(t *testing.T) { 163 | iter := ParseString(`{"field1": "hello", "field2": "world"}`) 164 | struct_ := StructOfString{} 165 | iter.Read(&struct_) 166 | if struct_.field1 != "hello" { 167 | fmt.Println(iter.Error) 168 | t.Fatal(struct_.field1) 169 | } 170 | if struct_.field2 != "world" { 171 | fmt.Println(iter.Error) 172 | t.Fatal(struct_.field1) 173 | } 174 | } 175 | 176 | type StructOfStringPtr struct { 177 | field1 *string 178 | field2 *string 179 | } 180 | 181 | func Test_reflect_struct_string_ptr(t *testing.T) { 182 | iter := ParseString(`{"field1": null, "field2": "world"}`) 183 | struct_ := StructOfStringPtr{} 184 | iter.Read(&struct_) 185 | if struct_.field1 != nil { 186 | fmt.Println(iter.Error) 187 | t.Fatal(struct_.field1) 188 | } 189 | if *struct_.field2 != "world" { 190 | fmt.Println(iter.Error) 191 | t.Fatal(struct_.field2) 192 | } 193 | } 194 | 195 | 196 | type StructOfTag struct { 197 | field1 string `json:"field-1"` 198 | field2 string `json:"-"` 199 | field3 int `json:",string"` 200 | } 201 | 202 | func Test_reflect_struct_tag_field(t *testing.T) { 203 | iter := ParseString(`{"field-1": "hello", "field2": "", "field3": "100"}`) 204 | struct_ := StructOfTag{field2: "world"} 205 | iter.Read(&struct_) 206 | if struct_.field1 != "hello" { 207 | fmt.Println(iter.Error) 208 | t.Fatal(struct_.field1) 209 | } 210 | if struct_.field2 != "world" { 211 | fmt.Println(iter.Error) 212 | t.Fatal(struct_.field2) 213 | } 214 | if struct_.field3 != 100 { 215 | fmt.Println(iter.Error) 216 | t.Fatal(struct_.field3) 217 | } 218 | } 219 | 220 | func Test_reflect_slice(t *testing.T) { 221 | iter := ParseString(`["hello", "world"]`) 222 | slice := make([]string, 0, 1) 223 | iter.Read(&slice) 224 | if len(slice) != 2 { 225 | fmt.Println(iter.Error) 226 | t.Fatal(len(slice)) 227 | } 228 | if slice[0] != "hello" { 229 | fmt.Println(iter.Error) 230 | t.Fatal(slice[0]) 231 | } 232 | if slice[1] != "world" { 233 | fmt.Println(iter.Error) 234 | t.Fatal(slice[1]) 235 | } 236 | } 237 | 238 | func Test_reflect_nested(t *testing.T) { 239 | iter := ParseString(`[{"field1": "hello"}, null, {"field2": "world"}]`) 240 | slice := []*StructOfString{} 241 | iter.Read(&slice) 242 | if len(slice) != 3 { 243 | fmt.Println(iter.Error) 244 | t.Fatal(len(slice)) 245 | } 246 | if slice[0].field1 != "hello" { 247 | fmt.Println(iter.Error) 248 | t.Fatal(slice[0]) 249 | } 250 | if slice[1] != nil { 251 | fmt.Println(iter.Error) 252 | t.Fatal(slice[1]) 253 | } 254 | if slice[2].field2 != "world" { 255 | fmt.Println(iter.Error) 256 | t.Fatal(slice[1]) 257 | } 258 | } 259 | 260 | func Benchmark_jsoniter_reflect(b *testing.B) { 261 | b.ReportAllocs() 262 | for n := 0; n < b.N; n++ { 263 | iter := ParseString(`{"field3": "100"}`) 264 | struct_ := StructOfTag{} 265 | iter.Read(&struct_) 266 | //iter := ParseString(`["hello", "world"]`) 267 | //array := make([]string, 0, 1) 268 | //iter.Read(&array) 269 | } 270 | } 271 | 272 | func Benchmark_jsoniter_direct(b *testing.B) { 273 | b.ReportAllocs() 274 | for n := 0; n < b.N; n++ { 275 | //iter := ParseString(`{"field1": "hello", "field2": "world"}`) 276 | //struct_ := StructOfString{} 277 | //for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { 278 | // switch field { 279 | // case "field1": 280 | // struct_.field1 = iter.ReadString() 281 | // case "field2": 282 | // struct_.field2 = iter.ReadString() 283 | // default: 284 | // iter.Skip() 285 | // } 286 | //} 287 | iter := ParseString(`["hello", "world"]`) 288 | array := make([]string, 0, 2) 289 | for iter.ReadArray() { 290 | array = append(array, iter.ReadString()) 291 | } 292 | } 293 | } 294 | 295 | func Benchmark_json_reflect(b *testing.B) { 296 | b.ReportAllocs() 297 | for n := 0; n < b.N; n++ { 298 | struct_ := StructOfTag{} 299 | json.Unmarshal([]byte(`{"field3": "100"}`), &struct_) 300 | //array := make([]string, 0, 2) 301 | //json.Unmarshal([]byte(`["hello", "world"]`), &array) 302 | } 303 | } -------------------------------------------------------------------------------- /jsoniter_reflect.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "reflect" 5 | "errors" 6 | "fmt" 7 | "unsafe" 8 | "sync/atomic" 9 | "strings" 10 | "io" 11 | ) 12 | 13 | type Decoder interface { 14 | decode(ptr unsafe.Pointer, iter *Iterator) 15 | } 16 | 17 | type stringDecoder struct { 18 | } 19 | 20 | func (decoder *stringDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 21 | *((*string)(ptr)) = iter.ReadString() 22 | } 23 | 24 | type intDecoder struct { 25 | } 26 | 27 | func (decoder *intDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 28 | *((*int)(ptr)) = iter.ReadInt() 29 | } 30 | 31 | type int8Decoder struct { 32 | } 33 | 34 | func (decoder *int8Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 35 | *((*int8)(ptr)) = iter.ReadInt8() 36 | } 37 | 38 | type int16Decoder struct { 39 | } 40 | 41 | func (decoder *int16Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 42 | *((*int16)(ptr)) = iter.ReadInt16() 43 | } 44 | 45 | type int32Decoder struct { 46 | } 47 | 48 | func (decoder *int32Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 49 | *((*int32)(ptr)) = iter.ReadInt32() 50 | } 51 | 52 | type int64Decoder struct { 53 | } 54 | 55 | func (decoder *int64Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 56 | *((*int64)(ptr)) = iter.ReadInt64() 57 | } 58 | 59 | type uintDecoder struct { 60 | } 61 | 62 | func (decoder *uintDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 63 | *((*uint)(ptr)) = iter.ReadUint() 64 | } 65 | 66 | type uint8Decoder struct { 67 | } 68 | 69 | func (decoder *uint8Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 70 | *((*uint8)(ptr)) = iter.ReadUint8() 71 | } 72 | 73 | type uint16Decoder struct { 74 | } 75 | 76 | func (decoder *uint16Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 77 | *((*uint16)(ptr)) = iter.ReadUint16() 78 | } 79 | 80 | type uint32Decoder struct { 81 | } 82 | 83 | func (decoder *uint32Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 84 | *((*uint32)(ptr)) = iter.ReadUint32() 85 | } 86 | 87 | type uint64Decoder struct { 88 | } 89 | 90 | func (decoder *uint64Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 91 | *((*uint64)(ptr)) = iter.ReadUint64() 92 | } 93 | 94 | type float32Decoder struct { 95 | } 96 | 97 | func (decoder *float32Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 98 | *((*float32)(ptr)) = iter.ReadFloat32() 99 | } 100 | 101 | type float64Decoder struct { 102 | } 103 | 104 | func (decoder *float64Decoder) decode(ptr unsafe.Pointer, iter *Iterator) { 105 | *((*float64)(ptr)) = iter.ReadFloat64() 106 | } 107 | 108 | type boolDecoder struct { 109 | } 110 | 111 | func (decoder *boolDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 112 | *((*bool)(ptr)) = iter.ReadBool() 113 | } 114 | 115 | type stringNumberDecoder struct { 116 | elemDecoder Decoder 117 | } 118 | 119 | func (decoder *stringNumberDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 120 | c := iter.readByte() 121 | if c != '"' { 122 | iter.ReportError("stringNumberDecoder", `expect "`) 123 | return 124 | } 125 | decoder.elemDecoder.decode(ptr, iter) 126 | if iter.Error != nil { 127 | return 128 | } 129 | c = iter.readByte() 130 | if c != '"' { 131 | iter.ReportError("stringNumberDecoder", `expect "`) 132 | return 133 | } 134 | } 135 | 136 | type optionalDecoder struct { 137 | valueType reflect.Type 138 | valueDecoder Decoder 139 | } 140 | 141 | func (decoder *optionalDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 142 | if iter.ReadNull() { 143 | *((*unsafe.Pointer)(ptr)) = nil 144 | } else { 145 | value := reflect.New(decoder.valueType) 146 | decoder.valueDecoder.decode(unsafe.Pointer(value.Pointer()), iter) 147 | *((*uintptr)(ptr)) = value.Pointer() 148 | } 149 | } 150 | 151 | type structDecoder struct { 152 | type_ reflect.Type 153 | fields map[string]Decoder 154 | } 155 | 156 | func (decoder *structDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 157 | for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { 158 | fieldDecoder := decoder.fields[field] 159 | if fieldDecoder == nil { 160 | iter.Skip() 161 | } else { 162 | fieldDecoder.decode(ptr, iter) 163 | } 164 | } 165 | if iter.Error != nil && iter.Error != io.EOF { 166 | iter.Error = fmt.Errorf("%v: %s", decoder.type_, iter.Error.Error()) 167 | } 168 | } 169 | 170 | type structFieldDecoder struct { 171 | field *reflect.StructField 172 | fieldDecoder Decoder 173 | } 174 | 175 | func (decoder *structFieldDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 176 | fieldPtr := uintptr(ptr) + decoder.field.Offset 177 | decoder.fieldDecoder.decode(unsafe.Pointer(fieldPtr), iter) 178 | if iter.Error != nil && iter.Error != io.EOF { 179 | iter.Error = fmt.Errorf("%s: %s", decoder.field.Name, iter.Error.Error()) 180 | } 181 | } 182 | 183 | type sliceDecoder struct { 184 | sliceType reflect.Type 185 | elemType reflect.Type 186 | elemDecoder Decoder 187 | } 188 | 189 | // sliceHeader is a safe version of SliceHeader used within this package. 190 | type sliceHeader struct { 191 | Data unsafe.Pointer 192 | Len int 193 | Cap int 194 | } 195 | 196 | func (decoder *sliceDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 197 | slice := (*sliceHeader)(ptr) 198 | slice.Len = 0 199 | for iter.ReadArray() { 200 | offset := uintptr(slice.Len) * decoder.elemType.Size() 201 | growOne(slice, decoder.sliceType, decoder.elemType) 202 | dataPtr := uintptr(slice.Data) + offset 203 | decoder.elemDecoder.decode(unsafe.Pointer(dataPtr), iter) 204 | } 205 | if iter.Error != nil && iter.Error != io.EOF { 206 | iter.Error = fmt.Errorf("%v: %s", decoder.sliceType, iter.Error.Error()) 207 | } 208 | } 209 | 210 | // grow grows the slice s so that it can hold extra more values, allocating 211 | // more capacity if needed. It also returns the old and new slice lengths. 212 | func growOne(slice *sliceHeader, sliceType reflect.Type, elementType reflect.Type) { 213 | newLen := slice.Len + 1 214 | if newLen <= slice.Cap { 215 | slice.Len = newLen 216 | return 217 | } 218 | newCap := slice.Cap 219 | if newCap == 0 { 220 | newCap = 1 221 | } else { 222 | for newCap < newLen { 223 | if slice.Len < 1024 { 224 | newCap += newCap 225 | } else { 226 | newCap += newCap / 4 227 | } 228 | } 229 | } 230 | dst := unsafe.Pointer(reflect.MakeSlice(sliceType, newLen, newCap).Pointer()) 231 | originalBytesCount := uintptr(slice.Len) * elementType.Size() 232 | srcPtr := (*[1 << 30]byte)(slice.Data) 233 | dstPtr := (*[1 << 30]byte)(dst) 234 | for i := uintptr(0); i < originalBytesCount; i++ { 235 | dstPtr[i] = srcPtr[i] 236 | } 237 | slice.Len = newLen 238 | slice.Cap = newCap 239 | slice.Data = dst 240 | } 241 | 242 | var DECODERS unsafe.Pointer 243 | 244 | func addDecoderToCache(cacheKey string, decoder Decoder) { 245 | retry := true 246 | for retry { 247 | ptr := atomic.LoadPointer(&DECODERS) 248 | cache := *(*map[string]Decoder)(ptr) 249 | copy := map[string]Decoder{} 250 | for k, v := range cache { 251 | copy[k] = v 252 | } 253 | copy[cacheKey] = decoder 254 | retry = !atomic.CompareAndSwapPointer(&DECODERS, ptr, unsafe.Pointer(©)) 255 | } 256 | } 257 | 258 | func getDecoderFromCache(cacheKey string) Decoder { 259 | ptr := atomic.LoadPointer(&DECODERS) 260 | cache := *(*map[string]Decoder)(ptr) 261 | return cache[cacheKey] 262 | } 263 | 264 | var typeDecoders map[string]Decoder 265 | var fieldDecoders map[string]Decoder 266 | 267 | func init() { 268 | typeDecoders = map[string]Decoder{} 269 | fieldDecoders = map[string]Decoder{} 270 | atomic.StorePointer(&DECODERS, unsafe.Pointer(&map[string]Decoder{})) 271 | } 272 | 273 | type DecoderFunc func(ptr unsafe.Pointer, iter *Iterator) 274 | 275 | type funcDecoder struct { 276 | func_ DecoderFunc 277 | } 278 | 279 | func (decoder *funcDecoder) decode(ptr unsafe.Pointer, iter *Iterator) { 280 | decoder.func_(ptr, iter) 281 | } 282 | 283 | func RegisterTypeDecoder(type_ string, func_ DecoderFunc) { 284 | typeDecoders[type_] = &funcDecoder{func_} 285 | } 286 | 287 | func RegisterFieldDecoder(type_ string, field string, func_ DecoderFunc) { 288 | fieldDecoders[fmt.Sprintf("%s/%s", type_, field)] = &funcDecoder{func_} 289 | } 290 | 291 | func ClearDecoders() { 292 | typeDecoders = map[string]Decoder{} 293 | fieldDecoders = map[string]Decoder{} 294 | } 295 | 296 | // emptyInterface is the header for an interface{} value. 297 | type emptyInterface struct { 298 | typ *struct{} 299 | word unsafe.Pointer 300 | } 301 | 302 | func (iter *Iterator) Read(obj interface{}) { 303 | type_ := reflect.TypeOf(obj) 304 | cacheKey := type_.String() 305 | cachedDecoder := getDecoderFromCache(cacheKey) 306 | if cachedDecoder == nil { 307 | decoder, err := decoderOfType(type_) 308 | if err != nil { 309 | iter.Error = err 310 | return 311 | } 312 | cachedDecoder = decoder 313 | addDecoderToCache(cacheKey, decoder) 314 | } 315 | e := (*emptyInterface)(unsafe.Pointer(&obj)) 316 | cachedDecoder.decode(e.word, iter) 317 | } 318 | 319 | type prefix string 320 | 321 | func (p prefix) addTo(decoder Decoder, err error) (Decoder, error) { 322 | if err != nil { 323 | return nil, fmt.Errorf("%s: %s", p, err.Error()) 324 | } 325 | return decoder, err 326 | } 327 | 328 | func decoderOfType(type_ reflect.Type) (Decoder, error) { 329 | switch type_.Kind() { 330 | case reflect.Ptr: 331 | return prefix("ptr").addTo(decoderOfPtr(type_.Elem())) 332 | default: 333 | return nil, errors.New("expect ptr") 334 | } 335 | } 336 | 337 | func decoderOfPtr(type_ reflect.Type) (Decoder, error) { 338 | typeDecoder := typeDecoders[type_.String()] 339 | if typeDecoder != nil { 340 | return typeDecoder, nil 341 | } 342 | switch type_.Kind() { 343 | case reflect.String: 344 | return &stringDecoder{}, nil 345 | case reflect.Int: 346 | return &intDecoder{}, nil 347 | case reflect.Int8: 348 | return &int8Decoder{}, nil 349 | case reflect.Int16: 350 | return &int16Decoder{}, nil 351 | case reflect.Int32: 352 | return &int32Decoder{}, nil 353 | case reflect.Int64: 354 | return &int64Decoder{}, nil 355 | case reflect.Uint: 356 | return &uintDecoder{}, nil 357 | case reflect.Uint8: 358 | return &uint8Decoder{}, nil 359 | case reflect.Uint16: 360 | return &uint16Decoder{}, nil 361 | case reflect.Uint32: 362 | return &uint32Decoder{}, nil 363 | case reflect.Uint64: 364 | return &uint64Decoder{}, nil 365 | case reflect.Float32: 366 | return &float32Decoder{}, nil 367 | case reflect.Float64: 368 | return &float64Decoder{}, nil 369 | case reflect.Bool: 370 | return &boolDecoder{}, nil 371 | case reflect.Struct: 372 | return decoderOfStruct(type_) 373 | case reflect.Slice: 374 | return prefix("[slice]").addTo(decoderOfSlice(type_)) 375 | case reflect.Ptr: 376 | return prefix("[optional]").addTo(decoderOfOptional(type_.Elem())) 377 | default: 378 | return nil, fmt.Errorf("unsupported type: %v", type_) 379 | } 380 | } 381 | 382 | func decoderOfOptional(type_ reflect.Type) (Decoder, error) { 383 | decoder, err := decoderOfPtr(type_) 384 | if err != nil { 385 | return nil, err 386 | } 387 | return &optionalDecoder{type_, decoder}, nil 388 | } 389 | 390 | func decoderOfStruct(type_ reflect.Type) (Decoder, error) { 391 | fields := map[string]Decoder{} 392 | for i := 0; i < type_.NumField(); i++ { 393 | field := type_.Field(i) 394 | fieldDecoderKey := fmt.Sprintf("%s/%s", type_.String(), field.Name) 395 | decoder := fieldDecoders[fieldDecoderKey] 396 | tagParts := strings.Split(field.Tag.Get("json"), ",") 397 | jsonFieldName := tagParts[0] 398 | if jsonFieldName == "" { 399 | jsonFieldName = field.Name 400 | } 401 | if decoder == nil { 402 | var err error 403 | decoder, err = decoderOfPtr(field.Type) 404 | if err != nil { 405 | return prefix(fmt.Sprintf("{%s}", field.Name)).addTo(decoder, err) 406 | } 407 | } 408 | if len(tagParts) > 1 && tagParts[1] == "string" { 409 | decoder = &stringNumberDecoder{decoder} 410 | } 411 | if jsonFieldName != "-" { 412 | fields[jsonFieldName] = &structFieldDecoder{&field, decoder} 413 | } 414 | } 415 | return &structDecoder{type_, fields}, nil 416 | } 417 | 418 | func decoderOfSlice(type_ reflect.Type) (Decoder, error) { 419 | decoder, err := decoderOfPtr(type_.Elem()) 420 | if err != nil { 421 | return nil, err 422 | } 423 | return &sliceDecoder{type_, type_.Elem(), decoder}, nil 424 | } 425 | -------------------------------------------------------------------------------- /jsoniter.go: -------------------------------------------------------------------------------- 1 | package jsoniter 2 | 3 | import ( 4 | "io" 5 | "fmt" 6 | "unicode/utf16" 7 | "strconv" 8 | "unsafe" 9 | ) 10 | 11 | var digits []byte 12 | 13 | func init() { 14 | digits = make([]byte, 256) 15 | for i := 0; i < len(digits); i++ { 16 | digits[i] = 255 17 | } 18 | for i := '0'; i <= '9'; i++ { 19 | digits[i] = byte(i - '0'); 20 | } 21 | for i := 'a'; i <= 'f'; i++ { 22 | digits[i] = byte((i - 'a') + 10); 23 | } 24 | for i := 'A'; i <= 'F'; i++ { 25 | digits[i] = byte((i - 'A') + 10); 26 | } 27 | } 28 | 29 | type Iterator struct { 30 | reader io.Reader 31 | buf []byte 32 | head int 33 | tail int 34 | Error error 35 | } 36 | 37 | func Parse(reader io.Reader, bufSize int) *Iterator { 38 | iter := &Iterator{ 39 | reader: reader, 40 | buf: make([]byte, bufSize), 41 | head: 0, 42 | tail: 0, 43 | } 44 | iter.skipWhitespaces() 45 | return iter 46 | } 47 | 48 | func ParseBytes(input []byte) *Iterator { 49 | iter := &Iterator{ 50 | reader: nil, 51 | buf: input, 52 | head: 0, 53 | tail: len(input), 54 | } 55 | iter.skipWhitespaces() 56 | return iter 57 | } 58 | 59 | func (iter *Iterator) Reuse(input []byte) *Iterator { 60 | // only for benchmarking 61 | iter.reader = nil 62 | iter.Error = nil 63 | iter.buf = input 64 | iter.head = 0 65 | iter.tail = len(input) 66 | iter.skipWhitespaces() 67 | return iter 68 | } 69 | 70 | func ParseString(input string) *Iterator { 71 | return ParseBytes([]byte(input)) 72 | } 73 | 74 | func (iter *Iterator) skipWhitespaces() { 75 | for { 76 | for i := iter.head; i < iter.tail; i++ { 77 | c := iter.buf[i] 78 | switch c { 79 | case ' ', '\n', '\t', 'r': 80 | continue 81 | } 82 | iter.head = i 83 | return 84 | } 85 | if !iter.loadMore() { 86 | return 87 | } 88 | } 89 | } 90 | 91 | func (iter *Iterator) nextToken() byte { 92 | // a variation of skip whitespaces, returning the next non-whitespace token 93 | for { 94 | for i := iter.head; i < iter.tail; i++ { 95 | c := iter.buf[i] 96 | switch c { 97 | case ' ', '\n', '\t', 'r': 98 | continue 99 | } 100 | iter.head = i+1 101 | return c 102 | } 103 | if !iter.loadMore() { 104 | return 0 105 | } 106 | } 107 | } 108 | 109 | func (iter *Iterator) ReportError(operation string, msg string) { 110 | if iter.Error != nil { 111 | return 112 | } 113 | peekStart := iter.head - 10 114 | if peekStart < 0 { 115 | peekStart = 0 116 | } 117 | iter.Error = fmt.Errorf("%s: %s, parsing %v ...%s... at %s", operation, msg, iter.head, 118 | string(iter.buf[peekStart: iter.head]), string(iter.buf[0:iter.tail])) 119 | } 120 | 121 | func (iter *Iterator) CurrentBuffer() string { 122 | peekStart := iter.head - 10 123 | if peekStart < 0 { 124 | peekStart = 0 125 | } 126 | return fmt.Sprintf("parsing %v ...%s... at %s", iter.head, 127 | string(iter.buf[peekStart: iter.head]), string(iter.buf[0:iter.tail])) 128 | } 129 | 130 | func (iter *Iterator) readByte() (ret byte) { 131 | if iter.head == iter.tail { 132 | if iter.loadMore() { 133 | ret = iter.buf[iter.head] 134 | iter.head++ 135 | return ret 136 | } 137 | } 138 | ret = iter.buf[iter.head] 139 | iter.head++ 140 | return ret 141 | } 142 | 143 | func (iter *Iterator) loadMore() bool { 144 | if iter.reader == nil { 145 | iter.Error = io.EOF 146 | return false 147 | } 148 | for { 149 | n, err := iter.reader.Read(iter.buf) 150 | if n == 0 { 151 | if err != nil { 152 | iter.Error = err 153 | return false 154 | } else { 155 | // n == 0, err == nil is not EOF 156 | continue 157 | } 158 | } else { 159 | iter.head = 0 160 | iter.tail = n 161 | return true 162 | } 163 | } 164 | } 165 | 166 | func (iter *Iterator) unreadByte() { 167 | if iter.head == 0 { 168 | iter.ReportError("unreadByte", "unread too many bytes") 169 | return 170 | } 171 | iter.head -= 1 172 | return 173 | } 174 | 175 | const maxUint64 = (1 << 64 - 1) 176 | const cutoffUint64 = maxUint64 / 10 + 1 177 | const maxUint32 = (1 << 32 - 1) 178 | const cutoffUint32 = maxUint32 / 10 + 1 179 | 180 | func (iter *Iterator) ReadUint() (ret uint) { 181 | val := iter.ReadUint64() 182 | converted := uint(val) 183 | if uint64(converted) != val { 184 | iter.ReportError("ReadUint", "int overflow") 185 | return 186 | } 187 | return converted 188 | } 189 | 190 | func (iter *Iterator) ReadUint8() (ret uint8) { 191 | val := iter.ReadUint64() 192 | converted := uint8(val) 193 | if uint64(converted) != val { 194 | iter.ReportError("ReadUint8", "int overflow") 195 | return 196 | } 197 | return converted 198 | } 199 | 200 | func (iter *Iterator) ReadUint16() (ret uint16) { 201 | val := iter.ReadUint64() 202 | converted := uint16(val) 203 | if uint64(converted) != val { 204 | iter.ReportError("ReadUint16", "int overflow") 205 | return 206 | } 207 | return converted 208 | } 209 | 210 | func (iter *Iterator) ReadUint32() (ret uint32) { 211 | val := iter.ReadUint64() 212 | converted := uint32(val) 213 | if uint64(converted) != val { 214 | iter.ReportError("ReadUint32", "int overflow") 215 | return 216 | } 217 | return converted 218 | } 219 | 220 | func (iter *Iterator) ReadUint64() (ret uint64) { 221 | c := iter.readByte() 222 | v := digits[c] 223 | if v == 0 { 224 | return 0 // single zero 225 | } 226 | if v == 255 { 227 | iter.ReportError("ReadUint64", "unexpected character") 228 | return 229 | } 230 | for { 231 | if ret >= cutoffUint64 { 232 | iter.ReportError("ReadUint64", "overflow") 233 | return 234 | } 235 | ret = ret * 10 + uint64(v) 236 | c = iter.readByte() 237 | v = digits[c] 238 | if v == 255 { 239 | iter.unreadByte() 240 | break 241 | } 242 | } 243 | return ret 244 | } 245 | 246 | func (iter *Iterator) ReadInt() (ret int) { 247 | val := iter.ReadInt64() 248 | converted := int(val) 249 | if int64(converted) != val { 250 | iter.ReportError("ReadInt", "int overflow") 251 | return 252 | } 253 | return converted 254 | } 255 | 256 | func (iter *Iterator) ReadInt8() (ret int8) { 257 | val := iter.ReadInt64() 258 | converted := int8(val) 259 | if int64(converted) != val { 260 | iter.ReportError("ReadInt8", "int overflow") 261 | return 262 | } 263 | return converted 264 | } 265 | 266 | func (iter *Iterator) ReadInt16() (ret int16) { 267 | val := iter.ReadInt64() 268 | converted := int16(val) 269 | if int64(converted) != val { 270 | iter.ReportError("ReadInt16", "int overflow") 271 | return 272 | } 273 | return converted 274 | } 275 | 276 | func (iter *Iterator) ReadInt32() (ret int32) { 277 | val := iter.ReadInt64() 278 | converted := int32(val) 279 | if int64(converted) != val { 280 | iter.ReportError("ReadInt32", "int overflow") 281 | return 282 | } 283 | return converted 284 | } 285 | 286 | func (iter *Iterator) ReadInt64() (ret int64) { 287 | c := iter.readByte() 288 | if iter.Error != nil { 289 | return 290 | } 291 | 292 | /* optional leading minus */ 293 | if c == '-' { 294 | n := iter.ReadUint64() 295 | return -int64(n) 296 | } else { 297 | iter.unreadByte() 298 | n := iter.ReadUint64() 299 | return int64(n) 300 | } 301 | } 302 | 303 | func (iter *Iterator) ReadString() (ret string) { 304 | return string(iter.ReadStringAsBytes()) 305 | } 306 | 307 | // adapted from: https://github.com/buger/jsonparser/blob/master/parser.go 308 | // Tries to find the end of string 309 | // Support if string contains escaped quote symbols. 310 | func (iter *Iterator) findStringEnd() (int, bool) { 311 | escaped := false 312 | for i := iter.head; i < iter.tail; i++ { 313 | c := iter.buf[i] 314 | if c == '"' { 315 | if !escaped { 316 | return i + 1, false 317 | } else { 318 | j := i - 1 319 | for { 320 | if j < iter.head || iter.buf[j] != '\\' { 321 | // even number of backslashes 322 | // either end of buffer, or " found 323 | return i + 1, true 324 | } 325 | j-- 326 | if j < iter.head || iter.buf[j] != '\\' { 327 | // odd number of backslashes 328 | // it is \" or \\\" 329 | break 330 | } 331 | j-- 332 | } 333 | } 334 | } else if c == '\\' { 335 | escaped = true 336 | } 337 | } 338 | j := iter.tail - 1 339 | for { 340 | if j < iter.head || iter.buf[j] != '\\' { 341 | // even number of backslashes 342 | // either end of buffer, or " found 343 | return -1, false // do not end with \ 344 | } 345 | j-- 346 | if j < iter.head || iter.buf[j] != '\\' { 347 | // odd number of backslashes 348 | // it is \" or \\\" 349 | break 350 | } 351 | j-- 352 | 353 | } 354 | return -1, true // end with \ 355 | } 356 | 357 | func (iter *Iterator) ReadStringAsBytes() (ret []byte) { 358 | c := iter.readByte() 359 | if c == 'n' { 360 | iter.skipUntilBreak() 361 | return 362 | } 363 | if c != '"' { 364 | iter.ReportError("ReadString", `expects " or n`) 365 | return 366 | } 367 | end, escaped := iter.findStringEnd() 368 | if end != -1 && !escaped { 369 | // fast path: reuse the underlying buffer 370 | ret = iter.buf[iter.head:end-1] 371 | iter.head = end 372 | return ret 373 | } 374 | str := make([]byte, 0, 8) 375 | for iter.Error == nil { 376 | c = iter.readByte() 377 | if c == '"' { 378 | return str 379 | } 380 | if c == '\\' { 381 | c = iter.readByte() 382 | if iter.Error != nil { 383 | return 384 | } 385 | switch c { 386 | case 'u': 387 | r := iter.readU4() 388 | if iter.Error != nil { 389 | return 390 | } 391 | if utf16.IsSurrogate(r) { 392 | c = iter.readByte() 393 | if iter.Error != nil { 394 | return 395 | } 396 | if c != '\\' { 397 | iter.ReportError("ReadString", 398 | `expects \u after utf16 surrogate, but \ not found`) 399 | return 400 | } 401 | c = iter.readByte() 402 | if iter.Error != nil { 403 | return 404 | } 405 | if c != 'u' { 406 | iter.ReportError("ReadString", 407 | `expects \u after utf16 surrogate, but \u not found`) 408 | return 409 | } 410 | r2 := iter.readU4() 411 | if iter.Error != nil { 412 | return 413 | } 414 | combined := utf16.DecodeRune(r, r2) 415 | str = appendRune(str, combined) 416 | } else { 417 | str = appendRune(str, r) 418 | } 419 | case '"': 420 | str = append(str, '"') 421 | case '\\': 422 | str = append(str, '\\') 423 | case '/': 424 | str = append(str, '/') 425 | case 'b': 426 | str = append(str, '\b') 427 | case 'f': 428 | str = append(str, '\f') 429 | case 'n': 430 | str = append(str, '\n') 431 | case 'r': 432 | str = append(str, '\r') 433 | case 't': 434 | str = append(str, '\t') 435 | default: 436 | iter.ReportError("ReadString", 437 | `invalid escape char after \`) 438 | return 439 | } 440 | } else { 441 | str = append(str, c) 442 | } 443 | } 444 | return 445 | } 446 | 447 | func (iter *Iterator) readU4() (ret rune) { 448 | for i := 0; i < 4; i++ { 449 | c := iter.readByte() 450 | if iter.Error != nil { 451 | return 452 | } 453 | if (c >= '0' && c <= '9') { 454 | if ret >= cutoffUint32 { 455 | iter.ReportError("readU4", "overflow") 456 | return 457 | } 458 | ret = ret * 16 + rune(c - '0') 459 | } else if ((c >= 'a' && c <= 'f') ) { 460 | if ret >= cutoffUint32 { 461 | iter.ReportError("readU4", "overflow") 462 | return 463 | } 464 | ret = ret * 16 + rune(c - 'a' + 10) 465 | } else { 466 | iter.ReportError("readU4", "expects 0~9 or a~f") 467 | return 468 | } 469 | } 470 | return ret 471 | } 472 | 473 | const ( 474 | t1 = 0x00 // 0000 0000 475 | tx = 0x80 // 1000 0000 476 | t2 = 0xC0 // 1100 0000 477 | t3 = 0xE0 // 1110 0000 478 | t4 = 0xF0 // 1111 0000 479 | t5 = 0xF8 // 1111 1000 480 | 481 | maskx = 0x3F // 0011 1111 482 | mask2 = 0x1F // 0001 1111 483 | mask3 = 0x0F // 0000 1111 484 | mask4 = 0x07 // 0000 0111 485 | 486 | rune1Max = 1 << 7 - 1 487 | rune2Max = 1 << 11 - 1 488 | rune3Max = 1 << 16 - 1 489 | 490 | surrogateMin = 0xD800 491 | surrogateMax = 0xDFFF 492 | 493 | MaxRune = '\U0010FFFF' // Maximum valid Unicode code point. 494 | RuneError = '\uFFFD' // the "error" Rune or "Unicode replacement character" 495 | ) 496 | 497 | func appendRune(p []byte, r rune) []byte { 498 | // Negative values are erroneous. Making it unsigned addresses the problem. 499 | switch i := uint32(r); { 500 | case i <= rune1Max: 501 | p = append(p, byte(r)) 502 | return p 503 | case i <= rune2Max: 504 | p = append(p, t2 | byte(r >> 6)) 505 | p = append(p, tx | byte(r) & maskx) 506 | return p 507 | case i > MaxRune, surrogateMin <= i && i <= surrogateMax: 508 | r = RuneError 509 | fallthrough 510 | case i <= rune3Max: 511 | p = append(p, t3 | byte(r >> 12)) 512 | p = append(p, tx | byte(r >> 6) & maskx) 513 | p = append(p, tx | byte(r) & maskx) 514 | return p 515 | default: 516 | p = append(p, t4 | byte(r >> 18)) 517 | p = append(p, tx | byte(r >> 12) & maskx) 518 | p = append(p, tx | byte(r >> 6) & maskx) 519 | p = append(p, tx | byte(r) & maskx) 520 | return p 521 | } 522 | } 523 | 524 | func (iter *Iterator) ReadArray() (ret bool) { 525 | c := iter.nextToken() 526 | if iter.Error != nil { 527 | return 528 | } 529 | switch c { 530 | case 'n': { 531 | iter.skipUntilBreak() 532 | return false // null 533 | } 534 | case '[': { 535 | c = iter.nextToken() 536 | if iter.Error != nil { 537 | return 538 | } 539 | if c == ']' { 540 | return false 541 | } else { 542 | iter.unreadByte() 543 | return true 544 | } 545 | } 546 | case ']': return false 547 | case ',': 548 | iter.skipWhitespaces() 549 | return true 550 | default: 551 | iter.ReportError("ReadArray", "expect [ or , or ] or n") 552 | return 553 | } 554 | } 555 | 556 | func (iter *Iterator) ReadArrayCB(cb func()) { 557 | c := iter.nextToken() 558 | if c == 'n' { 559 | iter.skipUntilBreak() 560 | return // null 561 | } 562 | if c != '[' { 563 | iter.ReportError("ReadArrayCB", "expect [ or n") 564 | return 565 | } 566 | c = iter.nextToken() 567 | if c == ']' { 568 | return // [] 569 | } else { 570 | iter.unreadByte() 571 | } 572 | for { 573 | if iter.Error != nil { 574 | return 575 | } 576 | cb() 577 | c = iter.nextToken() 578 | if c == ']' { 579 | return 580 | } 581 | if c != ',' { 582 | iter.ReportError("ReadArrayCB", "expect , or ]") 583 | return 584 | } 585 | iter.skipWhitespaces() 586 | } 587 | } 588 | 589 | func (iter *Iterator) ReadObjectCB(cb func(string)) { 590 | c := iter.nextToken() 591 | if c == 'n' { 592 | iter.skipUntilBreak() 593 | return // null 594 | } 595 | if c != '{' { 596 | iter.ReportError("ReadObjectCB", "expect { or n") 597 | return 598 | } 599 | c = iter.nextToken() 600 | if c == '}' { 601 | return // [] 602 | } else { 603 | iter.unreadByte() 604 | } 605 | for { 606 | iter.skipWhitespaces() 607 | field := iter.readObjectField() 608 | if iter.Error != nil { 609 | return 610 | } 611 | cb(field) 612 | c = iter.nextToken() 613 | if c == '}' { 614 | return // end of object 615 | } 616 | if c != ',' { 617 | iter.ReportError("ReadObjectCB", `expect ,`) 618 | return 619 | } 620 | } 621 | } 622 | 623 | func (iter *Iterator) ReadObject() (ret string) { 624 | c := iter.nextToken() 625 | if iter.Error != nil { 626 | return 627 | } 628 | switch c { 629 | case 'n': { 630 | iter.skipUntilBreak() 631 | if iter.Error != nil { 632 | return 633 | } 634 | return "" // null 635 | } 636 | case '{': { 637 | c = iter.nextToken() 638 | if iter.Error != nil { 639 | return 640 | } 641 | switch c { 642 | case '}': 643 | return "" // end of object 644 | case '"': 645 | iter.unreadByte() 646 | return iter.readObjectField() 647 | default: 648 | iter.ReportError("ReadObject", `expect " after {`) 649 | return 650 | } 651 | } 652 | case ',': 653 | iter.skipWhitespaces() 654 | return iter.readObjectField() 655 | case '}': 656 | return "" // end of object 657 | default: 658 | iter.ReportError("ReadObject", `expect { or , or } or n`) 659 | return 660 | } 661 | } 662 | 663 | func (iter *Iterator) readObjectField() (ret string) { 664 | str := iter.ReadStringAsBytes() 665 | field := *(*string)(unsafe.Pointer(&str)) 666 | c := iter.nextToken() 667 | if c != ':' { 668 | iter.ReportError("ReadObject", "expect : after object field") 669 | return 670 | } 671 | iter.skipWhitespaces() 672 | return field 673 | } 674 | 675 | func (iter *Iterator) ReadFloat32() (ret float32) { 676 | str := make([]byte, 0, 4) 677 | for c := iter.readByte(); iter.Error == nil; c = iter.readByte() { 678 | switch c { 679 | case '-', '+', '.', 'e', 'E', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': 680 | str = append(str, c) 681 | continue 682 | default: 683 | iter.unreadByte() 684 | } 685 | break 686 | } 687 | if iter.Error != nil && iter.Error != io.EOF { 688 | return 689 | } 690 | val, err := strconv.ParseFloat(*(*string)(unsafe.Pointer(&str)), 32) 691 | if err != nil { 692 | iter.Error = err 693 | return 694 | } 695 | return float32(val) 696 | } 697 | 698 | func (iter *Iterator) ReadFloat64() (ret float64) { 699 | str := make([]byte, 0, 4) 700 | for c := iter.readByte(); iter.Error == nil; c = iter.readByte() { 701 | switch c { 702 | case '-', '+', '.', 'e', 'E', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': 703 | str = append(str, c) 704 | continue 705 | default: 706 | iter.unreadByte() 707 | } 708 | break 709 | } 710 | if iter.Error != nil && iter.Error != io.EOF { 711 | return 712 | } 713 | val, err := strconv.ParseFloat(*(*string)(unsafe.Pointer(&str)), 64) 714 | if err != nil { 715 | iter.Error = err 716 | return 717 | } 718 | return val 719 | } 720 | 721 | func (iter *Iterator) ReadBool() (ret bool) { 722 | c := iter.readByte() 723 | if iter.Error != nil { 724 | return 725 | } 726 | switch c { 727 | case 't': 728 | iter.skipUntilBreak() 729 | if iter.Error != nil { 730 | return 731 | } 732 | return true 733 | case 'f': 734 | iter.skipUntilBreak() 735 | if iter.Error != nil { 736 | return 737 | } 738 | return false 739 | default: 740 | iter.ReportError("ReadBool", "expect t or f") 741 | return 742 | } 743 | } 744 | 745 | func (iter *Iterator) ReadNull() (ret bool) { 746 | c := iter.readByte() 747 | if c == 'n' { 748 | iter.skipUntilBreak() 749 | return true 750 | } 751 | iter.unreadByte() 752 | return false 753 | } 754 | 755 | func (iter *Iterator) Skip() { 756 | c := iter.readByte() 757 | switch c { 758 | case '"': 759 | iter.skipString() 760 | case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 't', 'f', 'n': 761 | iter.skipUntilBreak() 762 | case '[': 763 | iter.skipArray() 764 | case '{': 765 | iter.skipObject() 766 | default: 767 | iter.ReportError("Skip", fmt.Sprintf("do not know how to skip: %v", c)) 768 | return 769 | } 770 | } 771 | 772 | func (iter *Iterator) skipString() { 773 | for { 774 | end, escaped := iter.findStringEnd() 775 | if end == -1 { 776 | if !iter.loadMore() { 777 | return 778 | } 779 | if escaped { 780 | iter.head = 1 // skip the first char as last char read is \ 781 | } 782 | } else { 783 | iter.head = end 784 | return 785 | } 786 | } 787 | } 788 | 789 | func (iter *Iterator) skipArray() { 790 | level := 1 791 | for { 792 | for i := iter.head; i < iter.tail; i++ { 793 | switch iter.buf[i] { 794 | case '"': // If inside string, skip it 795 | iter.head = i + 1 796 | iter.skipString() 797 | i = iter.head - 1 // it will be i++ soon 798 | case '[': // If open symbol, increase level 799 | level++ 800 | case ']': // If close symbol, increase level 801 | level-- 802 | 803 | // If we have returned to the original level, we're done 804 | if level == 0 { 805 | iter.head = i + 1 806 | return 807 | } 808 | } 809 | } 810 | if (!iter.loadMore()) { 811 | return 812 | } 813 | } 814 | } 815 | 816 | func (iter *Iterator) skipObject() { 817 | level := 1 818 | for { 819 | for i := iter.head; i < iter.tail; i++ { 820 | switch iter.buf[i] { 821 | case '"': // If inside string, skip it 822 | iter.head = i + 1 823 | iter.skipString() 824 | i = iter.head - 1 // it will be i++ soon 825 | case '{': // If open symbol, increase level 826 | level++ 827 | case '}': // If close symbol, increase level 828 | level-- 829 | 830 | // If we have returned to the original level, we're done 831 | if level == 0 { 832 | iter.head = i + 1 833 | return 834 | } 835 | } 836 | } 837 | if (!iter.loadMore()) { 838 | return 839 | } 840 | } 841 | } 842 | 843 | func (iter *Iterator) skipUntilBreak() { 844 | // true, false, null, number 845 | for { 846 | for i := iter.head; i < iter.tail; i++ { 847 | c := iter.buf[i] 848 | switch c { 849 | case ' ', '\n', '\r', '\t', ',', '}', ']': 850 | iter.head = i 851 | return 852 | } 853 | } 854 | if (!iter.loadMore()) { 855 | return 856 | } 857 | } 858 | } --------------------------------------------------------------------------------