├── .gitignore ├── License.md ├── Readme.md ├── go.mod ├── go.sum ├── internal ├── finder │ └── finder.go └── imports │ └── imports.go └── json ├── scanner ├── scanner.go ├── scanner_test.go └── token.go ├── unmarshaler.go ├── unmarshaler.gotext ├── unmarshaler_test.go └── writer ├── writer.go └── writer_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | _tmp -------------------------------------------------------------------------------- /License.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Matt Mueller 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Marshaler (WIP) 2 | 3 | `Unmarshal` and `Marshal` generators for Go. For use in [Bud](https://github.com/livebud/bud). 4 | 5 | ## Example 6 | 7 | Given the following struct: 8 | 9 | ```go 10 | type Input struct { 11 | B string 12 | C int 13 | D float64 14 | E bool 15 | F map[string]string 16 | G []int 17 | H *string 18 | } 19 | ``` 20 | 21 | This library will generate an `UnmarshalJSON` function that can take arbitrary JSON and produce `Input`: 22 | 23 | ```go 24 | func UnmarshalJSON(json []byte, in *Input) error { 25 | // Generated code 26 | } 27 | ``` 28 | 29 | Then usage would look something like this: 30 | 31 | ```go 32 | func ReadBody(r io.ReadCloser) (*Input, error) { 33 | json, err := io.ReadAll(r) 34 | if err != nil { 35 | return nil, err 36 | } 37 | in := new(Input) 38 | if err := UnmarshalJSON(json, in); err != nil { 39 | return nil, err 40 | } 41 | return in, nil 42 | } 43 | ``` 44 | 45 | ## TODO 46 | 47 | This package is still very much WIP. There's a lot more work to do: 48 | 49 | - [x] Unmarshal nested structs 50 | - [x] Get tests running from a temporary directory 51 | - [ ] Unmarshal referenced types 52 | - [ ] Support the json tag 53 | - [ ] Support `Valid() error` that gets called while Unmarshaling 54 | - [ ] Pull in tests from other libraries 55 | - [ ] Encode nil maps and nil structs as empty objects 56 | - [ ] Fallback to `json.{Decode,Encode}` (?) 57 | - [ ] Bundle into Bud 58 | - [ ] Add MarshalJSON support using the writer 59 | - [x] Re-organize the package structure to allow more marshalers (e.g. form) 60 | 61 | If you have the itch, I'd very much appreciate your help! I plan to work on this here and there over the next couple months. Your PRs would speed up this timeline significantly. 62 | 63 | ## Why? 64 | 65 | - Nice speed improvement over the reflection-based alternatives. 66 | - Validate while you're unmarshaling. 67 | - Nil maps and structs can be changed to empty objects. 68 | - Code can be re-used for other formats like URL-encoded form data or protobufs. 69 | 70 | ## Development 71 | 72 | ``` 73 | git clone https://github.com/livebud/marshaler 74 | cd marshaler 75 | go mod tidy 76 | go test ./... 77 | ``` 78 | 79 | ## Prior Art 80 | 81 | - [megajson](https://github.com/benbjohnson/megajson): Simple, easy-to-understand code. Uses static analysis instead of build-time reflection. No longer in development. No built-in validation. No nested structure support. 82 | - [ffjson](https://github.com/pquerna/ffjson): More features. No updates since 2019. Recent issues unanswered. No built-in validation. Build-time reflection doesn't jibe well with Bud. 83 | - [go-codec](https://github.com/ugorji/go): Actively maintained. More features. Probably a better choice for the time being. No built-in validation. Build-time reflection doesn't jibe well with Bud. 84 | 85 | ## Thanks 86 | 87 | The scanner and writer were originally written by [Ben Johnson](https://twitter.com/benbjohnson) in [megajson](https://github.com/benbjohnson/megajson). 88 | 89 | ## License 90 | 91 | MIT 92 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/livebud/marshaler 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/hexops/valast v1.4.3 7 | github.com/lithammer/dedent v1.1.0 8 | github.com/matryer/is v1.4.0 9 | golang.org/x/mod v0.8.0 10 | ) 11 | 12 | require ( 13 | github.com/google/go-cmp v0.5.9 // indirect 14 | golang.org/x/sys v0.3.0 // indirect 15 | golang.org/x/tools v0.4.0 // indirect 16 | mvdan.cc/gofumpt v0.4.0 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= 3 | github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= 4 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 5 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 6 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 7 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 8 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 9 | github.com/hexops/autogold v0.8.1 h1:wvyd/bAJ+Dy+DcE09BoLk6r4Fa5R5W+O+GUzmR985WM= 10 | github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY= 11 | github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= 12 | github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= 13 | github.com/hexops/valast v1.4.3 h1:oBoGERMJh6UZdRc6cduE1CTPK+VAdXA59Y1HFgu3sm0= 14 | github.com/hexops/valast v1.4.3/go.mod h1:Iqx2kLj3Jn47wuXpj3wX40xn6F93QNFBHuiKBerkTGA= 15 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 16 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 17 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 18 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 19 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 20 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 21 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 22 | github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= 23 | github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= 24 | github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= 25 | github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= 26 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 27 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 28 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 29 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 30 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 31 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 32 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 33 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 34 | golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 35 | golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= 36 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 37 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 38 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 39 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 40 | golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= 41 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 43 | golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 44 | golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= 45 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 46 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 47 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 48 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 51 | golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 52 | golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= 53 | golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 54 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 55 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 56 | golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= 57 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 58 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 59 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 60 | golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 61 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 62 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 63 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 64 | golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= 65 | golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= 66 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 67 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 68 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 69 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 70 | mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= 71 | mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= 72 | -------------------------------------------------------------------------------- /internal/finder/finder.go: -------------------------------------------------------------------------------- 1 | package finder 2 | 3 | import ( 4 | "fmt" 5 | "go/ast" 6 | "go/build" 7 | "go/parser" 8 | "go/token" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "golang.org/x/mod/modfile" 14 | ) 15 | 16 | func New(dir string) *Finder { 17 | return &Finder{ 18 | dir: dir, 19 | } 20 | } 21 | 22 | type Finder struct { 23 | dir string 24 | } 25 | 26 | func (f *Finder) Find(importPath string, name string) (ast.Expr, error) { 27 | gomod, err := os.ReadFile(filepath.Join(f.dir, "go.mod")) 28 | if err != nil { 29 | return nil, err 30 | } 31 | modFile, err := modfile.Parse("go.mod", gomod, nil) 32 | if err != nil { 33 | return nil, err 34 | } 35 | modPath := modFile.Module.Mod.Path 36 | // If the import path has the modPath prefix, then it's a local import 37 | importPackage := f.importLocal 38 | if !strings.HasPrefix(importPath, modPath) { 39 | importPackage = f.importRemote 40 | } 41 | // Import the package 42 | pkg, err := importPackage(modFile, importPath, name) 43 | if err != nil { 44 | return nil, err 45 | } 46 | // Parse each valid Go file 47 | fset := token.NewFileSet() 48 | for _, filename := range pkg.GoFiles { 49 | filename = filepath.Join(f.dir, filename) 50 | code, err := os.ReadFile(filename) 51 | if err != nil { 52 | return nil, err 53 | } 54 | file, err := parser.ParseFile(fset, filename, code, parser.DeclarationErrors) 55 | if err != nil { 56 | return nil, err 57 | } 58 | // Look for the type spec 59 | for _, decl := range file.Decls { 60 | if gen, ok := decl.(*ast.GenDecl); ok { 61 | for _, spec := range gen.Specs { 62 | if ts, ok := spec.(*ast.TypeSpec); ok { 63 | if ts.Name.Name == name { 64 | return ts.Type, nil 65 | } 66 | } 67 | } 68 | } 69 | } 70 | } 71 | return nil, fmt.Errorf("finder:could not find type definition for %q.%s", importPath, name) 72 | } 73 | 74 | func (f *Finder) importLocal(modFile *modfile.File, importPath string, name string) (*build.Package, error) { 75 | dir := filepath.Join(f.dir, trimModulePath(modFile.Module.Mod.Path, importPath)) 76 | return build.Import(".", dir, build.ImportMode(0)) 77 | } 78 | 79 | func (f *Finder) importRemote(modFile *modfile.File, importPath string, name string) (*build.Package, error) { 80 | return nil, fmt.Errorf("find remote for %q.%s not implemneted yet", importPath, name) 81 | } 82 | 83 | func trimModulePath(modulePath string, importPath string) string { 84 | if modulePath == importPath { 85 | return "" 86 | } 87 | return strings.TrimPrefix(importPath, modulePath+"/") 88 | } 89 | -------------------------------------------------------------------------------- /internal/imports/imports.go: -------------------------------------------------------------------------------- 1 | package imports 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strconv" 7 | "strings" 8 | "unicode" 9 | "unicode/utf8" 10 | ) 11 | 12 | type Import struct { 13 | Path string 14 | Name string 15 | } 16 | 17 | type Imports []*Import 18 | 19 | func (i *Imports) Import(path string) (name string, err error) { 20 | name = assumedName(path) 21 | for _, imp := range *i { 22 | if imp.Name == name { 23 | if imp.Path != path { 24 | return "", fmt.Errorf("imports: name %s already used for %q", name, imp.Path) 25 | } 26 | return name, nil 27 | } 28 | } 29 | // Add the name 30 | *i = append(*i, &Import{ 31 | Path: path, 32 | Name: name, 33 | }) 34 | return name, nil 35 | } 36 | 37 | // assumedName returns the assumed name for the import path. It's pulled from: 38 | // https://cs.opensource.google/go/x/tools/+/refs/tags/v0.6.0:internal/imports/fix.go;l=1144 39 | func assumedName(importPath string) string { 40 | base := path.Base(importPath) 41 | if strings.HasPrefix(base, "v") { 42 | if _, err := strconv.Atoi(base[1:]); err == nil { 43 | dir := path.Dir(importPath) 44 | if dir != "." { 45 | base = path.Base(dir) 46 | } 47 | } 48 | } 49 | base = strings.TrimPrefix(base, "go-") 50 | if i := strings.IndexFunc(base, notIdentifier); i >= 0 { 51 | base = base[:i] 52 | } 53 | return base 54 | } 55 | 56 | // notIdentifier reports whether ch is an invalid identifier character. 57 | func notIdentifier(ch rune) bool { 58 | return !('a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || 59 | '0' <= ch && ch <= '9' || 60 | ch == '_' || 61 | ch >= utf8.RuneSelf && (unicode.IsLetter(ch) || unicode.IsDigit(ch))) 62 | } 63 | -------------------------------------------------------------------------------- /json/scanner/scanner.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strconv" 7 | "unicode/utf8" 8 | ) 9 | 10 | const ( 11 | // The size, in bytes, that is read from the reader at a time. 12 | bufSize = 4096 13 | ) 14 | 15 | // Scanner is a tokenizer for JSON input from an io.Reader. 16 | type Scanner interface { 17 | Pos() int 18 | Scan() (int, []byte, error) 19 | Expect(token int) ([]byte, error) 20 | Unscan(tok int, b []byte) 21 | ReadString(target *string) error 22 | ReadInt(target *int) error 23 | ReadInt64(target *int64) error 24 | ReadUint(target *uint) error 25 | ReadUint64(target *uint64) error 26 | ReadFloat32(target *float32) error 27 | ReadFloat64(target *float64) error 28 | ReadBool(target *bool) error 29 | ReadMap(target *map[string]interface{}) error 30 | ReadArray(target *[]interface{}) error 31 | } 32 | 33 | type scanner struct { 34 | r io.Reader 35 | c rune 36 | scratch [bufSize]byte 37 | buf [bufSize]byte 38 | buflen int 39 | idx int 40 | pos int 41 | tmpc rune 42 | tmp struct { 43 | tok int 44 | b []byte 45 | err error 46 | } 47 | } 48 | 49 | // NewScanner initializes a new scanner with a given reader. 50 | func NewScanner(r io.Reader) Scanner { 51 | s := &scanner{r: r, buflen: -1} 52 | return s 53 | } 54 | 55 | // Pos returns the current rune position of the scanner. 56 | func (s *scanner) Pos() int { 57 | return s.pos 58 | } 59 | 60 | // read retrieves the next rune from the reader. 61 | func (s *scanner) read() error { 62 | if s.tmpc > 0 { 63 | s.c = s.tmpc 64 | s.tmpc = 0 65 | return nil 66 | } 67 | 68 | // Read from the reader if the buffer is empty. 69 | if s.idx >= s.buflen { 70 | var err error 71 | if s.buflen, err = s.r.Read(s.buf[0:]); err != nil { 72 | return err 73 | } 74 | s.idx = 0 75 | } 76 | 77 | // Read a single byte and then determine if utf8 decoding is needed. 78 | b := s.buf[s.idx] 79 | if b < utf8.RuneSelf { 80 | s.c = rune(b) 81 | s.idx++ 82 | } else { 83 | // Read a new buffer if we don't have at least the max size of a UTF8 character. 84 | if s.idx+utf8.UTFMax >= s.buflen { 85 | s.buf[0] = b 86 | var err error 87 | if s.buflen, err = s.r.Read(s.buf[1:]); err != nil { 88 | return err 89 | } 90 | s.buflen += 1 91 | } 92 | 93 | var size int 94 | s.c, size = utf8.DecodeRune(s.buf[s.idx:]) 95 | s.idx += size 96 | } 97 | 98 | s.pos++ 99 | return nil 100 | } 101 | 102 | // unread places the current rune back on the reader. 103 | func (s *scanner) unread() { 104 | s.tmpc = s.c 105 | } 106 | 107 | // expect reads the next rune and checks that it matches. 108 | func (s *scanner) expect(c rune) error { 109 | if err := s.read(); err != nil { 110 | return err 111 | } else if s.c != c { 112 | return fmt.Errorf("unexpected char: %q", s.c) 113 | } 114 | return nil 115 | } 116 | 117 | // Scan returns the next JSON token from the reader. 118 | func (s *scanner) Scan() (int, []byte, error) { 119 | if s.tmp.tok != 0 { 120 | tok, b := s.tmp.tok, s.tmp.b 121 | s.tmp.tok, s.tmp.b = 0, nil 122 | return tok, b, nil 123 | } 124 | 125 | for { 126 | if err := s.read(); err != nil { 127 | return 0, nil, err 128 | } 129 | 130 | switch s.c { 131 | case '{': 132 | return TLBRACE, []byte{'{'}, nil 133 | case '}': 134 | return TRBRACE, []byte{'}'}, nil 135 | case '[': 136 | return TLBRACKET, []byte{'['}, nil 137 | case ']': 138 | return TRBRACKET, []byte{']'}, nil 139 | case ':': 140 | return TCOLON, []byte{':'}, nil 141 | case ',': 142 | return TCOMMA, []byte{','}, nil 143 | case '"': 144 | return s.scanString() 145 | case 't': 146 | return s.scanTrue() 147 | case 'f': 148 | return s.scanFalse() 149 | case 'n': 150 | return s.scanNull() 151 | } 152 | 153 | if (s.c >= '0' && s.c <= '9') || s.c == '-' { 154 | return s.scanNumber() 155 | } 156 | } 157 | } 158 | 159 | // Expect a specific token and return the byte array. 160 | func (s *scanner) Expect(token int) ([]byte, error) { 161 | tok, buf, err := s.Scan() 162 | if err != nil { 163 | return nil, err 164 | } 165 | if tok != token { 166 | return nil, fmt.Errorf("expected token %d, got %d", token, tok) 167 | } 168 | return buf, nil 169 | } 170 | 171 | // Unscan adds a token and byte array back onto the buffer to be read 172 | // on the next call to Scan(). 173 | func (s *scanner) Unscan(tok int, b []byte) { 174 | s.tmp.tok = tok 175 | s.tmp.b = b 176 | s.pos-- 177 | } 178 | 179 | // scanNumber reads a JSON number from the reader. 180 | func (s *scanner) scanNumber() (int, []byte, error) { 181 | var n int 182 | 183 | if s.c == '-' { 184 | s.scratch[n] = '-' 185 | n++ 186 | if err := s.read(); err != nil { 187 | return 0, nil, err 188 | } 189 | } 190 | 191 | // Read whole number. 192 | if err := s.scanDigits(&n); err == io.EOF { 193 | return TNUMBER, s.scratch[0:n], nil 194 | } else if err != nil { 195 | return 0, nil, err 196 | } 197 | n++ 198 | 199 | // Read period. 200 | if err := s.read(); err == io.EOF { 201 | return TNUMBER, s.scratch[0:n], nil 202 | } else if err != nil { 203 | return 0, nil, err 204 | } else if s.c != '.' { 205 | s.unread() 206 | // We can't just return. The number could be 1e-1 207 | return s.scanScientific(n) 208 | } 209 | s.scratch[n] = '.' 210 | n++ 211 | 212 | if err := s.read(); err != nil { 213 | return 0, nil, err 214 | } 215 | 216 | // Read fraction. 217 | if err := s.scanDigits(&n); err == io.EOF { 218 | return TNUMBER, s.scratch[0:n], nil 219 | } else if err != nil { 220 | return 0, nil, err 221 | } 222 | n++ 223 | 224 | return s.scanScientific(n) 225 | } 226 | 227 | func (s *scanner) scanScientific(n int) (int, []byte, error) { 228 | // Read scientific 229 | if err := s.read(); err == io.EOF { 230 | return TNUMBER, s.scratch[0:n], nil 231 | } else if err != nil { 232 | return 0, nil, err 233 | } else if s.c != 'e' && s.c != 'E' { 234 | s.unread() 235 | return TNUMBER, s.scratch[0:n], nil 236 | } 237 | s.scratch[n] = 'e' 238 | n++ 239 | 240 | // Read sign 241 | if err := s.read(); err == io.EOF { 242 | return TNUMBER, s.scratch[0:n], nil 243 | } else if err != nil { 244 | return 0, nil, err 245 | } else if s.c != '-' && s.c != '+' { 246 | s.unread() 247 | } else if s.c == '-' { // don't bother adding the + 248 | s.scratch[n] = '-' 249 | n++ 250 | } 251 | 252 | if err := s.read(); err != nil { 253 | return 0, nil, err 254 | } 255 | 256 | // Read whole number. 257 | if err := s.scanDigits(&n); err == io.EOF { 258 | return TNUMBER, s.scratch[0:n], nil 259 | } else if err != nil { 260 | fmt.Println("Bad?") 261 | return 0, nil, err 262 | } 263 | n++ 264 | 265 | return TNUMBER, s.scratch[0:n], nil 266 | } 267 | 268 | // scanDigits reads a series of digits from the reader. 269 | func (s *scanner) scanDigits(n *int) error { 270 | for { 271 | if s.c >= '0' && s.c <= '9' { 272 | s.scratch[*n] = byte(s.c) 273 | (*n)++ 274 | if err := s.read(); err != nil { 275 | return err 276 | } 277 | } else { 278 | s.unread() 279 | (*n)-- 280 | return nil 281 | } 282 | } 283 | } 284 | 285 | // scanString reads a quoted JSON string from the reader. 286 | func (s *scanner) scanString() (int, []byte, error) { 287 | var overflow []byte 288 | 289 | var n int 290 | for { 291 | if err := s.read(); err != nil { 292 | return 0, nil, err 293 | } 294 | switch s.c { 295 | case '\\': 296 | if err := s.read(); err != nil { 297 | return 0, nil, err 298 | } 299 | switch s.c { 300 | case '"': 301 | s.scratch[n] = '"' 302 | n++ 303 | case '\\': 304 | s.scratch[n] = '\\' 305 | n++ 306 | case '/': 307 | s.scratch[n] = '/' 308 | n++ 309 | case 'b': 310 | s.scratch[n] = '\b' 311 | n++ 312 | case 'f': 313 | s.scratch[n] = '\f' 314 | n++ 315 | case 'n': 316 | s.scratch[n] = '\n' 317 | n++ 318 | case 'r': 319 | s.scratch[n] = '\r' 320 | n++ 321 | case 't': 322 | s.scratch[n] = '\t' 323 | n++ 324 | case 'u': 325 | numeric := make([]byte, 4) 326 | numericCount := 0 327 | unicode_loop: 328 | for { 329 | if err := s.read(); err != nil { 330 | return 0, nil, err 331 | } 332 | switch { 333 | case s.c >= '0' && s.c <= '9' || s.c >= 'a' && s.c <= 'f' || s.c >= 'A' && s.c <= 'F': 334 | numeric[numericCount] = byte(s.c) 335 | numericCount++ 336 | if numericCount == 4 { 337 | var i int64 338 | var err error 339 | if i, err = strconv.ParseInt(string(numeric), 16, 32); err != nil { 340 | return 0, nil, err 341 | } 342 | if i < utf8.RuneSelf { 343 | s.scratch[n] = byte(i) 344 | n++ 345 | } else { 346 | encoded := utf8.EncodeRune(s.scratch[n:], rune(i)) 347 | n += encoded 348 | } 349 | break unicode_loop 350 | } 351 | default: 352 | s.unread() 353 | return 0, nil, fmt.Errorf("unexpected symbol in unicode escape: %c", s.c) 354 | } 355 | } 356 | default: 357 | return 0, nil, fmt.Errorf("invalid escape character: \\%c", s.c) 358 | } 359 | 360 | case '"': 361 | if len(overflow) == 0 { 362 | return TSTRING, s.scratch[0:n], nil 363 | } 364 | overflow = append(overflow, s.scratch[0:n]...) 365 | return TSTRING, overflow, nil 366 | 367 | default: 368 | if s.c < utf8.RuneSelf { 369 | if n == bufSize { 370 | overflow = append(overflow, s.scratch[0:n]...) 371 | n = 0 372 | } 373 | s.scratch[n] = byte(s.c) 374 | n++ 375 | } else { 376 | n += utf8.EncodeRune(s.scratch[n:], s.c) 377 | } 378 | } 379 | } 380 | } 381 | 382 | // scanTrue reads the "true" token. 383 | func (s *scanner) scanTrue() (int, []byte, error) { 384 | if err := s.expect('r'); err != nil { 385 | return 0, nil, err 386 | } 387 | if err := s.expect('u'); err != nil { 388 | return 0, nil, err 389 | } 390 | if err := s.expect('e'); err != nil { 391 | return 0, nil, err 392 | } 393 | return TTRUE, nil, nil 394 | } 395 | 396 | // scanFalse reads the "false" token. 397 | func (s *scanner) scanFalse() (int, []byte, error) { 398 | if err := s.expect('a'); err != nil { 399 | return 0, nil, err 400 | } 401 | if err := s.expect('l'); err != nil { 402 | return 0, nil, err 403 | } 404 | if err := s.expect('s'); err != nil { 405 | return 0, nil, err 406 | } 407 | if err := s.expect('e'); err != nil { 408 | return 0, nil, err 409 | } 410 | return TFALSE, nil, nil 411 | } 412 | 413 | // scanNull reads the "null" token. 414 | func (s *scanner) scanNull() (int, []byte, error) { 415 | if err := s.expect('u'); err != nil { 416 | return 0, nil, err 417 | } 418 | if err := s.expect('l'); err != nil { 419 | return 0, nil, err 420 | } 421 | if err := s.expect('l'); err != nil { 422 | return 0, nil, err 423 | } 424 | return TNULL, nil, nil 425 | } 426 | 427 | // ReadString reads a token into a string variable. 428 | func (s *scanner) ReadString(target *string) error { 429 | tok, b, err := s.Scan() 430 | if err != nil { 431 | return err 432 | } 433 | switch tok { 434 | case TSTRING: 435 | *target = string(b) 436 | case TNUMBER, TTRUE, TFALSE, TNULL: 437 | *target = "" 438 | default: 439 | return fmt.Errorf("unexpected %s at %d: %s; expected string", TokenName(tok), s.pos, string(b)) 440 | } 441 | return nil 442 | } 443 | 444 | // ReadInt reads a token into an int variable. 445 | func (s *scanner) ReadInt(target *int) error { 446 | tok, b, err := s.Scan() 447 | if err != nil { 448 | return err 449 | } 450 | switch tok { 451 | case TNUMBER: 452 | n, _ := strconv.ParseInt(string(b), 10, 64) 453 | *target = int(n) 454 | case TSTRING, TTRUE, TFALSE, TNULL: 455 | *target = 0 456 | default: 457 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 458 | } 459 | return nil 460 | } 461 | 462 | // ReadInt64 reads a token into an int64 variable. 463 | func (s *scanner) ReadInt64(target *int64) error { 464 | tok, b, err := s.Scan() 465 | if err != nil { 466 | return err 467 | } 468 | switch tok { 469 | case TNUMBER: 470 | n, _ := strconv.ParseInt(string(b), 10, 64) 471 | *target = n 472 | case TSTRING, TTRUE, TFALSE, TNULL: 473 | *target = 0 474 | default: 475 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 476 | } 477 | return nil 478 | } 479 | 480 | // ReadUint reads a token into an uint variable. 481 | func (s *scanner) ReadUint(target *uint) error { 482 | tok, b, err := s.Scan() 483 | if err != nil { 484 | return err 485 | } 486 | switch tok { 487 | case TNUMBER: 488 | n, _ := strconv.ParseUint(string(b), 10, 64) 489 | *target = uint(n) 490 | case TSTRING, TTRUE, TFALSE, TNULL: 491 | *target = 0 492 | default: 493 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 494 | } 495 | return nil 496 | } 497 | 498 | // ReadUint64 reads a token into an uint64 variable. 499 | func (s *scanner) ReadUint64(target *uint64) error { 500 | tok, b, err := s.Scan() 501 | if err != nil { 502 | return err 503 | } 504 | switch tok { 505 | case TNUMBER: 506 | n, _ := strconv.ParseUint(string(b), 10, 64) 507 | *target = n 508 | case TSTRING, TTRUE, TFALSE, TNULL: 509 | *target = 0 510 | default: 511 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 512 | } 513 | return nil 514 | } 515 | 516 | // ReadFloat32 reads a token into a float32 variable. 517 | func (s *scanner) ReadFloat32(target *float32) error { 518 | tok, b, err := s.Scan() 519 | if err != nil { 520 | return err 521 | } 522 | switch tok { 523 | case TNUMBER: 524 | n, _ := strconv.ParseFloat(string(b), 32) 525 | *target = float32(n) 526 | case TSTRING, TTRUE, TFALSE, TNULL: 527 | *target = 0 528 | default: 529 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 530 | } 531 | return nil 532 | } 533 | 534 | // ReadFloat64 reads a token into a float64 variable. 535 | func (s *scanner) ReadFloat64(target *float64) error { 536 | tok, b, err := s.Scan() 537 | if err != nil { 538 | return err 539 | } 540 | switch tok { 541 | case TNUMBER: 542 | n, _ := strconv.ParseFloat(string(b), 64) 543 | *target = n 544 | case TSTRING, TTRUE, TFALSE, TNULL: 545 | *target = 0 546 | default: 547 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 548 | } 549 | return nil 550 | } 551 | 552 | // ReadBool reads a token into a boolean variable. 553 | func (s *scanner) ReadBool(target *bool) error { 554 | tok, b, err := s.Scan() 555 | if err != nil { 556 | return err 557 | } 558 | switch tok { 559 | case TTRUE: 560 | *target = true 561 | case TFALSE, TSTRING, TNUMBER, TNULL: 562 | *target = false 563 | default: 564 | return fmt.Errorf("unexpected %s at %d: %s; expected number", TokenName(tok), s.pos, string(b)) 565 | } 566 | return nil 567 | } 568 | 569 | // ReadMap reads the next value into a map variable. 570 | func (s *scanner) ReadMap(target *map[string]interface{}) error { 571 | if tok, b, err := s.Scan(); err != nil { 572 | return err 573 | } else if tok == TNULL { 574 | *target = nil 575 | return nil 576 | } else if tok != TLBRACE { 577 | return fmt.Errorf("unexpected %s at %d: %s; expected '{'", TokenName(tok), s.Pos(), string(b)) 578 | } 579 | 580 | // Create a new map. 581 | *target = make(map[string]interface{}) 582 | v := *target 583 | 584 | // Loop over key/value pairs. 585 | index := 0 586 | for { 587 | // Read in key. 588 | var key string 589 | tok, b, err := s.Scan() 590 | if err != nil { 591 | return err 592 | } else if tok == TRBRACE { 593 | return nil 594 | } else if tok == TCOMMA { 595 | if index == 0 { 596 | return fmt.Errorf("unexpected comma at %d", s.Pos()) 597 | } 598 | if tok, b, err = s.Scan(); err != nil { 599 | return err 600 | } 601 | } 602 | 603 | if tok != TSTRING { 604 | return fmt.Errorf("unexpected %s at %d: %s; expected '{' or string", TokenName(tok), s.Pos(), string(b)) 605 | } else { 606 | key = string(b) 607 | } 608 | 609 | // Read in the colon. 610 | if tok, b, err := s.Scan(); err != nil { 611 | return err 612 | } else if tok != TCOLON { 613 | return fmt.Errorf("unexpected %s at %d: %s; expected colon", TokenName(tok), s.Pos(), string(b)) 614 | } 615 | 616 | // Read the next token. 617 | tok, b, err = s.Scan() 618 | if err != nil { 619 | return err 620 | } 621 | switch tok { 622 | case TSTRING: 623 | v[key] = string(b) 624 | case TNUMBER: 625 | v[key], _ = strconv.ParseFloat(string(b), 64) 626 | case TTRUE: 627 | v[key] = true 628 | case TFALSE: 629 | v[key] = false 630 | case TNULL: 631 | v[key] = nil 632 | case TLBRACE: 633 | s.Unscan(tok, b) 634 | m := make(map[string]interface{}) 635 | if err := s.ReadMap(&m); err != nil { 636 | return err 637 | } 638 | v[key] = m 639 | case TLBRACKET: 640 | s.Unscan(tok, b) 641 | var arr []interface{} 642 | if err := s.ReadArray(&arr); err != nil { 643 | return err 644 | } 645 | v[key] = arr 646 | default: 647 | return fmt.Errorf("unexpected %s at %d: %s", TokenName(tok), s.Pos(), string(b)) 648 | } 649 | 650 | index++ 651 | } 652 | } 653 | 654 | func (s *scanner) ReadArray(target *[]interface{}) error { 655 | if tok, b, err := s.Scan(); err != nil { 656 | return err 657 | } else if tok != TLBRACKET { 658 | return fmt.Errorf("unexpected %s at %d: %s; expected '['", TokenName(tok), s.Pos(), string(b)) 659 | } 660 | 661 | index := 0 662 | for { 663 | tok, b, err := s.Scan() 664 | if err != nil { 665 | return err 666 | } else if tok == TRBRACKET { 667 | return nil 668 | } else if tok == TCOMMA { 669 | if index == 0 { 670 | return fmt.Errorf("unexpected comma in array at %d", s.Pos()) 671 | } 672 | if tok, b, err = s.Scan(); err != nil { 673 | return err 674 | } 675 | } 676 | 677 | var v interface{} 678 | switch tok { 679 | case TSTRING: 680 | v = string(b) 681 | case TNUMBER: 682 | v, _ = strconv.ParseFloat(string(b), 64) 683 | case TTRUE: 684 | v = true 685 | case TFALSE: 686 | v = false 687 | case TNULL: 688 | v = nil 689 | case TLBRACE: 690 | s.Unscan(tok, b) 691 | m := make(map[string]interface{}) 692 | if err := s.ReadMap(&m); err != nil { 693 | return err 694 | } 695 | v = m 696 | case TLBRACKET: 697 | s.Unscan(tok, b) 698 | var arr []interface{} 699 | if err := s.ReadArray(&arr); err != nil { 700 | return err 701 | } 702 | v = arr 703 | default: 704 | return fmt.Errorf("unexpected %s at %d: %s", TokenName(tok), s.Pos(), string(b)) 705 | } 706 | *target = append(*target, v) 707 | 708 | index++ 709 | } 710 | } 711 | -------------------------------------------------------------------------------- /json/scanner/scanner_test.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "strconv" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/matryer/is" 11 | ) 12 | 13 | // Ensures that a positive number can be scanned. 14 | func TestScanPositiveNumber(t *testing.T) { 15 | is := is.New(t) 16 | tok, b, err := NewScanner(strings.NewReader("100")).Scan() 17 | is.NoErr(err) 18 | is.Equal(tok, TNUMBER) 19 | is.Equal(string(b), "100") 20 | } 21 | 22 | // Ensures that a negative number can be scanned. 23 | func TestScanNegativeNumber(t *testing.T) { 24 | is := is.New(t) 25 | tok, b, err := NewScanner(strings.NewReader("-1")).Scan() 26 | is.NoErr(err) 27 | is.Equal(tok, TNUMBER) 28 | is.Equal(string(b), "-1") 29 | } 30 | 31 | // Ensures that a fractional number can be scanned. 32 | func TestScanFloat(t *testing.T) { 33 | is := is.New(t) 34 | tok, b, err := NewScanner(strings.NewReader("120.12931")).Scan() 35 | is.NoErr(err) 36 | is.Equal(tok, TNUMBER) 37 | is.Equal(string(b), "120.12931") 38 | } 39 | 40 | // Ensures that a fractional number in scientific notation can be scanned. 41 | func TestScanFloatScientific(t *testing.T) { 42 | is := is.New(t) 43 | tok, b, err := NewScanner(strings.NewReader("10.1e01")).Scan() 44 | is.NoErr(err) 45 | is.Equal(tok, TNUMBER) 46 | is.Equal(string(b), "10.1e01") 47 | 48 | tok, b, err = NewScanner(strings.NewReader("10.1e-01")).Scan() 49 | is.NoErr(err) 50 | is.Equal(tok, TNUMBER) 51 | is.Equal(string(b), "10.1e-01") 52 | 53 | tok, b, err = NewScanner(strings.NewReader("10.1e+01")).Scan() 54 | is.NoErr(err) 55 | is.Equal(tok, TNUMBER) 56 | is.Equal(string(b), "10.1e01") 57 | f, _ := strconv.ParseFloat(string(b), 64) 58 | is.Equal(10.1e+01, f) 59 | 60 | tok, b, err = NewScanner(strings.NewReader("-1e1")).Scan() 61 | is.NoErr(err) 62 | is.Equal(tok, TNUMBER) 63 | is.Equal(string(b), "-1e1") 64 | } 65 | 66 | // Ensures that a quoted string can be scanned. 67 | func TestScanString(t *testing.T) { 68 | is := is.New(t) 69 | tok, b, err := NewScanner(strings.NewReader(`"hello world"`)).Scan() 70 | is.NoErr(err) 71 | is.Equal(tok, TSTRING) 72 | is.Equal(string(b), "hello world") 73 | } 74 | 75 | // Ensures that a quoted string with escaped characters can be scanned. 76 | func TestScanEscapedString(t *testing.T) { 77 | is := is.New(t) 78 | tok, b, err := NewScanner(strings.NewReader(`"\"\\\/\b\f\n\r\t"`)).Scan() 79 | is.NoErr(err) 80 | is.Equal(tok, TSTRING) 81 | is.Equal(string(b), "\"\\/\b\f\n\r\t") 82 | } 83 | 84 | // Ensures that escaped unicode sequences can be decoded. 85 | func TestScanEscapedUnicode(t *testing.T) { 86 | is := is.New(t) 87 | tok, b, err := NewScanner(strings.NewReader(`"\u0026 \u0424 \u03B4 \u03b4"`)).Scan() 88 | is.NoErr(err) 89 | is.Equal(tok, TSTRING) 90 | is.Equal(string(b), "& Ф δ δ") 91 | } 92 | 93 | // Ensures that a true value can be scanned. 94 | func TestScanTrue(t *testing.T) { 95 | is := is.New(t) 96 | tok, _, err := NewScanner(strings.NewReader(`true`)).Scan() 97 | is.NoErr(err) 98 | is.Equal(tok, TTRUE) 99 | } 100 | 101 | // Ensures that a false value can be scanned. 102 | func TestScanFalse(t *testing.T) { 103 | is := is.New(t) 104 | tok, _, err := NewScanner(strings.NewReader(`false`)).Scan() 105 | is.NoErr(err) 106 | is.Equal(tok, TFALSE) 107 | } 108 | 109 | // Ensures that a null value can be scanned. 110 | func TestScanNull(t *testing.T) { 111 | is := is.New(t) 112 | tok, _, err := NewScanner(strings.NewReader(`null`)).Scan() 113 | is.NoErr(err) 114 | is.Equal(tok, TNULL) 115 | } 116 | 117 | // Ensures that an EOF gets returned. 118 | func TestScanEOF(t *testing.T) { 119 | is := is.New(t) 120 | _, _, err := NewScanner(strings.NewReader(``)).Scan() 121 | is.Equal(err, io.EOF) 122 | } 123 | 124 | // Ensures that a string can be read into a field. 125 | func TestReadString(t *testing.T) { 126 | is := is.New(t) 127 | var v string 128 | err := NewScanner(strings.NewReader(`"foo"`)).ReadString(&v) 129 | is.NoErr(err) 130 | is.Equal(v, "foo") 131 | } 132 | 133 | // Ensures that strings largers than allocated buffer can be read. 134 | func TestReadHugeString(t *testing.T) { 135 | is := is.New(t) 136 | var v string 137 | huge := strings.Repeat("s", bufSize*3) 138 | err := NewScanner(strings.NewReader(`"` + huge + `"`)).ReadString(&v) 139 | is.NoErr(err) 140 | is.Equal(v, huge) 141 | } 142 | 143 | // Ensures that a non-string value is read into a string field as blank. 144 | func TestReadNonStringAsString(t *testing.T) { 145 | is := is.New(t) 146 | var v string 147 | err := NewScanner(strings.NewReader(`12`)).ReadString(&v) 148 | is.NoErr(err) 149 | is.Equal(v, "") 150 | } 151 | 152 | // Ensures that a non-value returns a read error. 153 | func TestReadNonValueAsString(t *testing.T) { 154 | is := is.New(t) 155 | var v string 156 | err := NewScanner(strings.NewReader(`{`)).ReadString(&v) 157 | is.True(err != nil) 158 | // TODO: test error 159 | } 160 | 161 | // Ensures that an int can be read into a field. 162 | func TestReadInt(t *testing.T) { 163 | is := is.New(t) 164 | var v int 165 | err := NewScanner(strings.NewReader(`100`)).ReadInt(&v) 166 | is.NoErr(err) 167 | is.Equal(v, 100) 168 | } 169 | 170 | // Ensures that a non-number value is read into an int field as zero. 171 | func TestReadNonNumberAsInt(t *testing.T) { 172 | is := is.New(t) 173 | var v int 174 | err := NewScanner(strings.NewReader(`"foo"`)).ReadInt(&v) 175 | is.NoErr(err) 176 | is.Equal(v, 0) 177 | } 178 | 179 | // Ensures that an int64 can be read into a field. 180 | func TestReadInt64(t *testing.T) { 181 | is := is.New(t) 182 | var v int64 183 | err := NewScanner(strings.NewReader(`-100`)).ReadInt64(&v) 184 | is.NoErr(err) 185 | is.Equal(v, int64(-100)) 186 | } 187 | 188 | // Ensures that a uint can be read into a field. 189 | func TestReadUint(t *testing.T) { 190 | is := is.New(t) 191 | var v uint 192 | err := NewScanner(strings.NewReader(`100`)).ReadUint(&v) 193 | is.NoErr(err) 194 | is.Equal(v, uint(100)) 195 | } 196 | 197 | // Ensures that an uint64 can be read into a field. 198 | func TestReadUint64(t *testing.T) { 199 | is := is.New(t) 200 | var v uint64 201 | err := NewScanner(strings.NewReader(`1024`)).ReadUint64(&v) 202 | is.NoErr(err) 203 | is.Equal(v, uint64(1024)) 204 | } 205 | 206 | // Ensures that a float32 can be read into a field. 207 | func TestReadFloat32(t *testing.T) { 208 | is := is.New(t) 209 | var v float32 210 | err := NewScanner(strings.NewReader(`1293.123`)).ReadFloat32(&v) 211 | is.NoErr(err) 212 | is.Equal(v, float32(1293.123)) 213 | } 214 | 215 | // Ensures that a float64 can be read into a field. 216 | func TestReadFloat64(t *testing.T) { 217 | is := is.New(t) 218 | var v float64 219 | err := NewScanner(strings.NewReader(`9871293.414123`)).ReadFloat64(&v) 220 | is.NoErr(err) 221 | is.Equal(v, 9871293.414123) 222 | } 223 | 224 | // Ensures that a boolean can be read into a field. 225 | func TestReadBoolTrue(t *testing.T) { 226 | is := is.New(t) 227 | var v bool 228 | err := NewScanner(strings.NewReader(`true`)).ReadBool(&v) 229 | is.NoErr(err) 230 | is.Equal(v, true) 231 | } 232 | 233 | // Ensures whitespace between tokens are ignored. 234 | func TestScanIgnoreWhitespace(t *testing.T) { 235 | is := is.New(t) 236 | s := NewScanner(strings.NewReader(" 100 true false ")) 237 | 238 | tok, _, err := s.Scan() 239 | is.NoErr(err) 240 | is.Equal(tok, TNUMBER) 241 | 242 | tok, _, err = s.Scan() 243 | is.NoErr(err) 244 | is.Equal(tok, TTRUE) 245 | 246 | tok, _, err = s.Scan() 247 | is.NoErr(err) 248 | is.Equal(tok, TFALSE) 249 | 250 | tok, _, err = s.Scan() 251 | is.Equal(err, io.EOF) 252 | is.Equal(tok, 0) 253 | } 254 | 255 | // Ensures that a map can be read into a field. 256 | func TestReadMap(t *testing.T) { 257 | is := is.New(t) 258 | var v map[string]interface{} 259 | err := NewScanner(strings.NewReader(`{"foo":"bar", "bat":1293,"truex":true,"falsex":false,"nullx":null,"nested":{"xxx":"yyy"}}`)).ReadMap(&v) 260 | is.NoErr(err) 261 | is.Equal(v["foo"], "bar") 262 | is.Equal(v["bat"], float64(1293)) 263 | is.Equal(v["truex"], true) 264 | is.Equal(v["falsex"], false) 265 | _, exists := v["nullx"] 266 | is.Equal(v["nullx"], nil) 267 | is.True(exists) 268 | is.True(v["nested"] != nil) 269 | nested := v["nested"].(map[string]interface{}) 270 | is.Equal(nested["xxx"], "yyy") 271 | } 272 | 273 | // Ensures that a map with arrays can be read into a field. 274 | func TestReadMapWithArray(t *testing.T) { 275 | is := is.New(t) 276 | var v map[string]interface{} 277 | err := NewScanner(strings.NewReader(`{"foo":["bar", 42]}`)).ReadMap(&v) 278 | is.NoErr(err) 279 | arr := v["foo"].([]interface{}) 280 | t.Logf("got=%#v", v) 281 | is.Equal("bar", arr[0].(string)) 282 | is.Equal(42.0, arr[1].(float64)) 283 | } 284 | 285 | func BenchmarkScanNumber(b *testing.B) { 286 | withBuffer(b, "100", func(buf []byte) { 287 | s := NewScanner(bytes.NewBuffer(buf)) 288 | for i := 0; i < b.N; i++ { 289 | if _, _, err := s.Scan(); err == io.EOF { 290 | s = NewScanner(bytes.NewBuffer(buf)) 291 | } else if err != nil { 292 | b.Fatal("scan error:", err) 293 | } 294 | } 295 | }) 296 | } 297 | 298 | func BenchmarkScanString(b *testing.B) { 299 | withBuffer(b, `"01234567"`, func(buf []byte) { 300 | s := NewScanner(bytes.NewBuffer(buf)) 301 | for i := 0; i < b.N; i++ { 302 | if _, _, err := s.Scan(); err == io.EOF { 303 | s = NewScanner(bytes.NewBuffer(buf)) 304 | } else if err != nil { 305 | b.Fatal("scan error:", err) 306 | } 307 | } 308 | }) 309 | } 310 | 311 | func BenchmarkScanLongString(b *testing.B) { 312 | withBuffer(b, `"foo foo foo foo foo foo foo foo foo foo foo foo foo foo"`, func(buf []byte) { 313 | s := NewScanner(bytes.NewBuffer(buf)) 314 | for i := 0; i < b.N; i++ { 315 | if _, _, err := s.Scan(); err == io.EOF { 316 | s = NewScanner(bytes.NewBuffer(buf)) 317 | } else if err != nil { 318 | b.Fatal("scan error:", err) 319 | } 320 | } 321 | }) 322 | } 323 | 324 | func BenchmarkScanEscapedString(b *testing.B) { 325 | withBuffer(b, `"\"\\\/\b\f\n\r\t"`, func(buf []byte) { 326 | s := NewScanner(bytes.NewBuffer(buf)) 327 | for i := 0; i < b.N; i++ { 328 | if _, _, err := s.Scan(); err == io.EOF { 329 | s = NewScanner(bytes.NewBuffer(buf)) 330 | } else if err != nil { 331 | b.Fatal("scan error:", err) 332 | } 333 | } 334 | }) 335 | } 336 | 337 | func BenchmarkReadString(b *testing.B) { 338 | withBuffer(b, `"01234567"`, func(buf []byte) { 339 | var v string 340 | s := NewScanner(bytes.NewBuffer(buf)) 341 | for i := 0; i < b.N; i++ { 342 | if err := s.ReadString(&v); err == io.EOF { 343 | s = NewScanner(bytes.NewBuffer(buf)) 344 | } else if err != nil { 345 | b.Fatal("scan error:", err) 346 | } 347 | } 348 | }) 349 | } 350 | 351 | func BenchmarkReadLongString(b *testing.B) { 352 | withBuffer(b, `"foo foo foo foo foo foo foo foo foo foo foo foo foo foo"`, func(buf []byte) { 353 | var v string 354 | s := NewScanner(bytes.NewBuffer(buf)) 355 | for i := 0; i < b.N; i++ { 356 | if err := s.ReadString(&v); err == io.EOF { 357 | s = NewScanner(bytes.NewBuffer(buf)) 358 | } else if err != nil { 359 | b.Fatal("scan error:", err) 360 | } 361 | } 362 | }) 363 | } 364 | 365 | func BenchmarkReadInt(b *testing.B) { 366 | withBuffer(b, `"100"`, func(buf []byte) { 367 | var v int 368 | s := NewScanner(bytes.NewBuffer(buf)) 369 | for i := 0; i < b.N; i++ { 370 | if err := s.ReadInt(&v); err == io.EOF { 371 | s = NewScanner(bytes.NewBuffer(buf)) 372 | } else if err != nil { 373 | b.Fatal("scan error:", err) 374 | } 375 | } 376 | }) 377 | } 378 | 379 | func BenchmarkReadFloat64(b *testing.B) { 380 | withBuffer(b, `"9871293.414123"`, func(buf []byte) { 381 | var v float64 382 | s := NewScanner(bytes.NewBuffer(buf)) 383 | for i := 0; i < b.N; i++ { 384 | if err := s.ReadFloat64(&v); err == io.EOF { 385 | s = NewScanner(bytes.NewBuffer(buf)) 386 | } else if err != nil { 387 | b.Fatal("scan error:", err) 388 | } 389 | } 390 | }) 391 | } 392 | 393 | func BenchmarkReadBool(b *testing.B) { 394 | withBuffer(b, `true`, func(buf []byte) { 395 | var v bool 396 | s := NewScanner(bytes.NewBuffer(buf)) 397 | for i := 0; i < b.N; i++ { 398 | if err := s.ReadBool(&v); err == io.EOF { 399 | s = NewScanner(bytes.NewBuffer(buf)) 400 | } else if err != nil { 401 | b.Fatal("scan error:", err) 402 | } 403 | } 404 | }) 405 | } 406 | 407 | func withBuffer(b *testing.B, value string, fn func([]byte)) { 408 | b.StopTimer() 409 | var str string 410 | for i := 0; i < 1000; i++ { 411 | str += value + " " 412 | } 413 | b.StartTimer() 414 | 415 | fn([]byte(str)) 416 | 417 | b.SetBytes(int64(len(value))) 418 | } 419 | -------------------------------------------------------------------------------- /json/scanner/token.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | const ( 4 | TSTRING = iota + 1 5 | TNUMBER 6 | TTRUE 7 | TFALSE 8 | TNULL 9 | TLBRACE 10 | TRBRACE 11 | TLBRACKET 12 | TRBRACKET 13 | TCOLON 14 | TCOMMA 15 | ) 16 | 17 | var names = map[int]string{ 18 | TSTRING: "string", 19 | TNUMBER: "number", 20 | TTRUE: "true", 21 | TFALSE: "false", 22 | TNULL: "null", 23 | TLBRACE: "left brace", 24 | TRBRACE: "right brace", 25 | TLBRACKET: "left bracket", 26 | TRBRACKET: "right bracket", 27 | TCOLON: "colon", 28 | TCOMMA: "comma", 29 | } 30 | 31 | // TokenName returns a human readable version of the token. 32 | func TokenName(tok int) string { 33 | return names[tok] 34 | } 35 | -------------------------------------------------------------------------------- /json/unmarshaler.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "fmt" 7 | "go/ast" 8 | "go/format" 9 | "strconv" 10 | "strings" 11 | "text/template" 12 | ) 13 | 14 | type Unmarshaler struct { 15 | // Import path we're generating code into 16 | TargetPath string 17 | // Find the type spec for the given import path and name 18 | Find func(importPath string, name string) (ast.Expr, error) 19 | // Add an import to the generated code 20 | Import func(path string) (name string, err error) 21 | } 22 | 23 | //go:embed unmarshaler.gotext 24 | var unmarshalerTemplate string 25 | 26 | var generator = template.Must(template.New("unmarshaler").Parse(unmarshalerTemplate)) 27 | 28 | var generatorImports = []string{ 29 | "fmt", 30 | "github.com/livebud/marshaler/json/scanner", 31 | "bytes", 32 | } 33 | 34 | // TODO: allow the import path name to be customized 35 | func (u *Unmarshaler) typeName(importPath, name string) (string, error) { 36 | if u.TargetPath == importPath { 37 | return name, nil 38 | } 39 | importName, err := u.Import(importPath) 40 | if err != nil { 41 | return "", err 42 | } 43 | return importName + "." + name, nil 44 | } 45 | 46 | func (u *Unmarshaler) Generate(importPath, name string) ([]byte, error) { 47 | expr, err := u.Find(importPath, name) 48 | if err != nil { 49 | return nil, err 50 | } 51 | schema, err := fromExpr(expr, 0, "in") 52 | if err != nil { 53 | return nil, err 54 | } 55 | for _, importPath := range generatorImports { 56 | if _, err := u.Import(importPath); err != nil { 57 | return nil, err 58 | } 59 | } 60 | typeName, err := u.typeName(importPath, name) 61 | if err != nil { 62 | return nil, err 63 | } 64 | state := State{ 65 | Schema: schema, 66 | Name: typeName, 67 | } 68 | code := new(bytes.Buffer) 69 | if err := generator.Execute(code, state); err != nil { 70 | return nil, err 71 | } 72 | return format.Source(code.Bytes()) 73 | } 74 | 75 | func fromExpr(x ast.Expr, depth int, target string) (Type, error) { 76 | switch x := x.(type) { 77 | case *ast.Ident: 78 | return fromIdent(x, depth, target) 79 | case *ast.StructType: 80 | return fromStruct(x, depth, target) 81 | case *ast.MapType: 82 | return fromMap(x, depth, target) 83 | case *ast.ArrayType: 84 | return fromArray(x, depth, target) 85 | case *ast.StarExpr: 86 | return fromStar(x, depth, target) 87 | default: 88 | return nil, fmt.Errorf("fromExpr: %T not implemented", x) 89 | } 90 | } 91 | 92 | func fromStruct(s *ast.StructType, depth int, target string) (*Struct, error) { 93 | var fields []StructField 94 | for _, f := range s.Fields.List { 95 | dataType, err := fromExpr(f.Type, depth+1, "&"+target+"."+f.Names[0].Name) 96 | if err != nil { 97 | return nil, err 98 | } 99 | fields = append(fields, StructField{ 100 | Key: f.Names[0].Name, 101 | Type: dataType, 102 | }) 103 | } 104 | return &Struct{fields, depth, target}, nil 105 | } 106 | 107 | func fromIdent(i *ast.Ident, depth int, target string) (Type, error) { 108 | switch i.Name { 109 | case "string": 110 | return String{depth, target}, nil 111 | case "int": 112 | return Int{depth, target}, nil 113 | case "float64": 114 | return Float64{depth, target}, nil 115 | case "bool": 116 | return Bool{depth, target}, nil 117 | } 118 | return nil, fmt.Errorf("fromIdent: %q not implemented", i.Name) 119 | } 120 | 121 | func fromMap(m *ast.MapType, depth int, target string) (*Map, error) { 122 | keyType, err := fromExpr(m.Key, depth+1, target) 123 | if err != nil { 124 | return nil, err 125 | } 126 | // Static target because it's defined in the template 127 | valueType, err := fromExpr(m.Value, depth+1, "&val"+strconv.Itoa(depth)) 128 | if err != nil { 129 | return nil, err 130 | } 131 | // For maps, we pull the value out of the target first and you Go doesn't 132 | // support `val := &target["key"]`, so we do `val := target["key"]` and 133 | // then `&val` instead. 134 | newTarget := strings.TrimPrefix(target, "&") 135 | return &Map{keyType, valueType, depth, newTarget}, nil 136 | } 137 | 138 | func fromArray(a *ast.ArrayType, depth int, target string) (*Array, error) { 139 | // Static target because it's defined in the template 140 | dataType, err := fromExpr(a.Elt, depth+1, "&val"+strconv.Itoa(depth)) 141 | if err != nil { 142 | return nil, err 143 | } 144 | // For arrays, we pull the value out of the target first and you Go doesn't 145 | // support `&target := append(&target, val)`, so we do 146 | // `target := append(target, val)` instead. 147 | newTarget := strings.TrimPrefix(target, "&") 148 | return &Array{dataType, depth, newTarget}, nil 149 | } 150 | 151 | func fromStar(s *ast.StarExpr, depth int, target string) (*Star, error) { 152 | // Static target because it's defined in the template 153 | dataType, err := fromExpr(s.X, depth+1, "val"+strconv.Itoa(depth)) 154 | if err != nil { 155 | return nil, err 156 | } 157 | // For stars, we pull the value out of the target first and you Go doesn't 158 | // support `&target := val`, so we do `target := val` instead. 159 | newTarget := strings.TrimPrefix(target, "&") 160 | return &Star{dataType, depth, newTarget}, nil 161 | } 162 | 163 | type State struct { 164 | Schema Type 165 | Name string 166 | } 167 | 168 | type Type interface { 169 | Type() string 170 | String() string 171 | } 172 | 173 | func (String) Type() string { return "string" } 174 | func (Int) Type() string { return "int" } 175 | func (Float64) Type() string { return "float64" } 176 | func (Bool) Type() string { return "bool" } 177 | func (Struct) Type() string { return "struct" } 178 | func (Array) Type() string { return "array" } 179 | func (Map) Type() string { return "map" } 180 | func (Star) Type() string { return "star" } 181 | 182 | type String struct { 183 | Depth int 184 | Target string 185 | } 186 | 187 | type Int struct { 188 | Depth int 189 | Target string 190 | } 191 | 192 | type Float64 struct { 193 | Depth int 194 | Target string 195 | } 196 | 197 | type Bool struct { 198 | Depth int 199 | Target string 200 | } 201 | 202 | func (String) String() string { return "string" } 203 | func (Int) String() string { return "int" } 204 | func (Float64) String() string { return "float64" } 205 | func (Bool) String() string { return "bool" } 206 | 207 | type Struct struct { 208 | Fields []StructField 209 | Depth int 210 | Target string 211 | } 212 | 213 | func (s *Struct) String() string { 214 | out := new(strings.Builder) 215 | out.WriteString("struct {\n") 216 | for _, f := range s.Fields { 217 | out.WriteString(fmt.Sprintf(" %s %s\n", f.Key, f.Type.String())) 218 | } 219 | out.WriteString("}") 220 | return out.String() 221 | } 222 | 223 | type StructField struct { 224 | Key string 225 | Type Type 226 | } 227 | 228 | type Map struct { 229 | Key Type 230 | Value Type 231 | Depth int 232 | Target string 233 | } 234 | 235 | func (m Map) String() string { 236 | return fmt.Sprintf("map[%s]%s", m.Key.String(), m.Value.String()) 237 | } 238 | 239 | type Array struct { 240 | Elt Type 241 | Depth int 242 | Target string 243 | } 244 | 245 | func (s Array) String() string { 246 | return fmt.Sprintf("[]%s", s.Elt.String()) 247 | } 248 | 249 | type Star struct { 250 | X Type 251 | Depth int 252 | Target string 253 | } 254 | 255 | func (s Star) String() string { 256 | return fmt.Sprintf("*%s", s.X.String()) 257 | } 258 | -------------------------------------------------------------------------------- /json/unmarshaler.gotext: -------------------------------------------------------------------------------- 1 | {{- /* Switch between the templates based on the types */ -}} 2 | {{- define "type" }} 3 | {{- if eq .Type "string" }} 4 | {{- template "string" . }} 5 | {{- else if eq .Type "bool" }} 6 | {{- template "bool" . }} 7 | {{- else if eq .Type "int" }} 8 | {{- template "int" . }} 9 | {{- else if eq .Type "struct" }} 10 | {{- template "struct" . }} 11 | {{- else if eq .Type "map" }} 12 | {{- template "map" . }} 13 | {{- else if eq .Type "array" }} 14 | {{- template "array" . }} 15 | {{- else if eq .Type "float64" }} 16 | {{- template "float64" . }} 17 | {{- else if eq .Type "star" }} 18 | {{- template "star" . }} 19 | {{- else }} 20 | return fmt.Errorf("missing template for %q", `{{ .Type }}`) 21 | {{- end }} 22 | {{- end }} 23 | 24 | {{- /* String type */ -}} 25 | {{- define "string" }} 26 | if err := s.ReadString((*string)({{ .Target }})); err != nil { 27 | return err 28 | } 29 | {{- end }} 30 | 31 | {{- /* Bool type */ -}} 32 | {{- define "bool" }} 33 | if err := s.ReadBool((*bool)({{ .Target }})); err != nil { 34 | return err 35 | } 36 | {{- end -}} 37 | 38 | {{- /* Int type */ -}} 39 | {{- define "int" }} 40 | if err := s.ReadInt((*int)({{ .Target }})); err != nil { 41 | return err 42 | } 43 | {{- end }} 44 | 45 | {{- /* Float64 type */ -}} 46 | {{- define "float64" }} 47 | if err := s.ReadFloat64((*float64)({{ .Target }})); err != nil { 48 | return err 49 | } 50 | {{- end }} 51 | 52 | {{- /* Struct type */ -}} 53 | {{- define "struct" }} 54 | // Scanning struct 55 | if _, err := s.Expect(scanner.TLBRACE); err != nil { 56 | return err 57 | } 58 | for { 59 | tok, buf, err := s.Scan() 60 | if err != nil { 61 | return err 62 | } 63 | key := string(buf) 64 | // We're expecting either a string key or a closing brace 65 | if tok == scanner.TRBRACE { 66 | break 67 | } else if tok != scanner.TSTRING { 68 | return fmt.Errorf(`%d: expected "}" or string, got %q`, s.Pos(), scanner.TokenName(tok)) 69 | } 70 | switch key { 71 | {{- range $field := .Fields }} 72 | case `{{ $field.Key }}`: 73 | if _, err := s.Expect(scanner.TCOLON); err != nil { 74 | return err 75 | } 76 | {{ template "type" $field.Type }} 77 | {{ end }} 78 | default: 79 | return fmt.Errorf("unexpected key %q", key) 80 | } 81 | // Expect either a comma or a closing brace 82 | tok, _, err = s.Scan() 83 | if err != nil { 84 | return err 85 | } 86 | if tok == scanner.TRBRACE { 87 | break 88 | } else if tok != scanner.TCOMMA { 89 | return fmt.Errorf(`%d: expected "}" or ",", got %q`, s.Pos(), tok) 90 | } 91 | } // Scanned struct 92 | {{- end }} 93 | 94 | {{- /* Map type */ -}} 95 | {{- define "map" -}} 96 | if _, err := s.Expect(scanner.TLBRACE); err != nil { 97 | return err 98 | } 99 | for { 100 | tok, buf, err := s.Scan() 101 | if err != nil { 102 | return err 103 | } 104 | key := string(buf) 105 | // We're expecting either a string key or a closing brace 106 | if tok == scanner.TRBRACE { 107 | // We got the closing } 108 | break 109 | } else if tok != scanner.TSTRING { 110 | return fmt.Errorf(`%d: expected "}" or string, got %q`, s.Pos(), scanner.TokenName(tok)) 111 | } 112 | // Read the colon 113 | if _, err := s.Expect(scanner.TCOLON); err != nil { 114 | return err 115 | } 116 | // Read the value 117 | var val{{.Depth}} {{ .Value }} 118 | {{- template "type" .Value }} 119 | if {{ .Target }} == nil { 120 | {{ .Target }} = make({{ . }}) 121 | } 122 | {{ .Target }}[key] = val{{.Depth}} 123 | // Expect either a comma or a closing brace 124 | tok, _, err = s.Scan() 125 | if err != nil { 126 | return err 127 | } 128 | if tok == scanner.TRBRACE { 129 | // Got closing "}" 130 | break 131 | } else if tok != scanner.TCOMMA { 132 | return fmt.Errorf(`%d: expected "}" or ",", got %q`, s.Pos(), tok) 133 | } 134 | } 135 | {{- end }} 136 | 137 | {{- /* Array type */ -}} 138 | {{- define "array" }} 139 | if _, err := s.Expect(scanner.TLBRACKET); err != nil { 140 | return err 141 | } 142 | for { 143 | tok, buf, err := s.Scan() 144 | if err != nil { 145 | return err 146 | } 147 | if tok == scanner.TRBRACKET { 148 | break 149 | } 150 | // If it's not a ], then push the token back on 151 | s.Unscan(tok, buf) 152 | // Scan the token again with the proper reader 153 | var val{{.Depth}} {{ .Elt }} 154 | {{- template "type" .Elt }} 155 | {{ .Target }} = append({{ .Target }}, val{{.Depth}}) 156 | // Next is either a , or a ] 157 | tok, _, err = s.Scan() 158 | if err != nil { 159 | return err 160 | } 161 | if tok == scanner.TRBRACKET { 162 | break 163 | } else if tok != scanner.TCOMMA { 164 | return fmt.Errorf(`%d: expected "]" or ",", got %q`, s.Pos(), tok) 165 | } 166 | } 167 | {{- end }} 168 | 169 | {{- /* Star type */ -}} 170 | {{- define "star" }} 171 | val{{.Depth}} := new({{ .X }}) 172 | {{- template "type" .X }} 173 | {{ .Target }} = val{{.Depth}} 174 | {{- end }} 175 | 176 | {{- /* Generated Unmarshaler */ -}} 177 | // UnmarshalJSON unmarshals buf into in 178 | func UnmarshalJSON(buf []byte, in *{{ $.Name }}) (err error) { 179 | s := scanner.NewScanner(bytes.NewBuffer(buf)) 180 | _ = s 181 | _ = fmt.Errorf 182 | {{- template "type" $.Schema }} 183 | return nil 184 | } 185 | -------------------------------------------------------------------------------- /json/unmarshaler_test.go: -------------------------------------------------------------------------------- 1 | package json_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "io/fs" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | "text/template" 14 | 15 | _ "github.com/hexops/valast" 16 | 17 | "github.com/livebud/marshaler/internal/imports" 18 | 19 | "github.com/lithammer/dedent" 20 | "github.com/livebud/marshaler/internal/finder" 21 | "github.com/livebud/marshaler/json" 22 | "github.com/matryer/is" 23 | "golang.org/x/mod/modfile" 24 | ) 25 | 26 | func findGoMod(absdir string) (abs string, err error) { 27 | path := filepath.Join(absdir, "go.mod") 28 | // Check if this path exists, otherwise recursively traverse towards root 29 | if _, err = os.Stat(path); err != nil { 30 | if !errors.Is(err, fs.ErrNotExist) { 31 | return "", err 32 | } 33 | nextDir := filepath.Dir(absdir) 34 | if nextDir == absdir { 35 | return "", fs.ErrNotExist 36 | } 37 | return findGoMod(filepath.Dir(absdir)) 38 | } 39 | return filepath.EvalSymlinks(absdir) 40 | } 41 | 42 | func redent(s string) string { 43 | return strings.TrimSpace(dedent.Dedent(s)) + "\n" 44 | } 45 | 46 | func goRun(ctx context.Context, cacheDir, appDir string) (string, error) { 47 | cmd := exec.CommandContext(ctx, "go", "run", "-mod", "mod", ".") 48 | stdout := new(bytes.Buffer) 49 | cmd.Stdout = stdout 50 | cmd.Stderr = os.Stderr 51 | cmd.Stdin = os.Stdin 52 | cmd.Dir = appDir 53 | err := cmd.Run() 54 | if err != nil { 55 | return "", err 56 | } 57 | return stdout.String(), nil 58 | } 59 | 60 | type Test struct { 61 | Dir string 62 | Files map[string]string 63 | Input string 64 | Expect string 65 | } 66 | 67 | const goMod = ` 68 | module app.com 69 | 70 | require ( 71 | github.com/livebud/marshaler v0.0.0 72 | ) 73 | ` 74 | 75 | type State struct { 76 | Imports []*imports.Import 77 | Input string 78 | Unmarshal string 79 | } 80 | 81 | var mainGen = template.Must(template.New("main.go").Parse(` 82 | package main 83 | 84 | {{- if $.Imports }} 85 | 86 | import ( 87 | {{- range $import := $.Imports }} 88 | {{$import.Name}} "{{$import.Path}}" 89 | {{- end }} 90 | ) 91 | {{- end }} 92 | 93 | func main() { 94 | var in Input 95 | if err := UnmarshalJSON([]byte(` + "`" + `{{ .Input }}` + "`" + `), &in); err != nil { 96 | fmt.Fprintf(os.Stdout, "%s\n", err) 97 | return 98 | }; 99 | actual, err := json.Marshal(in) 100 | if err != nil { 101 | fmt.Fprintf(os.Stdout, "%s\n", err) 102 | return 103 | } 104 | fmt.Fprintf(os.Stdout, "%s", string(actual)) 105 | } 106 | 107 | {{ $.Unmarshal }} 108 | `)) 109 | 110 | func runTest(t testing.TB, test Test) { 111 | t.Helper() 112 | is := is.New(t) 113 | ctx := context.Background() 114 | if test.Dir == "" { 115 | test.Dir = t.TempDir() 116 | } 117 | if test.Files == nil { 118 | test.Files = map[string]string{} 119 | } 120 | if test.Files["go.mod"] == "" { 121 | test.Files["go.mod"] = goMod 122 | } 123 | // Parse the go.mod file 124 | modFile, err := modfile.Parse("go.mod", []byte(test.Files["go.mod"]), nil) 125 | is.NoErr(err) 126 | // Replace the marshaler dependency with the local version 127 | absDir, err := filepath.Abs(".") 128 | is.NoErr(err) 129 | myModDir, err := findGoMod(absDir) 130 | is.NoErr(err) 131 | is.NoErr(modFile.AddReplace("github.com/livebud/marshaler", "", myModDir, "")) 132 | test.Files["go.mod"] = string(modfile.Format(modFile.Syntax)) 133 | // Write the test files out 134 | for path, code := range test.Files { 135 | fullPath := filepath.Join(test.Dir, path) 136 | is.NoErr(os.MkdirAll(filepath.Dir(fullPath), 0755)) 137 | is.NoErr(os.WriteFile(fullPath, []byte(redent(code)), 0644)) 138 | } 139 | // Setup the marshaler 140 | finder := finder.New(test.Dir) 141 | imports := imports.Imports{} 142 | // Setup the unmarshaler 143 | unmarshaler := &json.Unmarshaler{ 144 | TargetPath: modFile.Module.Mod.Path, 145 | Find: finder.Find, 146 | Import: imports.Import, 147 | } 148 | // Generate the unmarshaler 149 | unmarshal, err := unmarshaler.Generate("app.com", "Input") 150 | if err != nil { 151 | is.Equal(err.Error(), test.Expect) 152 | return 153 | } 154 | // fmt.Println(string(unmarshal)) 155 | // Add main.go's imports 156 | _, err = imports.Import("fmt") 157 | is.NoErr(err) 158 | _, err = imports.Import("os") 159 | is.NoErr(err) 160 | _, err = imports.Import("encoding/json") 161 | is.NoErr(err) 162 | // Generate the main.go file 163 | mainGo := new(bytes.Buffer) 164 | is.NoErr(mainGen.Execute(mainGo, &State{ 165 | Imports: imports, 166 | Unmarshal: string(unmarshal), 167 | Input: test.Input, 168 | })) 169 | // Write the main.go file out 170 | mainPath := filepath.Join(test.Dir, "main.go") 171 | is.NoErr(os.WriteFile(mainPath, []byte(redent(mainGo.String())), 0644)) 172 | // Run the main.go file 173 | stdout, err := goRun(ctx, t.TempDir(), test.Dir) 174 | is.NoErr(err) 175 | is.Equal(stdout, test.Expect) 176 | } 177 | 178 | func TestString(t *testing.T) { 179 | runTest(t, Test{ 180 | Files: map[string]string{ 181 | "input.go": ` 182 | package main 183 | type Input string 184 | `, 185 | }, 186 | Input: `"hello"`, 187 | Expect: `"hello"`, 188 | }) 189 | } 190 | 191 | func TestEmptyStruct(t *testing.T) { 192 | runTest(t, Test{ 193 | Files: map[string]string{ 194 | "input.go": ` 195 | package main 196 | type Input struct { 197 | } 198 | `, 199 | }, 200 | Input: `{}`, 201 | Expect: `{}`, 202 | }) 203 | } 204 | 205 | func TestStruct(t *testing.T) { 206 | runTest(t, Test{ 207 | Files: map[string]string{ 208 | "input.go": ` 209 | package main 210 | type Input struct { 211 | B string 212 | C int 213 | D float64 214 | E bool 215 | F map[string]string 216 | G []int 217 | H *string 218 | } 219 | `, 220 | }, 221 | Input: `{"B":"foo","C":1,"D":1.1,"E":true,"F":{"foo":"bar"},"G":[1,2,3],"H":"hello"}`, 222 | Expect: `{"B":"foo","C":1,"D":1.1,"E":true,"F":{"foo":"bar"},"G":[1,2,3],"H":"hello"}`, 223 | }) 224 | } 225 | 226 | func TestNestedStruct(t *testing.T) { 227 | runTest(t, Test{ 228 | Files: map[string]string{ 229 | "input.go": ` 230 | package main 231 | type Input struct { 232 | B string 233 | C int 234 | D float64 235 | E bool 236 | F map[string]string 237 | G []int 238 | H *string 239 | I *struct{ 240 | B string 241 | C int 242 | D float64 243 | E bool 244 | F map[string]string 245 | G []int 246 | H *string 247 | } 248 | } 249 | `, 250 | }, 251 | Input: `{"B":"foo","C":1,"D":1.1,"E":true,"F":{"foo":"bar"},"G":[1,2,3],"H":"hello","I":{"B":"foo","C":1,"D":1.1,"E":true,"F":{"foo":"bar"},"G":[1,2,3],"H":"hello"}}`, 252 | Expect: `{"B":"foo","C":1,"D":1.1,"E":true,"F":{"foo":"bar"},"G":[1,2,3],"H":"hello","I":{"B":"foo","C":1,"D":1.1,"E":true,"F":{"foo":"bar"},"G":[1,2,3],"H":"hello"}}`, 253 | }) 254 | } 255 | -------------------------------------------------------------------------------- /json/writer/writer.go: -------------------------------------------------------------------------------- 1 | package writer 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "strconv" 7 | "unicode/utf8" 8 | ) 9 | 10 | const ( 11 | // The maximum size that a byte can be encoded as. 12 | maxByteEncodeSize = 6 13 | 14 | // The size, in bytes, that can be encoded at a time. 15 | bufSize = 16384 16 | 17 | // The max size, in bytes, that an encoded value can be. 18 | actualBufSize = (bufSize * maxByteEncodeSize) 19 | ) 20 | 21 | var hex = "0123456789abcdef" 22 | 23 | type Writer struct { 24 | w io.Writer 25 | buf [actualBufSize + 64]byte 26 | pos int 27 | } 28 | 29 | // NewWriter creates a new JSON writer. 30 | func NewWriter(w io.Writer) *Writer { 31 | return &Writer{w: w} 32 | } 33 | 34 | // Flush writes all data in the buffer to the writer. 35 | func (w *Writer) Flush() error { 36 | if w.pos > 0 { 37 | if _, err := w.w.Write(w.buf[0:w.pos]); err != nil { 38 | return err 39 | } 40 | w.pos = 0 41 | } 42 | return nil 43 | } 44 | 45 | // check verifies there is space in the buffer. 46 | func (w *Writer) check() error { 47 | if w.pos > actualBufSize { 48 | return w.Flush() 49 | } 50 | return nil 51 | } 52 | 53 | // writeByte writes a single byte to the buffer and increments the position. 54 | func (w *Writer) writeByte(c byte) { 55 | w.buf[w.pos] = c 56 | w.pos++ 57 | } 58 | 59 | // writeString writes a string to the buffer and increments the position. 60 | func (w *Writer) writeString(s string) { 61 | copy(w.buf[w.pos:], s) 62 | w.pos += len(s) 63 | } 64 | 65 | // WriteByte writes a single byte. 66 | func (w *Writer) WriteByte(c byte) error { 67 | if err := w.check(); err != nil { 68 | return err 69 | } 70 | w.buf[w.pos] = c 71 | w.pos++ 72 | return nil 73 | } 74 | 75 | // WriteString writes a JSON string to the writer. Parts of this function are 76 | // borrowed from the encoding/json package. 77 | func (w *Writer) WriteString(v string) error { 78 | bufsz := (actualBufSize - w.pos) / maxByteEncodeSize 79 | 80 | w.writeByte('"') 81 | for i := 0; i < len(v); i += bufsz { 82 | if i > 0 { 83 | bufsz = bufSize 84 | if err := w.Flush(); err != nil { 85 | return err 86 | } 87 | } 88 | 89 | // Extract substring. 90 | end := i + bufsz 91 | if end > len(v) { 92 | end = len(v) 93 | } 94 | bufend := end + utf8.UTFMax 95 | if bufend > len(v) { 96 | bufend = len(v) 97 | } 98 | sub := v[i:bufend] 99 | sublen := end - i 100 | 101 | prev := 0 102 | for j := 0; j < sublen; { 103 | if b := sub[j]; b < utf8.RuneSelf { 104 | if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' { 105 | j++ 106 | continue 107 | } 108 | if prev < j { 109 | w.writeString(sub[prev:j]) 110 | } 111 | switch b { 112 | case '\\': 113 | w.writeByte('\\') 114 | w.writeByte('\\') 115 | case '"': 116 | w.writeByte('\\') 117 | w.writeByte('"') 118 | case '\n': 119 | w.writeByte('\\') 120 | w.writeByte('n') 121 | case '\r': 122 | w.writeByte('\\') 123 | w.writeByte('r') 124 | default: 125 | // This encodes bytes < 0x20 except for \n and \r, 126 | // as well as < and >. The latter are escaped because they 127 | // can lead to security holes when user-controlled strings 128 | // are rendered into JSON and served to some browsers. 129 | w.writeByte('\\') 130 | w.writeByte('u') 131 | w.writeByte('0') 132 | w.writeByte('0') 133 | w.writeByte(hex[b>>4]) 134 | w.writeByte(hex[b&0xF]) 135 | } 136 | j++ 137 | prev = j 138 | continue 139 | } 140 | c, size := utf8.DecodeRuneInString(sub[j:]) 141 | if c == utf8.RuneError && size == 1 { 142 | //lint:ignore SA1019 this has been deprecated (TODO: replace) 143 | return &json.InvalidUTF8Error{S: v} 144 | } 145 | j += size 146 | 147 | // If we cross the buffer end then adjust the outer loop 148 | if j > bufsz { 149 | i += j - bufsz 150 | sublen += j - bufsz 151 | } 152 | } 153 | if prev < sublen { 154 | w.writeString(sub[prev:sublen]) 155 | } 156 | } 157 | w.writeByte('"') 158 | return nil 159 | } 160 | 161 | // WriteInt encodes and writes an integer. 162 | func (w *Writer) WriteInt(v int) error { 163 | return w.WriteInt64(int64(v)) 164 | } 165 | 166 | // WriteInt64 encodes and writes a 64-bit integer. 167 | func (w *Writer) WriteInt64(v int64) error { 168 | if err := w.check(); err != nil { 169 | return err 170 | } 171 | 172 | buf := strconv.AppendInt(w.buf[w.pos:w.pos], v, 10) 173 | w.pos += len(buf) 174 | return nil 175 | } 176 | 177 | // WriteUint encodes and writes an unsigned integer. 178 | func (w *Writer) WriteUint(v uint) error { 179 | return w.WriteUint64(uint64(v)) 180 | } 181 | 182 | // WriteUint encodes and writes an unsigned integer. 183 | func (w *Writer) WriteUint64(v uint64) error { 184 | if err := w.check(); err != nil { 185 | return err 186 | } 187 | 188 | buf := strconv.AppendUint(w.buf[w.pos:w.pos], v, 10) 189 | w.pos += len(buf) 190 | return nil 191 | } 192 | 193 | // WriteFloat32 encodes and writes a 32-bit float. 194 | func (w *Writer) WriteFloat32(v float32) error { 195 | if err := w.check(); err != nil { 196 | return err 197 | } 198 | buf := strconv.AppendFloat(w.buf[w.pos:w.pos], float64(v), 'g', -1, 32) 199 | w.pos += len(buf) 200 | return nil 201 | } 202 | 203 | // WriteFloat64 encodes and writes a 64-bit float. 204 | func (w *Writer) WriteFloat64(v float64) error { 205 | if err := w.check(); err != nil { 206 | return err 207 | } 208 | buf := strconv.AppendFloat(w.buf[w.pos:w.pos], v, 'g', -1, 64) 209 | w.pos += len(buf) 210 | return nil 211 | } 212 | 213 | // WriteBool writes a boolean. 214 | func (w *Writer) WriteBool(v bool) error { 215 | if err := w.check(); err != nil { 216 | return err 217 | } 218 | if v { 219 | w.buf[w.pos+0] = 't' 220 | w.buf[w.pos+1] = 'r' 221 | w.buf[w.pos+2] = 'u' 222 | w.buf[w.pos+3] = 'e' 223 | w.pos += 4 224 | } else { 225 | w.buf[w.pos+0] = 'f' 226 | w.buf[w.pos+1] = 'a' 227 | w.buf[w.pos+2] = 'l' 228 | w.buf[w.pos+3] = 's' 229 | w.buf[w.pos+4] = 'e' 230 | w.pos += 5 231 | } 232 | return nil 233 | } 234 | 235 | // WriteNull writes "null". 236 | func (w *Writer) WriteNull() error { 237 | if err := w.check(); err != nil { 238 | return err 239 | } 240 | w.buf[w.pos+0] = 'n' 241 | w.buf[w.pos+1] = 'u' 242 | w.buf[w.pos+2] = 'l' 243 | w.buf[w.pos+3] = 'l' 244 | w.pos += 4 245 | return nil 246 | } 247 | 248 | // WriteMap writes a map. 249 | func (w *Writer) WriteMap(v map[string]interface{}) error { 250 | if err := w.check(); err != nil { 251 | return err 252 | } 253 | 254 | w.buf[w.pos] = '{' 255 | w.pos++ 256 | 257 | var index int 258 | for key, value := range v { 259 | if index > 0 { 260 | w.buf[w.pos] = ',' 261 | w.pos++ 262 | } 263 | 264 | // Write key and colon. 265 | if err := w.WriteString(key); err != nil { 266 | return err 267 | } 268 | w.buf[w.pos] = ':' 269 | w.pos++ 270 | 271 | // Write value. 272 | if value == nil { 273 | if err := w.WriteNull(); err != nil { 274 | return err 275 | } 276 | 277 | } else { 278 | switch value := value.(type) { 279 | case string: 280 | if err := w.WriteString(value); err != nil { 281 | return err 282 | } 283 | case int: 284 | if err := w.WriteInt(value); err != nil { 285 | return err 286 | } 287 | case int64: 288 | if err := w.WriteInt64(value); err != nil { 289 | return err 290 | } 291 | case uint: 292 | if err := w.WriteUint(value); err != nil { 293 | return err 294 | } 295 | case uint64: 296 | if err := w.WriteUint64(value); err != nil { 297 | return err 298 | } 299 | case float32: 300 | if err := w.WriteFloat32(value); err != nil { 301 | return err 302 | } 303 | case float64: 304 | if err := w.WriteFloat64(value); err != nil { 305 | return err 306 | } 307 | case bool: 308 | if err := w.WriteBool(value); err != nil { 309 | return err 310 | } 311 | case map[string]interface{}: 312 | if err := w.WriteMap(value); err != nil { 313 | return err 314 | } 315 | } 316 | } 317 | 318 | index++ 319 | } 320 | 321 | w.buf[w.pos] = '}' 322 | w.pos++ 323 | 324 | return nil 325 | } 326 | -------------------------------------------------------------------------------- /json/writer/writer_test.go: -------------------------------------------------------------------------------- 1 | package writer 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/matryer/is" 9 | ) 10 | 11 | // Ensures that a string can be escaped and encoded. 12 | func TestWriteString(t *testing.T) { 13 | is := is.New(t) 14 | var b bytes.Buffer 15 | w := NewWriter(&b) 16 | w.WriteString("foo\t\n\r\"大") 17 | is.NoErr(w.Flush()) 18 | is.Equal(`"foo\u0009\n\r\"大"`, b.String()) 19 | } 20 | 21 | // Ensures that a large string can be escaped and encoded. 22 | func TestWriteStringLarge(t *testing.T) { 23 | is := is.New(t) 24 | var input, expected string 25 | for i := 0; i < 10000; i++ { 26 | input += "\t" 27 | expected += `\u0009` 28 | } 29 | input += "X" 30 | expected = "\"" + expected + "X\"" 31 | 32 | var b bytes.Buffer 33 | w := NewWriter(&b) 34 | err := w.WriteString(input) 35 | is.NoErr(w.Flush()) 36 | is.NoErr(err) 37 | is.Equal(len(expected), len(b.String())) 38 | if err == nil && len(expected) == len(b.String()) { 39 | is.Equal(expected, b.String()) 40 | } 41 | } 42 | 43 | // Ensures that a large unicode string can be escaped and encoded. 44 | func TestWriteStringLargeUnicode(t *testing.T) { 45 | is := is.New(t) 46 | var input, expected string 47 | for i := 0; i < 10000; i++ { 48 | input += "大" 49 | expected += "大" 50 | } 51 | expected = "\"" + expected + "\"" 52 | 53 | var b bytes.Buffer 54 | w := NewWriter(&b) 55 | err := w.WriteString(input) 56 | is.NoErr(w.Flush()) 57 | is.NoErr(err) 58 | is.Equal(len(expected), len(b.String())) 59 | if err == nil && len(expected) == len(b.String()) { 60 | is.Equal(expected, b.String()) 61 | } 62 | } 63 | 64 | // Ensures that a multiple strings can be encoded sequentially and share the same buffer. 65 | func TestWriteMultipleStrings(t *testing.T) { 66 | is := is.New(t) 67 | var b bytes.Buffer 68 | var expected string 69 | w := NewWriter(&b) 70 | 71 | for i := 0; i < 10000; i++ { 72 | err := w.WriteString("foo\t\n\r\"大\t") 73 | is.NoErr(err) 74 | expected += `"foo\u0009\n\r\"大\u0009"` 75 | } 76 | is.NoErr(w.Flush()) 77 | is.Equal(len(expected), len(b.String())) 78 | if len(expected) == len(b.String()) { 79 | is.Equal(expected, b.String()) 80 | } 81 | } 82 | 83 | // Ensures that a blank string can be encoded. 84 | func TestWriteBlankString(t *testing.T) { 85 | is := is.New(t) 86 | var b bytes.Buffer 87 | w := NewWriter(&b) 88 | w.WriteString("") 89 | is.NoErr(w.Flush()) 90 | is.Equal(b.String(), `""`) 91 | } 92 | 93 | func BenchmarkWriteRawBytes(b *testing.B) { 94 | s := "hello, world" 95 | var w bytes.Buffer 96 | for i := 0; i < b.N; i++ { 97 | if _, err := w.Write([]byte(s)); err != nil { 98 | b.Fatal("WriteRawBytes:", err) 99 | } 100 | } 101 | b.SetBytes(int64(len(s))) 102 | } 103 | 104 | func BenchmarkWriteString(b *testing.B) { 105 | var buf bytes.Buffer 106 | w := NewWriter(&buf) 107 | s := "hello, world" 108 | for i := 0; i < b.N; i++ { 109 | if err := w.WriteString(s); err != nil { 110 | b.Fatal("WriteString:", err) 111 | } 112 | } 113 | w.Flush() 114 | 115 | b.SetBytes(int64(len(s))) 116 | } 117 | 118 | // Ensures that an int can be written. 119 | func TestWriteInt(t *testing.T) { 120 | is := is.New(t) 121 | var b bytes.Buffer 122 | w := NewWriter(&b) 123 | is.NoErr(w.WriteInt(-100)) 124 | is.NoErr(w.Flush()) 125 | is.Equal(b.String(), `-100`) 126 | } 127 | 128 | // Ensures that a uint can be written. 129 | func TestWriteUint(t *testing.T) { 130 | is := is.New(t) 131 | var b bytes.Buffer 132 | w := NewWriter(&b) 133 | is.NoErr(w.WriteUint(uint(1230928137))) 134 | is.NoErr(w.Flush()) 135 | is.Equal(b.String(), `1230928137`) 136 | } 137 | 138 | func BenchmarkWriteInt(b *testing.B) { 139 | var buf bytes.Buffer 140 | w := NewWriter(&buf) 141 | v := -3 142 | for i := 0; i < b.N; i++ { 143 | if err := w.WriteInt(v); err != nil { 144 | b.Fatal("WriteInt:", err) 145 | } 146 | } 147 | w.Flush() 148 | b.SetBytes(int64(len("-3"))) 149 | } 150 | 151 | func BenchmarkWriteUint(b *testing.B) { 152 | var buf bytes.Buffer 153 | w := NewWriter(&buf) 154 | v := uint(30) 155 | for i := 0; i < b.N; i++ { 156 | if err := w.WriteUint(v); err != nil { 157 | b.Fatal("WriteUint:", err) 158 | } 159 | } 160 | b.SetBytes(int64(len("30"))) 161 | } 162 | 163 | // Ensures that a float32 can be written. 164 | func TestWriteFloat32(t *testing.T) { 165 | is := is.New(t) 166 | var b bytes.Buffer 167 | w := NewWriter(&b) 168 | is.NoErr(w.WriteFloat32(float32(2319.1921))) 169 | is.NoErr(w.Flush()) 170 | is.Equal(b.String(), `2319.1921`) 171 | } 172 | 173 | // Ensures that a float64 can be written. 174 | func TestWriteFloat64(t *testing.T) { 175 | is := is.New(t) 176 | var b bytes.Buffer 177 | w := NewWriter(&b) 178 | is.NoErr(w.WriteFloat64(2319123.1921918273)) 179 | is.NoErr(w.Flush()) 180 | is.Equal(b.String(), `2.319123192191827e+06`) 181 | } 182 | 183 | // Ensures that a simple map can be written. 184 | // func TestWriteSimpleMap(t *testing.T) { 185 | // is := is.New(t) 186 | // var b bytes.Buffer 187 | // w := NewWriter(&b) 188 | // m := map[string]interface{}{ 189 | // "foo": "bar", 190 | // "bat": "baz", 191 | // } 192 | // is.NoErr(w.WriteMap(m)) 193 | // is.NoErr(w.Flush()) 194 | // if b.String() != `{"bat":"baz","foo":"bar"}` { 195 | // t.Fatal("Invalid map encoding:", b.String()) 196 | // } 197 | // if b.String() != `{"bat":"baz","foo":"bar"}` { 198 | // t.Fatal("Invalid map encoding:", b.String()) 199 | // } 200 | // } 201 | 202 | // Ensures that a more complex map can be written. 203 | func TestWriteMap(t *testing.T) { 204 | is := is.New(t) 205 | var b bytes.Buffer 206 | w := NewWriter(&b) 207 | m := map[string]interface{}{ 208 | "stringx": "foo", 209 | "intx": 100, 210 | "int64x": int64(1023), 211 | "uintx": uint(100), 212 | "uint64x": uint64(1023), 213 | "float32x": float32(312.311), 214 | "float64x": float64(812731.19812), 215 | "truex": true, 216 | "falsex": false, 217 | "nullx": nil, 218 | } 219 | is.NoErr(w.WriteMap(m)) 220 | is.NoErr(w.Flush()) 221 | is.True(strings.Contains(b.String(), `"intx":100`)) 222 | is.True(strings.Contains(b.String(), `"int64x":1023`)) 223 | is.True(strings.Contains(b.String(), `"uint64x":1023`)) 224 | is.True(strings.Contains(b.String(), `"float32x":312.311`)) 225 | is.True(strings.Contains(b.String(), `"float64x":812731.19812`)) 226 | is.True(strings.Contains(b.String(), `"falsex":false`)) 227 | is.True(strings.Contains(b.String(), `"nullx":null`)) 228 | is.True(strings.Contains(b.String(), `"falsex":false`)) 229 | is.True(strings.Contains(b.String(), `"stringx":"foo"`)) 230 | is.True(strings.Contains(b.String(), `"uintx":100`)) 231 | is.True(strings.Contains(b.String(), `"truex":true`)) 232 | } 233 | 234 | // Ensures that a nested map can be written. 235 | func TestWriteNestedMap(t *testing.T) { 236 | is := is.New(t) 237 | var b bytes.Buffer 238 | w := NewWriter(&b) 239 | m := map[string]interface{}{ 240 | "foo": map[string]interface{}{"bar": "bat"}, 241 | } 242 | is.NoErr(w.WriteMap(m)) 243 | is.NoErr(w.Flush()) 244 | is.Equal(b.String(), `{"foo":{"bar":"bat"}}`) 245 | } 246 | 247 | func BenchmarkWriteFloat32(b *testing.B) { 248 | var buf bytes.Buffer 249 | w := NewWriter(&buf) 250 | v := float32(2319.1921) 251 | for i := 0; i < b.N; i++ { 252 | if err := w.WriteFloat32(v); err != nil { 253 | b.Fatal("WriteFloat32:", err) 254 | } 255 | } 256 | w.Flush() 257 | b.SetBytes(int64(len("2319.1921"))) 258 | } 259 | 260 | func BenchmarkWriteFloat64(b *testing.B) { 261 | var buf bytes.Buffer 262 | w := NewWriter(&buf) 263 | v := 2319123.1921918273 264 | for i := 0; i < b.N; i++ { 265 | if err := w.WriteFloat64(v); err != nil { 266 | b.Fatal("WriteFloat64:", err) 267 | } 268 | } 269 | w.Flush() 270 | b.SetBytes(int64(len(`2.319123192191827e+06`))) 271 | } 272 | 273 | // Ensures that a single byte can be written to the writer. 274 | func TestWriteByte(t *testing.T) { 275 | is := is.New(t) 276 | var b bytes.Buffer 277 | w := NewWriter(&b) 278 | is.NoErr(w.WriteByte(':')) 279 | is.NoErr(w.Flush()) 280 | is.Equal(b.String(), `:`) 281 | } 282 | 283 | // Ensures that a true boolean value can be written. 284 | func TestWriteTrue(t *testing.T) { 285 | is := is.New(t) 286 | var b bytes.Buffer 287 | w := NewWriter(&b) 288 | is.NoErr(w.WriteBool(true)) 289 | is.NoErr(w.Flush()) 290 | is.Equal(b.String(), `true`) 291 | } 292 | 293 | // Ensures that a false boolean value can be written. 294 | func TestWriteFalse(t *testing.T) { 295 | is := is.New(t) 296 | var b bytes.Buffer 297 | w := NewWriter(&b) 298 | is.NoErr(w.WriteBool(false)) 299 | is.NoErr(w.Flush()) 300 | is.Equal(b.String(), `false`) 301 | } 302 | 303 | func BenchmarkWriteBool(b *testing.B) { 304 | var buf bytes.Buffer 305 | w := NewWriter(&buf) 306 | for i := 0; i < b.N; i++ { 307 | if err := w.WriteBool(true); err != nil { 308 | b.Fatal("WriteBool:", err) 309 | } 310 | } 311 | w.Flush() 312 | b.SetBytes(int64(len(`true`))) 313 | } 314 | 315 | // Ensures that a null value can be written. 316 | func TestWriteNull(t *testing.T) { 317 | is := is.New(t) 318 | var b bytes.Buffer 319 | w := NewWriter(&b) 320 | is.NoErr(w.WriteNull()) 321 | is.NoErr(w.Flush()) 322 | is.Equal(b.String(), `null`) 323 | } 324 | 325 | func BenchmarkWriteNull(b *testing.B) { 326 | var buf bytes.Buffer 327 | w := NewWriter(&buf) 328 | for i := 0; i < b.N; i++ { 329 | if err := w.WriteNull(); err != nil { 330 | b.Fatal("WriteNull:", err) 331 | } 332 | } 333 | w.Flush() 334 | b.SetBytes(int64(len(`true`))) 335 | } 336 | --------------------------------------------------------------------------------