├── test ├── slice.go ├── demostruct.go ├── typed.go ├── typedslice.go ├── broken │ ├── nested.go │ └── nested2.go ├── embedded.go ├── Makefile ├── README ├── embedded2.go └── gen_test.go ├── Makefile ├── src ├── bi │ └── bi.go └── binidl │ └── binidl.go └── README.md /test/slice.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | type Sliced struct { 4 | A int 5 | B []int8 6 | } 7 | -------------------------------------------------------------------------------- /test/demostruct.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | type Demostruct struct { 4 | A int64 5 | B int32 6 | C [4]int16 7 | } 8 | -------------------------------------------------------------------------------- /test/typed.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | type Value int64 4 | 5 | type Something struct { 6 | V Value 7 | Q int32 8 | } 9 | -------------------------------------------------------------------------------- /test/typedslice.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | type SliceValue int64 4 | 5 | type HasASlice struct { 6 | V []SliceValue 7 | Q int32 8 | } 9 | -------------------------------------------------------------------------------- /test/broken/nested.go: -------------------------------------------------------------------------------- 1 | package nestedexample 2 | 3 | type Nested struct { 4 | A int 5 | C int 6 | B nest.Sometype 7 | D int 8 | E int 9 | } 10 | -------------------------------------------------------------------------------- /test/broken/nested2.go: -------------------------------------------------------------------------------- 1 | package nestedexample 2 | 3 | type Nested struct { 4 | A int 5 | C int 6 | B nest.Sometype 7 | B2 nest.Sometype 8 | D int 9 | E int 10 | } 11 | -------------------------------------------------------------------------------- /test/embedded.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | type IsEmbedded struct { 4 | X int 5 | Y int 6 | } 7 | 8 | type HasEmbedded struct { 9 | A int 10 | B int 11 | C IsEmbedded 12 | } 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all : src/bi/bi 2 | 3 | clean: 4 | rm src/bi/bi 5 | 6 | src/bi/bi: src/bi/bi.go src/binidl/binidl.go 7 | go tool fix $^ 8 | go tool vet $^ 9 | gofmt -s -w $^ 10 | (cd src ; GOPATH=$$PWD/.. go build -o bi/bi bi/bi.go) 11 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | GEN='../bin/bi' 2 | all: 3 | $(GEN) demostruct.go > demostruct_gen.go 4 | $(GEN) embedded.go > embedded_gen.go 5 | $(GEN) embedded2.go > embedded2_gen.go 6 | $(GEN) slice.go > slice_gen.go 7 | 8 | clean: 9 | /bin/rm *_gen.go 10 | 11 | -------------------------------------------------------------------------------- /test/README: -------------------------------------------------------------------------------- 1 | To run tests: 2 | 3 | make clean 4 | make 5 | go test 6 | 7 | (This is a little awkward because the code gen has to be run 8 | first in order to run the tests. 9 | 10 | Note that errors may arise when you run make *or* when you run 11 | go test!) 12 | -------------------------------------------------------------------------------- /test/embedded2.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | type IsEmbedded2 struct { 4 | X int 5 | Z AlsoEmbedded2 6 | Y int 7 | } 8 | 9 | type AlsoEmbedded2 struct { 10 | X int 11 | Y int 12 | } 13 | 14 | type HasEmbedded2 struct { 15 | A int 16 | B int 17 | C IsEmbedded2 18 | } 19 | -------------------------------------------------------------------------------- /src/bi/bi.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "binidl" 5 | "flag" 6 | "fmt" 7 | "os" 8 | ) 9 | 10 | func usage() { 11 | fmt.Println("usage: bi [-B] ") 12 | } 13 | 14 | var bigEndian *bool = flag.Bool("B", false, "Use big endian encoding (default: little)") 15 | 16 | func main() { 17 | flag.Parse() 18 | 19 | if flag.NArg() < 1 { 20 | usage() 21 | os.Exit(-1) 22 | } 23 | 24 | bi := binidl.NewBinidl(flag.Arg(0), *bigEndian) 25 | bi.PrintGo() 26 | } 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a work-in-progress stub generator for marshaling/unmarshaling go structs into the encoding/binary format. It handles many basic types, but if you get fancy, you'll break it. It does not handle strings. 2 | 3 | The reason this code exists, for a simple, statically-sized struct: 4 | 5 | ``` 6 | BenchmarkReflectionMarshal 1000000 2016 ns/op 7 | BenchmarkGeneratedMarshal 10000000 223 ns/op 8 | ``` 9 | 10 | Operation: Takes a file with go structs as input: 11 | 12 | ```go 13 | package duck 14 | 15 | type Quack struct { 16 | X int 17 | Y int 18 | } 19 | ``` 20 | 21 | Run as: 22 | 23 | ```sh 24 | bin/bi duck_decl.go > duck_marshal.go 25 | ``` 26 | and outputs go code that does the same thing that binary.Write would do, but faster: 27 | 28 | ```go 29 | package duck 30 | 31 | func (t *Quack) Marshal(w io.Writer) { 32 | var b [16]byte 33 | binary.LittleEndian.PutUint64(b[0:8], uint64(t.X)) 34 | binary.LittleEndian.PutUint64(b[8:16], uint64(t.Y)) 35 | w.Write(b[:]) 36 | } 37 | ``` 38 | You can use these stubs in your own code: 39 | 40 | ```go 41 | q := &Quack{X: 1, Y: 2} 42 | buf := new bytes.Buffer 43 | if err := q.Marshal(buf) { 44 | fmt.Println("Could not marshal data: ", err) 45 | // handle error appropriately, return, etc. 46 | } 47 | fmt.Println("Marshaled data: ", buf) 48 | 49 | q2 := &Quack{} 50 | if err := q2.Unmarshal(buf) { 51 | fmt.Println("Could not unmarshal buf: ", err) 52 | // handle error here 53 | } 54 | fmt.Println("q2: ", q2) 55 | ``` 56 | In addition to standard encoding/binary formats, gobin-codegen will output code to handle variable-length slice data within structs if you ask it to. It does so by first encoding the length of the slice as a varint, and then writing the members of the slice. Such a struct is not compatible with the standard encoding/binary, but will work if you know both sides use gobin-codegen. In this way, gobin-codegen can marshal []byte and other variable length data types. 57 | 58 | This is not production-quality code. Its optimizations are limited to small completely static structs - otherwise, the code it outputs will be both longer and slower than that shown above, but probably still 4x faster than the reflection-based marshaling. 59 | Generates code to handle marshaling and unmarshaling to/from 60 | encoding/binary. 61 | 62 | It probably isn't nearly as general as you'd like it to be. 63 | Don't depend on it working properly. Don't think that it 64 | handles corner cases well. You get the idea. :) 65 | 66 | NOTE: This library emits code for variable-length slices 67 | by prefixing them with a varint length. This is a departure 68 | from the encoding/binary format, which cannot handle variable 69 | length slices. If you use a slice type, your binary output 70 | will not be compatible with stock encoding/binary any more. 71 | 72 | use: 73 | ```sh 74 | export GOPATH=`/bin/pwd` 75 | go install bi 76 | bin/bi /path/to/your/go/struct/decl.go > generated_code.go 77 | ``` 78 | 79 | use generated_code.go as you see fit. 80 | -------------------------------------------------------------------------------- /test/gen_test.go: -------------------------------------------------------------------------------- 1 | package encodedemo 2 | 3 | import ( 4 | "testing" 5 | "bytes" 6 | "fmt" 7 | "encoding/binary" 8 | "encoding/gob" 9 | ) 10 | 11 | var d *Demostruct = &Demostruct{1, 2, [4]int16{9, 9, 9, 9}} 12 | var e *Demostruct = &Demostruct{0, 0, [4]int16{0, 0, 0, 0}} 13 | var buf *bytes.Buffer = new(bytes.Buffer) 14 | 15 | func TestE(t *testing.T) { 16 | buf.Reset() 17 | binary.Write(buf, binary.LittleEndian, d) 18 | fmt.Printf("bin: % x\n", buf.Bytes()) 19 | buf.Reset() 20 | d.Marshal(buf) 21 | fmt.Printf("gen: % x\n", buf.Bytes()) 22 | fmt.Println("") 23 | } 24 | 25 | func TestD(t *testing.T) { 26 | buf.Reset() 27 | binary.Write(buf, binary.LittleEndian, d) 28 | e.Unmarshal(buf) 29 | fmt.Println("Demostruct: ", e) 30 | } 31 | 32 | var s *Sliced = &Sliced{1, []int8{2, 3, 4}} 33 | func TestSliced(t *testing.T) { 34 | buf.Reset() 35 | s.Marshal(buf) 36 | fmt.Print("Marshaled: ", s, " into ") 37 | fmt.Printf("% x\n", buf.Bytes()) 38 | } 39 | func TestSliceUnmarshal(t *testing.T) { 40 | buf.Reset() 41 | s.Marshal(buf) 42 | s2 := &Sliced{0, nil} 43 | s2.Unmarshal(buf) 44 | fmt.Println("Unmarshaled: ", s2) 45 | } 46 | 47 | func TestEmbedded(t *testing.T) { 48 | x := &HasEmbedded{1, 2, IsEmbedded{3, 4}} 49 | buf.Reset() 50 | x.Marshal(buf) 51 | y := &HasEmbedded{} 52 | y.Unmarshal(buf) 53 | if (y.A != 1 || y.B != 2 || y.C.X != 3 || y.C.Y != 4) { 54 | t.Fatalf("Embedded struct test failed") 55 | } 56 | } 57 | 58 | 59 | func TestEmbedded2(t *testing.T) { 60 | x := HasEmbedded2{1, 2, IsEmbedded2{3, 61 | AlsoEmbedded2{4, 5}, 6}} 62 | buf.Reset() 63 | x.Marshal(buf) 64 | y := &HasEmbedded2{} 65 | y.Unmarshal(buf) 66 | if (x.A != y.A || x.B != y.B || x.C.X != y.C.X || 67 | x.C.Z.X != y.C.Z.X || x.C.Z.Y != y.C.Z.Y || 68 | x.C.Y != y.C.Y) { 69 | t.Fatalf("Structures not the same: ", x, y) 70 | } 71 | } 72 | 73 | func TestCached(t *testing.T) { 74 | buf.Reset() 75 | s.Marshal(buf) 76 | sc := &SlicedCache{} 77 | s2 := sc.Get() 78 | if s2 == nil { 79 | t.Fatalf("Eek - nil result from getting from object cache") 80 | } 81 | s2.Unmarshal(buf) 82 | sc.Put(s2) 83 | s3 := sc.Get() 84 | if s3 == nil { 85 | t.Fatalf("Eek - nil result on second get from object cache") 86 | } 87 | s3.Unmarshal(buf) 88 | } 89 | func BenchmarkReflectionMarshal(b *testing.B) { 90 | for i := 0; i < b.N; i++ { 91 | buf.Reset() 92 | binary.Write(buf, binary.LittleEndian, d) 93 | } 94 | } 95 | 96 | func BenchmarkGeneratedMarshal(b *testing.B) { 97 | for i := 0; i < b.N; i++ { 98 | buf.Reset() 99 | d.Marshal(buf) 100 | } 101 | } 102 | 103 | func BenchmarkGobMarshal(b *testing.B) { 104 | // Let's give gobs the benefit of the doubt here. 105 | enc := gob.NewEncoder(buf) 106 | buf.Reset() 107 | enc.Encode(d) 108 | b.ResetTimer() 109 | for i := 0; i < b.N; i++ { 110 | buf.Reset() 111 | enc.Encode(d) 112 | } 113 | } 114 | 115 | 116 | func BenchmarkReflectionUnmarshal(b *testing.B) { 117 | buf.Reset() 118 | d.Marshal(buf) 119 | by := buf.Bytes() 120 | buf2 := &bytes.Buffer{} 121 | for i := 0; i < b.N; i++ { 122 | buf2.Reset() 123 | buf2.Write(by) 124 | binary.Read(buf2, binary.LittleEndian, e) 125 | } 126 | } 127 | 128 | func BenchmarkGeneratedUnmarshal(b *testing.B) { 129 | buf.Reset() 130 | d.Marshal(buf) 131 | by := buf.Bytes() 132 | buf2 := &bytes.Buffer{} 133 | for i := 0; i < b.N; i++ { 134 | buf2.Reset() 135 | buf2.Write(by) 136 | e.Unmarshal(buf2) 137 | } 138 | } 139 | 140 | func BenchmarkGeneratedUnmarshalNew(b *testing.B) { 141 | buf.Reset() 142 | d.Marshal(buf) 143 | by := buf.Bytes() 144 | buf2 := &bytes.Buffer{} 145 | for i := 0; i < b.N; i++ { 146 | buf2.Reset() 147 | buf2.Write(by) 148 | e := &Demostruct{} 149 | e.Unmarshal(buf2) 150 | } 151 | } 152 | 153 | func BenchmarkGeneratedUnmarshalCached(b *testing.B) { 154 | c := &DemostructCache{} 155 | buf.Reset() 156 | d.Marshal(buf) 157 | by := buf.Bytes() 158 | buf2 := &bytes.Buffer{} 159 | for i := 0; i < b.N; i++ { 160 | buf2.Reset() 161 | buf2.Write(by) 162 | e := c.Get() 163 | e.Unmarshal(buf2) 164 | c.Put(e) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/binidl/binidl.go: -------------------------------------------------------------------------------- 1 | package binidl 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "go/ast" 7 | "go/parser" 8 | "go/printer" 9 | "go/token" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "strconv" 14 | "strings" 15 | ) 16 | 17 | type Binidl struct { 18 | ast *ast.File 19 | fset *token.FileSet 20 | bigEndian bool 21 | } 22 | 23 | const ( 24 | STATICMAX = 64 // Max size of an object for which we marshal into a stack-allocated local buffer 25 | ) 26 | 27 | func NewBinidl(filename string, bigEndian bool) *Binidl { 28 | fset := token.NewFileSet() 29 | ast, err := parser.ParseFile(fset, filename, nil, 0) // scanner.InsertSemis) 30 | if err != nil { 31 | fmt.Println("Error parsing", filename, ":", err) 32 | return nil 33 | } 34 | return &Binidl{ast, fset, bigEndian} 35 | } 36 | 37 | func setbs(b io.Writer, n int, es *EmitState) { 38 | if n != es.curBSize { 39 | fmt.Fprintf(b, "bs = b[:%d]\n", n) 40 | } 41 | es.curBSize = n 42 | } 43 | 44 | func unmarshalField(b io.Writer, fname, tname string, es *EmitState) { 45 | tconv := tname 46 | if mapped, ok := typemap[tname]; ok { 47 | tconv = mapped 48 | } 49 | 50 | ti, ok := typedb[tconv] 51 | if !ok { 52 | fmt.Fprintf(b, "%s.Unmarshal(wire)\n", fname) 53 | return 54 | } 55 | 56 | bstart := 0 57 | source := "bs" 58 | if es.resetBuffer { 59 | setbs(b, ti.Size, es) 60 | bstart = 0 61 | fmt.Fprintf(b, "if _, err := io.ReadAtLeast(wire, bs, %d); err != nil {\n", ti.Size) 62 | fmt.Fprintf(b, " return err\n") 63 | fmt.Fprintf(b, "}\n") 64 | } else { 65 | bstart = es.Bstart(ti.Size) 66 | if es.contiguous[es.crt] > 0 && bstart == 0 { 67 | setbs(b, es.contiguous[es.crt], es) 68 | fmt.Fprintf(b, "if _, err := io.ReadAtLeast(wire, bs, %d); err != nil {\n", es.contiguous[es.crt]) 69 | fmt.Fprintf(b, " return err\n") 70 | fmt.Fprintf(b, "}\n") 71 | } 72 | } 73 | 74 | if ildf, found := inlineDecode[ti.EncodesAs]; found { 75 | ild := ildf(source, bstart, es) 76 | fmt.Fprintf(b, "%s = %s(%s)\n", fname, tname, ild) 77 | } else { 78 | need_binary = true 79 | endian := "Little" 80 | if es.bigEndian { 81 | endian = "Big" 82 | } 83 | df := fmt.Sprintf(decodeFunc[ti.EncodesAs], endian) 84 | if es.resetBuffer { 85 | fmt.Fprintf(b, "%s = %s(%s(bs))\n", fname, tname, df) 86 | } else { 87 | fmt.Fprintf(b, "%s = %s(%s(b[%d:%d]))\n", fname, tname, df, bstart, bstart+ti.Size) 88 | } 89 | } 90 | if es.contiguous[es.crt] == es.staticOffset && es.staticOffset > 0 { 91 | es.crt++ 92 | es.staticOffset = 0 93 | } 94 | } 95 | 96 | func marshalField(b io.Writer, fname, tname string, es *EmitState) { 97 | if mapped, ok := typemap[tname]; ok { 98 | tname = mapped 99 | } 100 | ti, ok := typedb[tname] 101 | if !ok { 102 | fmt.Fprintf(b, "%s.Marshal(wire)\n", fname) 103 | return 104 | } 105 | 106 | encodefrom := "bs" 107 | bstart := 0 108 | if es.resetBuffer { 109 | setbs(b, ti.Size, es) 110 | bstart = 0 111 | } else { 112 | bstart = es.Bstart(ti.Size) 113 | if es.contiguous[es.crt] > 0 && bstart == 0 { 114 | setbs(b, es.contiguous[es.crt], es) 115 | } 116 | } 117 | 118 | ilef, found := inlineEncode[ti.EncodesAs] 119 | if found { 120 | fmt.Fprintf(b, "%s\n", ilef(encodefrom, bstart, fname, es)) 121 | } else { 122 | need_binary = true 123 | endian := "Little" 124 | if es.bigEndian { 125 | endian = "Big" 126 | } 127 | ef := fmt.Sprintf(encodeFunc[ti.EncodesAs], endian) 128 | if es.resetBuffer { 129 | fmt.Fprintf(b, "%s(bs, %s(%s))\n", ef, ti.EncodesAs, fname) 130 | } else { 131 | bend := bstart + ti.Size 132 | fmt.Fprintf(b, "%s(b[%d:%d], %s(%s))\n", ef, bstart, bend, ti.EncodesAs, fname) 133 | } 134 | } 135 | if es.resetBuffer || (es.contiguous[es.crt] == es.staticOffset && es.staticOffset > 0) { 136 | fmt.Fprintln(b, "wire.Write(bs)") 137 | } 138 | if es.contiguous[es.crt] == es.staticOffset && es.staticOffset > 0 { 139 | es.crt++ 140 | es.staticOffset = 0 141 | } 142 | } 143 | 144 | func walkContents(b io.Writer, st *ast.StructType, pred string, funcname string, fn func(io.Writer, string, string, *EmitState), es *EmitState) { 145 | for _, f := range st.Fields.List { 146 | for _, fNameEnt := range f.Names { 147 | newpred := pred + "." + fNameEnt.Name 148 | walkOne(b, f, newpred, funcname, fn, es) 149 | } 150 | } 151 | } 152 | 153 | const ( 154 | MARSHAL = iota 155 | UNMARSHAL 156 | ) 157 | 158 | type EmitState struct { 159 | op int // MARSHAL, UNMARSHAL 160 | nextIdx int 161 | staticOffset int 162 | alenIdx int 163 | curBSize int 164 | blen int 165 | bigEndian bool // TODO: This is duplicated now... integrate better. 166 | tmp32exists bool 167 | tmp64exists bool 168 | contiguous []int 169 | crt int 170 | resetBuffer bool 171 | } 172 | 173 | func (es *EmitState) getNewAlen() string { 174 | es.alenIdx++ 175 | return fmt.Sprintf("alen%d", es.alenIdx) 176 | } 177 | 178 | func (es *EmitState) getIndexStr() string { 179 | indexes := []string{"i", "j", "k"} 180 | repeats := (es.nextIdx / len(indexes)) + 1 181 | pos := es.nextIdx % len(indexes) 182 | es.nextIdx++ 183 | return strings.Repeat(indexes[pos], repeats) 184 | } 185 | 186 | func (es *EmitState) freeIndexStr() { 187 | es.nextIdx-- 188 | } 189 | 190 | func (es *EmitState) Bstart(n int) int { 191 | o := es.staticOffset 192 | es.staticOffset += n 193 | return o 194 | } 195 | 196 | var need_bufio = false 197 | var need_binary = false 198 | 199 | var typemap map[string]string = make(map[string]string) 200 | 201 | type TypeInfo struct { 202 | Name string 203 | Size int 204 | EncodesAs string 205 | } 206 | 207 | var encodeFunc map[string]string = map[string]string{ 208 | "uint64": "binary.%sEndian.PutUint64", 209 | "uint32": "binary.%sEndian.PutUint32", 210 | "uint16": "binary.%sEndian.PutUint16", 211 | } 212 | 213 | type encodefunc func(string, int, string, *EmitState) string 214 | 215 | func ilByteOut(b string, offset int, target string, es *EmitState) string { 216 | return fmt.Sprintf("%s[%d] = byte(%s)", b, offset, target) 217 | } 218 | 219 | func ilUint16Out(b string, offset int, target string, es *EmitState) string { 220 | if !es.bigEndian { 221 | return fmt.Sprintf("%s[%d] = byte(%s)\n%s[%d] = byte(%s >> 8)", 222 | b, offset, target, b, offset+1, target) 223 | } 224 | return fmt.Sprintf("%s[%d] = byte(%s >> 8)\n%s[%d] = byte(%s)", 225 | b, offset, target, b, offset+1, target) 226 | } 227 | 228 | func ilUint32Out(b string, offset int, target string, es *EmitState) string { 229 | tmp32 := "" 230 | if !es.tmp32exists { 231 | tmp32 = fmt.Sprintf("tmp32 := %s\n", target) 232 | es.tmp32exists = true 233 | } else { 234 | tmp32 = fmt.Sprintf("tmp32 = %s\n", target) 235 | } 236 | target = "tmp32" 237 | if !es.bigEndian { 238 | return fmt.Sprintf("%s%s[%d] = byte(%s)\n%s[%d] = byte(%s >> 8)\n%s[%d] = byte(%s >> 16)\n%s[%d] = byte(%s >> 24)", 239 | tmp32, b, offset, target, b, offset+1, target, b, offset+2, target, b, offset+3, target) 240 | } 241 | return fmt.Sprintf("%s%s[%d] = byte(%s >> 24)\n%s[%d] = byte(%s >> 16)\n%s[%d] = byte(%s >> 8)\n%s[%d] = byte(%s)", 242 | tmp32, b, offset, target, b, offset+1, target, b, offset+2, target, b, offset+3, target) 243 | } 244 | 245 | func ilUint64Out(b string, offset int, target string, es *EmitState) string { 246 | tmp64 := "" 247 | if !es.tmp64exists { 248 | tmp64 = fmt.Sprintf("tmp64 := %s\n", target) 249 | es.tmp64exists = true 250 | } else { 251 | tmp64 = fmt.Sprintf("tmp64 = %s\n", target) 252 | } 253 | target = "tmp64" 254 | if !es.bigEndian { 255 | return fmt.Sprintf("%s%s[%d] = byte(%s)\n%s[%d] = byte(%s >> 8)\n%s[%d] = byte(%s >> 16)\n%s[%d] = byte(%s >> 24)\n%s[%d] = byte(%s >> 32)\n%s[%d] = byte(%s >> 40)\n%s[%d] = byte(%s >> 48)\n%s[%d] = byte(%s >> 56)", 256 | tmp64, b, offset, target, b, offset+1, target, b, offset+2, target, b, offset+3, target, b, offset+4, target, b, offset+5, target, b, offset+6, target, b, offset+7, target) 257 | } 258 | return fmt.Sprintf("%s%s[%d] = byte(%s >> 56)\n%s[%d] = byte(%s >> 48)\n%s[%d] = byte(%s >> 40)\n%s[%d] = byte(%s >> 32)\n%s[%d] = byte(%s >> 24)\n%s[%d] = byte(%s >> 16)\n%s[%d] = byte(%s >> 8)\n%s[%d] = byte(%s)", 259 | tmp64, b, offset, target, b, offset+1, target, b, offset+2, target, b, offset+3, target, b, offset+4, target, b, offset+5, target, b, offset+6, target, b, offset+7, target) 260 | } 261 | 262 | 263 | var inlineEncode map[string]encodefunc = map[string]encodefunc{ 264 | "byte": ilByteOut, 265 | "uint16": ilUint16Out, 266 | "uint32": ilUint32Out, 267 | "uint64": ilUint64Out, 268 | } 269 | 270 | type decodefunc func(string, int, *EmitState) string 271 | 272 | func ilByte(b string, offset int, es *EmitState) string { 273 | return fmt.Sprintf("%s[%d]", b, offset) 274 | } 275 | 276 | func ilUint16(b string, offset int, es *EmitState) string { 277 | if es.bigEndian { 278 | return fmt.Sprintf("((uint16(%s[%d]) << 8) | uint16(%s[%d]))", b, offset, b, offset+1) 279 | } 280 | return fmt.Sprintf("(uint16(%s[%d]) | uint16(%s[%d]) << 8)", b, offset, b, offset+1) 281 | } 282 | 283 | func ilUint32(b string, offset int, es *EmitState) string { 284 | if es.bigEndian { 285 | return fmt.Sprintf("((uint32(%s[%d]) << 24) | (uint32(%s[%d]) << 16) | (uint32(%s[%d]) << 8) | uint32(%s[%d]))", b, offset, b, offset+1, b, offset+2, b, offset+3) 286 | } 287 | return fmt.Sprintf("(uint32(%s[%d]) | (uint32(%s[%d]) << 8) | (uint32(%s[%d]) << 16) | (uint32(%s[%d]) << 24))", b, offset, b, offset+1, b, offset+2, b, offset+3) 288 | } 289 | 290 | func ilUint64(b string, offset int, es *EmitState) string { 291 | if es.bigEndian { 292 | return fmt.Sprintf("((uint64(%s[%d]) << 56) | (uint64(%s[%d]) << 48) | (uint64(%s[%d]) << 40) | (uint64(%s[%d]) << 32) | (uint64(%s[%d]) << 24) | (uint64(%s[%d]) << 16) | (uint64(%s[%d]) << 8) | uint64(%s[%d]))", b, offset, b, offset+1, b, offset+2, b, offset+3, b, offset+4, b, offset+5, b, offset+6, b, offset+7) 293 | } 294 | return fmt.Sprintf("(uint64(%s[%d]) | (uint64(%s[%d]) << 8) | (uint64(%s[%d]) << 16) | (uint64(%s[%d]) << 24) | (uint64(%s[%d]) << 32) | (uint64(%s[%d]) << 40) | (uint64(%s[%d]) << 48) | (uint64(%s[%d]) << 56))", b, offset, b, offset+1, b, offset+2, b, offset+3, b, offset+4, b, offset+5, b, offset+6, b, offset+7) 295 | } 296 | 297 | 298 | var inlineDecode map[string]decodefunc = map[string]decodefunc{ 299 | "byte": ilByte, 300 | "uint16": ilUint16, 301 | "uint32": ilUint32, 302 | "uint64": ilUint64, 303 | } 304 | 305 | var decodeFunc map[string]string = map[string]string{ 306 | "uint64": "binary.%sEndian.Uint64", 307 | "uint32": "binary.%sEndian.Uint32", 308 | "uint16": "binary.%sEndian.Uint16", 309 | } 310 | 311 | var typedb map[string]TypeInfo = map[string]TypeInfo{ 312 | "int": {"int", 8, "uint64"}, 313 | "uint64": {"uint64", 8, "uint64"}, 314 | "int64": {"int64", 8, "uint64"}, 315 | "int32": {"int32", 4, "uint32"}, 316 | "uint32": {"uint32", 4, "uint32"}, 317 | "int16": {"int16", 2, "uint16"}, 318 | "uint16": {"uint16", 2, "uint16"}, 319 | "int8": {"int8", 1, "byte"}, 320 | "uint8": {"uint8", 1, "byte"}, 321 | "byte": {"byte", 1, "byte"}, 322 | } 323 | 324 | func walkOne(b io.Writer, f *ast.Field, pred string, funcname string, fn func(io.Writer, string, string, *EmitState), es *EmitState) { 325 | switch f.Type.(type) { 326 | case *ast.Ident: 327 | t := f.Type.(*ast.Ident) 328 | _, is_mapped := typemap[t.Name] 329 | _, simple := simpleStructMap[t.Name] 330 | 331 | if dispatchTo, ok := globalDeclMap[t.Name]; !is_mapped && ok && simple { 332 | if strucType, ok := dispatchTo.Type.(*ast.StructType); ok { 333 | walkContents(b, strucType, pred, funcname, fn, es) 334 | } else { 335 | panic("Eek, a type I don't handle properly") 336 | } 337 | } else { 338 | fn(b, pred, t.Name, es) 339 | } 340 | case *ast.SelectorExpr: 341 | fmt.Fprintf(b, "%s.%s(wire)\n", pred, funcname) 342 | case *ast.ArrayType: 343 | s := f.Type.(*ast.ArrayType) 344 | i := es.getIndexStr() 345 | arrayLen := 0 346 | alenid := es.getNewAlen() 347 | if s.Len == nil { 348 | // If we are unmarshaling we need to allocate. 349 | need_binary = true 350 | if es.op == UNMARSHAL { 351 | fmt.Fprintf(b, "%s, err := binary.ReadVarint(wire)\n", alenid) 352 | fmt.Fprintf(b, "if err != nil {\n") 353 | fmt.Fprintf(b, "return err\n") 354 | fmt.Fprintf(b, "}\n") 355 | if se, ok := s.Elt.(*ast.SelectorExpr); ok { 356 | fmt.Fprintf(b, "%s = make([]%s.%s, %s)\n", pred, se.X, se.Sel, alenid) 357 | } else { 358 | fmt.Fprintf(b, "%s = make([]%s, %s)\n", pred, s.Elt, alenid) 359 | } 360 | } else { 361 | //setbs(b, 10, es, false) 362 | fmt.Fprintf(b, "bs = b[:]\n") 363 | es.curBSize = es.blen 364 | fmt.Fprintf(b, "%s := int64(len(%s))\n", alenid, pred) 365 | fmt.Fprintf(b, "if wlen := binary.PutVarint(bs, %s); wlen >= 0 {\n", alenid) 366 | fmt.Fprintf(b, "wire.Write(b[0:wlen])\n") 367 | fmt.Fprintf(b, "}\n") 368 | } 369 | fmt.Fprintf(b, "for %s := int64(0); %s < %s; %s++ {\n", i, i, alenid, i) 370 | fsub := fmt.Sprintf("%s[%s]", pred, i) 371 | pseudofield := &ast.Field{Type: s.Elt} 372 | es.resetBuffer = true 373 | walkOne(b, pseudofield, fsub, funcname, fn, es) 374 | es.resetBuffer = false 375 | fmt.Fprintln(b, "}") 376 | } else { 377 | e, ok := s.Len.(*ast.BasicLit) 378 | if !ok { 379 | panic("Bad literal in array decl") 380 | } 381 | var err error 382 | arrayLen, err = strconv.Atoi(e.Value) 383 | if err != nil { 384 | panic("Bad array length value. Must be a simple int.") 385 | } 386 | pseudofield := &ast.Field{Type: s.Elt} 387 | for idx := 0; idx < arrayLen; idx++ { 388 | fsub := fmt.Sprintf("%s[%d]", pred, idx) 389 | walkOne(b, pseudofield, fsub, funcname, fn, es) 390 | } 391 | } 392 | es.freeIndexStr() 393 | default: 394 | fmt.Println("Unknown type: ", f) 395 | panic("Unknown type in struct") 396 | } 397 | } 398 | 399 | type StructInfo struct { 400 | size int 401 | maxSize int 402 | maxContiguous int 403 | contiguous []int 404 | varLen bool 405 | mustDispatch bool 406 | totalSize int // Including embedded types, if known 407 | } 408 | 409 | var structInfoMap map[string]*StructInfo 410 | var simpleStructMap map[string]*StructInfo 411 | 412 | func mergeInfo(parent, child *StructInfo, childcount int) { 413 | crt := len(parent.contiguous) - 1 414 | if !child.mustDispatch && !child.varLen { 415 | parent.contiguous[crt] += childcount * child.size 416 | } else { 417 | parent.contiguous[crt] += child.contiguous[0] 418 | } 419 | if parent.contiguous[crt] > parent.maxContiguous { 420 | parent.maxContiguous = parent.contiguous[crt] 421 | } 422 | if (child.mustDispatch || child.varLen) && parent.contiguous[crt] > 0 { 423 | parent.contiguous = append(parent.contiguous, 0) 424 | } 425 | 426 | if child.maxSize > parent.maxSize { 427 | parent.maxSize = child.maxSize 428 | } 429 | parent.varLen = parent.varLen || child.varLen 430 | parent.mustDispatch = parent.mustDispatch || child.mustDispatch 431 | 432 | if childcount > 0 { 433 | parent.size += child.size * childcount 434 | parent.totalSize += child.totalSize * childcount 435 | } 436 | 437 | } 438 | 439 | // Wouldn't it be nice to cache a lot of this? :-) 440 | func analyze(n interface{}) (info *StructInfo) { 441 | info = new(StructInfo) 442 | info.contiguous = make([]int, 1) 443 | switch n.(type) { 444 | case *ast.StructType: 445 | st := n.(*ast.StructType) 446 | for _, field := range st.Fields.List { 447 | for _ = range field.Names { 448 | mergeInfo(info, analyze(field), 1) 449 | } 450 | } 451 | case *ast.Field: 452 | f := n.(*ast.Field) 453 | switch f.Type.(type) { 454 | case *ast.Ident: 455 | tname := f.Type.(*ast.Ident).Name 456 | if mapped, ok := typemap[tname]; ok { 457 | tname = mapped 458 | } 459 | if tinfo, ok := typedb[tname]; ok { 460 | info.maxSize = tinfo.Size 461 | info.size = tinfo.Size 462 | } else { 463 | seinfo := analyzeType(tname) 464 | if seinfo != nil && seinfo.mustDispatch == false && seinfo.varLen == false { 465 | mergeInfo(info, seinfo, 1) 466 | simpleStructMap[tname] = seinfo 467 | } else { 468 | info.mustDispatch = true 469 | } 470 | } 471 | case *ast.SelectorExpr: 472 | info.mustDispatch = true 473 | case *ast.ArrayType: 474 | s := f.Type.(*ast.ArrayType) 475 | arraylen := 0 476 | if s.Len == nil { 477 | // If we are unmarshaling we need to allocate. 478 | info.varLen = true 479 | need_bufio = true // eventually just in info 480 | } else { 481 | e, ok := s.Len.(*ast.BasicLit) 482 | if !ok { 483 | panic("Bad literal in array decl") 484 | } 485 | var err error 486 | arraylen, err = strconv.Atoi(e.Value) 487 | if err != nil { 488 | panic("Bad array length value. Must be a simple int.") 489 | } 490 | } 491 | 492 | pseudofield := &ast.Field{Type: s.Elt} 493 | mergeInfo(info, analyze(pseudofield), arraylen) 494 | default: 495 | fmt.Println("Unknown type in struct: ", f) 496 | panic("Unknown type in struct") 497 | } 498 | default: 499 | panic("Unknown ast type") 500 | } 501 | return 502 | } 503 | 504 | func analyzeType(typeName string) (info *StructInfo) { 505 | ts, ok := globalDeclMap[typeName] 506 | if !ok { 507 | return nil 508 | } 509 | 510 | if st, ok := ts.Type.(*ast.StructType); ok { 511 | info = analyze(st) 512 | return info 513 | } 514 | 515 | if id, ok := ts.Type.(*ast.Ident); ok { 516 | tname := id.Name 517 | if ti, ok := typedb[tname]; ok { 518 | typemap[typeName] = tname 519 | info = &StructInfo{size: ti.Size, maxSize: ti.Size, maxContiguous: ti.Size, totalSize: ti.Size} 520 | return info 521 | } 522 | } 523 | panic("Can't handle decl: " + typeName) 524 | return 525 | } 526 | 527 | func (bi *Binidl) structmap(out io.Writer, ts *ast.TypeSpec) { 528 | typeName := ts.Name.Name 529 | st, ok := ts.Type.(*ast.StructType) 530 | if !ok { 531 | //fmt.Println("Type of type is ", reflect.TypeOf(ts.Type)) 532 | if id, ok := ts.Type.(*ast.Ident); ok { 533 | tname := id.Name 534 | if _, ok := typedb[tname]; ok { 535 | typemap[typeName] = tname 536 | return 537 | } 538 | } 539 | panic("Can't handle decl!") 540 | } 541 | info := analyze(st) 542 | //fmt.Println("Analysis result: ", info) 543 | 544 | fmt.Fprintf(out, "func (t *%s) BinarySize() (nbytes int, sizeKnown bool) {\n", typeName) 545 | if !info.varLen && !info.mustDispatch { 546 | fmt.Fprintf(out, " return %d, true\n", info.size) 547 | } else { 548 | fmt.Fprintf(out, "return 0, false\n") 549 | } 550 | fmt.Fprintln(out, "}") 551 | 552 | fmt.Fprintf(out, "type %sCache struct {\n", typeName) 553 | fmt.Fprintf(out, " mu sync.Mutex\n") 554 | fmt.Fprintf(out, " cache []*%s\n", typeName) 555 | fmt.Fprintf(out, "}\n\n") 556 | fmt.Fprintf(out, "func New%sCache() *%sCache {\nc := &%sCache{}\nc.cache = make([]*%s, 0)\nreturn c\n}\n\n", typeName, typeName, typeName, typeName) 557 | 558 | fmt.Fprintf(out, "func (p *%sCache) Get() *%s {\n", typeName, typeName) 559 | fmt.Fprintf(out, "var t *%s\n", typeName) 560 | fmt.Fprintf(out, "p.mu.Lock()\n") 561 | fmt.Fprintf(out, "if (len(p.cache) > 0) {\n") 562 | fmt.Fprintf(out, " t = p.cache[len(p.cache)-1]\n") 563 | fmt.Fprintf(out, " p.cache = p.cache[0:(len(p.cache)-1)]\n") 564 | fmt.Fprintf(out, "}\n") 565 | fmt.Fprintf(out, "p.mu.Unlock()\n") 566 | fmt.Fprintf(out, "if t == nil { t = &%s{} }\n", typeName) 567 | fmt.Fprintf(out, "return t") 568 | fmt.Fprintf(out, "}\n") 569 | 570 | // Currently relying on re-allocating any variable length arrays to handle 571 | // properly zeroing out reused data structures. 572 | fmt.Fprintf(out, "func (p *%sCache) Put(t *%s) {\n", typeName, typeName) 573 | fmt.Fprintf(out, "p.mu.Lock()\n") 574 | fmt.Fprintf(out, "p.cache = append(p.cache, t)\n") 575 | fmt.Fprintf(out, "p.mu.Unlock()\n") 576 | fmt.Fprintf(out, "}\n") 577 | 578 | blen := info.maxContiguous 579 | if info.varLen && blen < 10 { 580 | blen = 10 581 | } 582 | 583 | mes := &EmitState{bigEndian: bi.bigEndian, op: MARSHAL, contiguous: info.contiguous, blen: blen} 584 | 585 | fmt.Fprintf(out, "func (t *%s) Marshal(wire io.Writer) {\n", typeName) 586 | if (blen > 0) { 587 | fmt.Fprintf(out, "var b [%d]byte\n", blen) 588 | fmt.Fprintf(out, "var bs []byte\n") 589 | mes.curBSize = 0 590 | } 591 | walkContents(out, st, "t", "Marshal", marshalField, mes) 592 | fmt.Fprintf(out, "}\n\n") 593 | 594 | ues := &EmitState{bigEndian: bi.bigEndian, op: UNMARSHAL, contiguous: info.contiguous, blen: blen} 595 | paramname := "wire" 596 | if info.varLen { 597 | paramname = "rr" 598 | } 599 | fmt.Fprintf(out, "func (t *%s) Unmarshal(%s io.Reader) error {\n", typeName, paramname) 600 | if info.varLen { 601 | fmt.Fprintln(out, 602 | `var wire byteReader 603 | var ok bool 604 | if wire, ok = rr.(byteReader); !ok { 605 | wire = bufio.NewReader(rr) 606 | }`) 607 | } 608 | if blen > 0 { 609 | fmt.Fprintf(out, "var b [%d]byte\n", blen) 610 | fmt.Fprintf(out, "var bs []byte\n") 611 | } 612 | walkContents(out, st, "t", "Unmarshal", unmarshalField, ues) 613 | fmt.Fprintf(out, "return nil\n}\n\n") 614 | } 615 | 616 | var globalDeclMap map[string]*ast.TypeSpec = make(map[string]*ast.TypeSpec) 617 | 618 | func createGlobalDeclMap(decls []ast.Decl) { 619 | for _, d := range decls { 620 | if decl, ok := d.(*ast.GenDecl); ok && decl.Tok == token.TYPE { 621 | ts := decl.Specs[0].(*ast.TypeSpec) 622 | globalDeclMap[ts.Name.Name] = ts 623 | } 624 | } 625 | } 626 | 627 | func (bf *Binidl) PrintGo() { 628 | createGlobalDeclMap(bf.ast.Decls) // still a temporary hack 629 | rest := new(bytes.Buffer) 630 | simpleStructMap = make(map[string]*StructInfo) 631 | for _, d := range globalDeclMap { 632 | bf.structmap(rest, d) 633 | } 634 | 635 | tf, err := ioutil.TempFile("", "gobin-codegen") 636 | if err != nil { 637 | panic(err) 638 | } 639 | tfname := tf.Name() 640 | defer os.Remove(tfname) 641 | defer tf.Close() 642 | 643 | defer func() { 644 | if r := recover(); r != nil { 645 | fmt.Println("Panic when parsing generated output: ", r) 646 | fmt.Println("Generated output in temporary file ", tfname) 647 | os.Exit(-1) 648 | } 649 | }() 650 | 651 | fmt.Fprintln(tf, "package", bf.ast.Name.Name) 652 | imports := []string{"io", "sync"} 653 | if need_bufio { 654 | imports = append(imports, "bufio") 655 | } 656 | if need_binary { 657 | imports = append(imports, "encoding/binary") 658 | } 659 | fmt.Fprintln(tf, "import (") 660 | for _, imp := range imports { 661 | fmt.Fprintf(tf, "\"%s\"\n", imp) 662 | } 663 | fmt.Fprintln(tf, ")") 664 | if need_bufio { 665 | fmt.Fprintln(tf, `type byteReader interface { 666 | io.Reader 667 | ReadByte() (c byte, err error) 668 | }`) 669 | } 670 | // Output and then gofmt it to make it pretty and shiny. And readable. 671 | rest.WriteTo(tf) 672 | tf.Sync() 673 | 674 | fset := token.NewFileSet() 675 | ast, err := parser.ParseFile(fset, tfname, nil, 0) 676 | if err != nil { 677 | panic(err.Error()) 678 | } 679 | printer.Fprint(os.Stdout, fset, ast) 680 | } 681 | --------------------------------------------------------------------------------