├── end-to-end ├── Dockerfile ├── Makefile └── main.go ├── go.mod ├── common.go ├── serialize_test.go ├── .github └── workflows │ └── go.yml ├── homogenous_test.go ├── single_test.go ├── alignment_test.go ├── LICENSE ├── relocate_test.go ├── go.sum ├── single.go ├── delim.go ├── README.md ├── relocate.go ├── delim_test.go ├── bench ├── bench_test.go └── summarize.go ├── heterogeneous_test.go ├── descriptor.go ├── descriptor_test.go ├── heterogeneous.go ├── homogenous.go └── serialize.go /end-to-end/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.7 2 | RUN go get github.com/alexflint/go-memdump 3 | ADD main.go / 4 | CMD go run /main.go 5 | -------------------------------------------------------------------------------- /end-to-end/Makefile: -------------------------------------------------------------------------------- 1 | default: test 2 | 3 | image: 4 | docker build -t alexflint/memdump-end-to-end . 5 | 6 | test: image 7 | docker run -it --rm alexflint/memdump-end-to-end 8 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/alexflint/go-memdump 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/alexflint/go-arg v1.4.3 7 | github.com/stretchr/testify v1.7.5 8 | ) 9 | 10 | require ( 11 | github.com/alexflint/go-scalar v1.1.0 // indirect 12 | github.com/davecgh/go-spew v1.1.1 // indirect 13 | github.com/pmezard/go-difflib v1.0.0 // indirect 14 | gopkg.in/yaml.v3 v3.0.1 // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import "errors" 4 | 5 | // Protocols numbers used: (do not re-use) 6 | // 1: homogeneous protocol, April 20, 2016 7 | // 2: heterogeneous protocol, April 20, 2016 8 | 9 | const ( 10 | homogeneousProtocol int32 = 1 11 | heterogeneousProtocol int32 = 2 12 | ) 13 | 14 | var ( 15 | // ErrIncompatibleLayout is returned by decoders when the object on the wire has 16 | // an in-memory layout that is not compatible with the requested Go type. 17 | ErrIncompatibleLayout = errors.New("attempted to load data with incompatible layout") 18 | ) 19 | -------------------------------------------------------------------------------- /end-to-end/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "reflect" 8 | 9 | memdump "github.com/alexflint/go-memdump" 10 | ) 11 | 12 | type T struct { 13 | X int 14 | Y string 15 | } 16 | 17 | func main() { 18 | var b bytes.Buffer 19 | 20 | in := T{X: 123, Y: "xyz"} 21 | err := memdump.Encode(&b, &in) 22 | if err != nil { 23 | fmt.Println(err) 24 | os.Exit(1) 25 | } 26 | 27 | var out *T 28 | err = memdump.Decode(&b, &out) 29 | if err != nil { 30 | fmt.Println(err) 31 | os.Exit(1) 32 | } 33 | 34 | if !reflect.DeepEqual(in, *out) { 35 | fmt.Printf("objects were not equal: %v vs %v\n", in, *out) 36 | os.Exit(1) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /serialize_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func assertSerialize(t *testing.T, obj interface{}) { 13 | } 14 | 15 | func TestSerialize_Struct(t *testing.T) { 16 | type T struct { 17 | X int 18 | y uint8 19 | } 20 | obj := T{ 21 | X: 3, 22 | y: 7, 23 | } 24 | 25 | var b bytes.Buffer 26 | enc := newMemEncoder(&b) 27 | ptrs, err := enc.Encode(&obj) 28 | require.NoError(t, err) 29 | 30 | obj2, err := relocate(b.Bytes(), ptrs, 0, reflect.TypeOf(obj)) 31 | require.NoError(t, err) 32 | assert.EqualValues(t, &obj, obj2) 33 | } 34 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build_and_test: 12 | name: Build and test 13 | 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | go: ['1.17', '1.18'] 18 | os: ['ubuntu-latest', 'windows-latest', 'macos-latest'] 19 | 20 | runs-on: ${{ matrix.os }} 21 | 22 | steps: 23 | - id: go 24 | name: Set up Go 25 | uses: actions/setup-go@v1 26 | with: 27 | go-version: ${{ matrix.go }} 28 | 29 | - name: Checkout 30 | uses: actions/checkout@v2 31 | 32 | - name: Build 33 | run: go build -v . 34 | 35 | - name: Test 36 | run: go test -v . 37 | -------------------------------------------------------------------------------- /homogenous_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestHomogenous_Struct(t *testing.T) { 13 | type T struct { 14 | X int 15 | Y string 16 | } 17 | src := []T{{1, "s1"}, {2, "s2"}, {3, "s3"}} 18 | 19 | var b bytes.Buffer 20 | 21 | enc := NewEncoder(&b) 22 | for i, x := range src { 23 | t.Logf("encoding object %d", i) 24 | err := enc.Encode(&x) 25 | require.NoError(t, err) 26 | } 27 | 28 | dec := NewDecoder(&b) 29 | var dest []T 30 | for i := 0; ; i++ { 31 | t.Logf("decoding object %d", i) 32 | var x T 33 | err := dec.Decode(&x) 34 | if err == io.EOF { 35 | break 36 | } 37 | assert.NoError(t, err) 38 | dest = append(dest, x) 39 | } 40 | 41 | assert.EqualValues(t, src, dest) 42 | } 43 | -------------------------------------------------------------------------------- /single_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestSingle(t *testing.T) { 12 | type T struct { 13 | X int 14 | Y string 15 | Ts []*T 16 | } 17 | src := T{ 18 | X: 123, 19 | Y: "abc", 20 | Ts: []*T{ 21 | {4, "x", nil}, 22 | {5, "y", nil}, 23 | }, 24 | } 25 | 26 | var b bytes.Buffer 27 | 28 | err := Encode(&b, &src) 29 | require.NoError(t, err) 30 | 31 | var dest *T 32 | err = Decode(&b, &dest) 33 | require.NoError(t, err) 34 | 35 | assert.EqualValues(t, src, *dest) 36 | } 37 | 38 | func TestEncode_PanicsForNonPointer(t *testing.T) { 39 | var x struct{} 40 | var b bytes.Buffer 41 | assert.Panics(t, func() { Encode(&b, x) }) 42 | } 43 | 44 | func TestDecode_PanicsForNonPointerToPointer(t *testing.T) { 45 | var x struct{} 46 | var b bytes.Buffer 47 | assert.Panics(t, func() { Decode(&b, x) }) 48 | assert.Panics(t, func() { Decode(&b, &x) }) 49 | } 50 | -------------------------------------------------------------------------------- /alignment_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type byteAndInt struct { 13 | b *byte // should be aligned to a 1-byte boundary 14 | i *int // should be aligned to an 8-byte boundary 15 | } 16 | 17 | func newByte(b byte) *byte { 18 | return &b 19 | } 20 | 21 | func newInt(i int) *int { 22 | return &i 23 | } 24 | 25 | func assertAligned(t *testing.T, v reflect.Value) { 26 | addr := v.UnsafeAddr() 27 | 28 | assert.Zero(t, addr%uintptr(v.Type().Align()), 29 | "alignment of %v was off by %d", v.Type(), addr%uintptr(v.Type().Align())) 30 | } 31 | 32 | func TestAlignment(t *testing.T) { 33 | in := byteAndInt{ 34 | b: newByte(3), 35 | i: newInt(4), 36 | } 37 | 38 | var buf bytes.Buffer 39 | err := Encode(&buf, &in) 40 | require.NoError(t, err) 41 | 42 | var out *byteAndInt 43 | err = Decode(&buf, &out) 44 | require.NoError(t, err) 45 | 46 | assertAligned(t, reflect.ValueOf(out.b).Elem()) 47 | assertAligned(t, reflect.ValueOf(out.i).Elem()) 48 | } 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Alex Flint 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | -------------------------------------------------------------------------------- /relocate_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestLocations(t *testing.T) { 13 | expected := locations{ 14 | Main: 2, 15 | Pointers: []int64{3, 4}, 16 | } 17 | 18 | var b bytes.Buffer 19 | 20 | err := encodeLocations(&b, &expected) 21 | require.NoError(t, err) 22 | 23 | var actual locations 24 | err = decodeLocations(&b, &actual) 25 | require.NoError(t, err) 26 | 27 | assert.Equal(t, expected, actual) 28 | } 29 | 30 | func TestRelocate_EmptyBuffer(t *testing.T) { 31 | var buf []byte 32 | _, err := relocate(buf, nil, 0, reflect.TypeOf(0)) 33 | assert.Error(t, err) 34 | } 35 | 36 | func TestRelocate_MainOutOfBounds(t *testing.T) { 37 | buf := []byte{1, 2, 3} 38 | _, err := relocate(buf, nil, 100, reflect.TypeOf(0)) 39 | assert.Error(t, err) 40 | } 41 | 42 | func TestRelocate_PointerOutOfBounds(t *testing.T) { 43 | buf := []byte{1, 2, 3} 44 | _, err := relocate(buf, []int64{100}, 0, reflect.TypeOf(0)) 45 | assert.Error(t, err) 46 | } 47 | 48 | func BenchmarkLocations(b *testing.B) { 49 | in := locations{Pointers: make([]int64, 200000)} 50 | var out locations 51 | 52 | var buf bytes.Buffer 53 | err := encodeLocations(&buf, &in) 54 | require.NoError(b, err) 55 | 56 | b.ResetTimer() 57 | for i := 0; i < b.N; i++ { 58 | r := bytes.NewBuffer(buf.Bytes()) 59 | err = decodeLocations(r, &out) 60 | require.NoError(b, err) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo= 2 | github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA= 3 | github.com/alexflint/go-scalar v1.1.0 h1:aaAouLLzI9TChcPXotr6gUhq+Scr8rl0P9P4PnltbhM= 4 | github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 12 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 13 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 14 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 15 | github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= 16 | github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 17 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 19 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 20 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 21 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | -------------------------------------------------------------------------------- /single.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "reflect" 9 | ) 10 | 11 | // Encode writes a memdump of the provided object to output. You must 12 | // pass a pointer to the object you wish to encode. 13 | func Encode(w io.Writer, obj interface{}) error { 14 | t := reflect.TypeOf(obj) 15 | if t.Kind() != reflect.Ptr { 16 | panic(fmt.Sprintf("expected a pointer but got %T", obj)) 17 | } 18 | 19 | // write the object data to a temporary buffer 20 | var buf bytes.Buffer 21 | mem := newMemEncoder(&buf) 22 | ptrs, err := mem.Encode(obj) 23 | if err != nil { 24 | return fmt.Errorf("error while walking data: %v", err) 25 | } 26 | 27 | // write the locations at the top 28 | err = encodeLocations(w, &locations{Pointers: ptrs}) 29 | if err != nil { 30 | return fmt.Errorf("error writing location segment: %v", err) 31 | } 32 | 33 | // now write the data segment 34 | _, err = buf.WriteTo(w) 35 | if err != nil { 36 | return fmt.Errorf("error writing data segment: %v", err) 37 | } 38 | 39 | return nil 40 | } 41 | 42 | // Decode reads an object of the specified type from the input 43 | // and stores a pointer to it at the location specified by ptrptr, 44 | // which must be a pointer to a pointer. If you originally called 45 | // Encode with parameter *T then you should pass **T to Decode. 46 | func Decode(r io.Reader, ptrptr interface{}) error { 47 | v := reflect.ValueOf(ptrptr) 48 | t := v.Type() 49 | if t.Kind() != reflect.Ptr || t.Elem().Kind() != reflect.Ptr { 50 | panic(fmt.Sprintf("expected a pointer to a pointer but got %v", v.Type())) 51 | } 52 | 53 | // read the locations 54 | var loc locations 55 | err := decodeLocations(r, &loc) 56 | if err != nil { 57 | return fmt.Errorf("error decoding relocation data: %v", err) 58 | } 59 | 60 | buf, err := ioutil.ReadAll(r) 61 | if err != nil { 62 | return fmt.Errorf("error reading data segment: %v", err) 63 | } 64 | 65 | // relocate the data 66 | out, err := relocate(buf, loc.Pointers, loc.Main, v.Type().Elem().Elem()) 67 | if err != nil { 68 | return fmt.Errorf("error relocating data: %v", err) 69 | } 70 | 71 | v.Elem().Set(reflect.ValueOf(out)) 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /delim.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | ) 7 | 8 | // delim is used to recognize the end of the memory dump 9 | var delim = []byte{ 10 | 130, 14, 133, 49, 108, 178, 125, 95, 11 | 35, 126, 41, 129, 229, 48, 16, 94, 12 | } 13 | 14 | // ErrBufferTooSmall is returned by delimetedReader if the input buffer 15 | // is smaller than the length of the delimeter 16 | var ErrBufferTooSmall = errors.New( 17 | "cannot read into buffer of size less than 16 bytes (due to delim)") 18 | 19 | // ErrUnexpectedEOF is returned by delimetedReader if EOF is reached 20 | // before finding a delimeter 21 | var ErrUnexpectedEOF = errors.New( 22 | "got EOF before finding the delimeter") 23 | 24 | // DelimitedReader reads delimited segments 25 | type DelimitedReader struct { 26 | r io.Reader 27 | buf []byte 28 | begin int 29 | end int 30 | } 31 | 32 | // NewDelimitedReader creates a reader for delimited segments 33 | func NewDelimitedReader(r io.Reader) *DelimitedReader { 34 | return &DelimitedReader{ 35 | r: r, 36 | } 37 | } 38 | 39 | // Next returns the next segment, or (nil, io.EOF) if there are no more segments. 40 | // The data is only valid until the next call to Next(). 41 | func (r *DelimitedReader) Next() ([]byte, error) { 42 | var state, offset int 43 | for { 44 | // look for the next delimiter 45 | for i, b := range r.buf[r.begin+offset : r.end] { 46 | if b != delim[state] { 47 | state = 0 48 | } 49 | // do not use "else" here because we updated state above 50 | if b == delim[state] { 51 | state++ 52 | if state == len(delim) { 53 | out := r.buf[r.begin : r.begin+offset+i-len(delim)+1] 54 | r.begin = r.begin + offset + i + 1 55 | return out, nil 56 | } 57 | } 58 | } 59 | offset = r.end - r.begin 60 | 61 | // allocate a larger buffer 62 | if r.buf == nil { 63 | r.buf = make([]byte, 16384) 64 | } else { 65 | var newbuf []byte 66 | newbuf = make([]byte, 4*len(r.buf)) 67 | copy(newbuf, r.buf[r.begin:r.end]) 68 | r.end -= r.begin 69 | r.begin = 0 70 | r.buf = newbuf 71 | } 72 | 73 | // fill the rest of the buffer 74 | n, err := r.r.Read(r.buf[r.end:]) 75 | r.end += n 76 | 77 | // check for exit conditions 78 | if n == 0 && err == io.EOF { 79 | if offset == 0 { 80 | return nil, io.EOF 81 | } 82 | return nil, ErrUnexpectedEOF 83 | } else if err != nil { 84 | return nil, err 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Documentation](https://godoc.org/github.com/alexflint/go-memdump?status.svg)](https://godoc.org/github.com/alexflint/go-memdump) 2 | [![Build Status](https://github.com/alexflint/go-memdump/workflows/Go/badge.svg)](https://github.com/alexflint/go-memdump/actions) 3 | [![Coverage Status](https://coveralls.io/repos/github/alexflint/go-memdump/badge.svg?branch=master)](https://coveralls.io/github/alexflint/go-memdump?branch=master) 4 | [![Report Card](https://goreportcard.com/badge/github.com/alexflint/go-memdump)](https://goreportcard.com/report/github.com/alexflint/go-memdump) 5 | 6 | ## Very fast, very unsafe serialization for Go 7 | 8 | This package provides a fast way to load large amounts of data into Go structs. Memdump can load datasets containing millions of small structs at over 1 GB/s (compared to ~30 MB/s for gob or json). 9 | 10 | The price you pay is: 11 | - you cannot load structs that contain maps or interfaces 12 | - your data is not portable across machine architectures (64 bit vs 32 bit, big-endian vs small-endian) 13 | 14 | ### Benchmarks 15 | 16 | The benchmarks were measured by encoding and decoding a tree containing 2,097,151 small structs. Code is under the bench dir. 17 | 18 | **Decode** 19 | ``` 20 | gob 28.17 MB/s (39.8 MB in 1.41s) 21 | json 30.17 MB/s (113.8 MB in 3.77s) 22 | memdump 1031.54 MB/s (113.2 MB in 0.11s) 23 | ``` 24 | 25 | **Encode** 26 | ``` 27 | gob 37.07 MB/s (39.8 MB in 1.07s) 28 | json 77.20 MB/s (113.8 MB in 1.47s) 29 | memdump 61.25 MB/s (113.2 MB in 1.85s) 30 | ``` 31 | 32 | To reproduce these results: 33 | ```shell 34 | $ go run ./bench/summarize.go 35 | ``` 36 | 37 | ## Quick start 38 | 39 | ```shell 40 | go get github.com/alexflint/go-memdump 41 | ``` 42 | 43 | Write data to a file: 44 | 45 | ```go 46 | type data struct { 47 | Foo string 48 | Bar int 49 | } 50 | 51 | w, err := os.Create("/tmp/data.memdump") 52 | if err != nil { 53 | ... 54 | } 55 | 56 | // note that you must pass a pointer when encoding 57 | mydata := data{Foo: "abc", Bar: 123} 58 | memdump.Encode(w, &mydata) 59 | ``` 60 | 61 | Load data from a file: 62 | 63 | ```go 64 | r, err := os.Open("/tmp/data.memdump") 65 | if err != nil { 66 | ... 67 | } 68 | 69 | // note that you muss pass a pointer to a pointer when decoding 70 | var mydata *data 71 | memdump.Decode(r, &mydata) 72 | ``` 73 | -------------------------------------------------------------------------------- /relocate.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "reflect" 8 | "unsafe" 9 | ) 10 | 11 | // locations contains the locations of pointers in a data segments 12 | type locations struct { 13 | Main int64 // Main contains the offset of the primary object 14 | Pointers []int64 // Pointers contains the offset of each pointer 15 | } 16 | 17 | func encodeLocations(w io.Writer, f *locations) error { 18 | err := binary.Write(w, binary.LittleEndian, int64(len(f.Pointers))) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | err = binary.Write(w, binary.LittleEndian, f.Main) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | err = binary.Write(w, binary.LittleEndian, f.Pointers) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | return nil 34 | } 35 | 36 | func decodeLocations(r io.Reader, f *locations) error { 37 | // read the number of pointers 38 | var n int64 39 | err := binary.Read(r, binary.LittleEndian, &n) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | // read the main offset 45 | err = binary.Read(r, binary.LittleEndian, &f.Main) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | // read the list of pointers 51 | f.Pointers = make([]int64, n) 52 | err = binary.Read(r, binary.LittleEndian, f.Pointers) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return nil 58 | } 59 | 60 | // relocate adds the base address to each pointer in the buffer, then reinterprets 61 | // the buffer as an object of type t. 62 | func relocate(buf []byte, ptrs []int64, main int64, t reflect.Type) (interface{}, error) { 63 | if len(buf) == 0 { 64 | return nil, fmt.Errorf("cannot relocate an empty buffer") 65 | } 66 | 67 | // If the buffer is not aligned then we have to move the 68 | // whole thing. We can assume that a freshly allocated 69 | // buffer from make() is 8-byte aligned. 70 | if uintptr(unsafe.Pointer(&buf[0]))%uintptr(t.Align()) != 0 { 71 | buf2 := make([]byte, len(buf)) 72 | copy(buf2, buf) 73 | buf = buf2 74 | } 75 | 76 | base := uintptr(unsafe.Pointer(&buf[0])) 77 | for i, loc := range ptrs { 78 | if loc < 0 || loc >= int64(len(buf)) { 79 | return nil, fmt.Errorf("pointer %d was out of range: %d (buffer len=%d)", i, loc, len(buf)) 80 | } 81 | v := (*uintptr)(unsafe.Pointer(&buf[loc])) 82 | *v += base 83 | } 84 | if main < 0 || main >= int64(len(buf)) { 85 | return nil, fmt.Errorf("main offset was out of range: %d (buffer len=%d)", main, len(buf)) 86 | } 87 | return reflect.NewAt(t, unsafe.Pointer(&buf[main])).Interface(), nil 88 | } 89 | -------------------------------------------------------------------------------- /delim_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func join(bufs ...[]byte) []byte { 12 | return bytes.Join(bufs, nil) 13 | } 14 | 15 | func TestDelimitedReader_Simple(t *testing.T) { 16 | data := join([]byte("abc"), delim, []byte("defggg"), delim) 17 | r := NewDelimitedReader(bytes.NewReader(data)) 18 | 19 | seg, err := r.Next() 20 | assert.NoError(t, err) 21 | assert.Equal(t, "abc", string(seg)) 22 | 23 | seg, err = r.Next() 24 | assert.NoError(t, err) 25 | assert.Equal(t, "defggg", string(seg)) 26 | 27 | seg, err = r.Next() 28 | assert.Equal(t, io.EOF, err) 29 | assert.Nil(t, seg) 30 | } 31 | 32 | func TestDelimitedReader_Long(t *testing.T) { 33 | data := make([]byte, 32768) 34 | for i := range data { 35 | data[i] = 0 36 | } 37 | copy(data[len(data)-len(delim):], delim) 38 | r := NewDelimitedReader(bytes.NewReader(data)) 39 | 40 | seg, err := r.Next() 41 | assert.NoError(t, err) 42 | assert.Len(t, seg, len(data)-len(delim)) 43 | 44 | seg, err = r.Next() 45 | assert.Equal(t, io.EOF, err) 46 | assert.Nil(t, seg) 47 | } 48 | 49 | func TestDelimitedReader_SimpleThenEmpty(t *testing.T) { 50 | data := join([]byte("abc"), delim, delim) 51 | r := NewDelimitedReader(bytes.NewReader(data)) 52 | 53 | seg, err := r.Next() 54 | assert.NoError(t, err) 55 | assert.Equal(t, "abc", string(seg)) 56 | 57 | seg, err = r.Next() 58 | assert.NoError(t, err) 59 | assert.NotNil(t, seg) 60 | assert.Equal(t, "", string(seg)) 61 | 62 | seg, err = r.Next() 63 | assert.Equal(t, io.EOF, err) 64 | assert.Nil(t, seg) 65 | } 66 | 67 | func TestDelimitedReader_Empty(t *testing.T) { 68 | r := NewDelimitedReader(bytes.NewReader(nil)) 69 | seg, err := r.Next() 70 | assert.Equal(t, io.EOF, err) 71 | assert.Nil(t, seg) 72 | 73 | // same thing again 74 | seg, err = r.Next() 75 | assert.Equal(t, io.EOF, err) 76 | assert.Nil(t, seg) 77 | } 78 | 79 | func TestDelimitedReader_Unterminated(t *testing.T) { 80 | r := NewDelimitedReader(bytes.NewReader([]byte("abc"))) 81 | _, err := r.Next() 82 | assert.Equal(t, ErrUnexpectedEOF, err) 83 | } 84 | 85 | func TestDelimitedReader_ReadAfterEOF(t *testing.T) { 86 | data := join([]byte("abc"), delim) 87 | r := NewDelimitedReader(bytes.NewReader(data)) 88 | 89 | // first read: should not return error 90 | _, err := r.Next() 91 | assert.NoError(t, err) 92 | 93 | // second read: should return EOF 94 | _, err = r.Next() 95 | assert.Equal(t, io.EOF, err) 96 | 97 | // third read: should return EOF again 98 | _, err = r.Next() 99 | assert.Equal(t, io.EOF, err) 100 | } 101 | -------------------------------------------------------------------------------- /bench/bench_test.go: -------------------------------------------------------------------------------- 1 | // Package bench contains benchmarks that compare memdump with other 2 | // serialization packages. This code is in a separate package to avoid 3 | // introducing unnecessary dependencies to memdump. 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "encoding/gob" 10 | "encoding/json" 11 | "fmt" 12 | "testing" 13 | 14 | memdump "github.com/alexflint/go-memdump" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | const ( 19 | minDepth = 16 20 | maxDepth = 16 21 | degree = 2 22 | ) 23 | 24 | func BenchmarkHomogeneousMemdump(b *testing.B) { 25 | var bufs [][]byte 26 | for i := minDepth; i <= maxDepth; i++ { 27 | in := generateTree(i, degree) 28 | var buf bytes.Buffer 29 | enc := memdump.NewEncoder(&buf) 30 | err := enc.Encode(in) 31 | require.NoError(b, err) 32 | bufs = append(bufs, buf.Bytes()) 33 | } 34 | 35 | for d, buf := range bufs { 36 | b.Run(fmt.Sprintf("depth=%d", d), func(b *testing.B) { 37 | for i := 0; i < b.N; i++ { 38 | dec := memdump.NewDecoder(bytes.NewBuffer(buf)) 39 | var out treeNode 40 | err := dec.Decode(&out) 41 | require.NoError(b, err) 42 | } 43 | }) 44 | } 45 | } 46 | 47 | func BenchmarkSingleMemdump(b *testing.B) { 48 | var bufs [][]byte 49 | for i := minDepth; i <= maxDepth; i++ { 50 | in := generateTree(i, degree) 51 | var buf bytes.Buffer 52 | err := memdump.Encode(&buf, in) 53 | require.NoError(b, err) 54 | bufs = append(bufs, buf.Bytes()) 55 | } 56 | 57 | for d, buf := range bufs { 58 | b.Run(fmt.Sprintf("depth=%d", d), func(b *testing.B) { 59 | for i := 0; i < b.N; i++ { 60 | var out *treeNode 61 | err := memdump.Decode(bytes.NewBuffer(buf), &out) 62 | require.NoError(b, err) 63 | } 64 | }) 65 | } 66 | } 67 | 68 | func BenchmarkGob(b *testing.B) { 69 | var bufs [][]byte 70 | for i := minDepth; i <= maxDepth; i++ { 71 | in := generateTree(i, degree) 72 | var buf bytes.Buffer 73 | enc := gob.NewEncoder(&buf) 74 | err := enc.Encode(in) 75 | require.NoError(b, err) 76 | bufs = append(bufs, buf.Bytes()) 77 | } 78 | 79 | for d, buf := range bufs { 80 | b.Run(fmt.Sprintf("depth=%d", d), func(b *testing.B) { 81 | for i := 0; i < b.N; i++ { 82 | dec := gob.NewDecoder(bytes.NewBuffer(buf)) 83 | var out treeNode 84 | err := dec.Decode(&out) 85 | require.NoError(b, err) 86 | } 87 | }) 88 | } 89 | } 90 | 91 | func BenchmarkJSON(b *testing.B) { 92 | var bufs [][]byte 93 | for i := minDepth; i <= maxDepth; i++ { 94 | in := generateTree(i, degree) 95 | buf, err := json.Marshal(in) 96 | require.NoError(b, err) 97 | bufs = append(bufs, buf) 98 | } 99 | 100 | for d, buf := range bufs { 101 | b.Run(fmt.Sprintf("depth=%d", d), func(b *testing.B) { 102 | for i := 0; i < b.N; i++ { 103 | var out treeNode 104 | err := json.Unmarshal(buf, &out) 105 | require.NoError(b, err) 106 | } 107 | }) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /heterogeneous_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func testEncodeDecode(t *testing.T, src interface{}, dest interface{}) { 13 | var b bytes.Buffer 14 | 15 | enc := NewHeterogeneousEncoder(&b) 16 | err := enc.Encode(src) 17 | require.NoError(t, err) 18 | 19 | dec := NewHeterogeneousDecoder(&b) 20 | err = dec.Decode(dest) 21 | require.NoError(t, err) 22 | } 23 | 24 | func TestHeterogeneous_Int(t *testing.T) { 25 | var dest int 26 | var src int = 3 27 | testEncodeDecode(t, &src, &dest) 28 | assert.EqualValues(t, src, dest) 29 | } 30 | 31 | func TestHeterogeneous_String(t *testing.T) { 32 | var dest string 33 | src := "abc" 34 | testEncodeDecode(t, &src, &dest) 35 | assert.EqualValues(t, src, dest) 36 | } 37 | 38 | func TestHeterogeneous_Slice(t *testing.T) { 39 | var dest []int16 40 | src := []int16{5, 4, 3} 41 | testEncodeDecode(t, &src, &dest) 42 | assert.EqualValues(t, src, dest) 43 | } 44 | 45 | func TestHeterogeneous_Large(t *testing.T) { 46 | type T struct { 47 | A string 48 | B int 49 | } 50 | src := make([]T, 1000) 51 | for i := range src { 52 | src[i].A = strings.Repeat("123", 100) 53 | src[i].B = 123 54 | } 55 | var dest []T 56 | testEncodeDecode(t, &src, &dest) 57 | assert.EqualValues(t, src, dest) 58 | } 59 | 60 | func TestHeterogeneous_Struct(t *testing.T) { 61 | type U struct { 62 | W string 63 | X *string 64 | } 65 | type T struct { 66 | U 67 | A int 68 | B byte 69 | C []U 70 | D []*U 71 | E *[]U 72 | F *U 73 | G *T 74 | } 75 | 76 | s := "hello" 77 | u := U{W: s, X: &s} 78 | src := T{ 79 | U: u, 80 | A: 5, 81 | B: 6, 82 | C: []U{u}, 83 | D: []*U{&u}, 84 | F: &u, 85 | } 86 | src.E = &src.C 87 | src.G = &src 88 | 89 | var dest T 90 | testEncodeDecode(t, &src, &dest) 91 | assert.EqualValues(t, src, dest) 92 | } 93 | 94 | func TestHeterogeneous_Incompatible(t *testing.T) { 95 | type U struct { 96 | X int 97 | Y string 98 | } 99 | type V struct { 100 | X string 101 | Y int 102 | } 103 | 104 | src := U{3, "abc"} 105 | var dest V 106 | 107 | var b bytes.Buffer 108 | 109 | enc := NewHeterogeneousEncoder(&b) 110 | err := enc.Encode(&src) 111 | require.NoError(t, err) 112 | 113 | dec := NewHeterogeneousDecoder(&b) 114 | err = dec.Decode(&dest) 115 | assert.Equal(t, ErrIncompatibleLayout, err) 116 | } 117 | 118 | func TestHeterogeneousEncodeUnsupportedTypes(t *testing.T) { 119 | var buf bytes.Buffer 120 | enc := NewHeterogeneousEncoder(&buf) 121 | assert.Panics(t, func() { 122 | enc.Encode(&map[string]int{}) 123 | }) 124 | assert.Panics(t, func() { 125 | enc.Encode(func() {}) 126 | }) 127 | assert.Panics(t, func() { 128 | enc.Encode(make(chan int)) 129 | }) 130 | assert.Panics(t, func() { 131 | var x interface{} = "abc" 132 | enc.Encode(&x) 133 | }) 134 | } 135 | -------------------------------------------------------------------------------- /descriptor.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | ) 7 | 8 | // pointer - type 9 | // slice - type 10 | // array - type, length 11 | // string 12 | // int - size 13 | // float - size 14 | 15 | // A Descriptor describes a type such that if two types have the 16 | // same descriptor then their memory layout is identical. 17 | type descriptor []typ 18 | 19 | // a class contains information about a type 20 | type typ struct { 21 | Kind reflect.Kind // Kind is the kind of this type 22 | Size uintptr // Size is the size in bits, as per reflect.Value.Size 23 | Elem int // Elem is index of the underlying type for pointers, slices, and arrays 24 | Fields []field // Fields contains the fields for structs 25 | } 26 | 27 | type field struct { 28 | Name string // Name is the name of this field, or its memdump tag if present 29 | Offset uintptr // Offset is the position of this field relative to the beginning of the struct 30 | Type int // ID is the index of the type of this field in the descripto 31 | } 32 | 33 | // descriptorsEqual compares two descriptors 34 | func descriptorsEqual(a, b descriptor) bool { 35 | if len(a) != len(b) { 36 | return false 37 | } 38 | for i := range a { 39 | if a[i].Kind != b[i].Kind { 40 | return false 41 | } 42 | if a[i].Size != b[i].Size { 43 | return false 44 | } 45 | if a[i].Elem != b[i].Elem { 46 | return false 47 | } 48 | if len(a[i].Fields) != len(b[i].Fields) { 49 | return false 50 | } 51 | for j := range a[i].Fields { 52 | if a[i].Fields[j].Name != b[i].Fields[j].Name { 53 | return false 54 | } 55 | if a[i].Fields[j].Offset != b[i].Fields[j].Offset { 56 | return false 57 | } 58 | if a[i].Fields[j].Type != b[i].Fields[j].Type { 59 | return false 60 | } 61 | } 62 | } 63 | return true 64 | } 65 | 66 | // describe computes the descriptor for a type 67 | func describe(t reflect.Type) descriptor { 68 | var nextID int 69 | var desc descriptor 70 | var queue []reflect.Type 71 | seen := make(map[reflect.Type]int) 72 | 73 | push := func(t reflect.Type) int { 74 | if id, found := seen[t]; found { 75 | return id 76 | } 77 | id := nextID 78 | seen[t] = id 79 | nextID++ 80 | queue = append(queue, t) 81 | return id 82 | } 83 | 84 | push(t) 85 | for len(queue) > 0 { 86 | cur := queue[0] 87 | queue = queue[1:] 88 | t := typ{ 89 | Size: cur.Size(), 90 | Kind: cur.Kind(), 91 | } 92 | 93 | switch cur.Kind() { 94 | case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map: 95 | panic(fmt.Sprintf("cannot compute descriptor for %v", cur.Kind())) 96 | case reflect.Array, reflect.Slice, reflect.Ptr: 97 | t.Elem = push(cur.Elem()) 98 | case reflect.Struct: 99 | for i := 0; i < cur.NumField(); i++ { 100 | f := cur.Field(i) 101 | if f.Type.Size() == 0 { 102 | continue 103 | } 104 | 105 | name := f.Name 106 | if tag := f.Tag.Get("memdump"); tag != "" { 107 | name = tag 108 | } 109 | t.Fields = append(t.Fields, field{ 110 | Name: name, 111 | Offset: f.Offset, 112 | Type: push(f.Type), 113 | }) 114 | } 115 | } 116 | 117 | desc = append(desc, t) 118 | } 119 | return desc 120 | } 121 | -------------------------------------------------------------------------------- /bench/summarize.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | "time" 10 | 11 | arg "github.com/alexflint/go-arg" 12 | memdump "github.com/alexflint/go-memdump" 13 | ) 14 | 15 | type pathComponent struct { 16 | S string 17 | R int 18 | } 19 | 20 | type treeNode struct { 21 | Label string 22 | Weight int 23 | Path []pathComponent 24 | Children []*treeNode 25 | } 26 | 27 | func generateTree(depth, degree int) *treeNode { 28 | tpl := treeNode{ 29 | Label: "label", 30 | Weight: 123, 31 | Path: []pathComponent{ 32 | {"abc", 4}, 33 | {"def", 5}, 34 | {"ghi", 6}, 35 | }, 36 | } 37 | 38 | root := tpl 39 | cur := []*treeNode{&root} 40 | var next []*treeNode 41 | for i := 1; i < depth; i++ { 42 | for _, n := range cur { 43 | for j := 0; j < degree; j++ { 44 | ch := tpl 45 | n.Children = append(n.Children, &ch) 46 | next = append(next, &ch) 47 | } 48 | } 49 | //log.Printf("at i=%d, len=%d", i, len(next)) 50 | cur, next = next, cur[:0] 51 | } 52 | return &root 53 | } 54 | 55 | type codec interface { 56 | name() string 57 | encode(t *treeNode) ([]byte, error) 58 | decode([]byte) error 59 | } 60 | 61 | type gobcodec struct{} 62 | 63 | func (gobcodec) name() string { return "gob" } 64 | 65 | func (gobcodec) encode(t *treeNode) ([]byte, error) { 66 | var b bytes.Buffer 67 | enc := gob.NewEncoder(&b) 68 | err := enc.Encode(t) 69 | if err != nil { 70 | return nil, err 71 | } 72 | return b.Bytes(), nil 73 | } 74 | 75 | func (gobcodec) decode(buf []byte) error { 76 | dec := gob.NewDecoder(bytes.NewBuffer(buf)) 77 | var t treeNode 78 | return dec.Decode(&t) 79 | } 80 | 81 | type jsoncodec struct{} 82 | 83 | func (jsoncodec) name() string { return "json" } 84 | 85 | func (jsoncodec) encode(t *treeNode) ([]byte, error) { 86 | return json.Marshal(t) 87 | } 88 | 89 | func (jsoncodec) decode(buf []byte) error { 90 | var t treeNode 91 | return json.Unmarshal(buf, &t) 92 | } 93 | 94 | type memdumpcodec struct{} 95 | 96 | func (memdumpcodec) name() string { return "memdump" } 97 | 98 | func (memdumpcodec) encode(t *treeNode) ([]byte, error) { 99 | var b bytes.Buffer 100 | err := memdump.Encode(&b, t) 101 | if err != nil { 102 | return nil, err 103 | } 104 | return b.Bytes(), nil 105 | } 106 | 107 | func (memdumpcodec) decode(buf []byte) error { 108 | var t *treeNode 109 | return memdump.Decode(bytes.NewBuffer(buf), &t) 110 | } 111 | 112 | func main() { 113 | var args struct { 114 | Repeat int 115 | Depth int 116 | Degree int 117 | } 118 | args.Repeat = 5 119 | args.Depth = 20 120 | args.Degree = 2 121 | arg.MustParse(&args) 122 | 123 | t := generateTree(args.Depth, args.Degree) 124 | 125 | fmt.Println("DECODE") 126 | for _, codec := range []codec{gobcodec{}, jsoncodec{}, memdumpcodec{}} { 127 | buf, err := codec.encode(t) 128 | if err != nil { 129 | fmt.Println(err) 130 | os.Exit(1) 131 | } 132 | 133 | begin := time.Now() 134 | for i := 0; i < args.Repeat; i++ { 135 | err := codec.decode(buf) 136 | if err != nil { 137 | fmt.Println(err) 138 | os.Exit(1) 139 | } 140 | } 141 | duration := time.Since(begin).Seconds() / float64(args.Repeat) 142 | 143 | bytesPerSec := float64(len(buf)) / duration 144 | 145 | fmt.Printf("%20s %8.2f MB/s (%.1f MB in %.2fs)\n", 146 | codec.name(), bytesPerSec/1000000., float64(len(buf))/1000000., duration) 147 | } 148 | 149 | fmt.Println("ENCODE") 150 | for _, codec := range []codec{gobcodec{}, jsoncodec{}, memdumpcodec{}} { 151 | buf, err := codec.encode(t) 152 | if err != nil { 153 | fmt.Println(err) 154 | os.Exit(1) 155 | } 156 | 157 | begin := time.Now() 158 | for i := 0; i < args.Repeat; i++ { 159 | codec.encode(t) 160 | } 161 | duration := time.Since(begin).Seconds() / float64(args.Repeat) 162 | 163 | bytesPerSec := float64(len(buf)) / duration 164 | 165 | fmt.Printf("%20s %8.2f MB/s (%.1f MB in %.2fs)\n", 166 | codec.name(), bytesPerSec/1000000., float64(len(buf))/1000000., duration) 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /descriptor_test.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func assertCompareDescriptors(t *testing.T, a interface{}, b interface{}, expected bool) { 11 | da := describe(reflect.TypeOf(a)) 12 | db := describe(reflect.TypeOf(b)) 13 | assert.Equal(t, expected, descriptorsEqual(da, db)) 14 | } 15 | 16 | func TestDescribeScalar(t *testing.T) { 17 | assertCompareDescriptors(t, "abc", "d", true) 18 | assertCompareDescriptors(t, 123, 456, true) 19 | assertCompareDescriptors(t, 1.2, 3.4, true) 20 | assertCompareDescriptors(t, uint64(12), uint64(34), true) 21 | 22 | assertCompareDescriptors(t, int16(12), int64(34), false) 23 | assertCompareDescriptors(t, uint64(12), int64(34), false) 24 | assertCompareDescriptors(t, uint64(12), "34", false) 25 | assertCompareDescriptors(t, uint64(12), 4.5, false) 26 | } 27 | 28 | func TestDescribePointer(t *testing.T) { 29 | var x int 30 | var y uint64 31 | var z string 32 | 33 | assertCompareDescriptors(t, &x, &x, true) 34 | assertCompareDescriptors(t, &y, &y, true) 35 | assertCompareDescriptors(t, &z, &z, true) 36 | 37 | assertCompareDescriptors(t, &x, &y, false) 38 | assertCompareDescriptors(t, &x, &z, false) 39 | assertCompareDescriptors(t, &x, x, false) 40 | 41 | xptr := &x 42 | xptrptr := &xptr 43 | 44 | assertCompareDescriptors(t, xptrptr, xptrptr, true) 45 | assertCompareDescriptors(t, xptr, xptrptr, false) 46 | assertCompareDescriptors(t, x, xptrptr, false) 47 | } 48 | 49 | func TestDescribeSlice(t *testing.T) { 50 | var x []int 51 | var y []uint64 52 | var z [][]int16 53 | 54 | assertCompareDescriptors(t, x, x, true) 55 | assertCompareDescriptors(t, y, y, true) 56 | assertCompareDescriptors(t, z, z, true) 57 | assertCompareDescriptors(t, &x, &x, true) 58 | assertCompareDescriptors(t, &y, &y, true) 59 | assertCompareDescriptors(t, &z, &z, true) 60 | 61 | assertCompareDescriptors(t, x, y, false) 62 | assertCompareDescriptors(t, &x, &y, false) 63 | assertCompareDescriptors(t, x, z, false) 64 | assertCompareDescriptors(t, &x, x, false) 65 | } 66 | 67 | func TestDescribeArray(t *testing.T) { 68 | var x [4]int 69 | var y [10]int 70 | var z [2][3]int16 71 | 72 | assertCompareDescriptors(t, x, x, true) 73 | assertCompareDescriptors(t, y, y, true) 74 | assertCompareDescriptors(t, z, z, true) 75 | assertCompareDescriptors(t, &x, &x, true) 76 | assertCompareDescriptors(t, &y, &y, true) 77 | assertCompareDescriptors(t, &z, &z, true) 78 | 79 | assertCompareDescriptors(t, x, y, false) 80 | assertCompareDescriptors(t, &x, &y, false) 81 | assertCompareDescriptors(t, x, z, false) 82 | assertCompareDescriptors(t, &x, x, false) 83 | } 84 | 85 | func TestDescribeStruct(t *testing.T) { 86 | type u struct { 87 | A string 88 | B int 89 | } 90 | type v struct { 91 | A string 92 | B int32 93 | } 94 | type w struct { 95 | u 96 | A string 97 | B int 98 | c v 99 | } 100 | 101 | assertCompareDescriptors(t, u{}, u{}, true) 102 | assertCompareDescriptors(t, v{}, v{}, true) 103 | assertCompareDescriptors(t, w{}, w{}, true) 104 | assertCompareDescriptors(t, &u{}, &u{}, true) 105 | assertCompareDescriptors(t, &v{}, &v{}, true) 106 | assertCompareDescriptors(t, &w{}, &w{}, true) 107 | 108 | assertCompareDescriptors(t, u{}, v{}, false) 109 | assertCompareDescriptors(t, u{}, w{}, false) 110 | assertCompareDescriptors(t, v{}, w{}, false) 111 | assertCompareDescriptors(t, u{}, &u{}, false) 112 | } 113 | 114 | func TestDescribeStructWithTags(t *testing.T) { 115 | type u struct { 116 | A string 117 | B int 118 | } 119 | type v struct { 120 | A string 121 | xyz int 122 | } 123 | type w struct { 124 | A string 125 | B int `memdump:"xyz"` 126 | } 127 | 128 | assertCompareDescriptors(t, u{}, v{}, false) 129 | assertCompareDescriptors(t, u{}, w{}, false) 130 | assertCompareDescriptors(t, v{}, w{}, true) 131 | } 132 | 133 | func TestDescriptorsEqual_DifferentElems(t *testing.T) { 134 | type u struct { 135 | A []string 136 | } 137 | type v struct { 138 | A []int 139 | } 140 | type w struct { 141 | A []int 142 | } 143 | 144 | assertCompareDescriptors(t, u{}, v{}, false) 145 | assertCompareDescriptors(t, u{}, w{}, false) 146 | assertCompareDescriptors(t, v{}, w{}, true) 147 | } 148 | 149 | func TestDescriptorsEqual_DifferentNumFields(t *testing.T) { 150 | type u struct { 151 | A string 152 | B string 153 | } 154 | type v struct { 155 | A string 156 | } 157 | type w struct { 158 | A string 159 | } 160 | 161 | assertCompareDescriptors(t, u{}, v{}, false) 162 | assertCompareDescriptors(t, u{}, w{}, false) 163 | assertCompareDescriptors(t, v{}, w{}, true) 164 | } 165 | 166 | func TestDescribe_PanicsOnMap(t *testing.T) { 167 | type T struct { 168 | A map[string]int 169 | } 170 | assert.Panics(t, func() { 171 | describe(reflect.TypeOf(T{})) 172 | }) 173 | } 174 | -------------------------------------------------------------------------------- /heterogeneous.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/gob" 7 | "fmt" 8 | "io" 9 | "reflect" 10 | ) 11 | 12 | type heterogeneousFooter struct { 13 | Pointers []int64 // Pointers contains the offset of each pointer 14 | Main int64 // Main contains the offset of the primary object 15 | Descriptor descriptor 16 | } 17 | 18 | // HeterogeneousEncoder writes memdumps to the provided writer 19 | type HeterogeneousEncoder struct { 20 | w io.Writer 21 | hasprotocol bool 22 | } 23 | 24 | // NewHeterogeneousEncoder creates an HeterogeneousEncoder that writes memdumps to the provided writer 25 | func NewHeterogeneousEncoder(w io.Writer) *HeterogeneousEncoder { 26 | return &HeterogeneousEncoder{ 27 | w: w, 28 | } 29 | } 30 | 31 | // Encode writes a memdump of the provided object to output. You must pass a 32 | // pointer to the object you wish to encode. To encode a pointer, pass a 33 | // double-pointer. 34 | func (e *HeterogeneousEncoder) Encode(obj interface{}) error { 35 | t := reflect.TypeOf(obj) 36 | if t.Kind() != reflect.Ptr { 37 | panic(fmt.Sprintf("expected a pointer but got %T", obj)) 38 | } 39 | 40 | // write a protocol heterogeneousProtocol number 41 | if !e.hasprotocol { 42 | err := binary.Write(e.w, binary.LittleEndian, heterogeneousProtocol) 43 | if err != nil { 44 | return fmt.Errorf("error writing protocol: %v", err) 45 | } 46 | e.hasprotocol = true 47 | } 48 | 49 | // first segment: write the object data 50 | mem := newMemEncoder(e.w) 51 | ptrs, err := mem.Encode(obj) 52 | if err != nil { 53 | return fmt.Errorf("error writing data segment: %v", err) 54 | } 55 | 56 | // write delimiter 57 | _, err = e.w.Write(delim) 58 | if err != nil { 59 | return fmt.Errorf("error writing delimiter: %v", err) 60 | } 61 | 62 | // second segment: write the metadata 63 | gob := gob.NewEncoder(e.w) 64 | err = gob.Encode(heterogeneousFooter{ 65 | Pointers: ptrs, 66 | Descriptor: describe(t.Elem()), 67 | }) 68 | if err != nil { 69 | return fmt.Errorf("error writing heterogeneousFooter: %v", err) 70 | } 71 | 72 | // write delimiter 73 | _, err = e.w.Write(delim) 74 | if err != nil { 75 | return fmt.Errorf("error writing delimiter: %v", err) 76 | } 77 | return nil 78 | } 79 | 80 | // HeterogeneousDecoder reads memdumps from the provided reader 81 | type HeterogeneousDecoder struct { 82 | r io.Reader 83 | dr *DelimitedReader 84 | hasprotocol bool 85 | } 86 | 87 | // NewHeterogeneousDecoder creates a HeterogeneousDecoder that reads memdumps 88 | func NewHeterogeneousDecoder(r io.Reader) *HeterogeneousDecoder { 89 | return &HeterogeneousDecoder{ 90 | r: r, 91 | dr: NewDelimitedReader(r), 92 | } 93 | } 94 | 95 | // Decode reads an object of the specified type from the input. 96 | // The object passed to Decode must be a pointer to the type 97 | // was originally passed to Encode(). 98 | func (d *HeterogeneousDecoder) Decode(dest interface{}) error { 99 | t := reflect.TypeOf(dest) 100 | if t.Kind() != reflect.Ptr { 101 | panic(fmt.Sprintf("expected a pointer but got %T", dest)) 102 | } 103 | 104 | ptr, err := d.DecodePtr(t.Elem()) 105 | if err != nil { 106 | return err 107 | } 108 | reflect.ValueOf(dest).Elem().Set(reflect.ValueOf(ptr).Elem()) 109 | return nil 110 | } 111 | 112 | // DecodePtr reads an object of the specified type from the input 113 | // and returns a pointer to it. The provided type must be the result 114 | // of calling reflect.TypeOf(x) where x is the object originally 115 | // passed to Encode(). The return valoue will be of type *x 116 | func (d *HeterogeneousDecoder) DecodePtr(typ reflect.Type) (interface{}, error) { 117 | // read protocol 118 | if !d.hasprotocol { 119 | var protocol int32 120 | err := binary.Read(d.r, binary.LittleEndian, &protocol) 121 | if err != nil { 122 | return nil, fmt.Errorf("error reading protocol: %v", err) 123 | } 124 | if protocol != heterogeneousProtocol { 125 | return nil, fmt.Errorf("invalid protocol %d", protocol) 126 | } 127 | d.hasprotocol = true 128 | } 129 | 130 | // first segment: read the memory buffer 131 | dataseg, err := d.dr.Next() 132 | if len(dataseg) == 0 && err == io.EOF { 133 | return nil, io.EOF 134 | } 135 | if err != nil { 136 | return nil, fmt.Errorf("error reading data segment: %v", err) 137 | } 138 | 139 | // read the footer 140 | footerseg, err := d.dr.Next() 141 | if err != nil { 142 | return nil, fmt.Errorf("error reading footer segment: %v", err) 143 | } 144 | 145 | // decode footer 146 | var f heterogeneousFooter 147 | dec := gob.NewDecoder(bytes.NewBuffer(footerseg)) 148 | err = dec.Decode(&f) 149 | if err != nil { 150 | return nil, fmt.Errorf("error decoding footer: %v", err) 151 | } 152 | 153 | // compare descriptors 154 | descr := describe(typ) 155 | if !descriptorsEqual(descr, f.Descriptor) { 156 | return nil, ErrIncompatibleLayout 157 | } 158 | 159 | // relocate the data 160 | return relocate(dataseg, f.Pointers, f.Main, typ) 161 | } 162 | -------------------------------------------------------------------------------- /homogenous.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "fmt" 7 | "io" 8 | "reflect" 9 | ) 10 | 11 | // header is gob-encoded in the first segment 12 | type header struct { 13 | Protocol int32 14 | Descriptor descriptor 15 | } 16 | 17 | // Encoder writes memdumps to the provided writer 18 | type Encoder struct { 19 | w io.Writer 20 | t reflect.Type 21 | } 22 | 23 | // NewEncoder creates an Encoder that writes memdumps to the provided writer. 24 | // Each object passed to Encode must be of the same type. 25 | func NewEncoder(w io.Writer) *Encoder { 26 | return &Encoder{ 27 | w: w, 28 | } 29 | } 30 | 31 | // Encode writes a memdump of the provided object to output. You must pass a 32 | // pointer to the object you wish to encode. (To encode a pointer, pass a 33 | // pointer to a pointer.) 34 | func (e *Encoder) Encode(obj interface{}) error { 35 | t := reflect.TypeOf(obj) 36 | if t.Kind() != reflect.Ptr { 37 | panic(fmt.Sprintf("expected a pointer but got %T", obj)) 38 | } 39 | if e.t != nil && e.t != t { 40 | panic(fmt.Sprintf("each call to Encode should pass the same type, but got %v then %v", e.t, t)) 41 | } 42 | 43 | if e.t == nil { 44 | // write the header 45 | gob := gob.NewEncoder(e.w) 46 | err := gob.Encode(header{ 47 | Protocol: homogeneousProtocol, 48 | Descriptor: describe(t.Elem()), 49 | }) 50 | if err != nil { 51 | return fmt.Errorf("error writing footer: %v", err) 52 | } 53 | 54 | e.t = t 55 | _, err = e.w.Write(delim) 56 | if err != nil { 57 | return fmt.Errorf("error writing delimeter: %v", err) 58 | } 59 | } 60 | 61 | // first segment: write the object data 62 | mem := newMemEncoder(e.w) 63 | ptrs, err := mem.Encode(obj) 64 | if err != nil { 65 | return fmt.Errorf("error writing data segment: %v", err) 66 | } 67 | 68 | // write delimiter 69 | _, err = e.w.Write(delim) 70 | if err != nil { 71 | return fmt.Errorf("error writing delimiter: %v", err) 72 | } 73 | 74 | // second segment: write the footer 75 | err = encodeLocations(e.w, &locations{Pointers: ptrs}) 76 | if err != nil { 77 | return fmt.Errorf("error writing footer: %v", err) 78 | } 79 | 80 | // write delimiter 81 | _, err = e.w.Write(delim) 82 | if err != nil { 83 | return fmt.Errorf("error writing delimiter: %v", err) 84 | } 85 | return nil 86 | } 87 | 88 | // Decoder reads memdumps from the provided reader 89 | type Decoder struct { 90 | dr *DelimitedReader 91 | t reflect.Type 92 | } 93 | 94 | // NewDecoder creates a Decoder that reads memdumps 95 | func NewDecoder(r io.Reader) *Decoder { 96 | return &Decoder{ 97 | dr: NewDelimitedReader(r), 98 | } 99 | } 100 | 101 | // Decode reads an object of the specified type from the input. 102 | // The object passed to Decode must be a pointer to the type 103 | // was originally passed to Encode(). 104 | func (d *Decoder) Decode(dest interface{}) error { 105 | t := reflect.TypeOf(dest) 106 | if t.Kind() != reflect.Ptr { 107 | panic(fmt.Sprintf("expected a pointer but got %T", dest)) 108 | } 109 | 110 | ptr, err := d.DecodePtr(t.Elem()) 111 | if err != nil { 112 | return err 113 | } 114 | reflect.ValueOf(dest).Elem().Set(reflect.ValueOf(ptr).Elem()) 115 | return nil 116 | } 117 | 118 | // DecodePtr reads an object of the specified type from the input 119 | // and returns a pointer to it. The provided type must be the result 120 | // of calling reflect.TypeOf(x) where x is the object originally 121 | // passed to Encode(). The return valoue will be of type *x 122 | func (d *Decoder) DecodePtr(t reflect.Type) (interface{}, error) { 123 | if d.t != nil && d.t != t { 124 | panic(fmt.Sprintf("each call to Encode should pass the same type, but got %v then %v", d.t, t)) 125 | } 126 | 127 | // read the header 128 | if d.t == nil { 129 | // decode the descriptor 130 | seg, err := d.dr.Next() 131 | if err != nil { 132 | return nil, fmt.Errorf("error reading header segment: %v", err) 133 | } 134 | 135 | var header header 136 | dec := gob.NewDecoder(bytes.NewBuffer(seg)) 137 | err = dec.Decode(&header) 138 | if err != nil { 139 | return nil, fmt.Errorf("error decoding header: %v", err) 140 | } 141 | 142 | // compare descriptors 143 | expectedDescr := describe(t) 144 | if !descriptorsEqual(expectedDescr, header.Descriptor) { 145 | return nil, ErrIncompatibleLayout 146 | } 147 | 148 | d.t = t 149 | } 150 | 151 | // read the data 152 | dataseg, err := d.dr.Next() 153 | if len(dataseg) == 0 && err == io.EOF { 154 | return nil, io.EOF 155 | } 156 | if err != nil { 157 | return nil, fmt.Errorf("error reading data segment: %v", err) 158 | } 159 | 160 | // read the footer 161 | footerseg, err := d.dr.Next() 162 | if err != nil { 163 | return nil, fmt.Errorf("error decoding footer: %v", err) 164 | } 165 | 166 | // decode footer 167 | var f locations 168 | err = decodeLocations(bytes.NewBuffer(footerseg), &f) 169 | if err != nil { 170 | return nil, fmt.Errorf("error decoding footer: %v", err) 171 | } 172 | 173 | // relocate the data 174 | return relocate(dataseg, f.Pointers, f.Main, t) 175 | } 176 | -------------------------------------------------------------------------------- /serialize.go: -------------------------------------------------------------------------------- 1 | package memdump 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "reflect" 8 | "sort" 9 | "sync" 10 | "unsafe" 11 | ) 12 | 13 | // uintptrSize is the size in bytes of uintptr 14 | const uintptrSize = unsafe.Sizeof(uintptr(0)) 15 | 16 | // byteType is the reflect.Type of byte 17 | var ( 18 | byteType = reflect.TypeOf(byte(0)) 19 | typeCache = make(map[reflect.Type]*typeInfo) 20 | typeCacheLock sync.Mutex 21 | ) 22 | 23 | // block represents a value to be written to the stream 24 | type block struct { 25 | src reflect.Value 26 | dest uintptr 27 | } 28 | 29 | // pointer represents the location of a pointer in a type 30 | type pointer struct { 31 | offset uintptr 32 | typ reflect.Type 33 | } 34 | 35 | // typeInfo represents the location of the pointers in a type 36 | type typeInfo struct { 37 | pointers []pointer 38 | } 39 | 40 | // asBytes gets a byte slice with data pointer set to the address of the 41 | // assigned value and length set to the sizeof the value. 42 | func asBytes(v reflect.Value) []byte { 43 | size := int(v.Type().Size()) 44 | hdr := reflect.SliceHeader{ 45 | Data: v.Addr().Pointer(), 46 | Len: size, 47 | Cap: size, 48 | } 49 | return *(*[]byte)(unsafe.Pointer(&hdr)) 50 | } 51 | 52 | // isNil determines whether the pointer contained within v is nil. 53 | // This is equivalent to checking x==nil, except for strings, where 54 | // this method checks the data pointer inside the string header. 55 | func isNil(v reflect.Value) bool { 56 | if v.Kind() == reflect.String { 57 | hdr := (*reflect.StringHeader)(unsafe.Pointer(v.Addr().Pointer())) 58 | return hdr.Data == 0 59 | } 60 | return v.IsNil() 61 | } 62 | 63 | func readPointer(v reflect.Value) uintptr { 64 | if v.Kind() == reflect.String { 65 | hdr := (*reflect.StringHeader)(unsafe.Pointer(v.Addr().Pointer())) 66 | return hdr.Data 67 | } 68 | return v.Pointer() 69 | } 70 | 71 | type byOffset []pointer 72 | 73 | func (xs byOffset) Len() int { return len(xs) } 74 | func (xs byOffset) Swap(i, j int) { xs[i], xs[j] = xs[j], xs[i] } 75 | func (xs byOffset) Less(i, j int) bool { return xs[i].offset < xs[j].offset } 76 | 77 | type countingWriter struct { 78 | w io.Writer 79 | offset int 80 | } 81 | 82 | func (w *countingWriter) Write(buf []byte) (int, error) { 83 | n, err := w.w.Write(buf) 84 | w.offset += n 85 | return n, err 86 | } 87 | 88 | // memEncoder writes the in-memory representation of an object, together 89 | // with all referenced objects. 90 | type memEncoder struct { 91 | w countingWriter 92 | } 93 | 94 | func newMemEncoder(w io.Writer) *memEncoder { 95 | return &memEncoder{ 96 | w: countingWriter{w: w}, 97 | } 98 | } 99 | 100 | // memEncoderState contains the state that is local to a single Encode() call. 101 | type memEncoderState struct { 102 | ptrLocs []int64 103 | next uintptr 104 | } 105 | 106 | // alloc makes room for N objects of the specified type, and returns the 107 | // base offset for that object. It deals correctly with alignment. 108 | func (e *memEncoderState) alloc(t reflect.Type, n int) uintptr { 109 | align := uintptr(t.Align()) 110 | if e.next%align != 0 { 111 | e.next += align - (e.next % align) 112 | } 113 | cur := e.next 114 | e.next += t.Size() * uintptr(n) 115 | return cur 116 | } 117 | 118 | // arrayFromString gets a fixed-size array representing the bytes pointed to 119 | // by a string. 120 | func arrayFromString(strval reflect.Value) reflect.Value { 121 | if strval.Kind() != reflect.String { 122 | panic(fmt.Sprintf("expected string type but got %s", strval.Type())) 123 | } 124 | 125 | hdr := (*reflect.StringHeader)(unsafe.Pointer(strval.Addr().Pointer())) 126 | typ := reflect.ArrayOf(hdr.Len, byteType) 127 | return reflect.NewAt(typ, unsafe.Pointer(hdr.Data)).Elem() 128 | } 129 | 130 | // arrayFromSlice gets a fixed-size array representing the data pointed to by 131 | // a slice 132 | func arrayFromSlice(sliceval reflect.Value) reflect.Value { 133 | if sliceval.Kind() != reflect.Slice { 134 | panic(fmt.Sprintf("expected string type but got %s", sliceval.Type())) 135 | } 136 | 137 | hdr := (*reflect.SliceHeader)(unsafe.Pointer(sliceval.Addr().Pointer())) 138 | typ := reflect.ArrayOf(hdr.Len, sliceval.Type().Elem()) 139 | return reflect.NewAt(typ, unsafe.Pointer(hdr.Data)).Elem() 140 | } 141 | 142 | // Encode writes the in-memory representation of the object pointed to by ptr. It 143 | // returns the offset of each pointer and an error. 144 | func (e *memEncoder) Encode(ptr interface{}) ([]int64, error) { 145 | var state memEncoderState 146 | 147 | ptrval := reflect.ValueOf(ptr) 148 | objval := ptrval.Elem() 149 | cache := make(map[uintptr]uintptr) 150 | queue := []block{{ 151 | src: objval, 152 | dest: state.alloc(objval.Type(), 1), 153 | }} 154 | 155 | for len(queue) > 0 { 156 | cur := queue[0] 157 | queue = queue[1:] 158 | 159 | blockaddr := cur.src.Addr() 160 | blockbytes := asBytes(cur.src) 161 | 162 | // check the position of the writer 163 | if cur.dest < uintptr(e.w.offset) { 164 | panic(fmt.Sprintf("block.dest=%d but writer is at %d", cur.dest, e.w.offset)) 165 | } 166 | 167 | // for byte-alignment purposes we may need to fill some bytes 168 | if fill := cur.dest - uintptr(e.w.offset); fill > 0 { 169 | _, err := e.w.Write(make([]byte, fill)) 170 | if err != nil { 171 | return nil, err 172 | } 173 | } 174 | 175 | // look up info about this type 176 | info := lookupType(cur.src.Type()) 177 | 178 | // add each referenced object to the queue 179 | var blockpos uintptr 180 | for _, ptr := range info.pointers { 181 | _, err := e.w.Write(blockbytes[blockpos:ptr.offset]) 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | ptrdata := unsafe.Pointer(blockaddr.Pointer() + ptr.offset) 187 | ptrval := reflect.NewAt(ptr.typ, ptrdata).Elem() 188 | 189 | var dest uintptr 190 | if !isNil(ptrval) { 191 | state.ptrLocs = append(state.ptrLocs, int64(cur.dest+ptr.offset)) 192 | 193 | var found bool 194 | dest, found = cache[readPointer(ptrval)] 195 | if !found { 196 | switch ptr.typ.Kind() { 197 | case reflect.Ptr: 198 | dest = state.alloc(ptr.typ.Elem(), 1) 199 | queue = append(queue, block{ 200 | src: ptrval.Elem(), 201 | dest: dest, 202 | }) 203 | case reflect.Slice: 204 | dest = state.alloc(ptr.typ.Elem(), ptrval.Len()) 205 | arr := arrayFromSlice(ptrval) 206 | queue = append(queue, block{ 207 | src: arr, 208 | dest: dest, 209 | }) 210 | case reflect.String: 211 | dest = state.alloc(byteType, ptrval.Len()) 212 | arr := arrayFromString(ptrval) 213 | queue = append(queue, block{ 214 | src: arr, 215 | dest: dest, 216 | }) 217 | } 218 | cache[readPointer(ptrval)] = dest 219 | } 220 | } 221 | 222 | err = binary.Write(&e.w, binary.LittleEndian, uint64(dest)) 223 | if err != nil { 224 | return nil, err 225 | } 226 | blockpos = ptr.offset + uintptrSize 227 | } 228 | 229 | _, err := e.w.Write(blockbytes[blockpos:]) 230 | if err != nil { 231 | return nil, err 232 | } 233 | } 234 | 235 | return state.ptrLocs, nil 236 | } 237 | 238 | // pointerFinder gets the byte offset of each pointer in an object. It 239 | // only considers the immediate value of an object (i.e. the bytes that 240 | // would be copied in a simple assignment). It does not follow pointers 241 | // to other objects. 242 | type pointerFinder struct { 243 | pointers []pointer 244 | } 245 | 246 | func (f *pointerFinder) visit(t reflect.Type, base uintptr) { 247 | switch t.Kind() { 248 | case reflect.Ptr, reflect.String, reflect.Slice: 249 | // these four types all store one pointer at offset zero 250 | f.pointers = append(f.pointers, pointer{ 251 | offset: base, 252 | typ: t, 253 | }) 254 | case reflect.Struct: 255 | for i := 0; i < t.NumField(); i++ { 256 | field := t.Field(i) 257 | f.visit(field.Type, base+field.Offset) 258 | } 259 | case reflect.Array: 260 | elemSize := t.Elem().Size() 261 | elemPtrs := lookupType(t.Elem()).pointers 262 | for _, elemPtr := range elemPtrs { 263 | for i := 0; i < t.Len(); i++ { 264 | f.pointers = append(f.pointers, pointer{ 265 | offset: base + uintptr(i)*elemSize + elemPtr.offset, 266 | typ: elemPtr.typ, 267 | }) 268 | } 269 | } 270 | case reflect.Map, reflect.Chan, reflect.Interface, reflect.UnsafePointer, reflect.Func: 271 | panic(fmt.Sprintf("cannot serialize objects of %v kind (got %v)", t.Kind(), t)) 272 | } 273 | } 274 | 275 | // lookupType gets the type info for t. 276 | func lookupType(t reflect.Type) *typeInfo { 277 | typeCacheLock.Lock() 278 | info, found := typeCache[t] 279 | typeCacheLock.Unlock() 280 | 281 | if !found { 282 | var f pointerFinder 283 | f.visit(t, 0) 284 | info = &typeInfo{pointers: f.pointers} 285 | sort.Sort(byOffset(info.pointers)) 286 | 287 | typeCacheLock.Lock() 288 | typeCache[t] = info 289 | typeCacheLock.Unlock() 290 | } 291 | 292 | return info 293 | } 294 | --------------------------------------------------------------------------------