├── fixtures ├── intset_16.rdb ├── intset_32.rdb ├── intset_64.rdb ├── dictionary.rdb ├── linkedlist.rdb ├── regular_set.rdb ├── empty_database.rdb ├── integer_keys.rdb ├── hash_as_ziplist.rdb ├── keys_with_expiry.rdb ├── multiple_databases.rdb ├── regular_sorted_set.rdb ├── rdb_v7_list_quicklist.rdb ├── sorted_set_as_ziplist.rdb ├── ziplist_with_integers.rdb ├── keys_with_mixed_expiry.rdb ├── zipmap_with_big_values.rdb ├── uncompressible_string_keys.rdb ├── rdb_version_5_with_checksum.rdb ├── ziplist_that_doesnt_compress.rdb ├── zipmap_that_compresses_easily.rdb ├── zipmap_that_doesnt_compress.rdb ├── easily_compressible_string_key.rdb └── ziplist_that_compresses_easily.rdb ├── .travis.yml ├── .gitignore ├── README.md ├── LICENCE ├── encoder_test.go ├── slice_buffer.go ├── nopdecoder └── nop_decoder.go ├── examples └── diff.go ├── encoder.go ├── crc64 └── crc64.go ├── decoder_test.go └── decoder.go /fixtures/intset_16.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/intset_16.rdb -------------------------------------------------------------------------------- /fixtures/intset_32.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/intset_32.rdb -------------------------------------------------------------------------------- /fixtures/intset_64.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/intset_64.rdb -------------------------------------------------------------------------------- /fixtures/dictionary.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/dictionary.rdb -------------------------------------------------------------------------------- /fixtures/linkedlist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/linkedlist.rdb -------------------------------------------------------------------------------- /fixtures/regular_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/regular_set.rdb -------------------------------------------------------------------------------- /fixtures/empty_database.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/empty_database.rdb -------------------------------------------------------------------------------- /fixtures/integer_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/integer_keys.rdb -------------------------------------------------------------------------------- /fixtures/hash_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/hash_as_ziplist.rdb -------------------------------------------------------------------------------- /fixtures/keys_with_expiry.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/keys_with_expiry.rdb -------------------------------------------------------------------------------- /fixtures/multiple_databases.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/multiple_databases.rdb -------------------------------------------------------------------------------- /fixtures/regular_sorted_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/regular_sorted_set.rdb -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.1 4 | - tip 5 | before_install: 6 | - go get gopkg.in/check.v1 7 | -------------------------------------------------------------------------------- /fixtures/rdb_v7_list_quicklist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/rdb_v7_list_quicklist.rdb -------------------------------------------------------------------------------- /fixtures/sorted_set_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/sorted_set_as_ziplist.rdb -------------------------------------------------------------------------------- /fixtures/ziplist_with_integers.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/ziplist_with_integers.rdb -------------------------------------------------------------------------------- /fixtures/keys_with_mixed_expiry.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/keys_with_mixed_expiry.rdb -------------------------------------------------------------------------------- /fixtures/zipmap_with_big_values.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/zipmap_with_big_values.rdb -------------------------------------------------------------------------------- /fixtures/uncompressible_string_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/uncompressible_string_keys.rdb -------------------------------------------------------------------------------- /fixtures/rdb_version_5_with_checksum.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/rdb_version_5_with_checksum.rdb -------------------------------------------------------------------------------- /fixtures/ziplist_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/ziplist_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /fixtures/zipmap_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/zipmap_that_compresses_easily.rdb -------------------------------------------------------------------------------- /fixtures/zipmap_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/zipmap_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /fixtures/easily_compressible_string_key.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/easily_compressible_string_key.rdb -------------------------------------------------------------------------------- /fixtures/ziplist_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tent/rdb/HEAD/fixtures/ziplist_that_compresses_easily.rdb -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | 24 | # Project-specific files 25 | diff 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rdb [![Build Status](https://travis-ci.org/cupcake/rdb.png?branch=master)](https://travis-ci.org/cupcake/rdb) 2 | 3 | rdb is a Go package that implements parsing and encoding of the 4 | [Redis](http://redis.io) [RDB file 5 | format](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_File_Format.textile). 6 | 7 | This package was heavily inspired by 8 | [redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools) by 9 | [Sripathi Krishnan](https://github.com/sripathikrishnan). 10 | 11 | [**Documentation**](http://godoc.org/github.com/cupcake/rdb) 12 | 13 | ## Installation 14 | 15 | ``` 16 | go get github.com/cupcake/rdb 17 | ``` 18 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Jonathan Rudenberg 2 | Copyright (c) 2012 Sripathi Krishnan 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining 5 | a copy of this software and associated documentation files (the 6 | "Software"), to deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, merge, publish, 8 | distribute, sublicense, and/or sell copies of the Software, and to 9 | permit persons to whom the Software is furnished to do so, subject to 10 | the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /encoder_test.go: -------------------------------------------------------------------------------- 1 | package rdb_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | 7 | "github.com/cupcake/rdb" 8 | . "gopkg.in/check.v1" 9 | ) 10 | 11 | type EncoderSuite struct{} 12 | 13 | var _ = Suite(&EncoderSuite{}) 14 | 15 | var stringEncodingTests = []struct { 16 | str string 17 | res string 18 | }{ 19 | {"0", "AMAABgAOrc/4DQU/mw=="}, 20 | {"127", "AMB/BgCbWIOxpwH5hw=="}, 21 | {"-128", "AMCABgAPi1rt2llnSg=="}, 22 | {"128", "AMGAAAYAfZfbNeWad/Y="}, 23 | {"-129", "AMF//wYAgY3qqKHVuBM="}, 24 | {"32767", "AMH/fwYA37dfWuKh6bg="}, 25 | {"-32768", "AMEAgAYAI61ux6buJl0="}, 26 | {"-32768", "AMEAgAYAI61ux6buJl0="}, 27 | {"2147483647", "AML///9/BgC6mY0eFXuRMg=="}, 28 | {"-2147483648", "AMIAAACABgBRou++xgC9FA=="}, 29 | {"a", "AAFhBgApE4cbemNBJw=="}, 30 | } 31 | 32 | func (e *EncoderSuite) TestStringEncoding(c *C) { 33 | buf := &bytes.Buffer{} 34 | for _, t := range stringEncodingTests { 35 | e := rdb.NewEncoder(buf) 36 | e.EncodeType(rdb.TypeString) 37 | e.EncodeString([]byte(t.str)) 38 | e.EncodeDumpFooter() 39 | expected, _ := base64.StdEncoding.DecodeString(t.res) 40 | c.Assert(buf.Bytes(), DeepEquals, expected, Commentf("%s - expected: %x, actual: %x", t.str, expected, buf.Bytes())) 41 | buf.Reset() 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /slice_buffer.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | ) 7 | 8 | type sliceBuffer struct { 9 | s []byte 10 | i int 11 | } 12 | 13 | func newSliceBuffer(s []byte) *sliceBuffer { 14 | return &sliceBuffer{s, 0} 15 | } 16 | 17 | func (s *sliceBuffer) Slice(n int) ([]byte, error) { 18 | if s.i+n > len(s.s) { 19 | return nil, io.EOF 20 | } 21 | b := s.s[s.i : s.i+n] 22 | s.i += n 23 | return b, nil 24 | } 25 | 26 | func (s *sliceBuffer) ReadByte() (byte, error) { 27 | if s.i >= len(s.s) { 28 | return 0, io.EOF 29 | } 30 | b := s.s[s.i] 31 | s.i++ 32 | return b, nil 33 | } 34 | 35 | func (s *sliceBuffer) Read(b []byte) (int, error) { 36 | if len(b) == 0 { 37 | return 0, nil 38 | } 39 | if s.i >= len(s.s) { 40 | return 0, io.EOF 41 | } 42 | n := copy(b, s.s[s.i:]) 43 | s.i += n 44 | return n, nil 45 | } 46 | 47 | func (s *sliceBuffer) Seek(offset int64, whence int) (int64, error) { 48 | var abs int64 49 | switch whence { 50 | case 0: 51 | abs = offset 52 | case 1: 53 | abs = int64(s.i) + offset 54 | case 2: 55 | abs = int64(len(s.s)) + offset 56 | default: 57 | return 0, errors.New("invalid whence") 58 | } 59 | if abs < 0 { 60 | return 0, errors.New("negative position") 61 | } 62 | if abs >= 1<<31 { 63 | return 0, errors.New("position out of range") 64 | } 65 | s.i = int(abs) 66 | return abs, nil 67 | } 68 | -------------------------------------------------------------------------------- /nopdecoder/nop_decoder.go: -------------------------------------------------------------------------------- 1 | package nopdecoder 2 | 3 | // NopDecoder may be embedded in a real Decoder to avoid implementing methods. 4 | type NopDecoder struct{} 5 | 6 | func (d NopDecoder) StartRDB() {} 7 | func (d NopDecoder) StartDatabase(n int) {} 8 | func (d NopDecoder) Aux(key, value []byte) {} 9 | func (d NopDecoder) ResizeDatabase(dbSize, expiresSize uint32) {} 10 | func (d NopDecoder) EndDatabase(n int) {} 11 | func (d NopDecoder) EndRDB() {} 12 | func (d NopDecoder) Set(key, value []byte, expiry int64) {} 13 | func (d NopDecoder) StartHash(key []byte, length, expiry int64) {} 14 | func (d NopDecoder) Hset(key, field, value []byte) {} 15 | func (d NopDecoder) EndHash(key []byte) {} 16 | func (d NopDecoder) StartSet(key []byte, cardinality, expiry int64) {} 17 | func (d NopDecoder) Sadd(key, member []byte) {} 18 | func (d NopDecoder) EndSet(key []byte) {} 19 | func (d NopDecoder) StartList(key []byte, length, expiry int64) {} 20 | func (d NopDecoder) Rpush(key, value []byte) {} 21 | func (d NopDecoder) EndList(key []byte) {} 22 | func (d NopDecoder) StartZSet(key []byte, cardinality, expiry int64) {} 23 | func (d NopDecoder) Zadd(key []byte, score float64, member []byte) {} 24 | func (d NopDecoder) EndZSet(key []byte) {} 25 | -------------------------------------------------------------------------------- /examples/diff.go: -------------------------------------------------------------------------------- 1 | // This is a very basic example of a program that implements rdb.decoder and 2 | // outputs a human readable diffable dump of the rdb file. 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | 9 | "github.com/cupcake/rdb" 10 | "github.com/cupcake/rdb/nopdecoder" 11 | ) 12 | 13 | type decoder struct { 14 | db int 15 | i int 16 | nopdecoder.NopDecoder 17 | } 18 | 19 | func (p *decoder) StartDatabase(n int) { 20 | p.db = n 21 | } 22 | 23 | func (p *decoder) Set(key, value []byte, expiry int64) { 24 | fmt.Printf("db=%d %q -> %q\n", p.db, key, value) 25 | } 26 | 27 | func (p *decoder) Hset(key, field, value []byte) { 28 | fmt.Printf("db=%d %q . %q -> %q\n", p.db, key, field, value) 29 | } 30 | 31 | func (p *decoder) Sadd(key, member []byte) { 32 | fmt.Printf("db=%d %q { %q }\n", p.db, key, member) 33 | } 34 | 35 | func (p *decoder) StartList(key []byte, length, expiry int64) { 36 | p.i = 0 37 | } 38 | 39 | func (p *decoder) Rpush(key, value []byte) { 40 | fmt.Printf("db=%d %q[%d] -> %q\n", p.db, key, p.i, value) 41 | p.i++ 42 | } 43 | 44 | func (p *decoder) StartZSet(key []byte, cardinality, expiry int64) { 45 | p.i = 0 46 | } 47 | 48 | func (p *decoder) Zadd(key []byte, score float64, member []byte) { 49 | fmt.Printf("db=%d %q[%d] -> {%q, score=%g}\n", p.db, key, p.i, member, score) 50 | p.i++ 51 | } 52 | 53 | func maybeFatal(err error) { 54 | if err != nil { 55 | fmt.Printf("Fatal error: %s\n", err) 56 | os.Exit(1) 57 | } 58 | } 59 | 60 | func main() { 61 | f, err := os.Open(os.Args[1]) 62 | maybeFatal(err) 63 | err = rdb.Decode(f, &decoder{}) 64 | maybeFatal(err) 65 | } 66 | -------------------------------------------------------------------------------- /encoder.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "hash" 7 | "io" 8 | "math" 9 | "strconv" 10 | 11 | "github.com/cupcake/rdb/crc64" 12 | ) 13 | 14 | const Version = 6 15 | 16 | type Encoder struct { 17 | w io.Writer 18 | crc hash.Hash 19 | } 20 | 21 | func NewEncoder(w io.Writer) *Encoder { 22 | e := &Encoder{crc: crc64.New()} 23 | e.w = io.MultiWriter(w, e.crc) 24 | return e 25 | } 26 | 27 | func (e *Encoder) EncodeHeader() error { 28 | _, err := fmt.Fprintf(e.w, "REDIS%04d", Version) 29 | return err 30 | } 31 | 32 | func (e *Encoder) EncodeFooter() error { 33 | e.w.Write([]byte{rdbFlagEOF}) 34 | _, err := e.w.Write(e.crc.Sum(nil)) 35 | return err 36 | } 37 | 38 | func (e *Encoder) EncodeDumpFooter() error { 39 | binary.Write(e.w, binary.LittleEndian, uint16(Version)) 40 | _, err := e.w.Write(e.crc.Sum(nil)) 41 | return err 42 | } 43 | 44 | func (e *Encoder) EncodeDatabase(n int) error { 45 | e.w.Write([]byte{rdbFlagSelectDB}) 46 | return e.EncodeLength(uint32(n)) 47 | } 48 | 49 | func (e *Encoder) EncodeExpiry(expiry uint64) error { 50 | b := make([]byte, 9) 51 | b[0] = rdbFlagExpiryMS 52 | binary.LittleEndian.PutUint64(b[1:], expiry) 53 | _, err := e.w.Write(b) 54 | return err 55 | } 56 | 57 | func (e *Encoder) EncodeType(v ValueType) error { 58 | _, err := e.w.Write([]byte{byte(v)}) 59 | return err 60 | } 61 | 62 | func (e *Encoder) EncodeString(s []byte) error { 63 | written, err := e.encodeIntString(s) 64 | if written { 65 | return err 66 | } 67 | e.EncodeLength(uint32(len(s))) 68 | _, err = e.w.Write(s) 69 | return err 70 | } 71 | 72 | func (e *Encoder) EncodeLength(l uint32) (err error) { 73 | switch { 74 | case l < 1<<6: 75 | _, err = e.w.Write([]byte{byte(l)}) 76 | case l < 1<<14: 77 | _, err = e.w.Write([]byte{byte(l>>8) | rdb14bitLen<<6, byte(l)}) 78 | default: 79 | b := make([]byte, 5) 80 | b[0] = rdb32bitLen << 6 81 | binary.BigEndian.PutUint32(b[1:], l) 82 | _, err = e.w.Write(b) 83 | } 84 | return 85 | } 86 | 87 | func (e *Encoder) EncodeFloat(f float64) (err error) { 88 | switch { 89 | case math.IsNaN(f): 90 | _, err = e.w.Write([]byte{253}) 91 | case math.IsInf(f, 1): 92 | _, err = e.w.Write([]byte{254}) 93 | case math.IsInf(f, -1): 94 | _, err = e.w.Write([]byte{255}) 95 | default: 96 | b := []byte(strconv.FormatFloat(f, 'g', 17, 64)) 97 | e.w.Write([]byte{byte(len(b))}) 98 | _, err = e.w.Write(b) 99 | } 100 | return 101 | } 102 | 103 | func (e *Encoder) encodeIntString(b []byte) (written bool, err error) { 104 | s := string(b) 105 | i, err := strconv.ParseInt(s, 10, 32) 106 | if err != nil { 107 | return 108 | } 109 | // if the stringified parsed int isn't exactly the same, we can't encode it as an int 110 | if s != strconv.FormatInt(i, 10) { 111 | return 112 | } 113 | switch { 114 | case i >= math.MinInt8 && i <= math.MaxInt8: 115 | _, err = e.w.Write([]byte{rdbEncVal << 6, byte(int8(i))}) 116 | case i >= math.MinInt16 && i <= math.MaxInt16: 117 | b := make([]byte, 3) 118 | b[0] = rdbEncVal<<6 | rdbEncInt16 119 | binary.LittleEndian.PutUint16(b[1:], uint16(int16(i))) 120 | _, err = e.w.Write(b) 121 | case i >= math.MinInt32 && i <= math.MaxInt32: 122 | b := make([]byte, 5) 123 | b[0] = rdbEncVal<<6 | rdbEncInt32 124 | binary.LittleEndian.PutUint32(b[1:], uint32(int32(i))) 125 | _, err = e.w.Write(b) 126 | default: 127 | return 128 | } 129 | return true, err 130 | } 131 | -------------------------------------------------------------------------------- /crc64/crc64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2009 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package crc64 implements the Jones coefficients with an init value of 0. 6 | package crc64 7 | 8 | import "hash" 9 | 10 | // Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. 11 | // 12 | // Specification of this CRC64 variant follows: 13 | // Name: crc-64-jones 14 | // Width: 64 bits 15 | // Poly: 0xad93d23594c935a9 16 | // Reflected In: True 17 | // Xor_In: 0xffffffffffffffff 18 | // Reflected_Out: True 19 | // Xor_Out: 0x0 20 | 21 | var table = [256]uint64{0x0000000000000000, 0x7ad870c830358979, 0xf5b0e190606b12f2, 0x8f689158505e9b8b, 0xc038e5739841b68f, 0xbae095bba8743ff6, 0x358804e3f82aa47d, 0x4f50742bc81f2d04, 0xab28ecb46814fe75, 0xd1f09c7c5821770c, 0x5e980d24087fec87, 0x24407dec384a65fe, 0x6b1009c7f05548fa, 0x11c8790fc060c183, 0x9ea0e857903e5a08, 0xe478989fa00bd371, 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8, 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a, 0xbd301a4810ffd90e, 0xc7e86a8020ca5077, 0x4880fbd87094cbfc, 0x32588b1040a14285, 0xd620138fe0aa91f4, 0xacf86347d09f188d, 0x2390f21f80c18306, 0x594882d7b0f40a7f, 0x1618f6fc78eb277b, 0x6cc0863448deae02, 0xe3a8176c18803589, 0x997067a428b5bcf0, 0xfa11fe77117cdf02, 0x80c98ebf2149567b, 0x0fa11fe77117cdf0, 0x75796f2f41224489, 0x3a291b04893d698d, 0x40f16bccb908e0f4, 0xcf99fa94e9567b7f, 0xb5418a5cd963f206, 0x513912c379682177, 0x2be1620b495da80e, 0xa489f35319033385, 0xde51839b2936bafc, 0x9101f7b0e12997f8, 0xebd98778d11c1e81, 0x64b116208142850a, 0x1e6966e8b1770c73, 0x8719014c99c2b083, 0xfdc17184a9f739fa, 0x72a9e0dcf9a9a271, 0x08719014c99c2b08, 0x4721e43f0183060c, 0x3df994f731b68f75, 0xb29105af61e814fe, 0xc849756751dd9d87, 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f, 0xd9810c6891bd5c04, 0xa3597ca0a188d57d, 0xec09088b6997f879, 0x96d1784359a27100, 0x19b9e91b09fcea8b, 0x636199d339c963f2, 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416, 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4, 0x1f423fcee22f9be0, 0x659a4f06d21a1299, 0xeaf2de5e82448912, 0x902aae96b271006b, 0x74523609127ad31a, 0x0e8a46c1224f5a63, 0x81e2d7997211c1e8, 0xfb3aa75142244891, 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec, 0x41da32eaea507767, 0x3b024222da65fe1e, 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97, 0x57c2c41692bb501c, 0x2d1ab4dea28ed965, 0x624ac0f56a91f461, 0x1892b03d5aa47d18, 0x97fa21650afae693, 0xed2251ad3acf6fea, 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2, 0xfcea28a2faafae69, 0x8632586aca9a2710, 0xc9622c4102850a14, 0xb3ba5c8932b0836d, 0x3cd2cdd162ee18e6, 0x460abd1952db919f, 0x256b24ca6b12f26d, 0x5fb354025b277b14, 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6, 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b, 0x10e3202993385610, 0x6a3b50e1a30ddf69, 0x8e43c87e03060c18, 0xf49bb8b633338561, 0x7bf329ee636d1eea, 0x012b592653589793, 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee, 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c, 0x5863dbf1e3ac9dec, 0x22bbab39d3991495, 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667, 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a, 0x6debdf121b863991, 0x1733afda2bb3b0e8, 0xf34b37458bb86399, 0x8993478dbb8deae0, 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812, 0x3373d23613f9d516, 0x49aba2fe23cc5c6f, 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d, 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc, 0x601c72b9cc20db47, 0x1ac40271fc15523e, 0x5594765a340a7f3a, 0x2f4c0692043ff643, 0xa02497ca54616dc8, 0xdafce7026454e4b1, 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9, 0xcb349e0da4342532, 0xb1eceec59401ac4b, 0xfebc9aee5c1e814f, 0x8464ea266c2b0836, 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4, 0xe8a46c1224f5a634, 0x927c1cda14c02f4d, 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf, 0x289c8961bcb410bb, 0x5244f9a98c8199c2, 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30, 0x438c80a64ce15841, 0x3954f06e7cd4d138, 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca, 0x83b465d5d4a0eece, 0xf96c151de49567b7, 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545, 0x6fbd6d5ebd3716b7, 0x15651d968d029fce, 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c, 0xaf85882d2576a038, 0xd55df8e515432941, 0x5a3569bd451db2ca, 0x20ed197575283bb3, 0xc49581ead523e8c2, 0xbe4df122e51661bb, 0x3125607ab548fa30, 0x4bfd10b2857d7349, 0x04ad64994d625e4d, 0x7e7514517d57d734, 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6, 0x12b5926535897936, 0x686de2ad05bcf04f, 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd, 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0, 0x273d9686cda3dd4b, 0x5de5e64efd965432, 0xb99d7ed15d9d8743, 0xc3450e196da80e3a, 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8, 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5, 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47, 0x4ad64994d625e4da, 0x300e395ce6106da3, 0xbf66a804b64ef628, 0xc5bed8cc867b7f51, 0x8aeeace74e645255, 0xf036dc2f7e51db2c, 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de, 0xe1fea520be311aaf, 0x9b26d5e88e0493d6, 0x144e44b0de5a085d, 0x6e963478ee6f8124, 0x21c640532670ac20, 0x5b1e309b16452559, 0xd476a1c3461bbed2, 0xaeaed10b762e37ab, 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222, 0xc26e573f3ef099a9, 0xb8b627f70ec510d0, 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad, 0x0256b24ca6b12f26, 0x788ec2849684a65f, 0x9cf65a1b368f752e, 0xe62e2ad306bafc57, 0x6946bb8b56e467dc, 0x139ecb4366d1eea5, 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8, 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a, 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1, 0x45775673a732292a, 0x3faf26bb9707a053, 0x70ff52905f188d57, 0x0a2722586f2d042e, 0x854fb3003f739fa5, 0xff97c3c80f4616dc, 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4, 0xee5fbac7cf26d75f, 0x9487ca0fff135e26, 0xdbd7be24370c7322, 0xa10fceec0739fa5b, 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9, 0xcdcf48d84fe75459, 0xb71738107fd2dd20, 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2, 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf, 0xf8474c3bb7cdf024, 0x829f3cf387f8795d, 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355, 0x935745fc4798b8de, 0xe98f353477ad31a7, 0xa6df411fbfb21ca3, 0xdc0731d78f8795da, 0x536fa08fdfd90e51, 0x29b7d047efec8728} 22 | 23 | func crc64(crc uint64, b []byte) uint64 { 24 | for _, v := range b { 25 | crc = table[byte(crc)^v] ^ (crc >> 8) 26 | } 27 | return crc 28 | } 29 | 30 | func Digest(b []byte) uint64 { 31 | return crc64(0, b) 32 | } 33 | 34 | type digest struct { 35 | crc uint64 36 | } 37 | 38 | func New() hash.Hash64 { 39 | return &digest{} 40 | } 41 | 42 | func (h *digest) Write(p []byte) (int, error) { 43 | h.crc = crc64(h.crc, p) 44 | return len(p), nil 45 | } 46 | 47 | // Encode in little endian 48 | func (d *digest) Sum(in []byte) []byte { 49 | s := d.Sum64() 50 | in = append(in, byte(s)) 51 | in = append(in, byte(s>>8)) 52 | in = append(in, byte(s>>16)) 53 | in = append(in, byte(s>>24)) 54 | in = append(in, byte(s>>32)) 55 | in = append(in, byte(s>>40)) 56 | in = append(in, byte(s>>48)) 57 | in = append(in, byte(s>>56)) 58 | return in 59 | } 60 | 61 | func (d *digest) Sum64() uint64 { return d.crc } 62 | func (d *digest) BlockSize() int { return 1 } 63 | func (d *digest) Size() int { return 8 } 64 | func (d *digest) Reset() { d.crc = 0 } 65 | -------------------------------------------------------------------------------- /decoder_test.go: -------------------------------------------------------------------------------- 1 | package rdb_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/cupcake/rdb" 10 | . "gopkg.in/check.v1" 11 | ) 12 | 13 | // Hook gocheck into the gotest runner. 14 | func Test(t *testing.T) { TestingT(t) } 15 | 16 | type DecoderSuite struct{} 17 | 18 | var _ = Suite(&DecoderSuite{}) 19 | 20 | func (s *DecoderSuite) TestEmptyRDB(c *C) { 21 | r := decodeRDB("empty_database") 22 | c.Assert(r.started, Equals, 1) 23 | c.Assert(r.ended, Equals, 1) 24 | c.Assert(len(r.dbs), Equals, 0) 25 | } 26 | 27 | func (s *DecoderSuite) TestMultipleDatabases(c *C) { 28 | r := decodeRDB("multiple_databases") 29 | c.Assert(len(r.dbs), Equals, 2) 30 | _, ok := r.dbs[1] 31 | c.Assert(ok, Equals, false) 32 | c.Assert(r.dbs[0]["key_in_zeroth_database"], Equals, "zero") 33 | c.Assert(r.dbs[2]["key_in_second_database"], Equals, "second") 34 | } 35 | 36 | func (s *DecoderSuite) TestExpiry(c *C) { 37 | r := decodeRDB("keys_with_expiry") 38 | c.Assert(r.expiries[0]["expires_ms_precision"], Equals, int64(1671963072573)) 39 | } 40 | 41 | func (s *DecoderSuite) TestMixedExpiry(c *C) { 42 | r := decodeRDB("keys_with_mixed_expiry") 43 | c.Assert(r.expiries[0]["key01"], Not(Equals), int64(0)) 44 | c.Assert(r.expiries[0]["key04"], Not(Equals), int64(0)) 45 | 46 | c.Assert(r.expiries[0]["key02"], Equals, int64(0)) 47 | c.Assert(r.expiries[0]["key03"], Equals, int64(0)) 48 | } 49 | 50 | func (s *DecoderSuite) TestIntegerKeys(c *C) { 51 | r := decodeRDB("integer_keys") 52 | c.Assert(r.dbs[0]["125"], Equals, "Positive 8 bit integer") 53 | c.Assert(r.dbs[0]["43947"], Equals, "Positive 16 bit integer") 54 | c.Assert(r.dbs[0]["183358245"], Equals, "Positive 32 bit integer") 55 | c.Assert(r.dbs[0]["-123"], Equals, "Negative 8 bit integer") 56 | c.Assert(r.dbs[0]["-29477"], Equals, "Negative 16 bit integer") 57 | c.Assert(r.dbs[0]["-183358245"], Equals, "Negative 32 bit integer") 58 | } 59 | 60 | func (s *DecoderSuite) TestStringKeyWithCompression(c *C) { 61 | r := decodeRDB("easily_compressible_string_key") 62 | c.Assert(r.dbs[0][strings.Repeat("a", 200)], Equals, "Key that redis should compress easily") 63 | } 64 | 65 | func (s *DecoderSuite) TestZipmapWithCompression(c *C) { 66 | r := decodeRDB("zipmap_that_compresses_easily") 67 | zm := r.dbs[0]["zipmap_compresses_easily"].(map[string]string) 68 | c.Assert(zm["a"], Equals, "aa") 69 | c.Assert(zm["aa"], Equals, "aaaa") 70 | c.Assert(zm["aaaaa"], Equals, "aaaaaaaaaaaaaa") 71 | } 72 | 73 | func (s *DecoderSuite) TestZipmap(c *C) { 74 | r := decodeRDB("zipmap_that_doesnt_compress") 75 | zm := r.dbs[0]["zimap_doesnt_compress"].(map[string]string) 76 | c.Assert(zm["MKD1G6"], Equals, "2") 77 | c.Assert(zm["YNNXK"], Equals, "F7TI") 78 | } 79 | 80 | func (s *DecoderSuite) TestZipmapWitBigValues(c *C) { 81 | r := decodeRDB("zipmap_with_big_values") 82 | zm := r.dbs[0]["zipmap_with_big_values"].(map[string]string) 83 | c.Assert(len(zm["253bytes"]), Equals, 253) 84 | c.Assert(len(zm["254bytes"]), Equals, 254) 85 | c.Assert(len(zm["255bytes"]), Equals, 255) 86 | c.Assert(len(zm["300bytes"]), Equals, 300) 87 | c.Assert(len(zm["20kbytes"]), Equals, 20000) 88 | } 89 | 90 | func (s *DecoderSuite) TestHashZiplist(c *C) { 91 | r := decodeRDB("hash_as_ziplist") 92 | zm := r.dbs[0]["zipmap_compresses_easily"].(map[string]string) 93 | c.Assert(zm["a"], Equals, "aa") 94 | c.Assert(zm["aa"], Equals, "aaaa") 95 | c.Assert(zm["aaaaa"], Equals, "aaaaaaaaaaaaaa") 96 | } 97 | 98 | func (s *DecoderSuite) TestDictionary(c *C) { 99 | r := decodeRDB("dictionary") 100 | d := r.dbs[0]["force_dictionary"].(map[string]string) 101 | c.Assert(len(d), Equals, 1000) 102 | c.Assert(d["ZMU5WEJDG7KU89AOG5LJT6K7HMNB3DEI43M6EYTJ83VRJ6XNXQ"], Equals, "T63SOS8DQJF0Q0VJEZ0D1IQFCYTIPSBOUIAI9SB0OV57MQR1FI") 103 | c.Assert(d["UHS5ESW4HLK8XOGTM39IK1SJEUGVV9WOPK6JYA5QBZSJU84491"], Equals, "6VULTCV52FXJ8MGVSFTZVAGK2JXZMGQ5F8OVJI0X6GEDDR27RZ") 104 | } 105 | 106 | func (s *DecoderSuite) TestZiplistWithCompression(c *C) { 107 | r := decodeRDB("ziplist_that_compresses_easily") 108 | for i, length := range []int{6, 12, 18, 24, 30, 36} { 109 | c.Assert(r.dbs[0]["ziplist_compresses_easily"].([]string)[i], Equals, strings.Repeat("a", length)) 110 | } 111 | } 112 | 113 | func (s *DecoderSuite) TestZiplist(c *C) { 114 | r := decodeRDB("ziplist_that_doesnt_compress") 115 | l := r.dbs[0]["ziplist_doesnt_compress"].([]string) 116 | c.Assert(l[0], Equals, "aj2410") 117 | c.Assert(l[1], Equals, "cc953a17a8e096e76a44169ad3f9ac87c5f8248a403274416179aa9fbd852344") 118 | } 119 | 120 | func (s *DecoderSuite) TestZiplistWithInts(c *C) { 121 | r := decodeRDB("ziplist_with_integers") 122 | expected := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "-2", "13", "25", "-61", "63", "16380", "-16000", "65535", "-65523", "4194304", "9223372036854775807"} 123 | for i, x := range expected { 124 | c.Assert(r.dbs[0]["ziplist_with_integers"].([]string)[i], Equals, x) 125 | } 126 | } 127 | 128 | func (s *DecoderSuite) TestIntSet16(c *C) { 129 | r := decodeRDB("intset_16") 130 | for i, x := range []string{"32764", "32765", "32766"} { 131 | c.Assert(r.dbs[0]["intset_16"].([]string)[i], Equals, x) 132 | } 133 | } 134 | 135 | func (s *DecoderSuite) TestIntSet32(c *C) { 136 | r := decodeRDB("intset_32") 137 | for i, x := range []string{"2147418108", "2147418109", "2147418110"} { 138 | c.Assert(r.dbs[0]["intset_32"].([]string)[i], Equals, x) 139 | } 140 | } 141 | 142 | func (s *DecoderSuite) TestIntSet64(c *C) { 143 | r := decodeRDB("intset_64") 144 | for i, x := range []string{"9223090557583032316", "9223090557583032317", "9223090557583032318"} { 145 | c.Assert(r.dbs[0]["intset_64"].([]string)[i], Equals, x) 146 | } 147 | } 148 | 149 | func (s *DecoderSuite) TestSet(c *C) { 150 | r := decodeRDB("regular_set") 151 | for i, x := range []string{"beta", "delta", "alpha", "phi", "gamma", "kappa"} { 152 | c.Assert(r.dbs[0]["regular_set"].([]string)[i], Equals, x) 153 | } 154 | } 155 | 156 | func (s *DecoderSuite) TestZSetZiplist(c *C) { 157 | r := decodeRDB("sorted_set_as_ziplist") 158 | z := r.dbs[0]["sorted_set_as_ziplist"].(map[string]float64) 159 | c.Assert(z["8b6ba6718a786daefa69438148361901"], Equals, float64(1)) 160 | c.Assert(z["cb7a24bb7528f934b841b34c3a73e0c7"], Equals, float64(2.37)) 161 | c.Assert(z["523af537946b79c4f8369ed39ba78605"], Equals, float64(3.423)) 162 | } 163 | 164 | func (s *DecoderSuite) TestRDBv5(c *C) { 165 | r := decodeRDB("rdb_version_5_with_checksum") 166 | c.Assert(r.dbs[0]["abcd"], Equals, "efgh") 167 | c.Assert(r.dbs[0]["foo"], Equals, "bar") 168 | c.Assert(r.dbs[0]["bar"], Equals, "baz") 169 | c.Assert(r.dbs[0]["abcdef"], Equals, "abcdef") 170 | c.Assert(r.dbs[0]["longerstring"], Equals, "thisisalongerstring.idontknowwhatitmeans") 171 | } 172 | 173 | func (s *DecoderSuite) TestRDBv7(c *C) { 174 | r := decodeRDB("rdb_v7_list_quicklist") 175 | c.Assert(r.aux["redis-ver"], Equals, "3.2.0") 176 | c.Assert(r.dbSize[0], Equals, uint32(1)) 177 | c.Assert(r.expiresSize[0], Equals, uint32(0)) 178 | z := r.dbs[0]["foo"].([]string) 179 | c.Assert(z[0], Equals, "bar") 180 | c.Assert(z[1], Equals, "baz") 181 | c.Assert(z[2], Equals, "boo") 182 | } 183 | 184 | func (s *DecoderSuite) TestDumpDecoder(c *C) { 185 | r := &FakeRedis{} 186 | err := rdb.DecodeDump([]byte("\u0000\xC0\n\u0006\u0000\xF8r?\xC5\xFB\xFB_("), 1, []byte("test"), 123, r) 187 | if err != nil { 188 | c.Error(err) 189 | } 190 | c.Assert(r.dbs[1]["test"], Equals, "10") 191 | } 192 | 193 | func decodeRDB(name string) *FakeRedis { 194 | r := &FakeRedis{} 195 | f, err := os.Open("fixtures/" + name + ".rdb") 196 | if err != nil { 197 | panic(err) 198 | } 199 | err = rdb.Decode(f, r) 200 | if err != nil { 201 | panic(err) 202 | } 203 | return r 204 | } 205 | 206 | type FakeRedis struct { 207 | dbs map[int]map[string]interface{} 208 | lengths map[int]map[string]int 209 | expiries map[int]map[string]int64 210 | dbSize map[int]uint32 211 | expiresSize map[int]uint32 212 | 213 | cdb int 214 | started int 215 | ended int 216 | 217 | aux map[string]string 218 | } 219 | 220 | func (r *FakeRedis) setExpiry(key []byte, expiry int64) { 221 | r.expiries[r.cdb][string(key)] = expiry 222 | } 223 | 224 | func (r *FakeRedis) setLength(key []byte, length int64) { 225 | r.lengths[r.cdb][string(key)] = int(length) 226 | } 227 | 228 | func (r *FakeRedis) getLength(key []byte) int { 229 | return int(r.lengths[r.cdb][string(key)]) 230 | } 231 | 232 | func (r *FakeRedis) db() map[string]interface{} { 233 | return r.dbs[r.cdb] 234 | } 235 | 236 | func (r *FakeRedis) StartRDB() { 237 | r.started++ 238 | r.dbs = make(map[int]map[string]interface{}) 239 | r.expiries = make(map[int]map[string]int64) 240 | r.lengths = make(map[int]map[string]int) 241 | r.aux = make(map[string]string) 242 | r.dbSize = make(map[int]uint32) 243 | r.expiresSize = make(map[int]uint32) 244 | } 245 | 246 | func (r *FakeRedis) StartDatabase(n int) { 247 | r.dbs[n] = make(map[string]interface{}) 248 | r.expiries[n] = make(map[string]int64) 249 | r.lengths[n] = make(map[string]int) 250 | r.cdb = n 251 | } 252 | 253 | func (r *FakeRedis) Set(key, value []byte, expiry int64) { 254 | r.setExpiry(key, expiry) 255 | r.db()[string(key)] = string(value) 256 | } 257 | 258 | func (r *FakeRedis) StartHash(key []byte, length, expiry int64) { 259 | r.setExpiry(key, expiry) 260 | r.setLength(key, length) 261 | r.db()[string(key)] = make(map[string]string) 262 | } 263 | 264 | func (r *FakeRedis) Hset(key, field, value []byte) { 265 | r.db()[string(key)].(map[string]string)[string(field)] = string(value) 266 | } 267 | 268 | func (r *FakeRedis) EndHash(key []byte) { 269 | actual := len(r.db()[string(key)].(map[string]string)) 270 | if actual != r.getLength(key) { 271 | panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) 272 | } 273 | } 274 | 275 | func (r *FakeRedis) StartSet(key []byte, cardinality, expiry int64) { 276 | r.setExpiry(key, expiry) 277 | r.setLength(key, cardinality) 278 | r.db()[string(key)] = make([]string, 0, cardinality) 279 | } 280 | 281 | func (r *FakeRedis) Sadd(key, member []byte) { 282 | r.db()[string(key)] = append(r.db()[string(key)].([]string), string(member)) 283 | } 284 | 285 | func (r *FakeRedis) EndSet(key []byte) { 286 | actual := len(r.db()[string(key)].([]string)) 287 | if actual != r.getLength(key) { 288 | panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) 289 | } 290 | } 291 | 292 | func (r *FakeRedis) StartList(key []byte, length, expiry int64) { 293 | r.setExpiry(key, expiry) 294 | r.setLength(key, length) 295 | cap := length 296 | if length < 0 { 297 | cap = 1 298 | } 299 | r.db()[string(key)] = make([]string, 0, cap) 300 | } 301 | 302 | func (r *FakeRedis) Rpush(key, value []byte) { 303 | r.db()[string(key)] = append(r.db()[string(key)].([]string), string(value)) 304 | } 305 | 306 | func (r *FakeRedis) EndList(key []byte) { 307 | actual := len(r.db()[string(key)].([]string)) 308 | if actual != r.getLength(key) && r.getLength(key) >= 0 { 309 | panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) 310 | } 311 | } 312 | 313 | func (r *FakeRedis) StartZSet(key []byte, cardinality, expiry int64) { 314 | r.setExpiry(key, expiry) 315 | r.setLength(key, cardinality) 316 | r.db()[string(key)] = make(map[string]float64) 317 | } 318 | 319 | func (r *FakeRedis) Zadd(key []byte, score float64, member []byte) { 320 | r.db()[string(key)].(map[string]float64)[string(member)] = score 321 | } 322 | 323 | func (r *FakeRedis) EndZSet(key []byte) { 324 | actual := len(r.db()[string(key)].(map[string]float64)) 325 | if actual != r.getLength(key) { 326 | panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) 327 | } 328 | } 329 | 330 | func (r *FakeRedis) EndDatabase(n int) { 331 | if n != r.cdb { 332 | panic(fmt.Sprintf("database end called with %d, expected %d", n, r.cdb)) 333 | } 334 | } 335 | 336 | func (r *FakeRedis) EndRDB() { 337 | r.ended++ 338 | } 339 | 340 | func (r *FakeRedis) Aux(key, value []byte) { 341 | r.aux[string(key)] = string(value) 342 | } 343 | 344 | func (r *FakeRedis) ResizeDatabase(dbSize, expiresSize uint32) { 345 | r.dbSize[r.cdb] = dbSize 346 | r.expiresSize[r.cdb] = expiresSize 347 | } 348 | -------------------------------------------------------------------------------- /decoder.go: -------------------------------------------------------------------------------- 1 | // Package rdb implements parsing and encoding of the Redis RDB file format. 2 | package rdb 3 | 4 | import ( 5 | "bufio" 6 | "bytes" 7 | "encoding/binary" 8 | "fmt" 9 | "io" 10 | "math" 11 | "strconv" 12 | 13 | "github.com/cupcake/rdb/crc64" 14 | ) 15 | 16 | // A Decoder must be implemented to parse a RDB file. 17 | type Decoder interface { 18 | // StartRDB is called when parsing of a valid RDB file starts. 19 | StartRDB() 20 | // StartDatabase is called when database n starts. 21 | // Once a database starts, another database will not start until EndDatabase is called. 22 | StartDatabase(n int) 23 | // AUX field 24 | Aux(key, value []byte) 25 | // ResizeDB hint 26 | ResizeDatabase(dbSize, expiresSize uint32) 27 | // Set is called once for each string key. 28 | Set(key, value []byte, expiry int64) 29 | // StartHash is called at the beginning of a hash. 30 | // Hset will be called exactly length times before EndHash. 31 | StartHash(key []byte, length, expiry int64) 32 | // Hset is called once for each field=value pair in a hash. 33 | Hset(key, field, value []byte) 34 | // EndHash is called when there are no more fields in a hash. 35 | EndHash(key []byte) 36 | // StartSet is called at the beginning of a set. 37 | // Sadd will be called exactly cardinality times before EndSet. 38 | StartSet(key []byte, cardinality, expiry int64) 39 | // Sadd is called once for each member of a set. 40 | Sadd(key, member []byte) 41 | // EndSet is called when there are no more fields in a set. 42 | EndSet(key []byte) 43 | // StartList is called at the beginning of a list. 44 | // Rpush will be called exactly length times before EndList. 45 | // If length of the list is not known, then length is -1 46 | StartList(key []byte, length, expiry int64) 47 | // Rpush is called once for each value in a list. 48 | Rpush(key, value []byte) 49 | // EndList is called when there are no more values in a list. 50 | EndList(key []byte) 51 | // StartZSet is called at the beginning of a sorted set. 52 | // Zadd will be called exactly cardinality times before EndZSet. 53 | StartZSet(key []byte, cardinality, expiry int64) 54 | // Zadd is called once for each member of a sorted set. 55 | Zadd(key []byte, score float64, member []byte) 56 | // EndZSet is called when there are no more members in a sorted set. 57 | EndZSet(key []byte) 58 | // EndDatabase is called at the end of a database. 59 | EndDatabase(n int) 60 | // EndRDB is called when parsing of the RDB file is complete. 61 | EndRDB() 62 | } 63 | 64 | // Decode parses a RDB file from r and calls the decode hooks on d. 65 | func Decode(r io.Reader, d Decoder) error { 66 | decoder := &decode{d, make([]byte, 8), bufio.NewReader(r)} 67 | return decoder.decode() 68 | } 69 | 70 | // Decode a byte slice from the Redis DUMP command. The dump does not contain the 71 | // database, key or expiry, so they must be included in the function call (but 72 | // can be zero values). 73 | func DecodeDump(dump []byte, db int, key []byte, expiry int64, d Decoder) error { 74 | err := verifyDump(dump) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | decoder := &decode{d, make([]byte, 8), bytes.NewReader(dump[1:])} 80 | decoder.event.StartRDB() 81 | decoder.event.StartDatabase(db) 82 | 83 | err = decoder.readObject(key, ValueType(dump[0]), expiry) 84 | 85 | decoder.event.EndDatabase(db) 86 | decoder.event.EndRDB() 87 | return err 88 | } 89 | 90 | type byteReader interface { 91 | io.Reader 92 | io.ByteReader 93 | } 94 | 95 | type decode struct { 96 | event Decoder 97 | intBuf []byte 98 | r byteReader 99 | } 100 | 101 | type ValueType byte 102 | 103 | const ( 104 | TypeString ValueType = 0 105 | TypeList ValueType = 1 106 | TypeSet ValueType = 2 107 | TypeZSet ValueType = 3 108 | TypeHash ValueType = 4 109 | 110 | TypeHashZipmap ValueType = 9 111 | TypeListZiplist ValueType = 10 112 | TypeSetIntset ValueType = 11 113 | TypeZSetZiplist ValueType = 12 114 | TypeHashZiplist ValueType = 13 115 | TypeListQuicklist ValueType = 14 116 | ) 117 | 118 | const ( 119 | rdb6bitLen = 0 120 | rdb14bitLen = 1 121 | rdb32bitLen = 2 122 | rdbEncVal = 3 123 | 124 | rdbFlagAux = 0xfa 125 | rdbFlagResizeDB = 0xfb 126 | rdbFlagExpiryMS = 0xfc 127 | rdbFlagExpiry = 0xfd 128 | rdbFlagSelectDB = 0xfe 129 | rdbFlagEOF = 0xff 130 | 131 | rdbEncInt8 = 0 132 | rdbEncInt16 = 1 133 | rdbEncInt32 = 2 134 | rdbEncLZF = 3 135 | 136 | rdbZiplist6bitlenString = 0 137 | rdbZiplist14bitlenString = 1 138 | rdbZiplist32bitlenString = 2 139 | 140 | rdbZiplistInt16 = 0xc0 141 | rdbZiplistInt32 = 0xd0 142 | rdbZiplistInt64 = 0xe0 143 | rdbZiplistInt24 = 0xf0 144 | rdbZiplistInt8 = 0xfe 145 | rdbZiplistInt4 = 15 146 | ) 147 | 148 | func (d *decode) decode() error { 149 | err := d.checkHeader() 150 | if err != nil { 151 | return err 152 | } 153 | 154 | d.event.StartRDB() 155 | 156 | var db uint32 157 | var expiry int64 158 | firstDB := true 159 | for { 160 | objType, err := d.r.ReadByte() 161 | if err != nil { 162 | return err 163 | } 164 | switch objType { 165 | case rdbFlagAux: 166 | auxKey, err := d.readString() 167 | if err != nil { 168 | return err 169 | } 170 | auxVal, err := d.readString() 171 | if err != nil { 172 | return err 173 | } 174 | d.event.Aux(auxKey, auxVal) 175 | case rdbFlagResizeDB: 176 | dbSize, _, err := d.readLength() 177 | if err != nil { 178 | return err 179 | } 180 | expiresSize, _, err := d.readLength() 181 | if err != nil { 182 | return err 183 | } 184 | d.event.ResizeDatabase(dbSize, expiresSize) 185 | case rdbFlagExpiryMS: 186 | _, err := io.ReadFull(d.r, d.intBuf) 187 | if err != nil { 188 | return err 189 | } 190 | expiry = int64(binary.LittleEndian.Uint64(d.intBuf)) 191 | case rdbFlagExpiry: 192 | _, err := io.ReadFull(d.r, d.intBuf[:4]) 193 | if err != nil { 194 | return err 195 | } 196 | expiry = int64(binary.LittleEndian.Uint32(d.intBuf)) * 1000 197 | case rdbFlagSelectDB: 198 | if !firstDB { 199 | d.event.EndDatabase(int(db)) 200 | } 201 | db, _, err = d.readLength() 202 | if err != nil { 203 | return err 204 | } 205 | d.event.StartDatabase(int(db)) 206 | case rdbFlagEOF: 207 | d.event.EndDatabase(int(db)) 208 | d.event.EndRDB() 209 | return nil 210 | default: 211 | key, err := d.readString() 212 | if err != nil { 213 | return err 214 | } 215 | err = d.readObject(key, ValueType(objType), expiry) 216 | if err != nil { 217 | return err 218 | } 219 | expiry = 0 220 | } 221 | } 222 | 223 | panic("not reached") 224 | } 225 | 226 | func (d *decode) readObject(key []byte, typ ValueType, expiry int64) error { 227 | switch typ { 228 | case TypeString: 229 | value, err := d.readString() 230 | if err != nil { 231 | return err 232 | } 233 | d.event.Set(key, value, expiry) 234 | case TypeList: 235 | length, _, err := d.readLength() 236 | if err != nil { 237 | return err 238 | } 239 | d.event.StartList(key, int64(length), expiry) 240 | for i := uint32(0); i < length; i++ { 241 | value, err := d.readString() 242 | if err != nil { 243 | return err 244 | } 245 | d.event.Rpush(key, value) 246 | } 247 | d.event.EndList(key) 248 | case TypeListQuicklist: 249 | length, _, err := d.readLength() 250 | if err != nil { 251 | return err 252 | } 253 | d.event.StartList(key, int64(-1), expiry) 254 | for i := uint32(0); i < length; i++ { 255 | d.readZiplist(key, 0, false) 256 | } 257 | d.event.EndList(key) 258 | case TypeSet: 259 | cardinality, _, err := d.readLength() 260 | if err != nil { 261 | return err 262 | } 263 | d.event.StartSet(key, int64(cardinality), expiry) 264 | for i := uint32(0); i < cardinality; i++ { 265 | member, err := d.readString() 266 | if err != nil { 267 | return err 268 | } 269 | d.event.Sadd(key, member) 270 | } 271 | d.event.EndSet(key) 272 | case TypeZSet: 273 | cardinality, _, err := d.readLength() 274 | if err != nil { 275 | return err 276 | } 277 | d.event.StartZSet(key, int64(cardinality), expiry) 278 | for i := uint32(0); i < cardinality; i++ { 279 | member, err := d.readString() 280 | if err != nil { 281 | return err 282 | } 283 | score, err := d.readFloat64() 284 | if err != nil { 285 | return err 286 | } 287 | d.event.Zadd(key, score, member) 288 | } 289 | d.event.EndZSet(key) 290 | case TypeHash: 291 | length, _, err := d.readLength() 292 | if err != nil { 293 | return err 294 | } 295 | d.event.StartHash(key, int64(length), expiry) 296 | for i := uint32(0); i < length; i++ { 297 | field, err := d.readString() 298 | if err != nil { 299 | return err 300 | } 301 | value, err := d.readString() 302 | if err != nil { 303 | return err 304 | } 305 | d.event.Hset(key, field, value) 306 | } 307 | d.event.EndHash(key) 308 | case TypeHashZipmap: 309 | return d.readZipmap(key, expiry) 310 | case TypeListZiplist: 311 | return d.readZiplist(key, expiry, true) 312 | case TypeSetIntset: 313 | return d.readIntset(key, expiry) 314 | case TypeZSetZiplist: 315 | return d.readZiplistZset(key, expiry) 316 | case TypeHashZiplist: 317 | return d.readZiplistHash(key, expiry) 318 | default: 319 | return fmt.Errorf("rdb: unknown object type %d for key %s", typ, key) 320 | } 321 | return nil 322 | } 323 | 324 | func (d *decode) readZipmap(key []byte, expiry int64) error { 325 | var length int 326 | zipmap, err := d.readString() 327 | if err != nil { 328 | return err 329 | } 330 | buf := newSliceBuffer(zipmap) 331 | lenByte, err := buf.ReadByte() 332 | if err != nil { 333 | return err 334 | } 335 | if lenByte >= 254 { // we need to count the items manually 336 | length, err = countZipmapItems(buf) 337 | length /= 2 338 | if err != nil { 339 | return err 340 | } 341 | } else { 342 | length = int(lenByte) 343 | } 344 | d.event.StartHash(key, int64(length), expiry) 345 | for i := 0; i < length; i++ { 346 | field, err := readZipmapItem(buf, false) 347 | if err != nil { 348 | return err 349 | } 350 | value, err := readZipmapItem(buf, true) 351 | if err != nil { 352 | return err 353 | } 354 | d.event.Hset(key, field, value) 355 | } 356 | d.event.EndHash(key) 357 | return nil 358 | } 359 | 360 | func readZipmapItem(buf *sliceBuffer, readFree bool) ([]byte, error) { 361 | length, free, err := readZipmapItemLength(buf, readFree) 362 | if err != nil { 363 | return nil, err 364 | } 365 | if length == -1 { 366 | return nil, nil 367 | } 368 | value, err := buf.Slice(length) 369 | if err != nil { 370 | return nil, err 371 | } 372 | _, err = buf.Seek(int64(free), 1) 373 | return value, err 374 | } 375 | 376 | func countZipmapItems(buf *sliceBuffer) (int, error) { 377 | n := 0 378 | for { 379 | strLen, free, err := readZipmapItemLength(buf, n%2 != 0) 380 | if err != nil { 381 | return 0, err 382 | } 383 | if strLen == -1 { 384 | break 385 | } 386 | _, err = buf.Seek(int64(strLen)+int64(free), 1) 387 | if err != nil { 388 | return 0, err 389 | } 390 | n++ 391 | } 392 | _, err := buf.Seek(0, 0) 393 | return n, err 394 | } 395 | 396 | func readZipmapItemLength(buf *sliceBuffer, readFree bool) (int, int, error) { 397 | b, err := buf.ReadByte() 398 | if err != nil { 399 | return 0, 0, err 400 | } 401 | switch b { 402 | case 253: 403 | s, err := buf.Slice(5) 404 | if err != nil { 405 | return 0, 0, err 406 | } 407 | return int(binary.BigEndian.Uint32(s)), int(s[4]), nil 408 | case 254: 409 | return 0, 0, fmt.Errorf("rdb: invalid zipmap item length") 410 | case 255: 411 | return -1, 0, nil 412 | } 413 | var free byte 414 | if readFree { 415 | free, err = buf.ReadByte() 416 | } 417 | return int(b), int(free), err 418 | } 419 | 420 | func (d *decode) readZiplist(key []byte, expiry int64, addListEvents bool) error { 421 | ziplist, err := d.readString() 422 | if err != nil { 423 | return err 424 | } 425 | buf := newSliceBuffer(ziplist) 426 | length, err := readZiplistLength(buf) 427 | if err != nil { 428 | return err 429 | } 430 | if addListEvents { 431 | d.event.StartList(key, length, expiry) 432 | } 433 | for i := int64(0); i < length; i++ { 434 | entry, err := readZiplistEntry(buf) 435 | if err != nil { 436 | return err 437 | } 438 | d.event.Rpush(key, entry) 439 | } 440 | if addListEvents { 441 | d.event.EndList(key) 442 | } 443 | return nil 444 | } 445 | 446 | func (d *decode) readZiplistZset(key []byte, expiry int64) error { 447 | ziplist, err := d.readString() 448 | if err != nil { 449 | return err 450 | } 451 | buf := newSliceBuffer(ziplist) 452 | cardinality, err := readZiplistLength(buf) 453 | if err != nil { 454 | return err 455 | } 456 | cardinality /= 2 457 | d.event.StartZSet(key, cardinality, expiry) 458 | for i := int64(0); i < cardinality; i++ { 459 | member, err := readZiplistEntry(buf) 460 | if err != nil { 461 | return err 462 | } 463 | scoreBytes, err := readZiplistEntry(buf) 464 | if err != nil { 465 | return err 466 | } 467 | score, err := strconv.ParseFloat(string(scoreBytes), 64) 468 | if err != nil { 469 | return err 470 | } 471 | d.event.Zadd(key, score, member) 472 | } 473 | d.event.EndZSet(key) 474 | return nil 475 | } 476 | 477 | func (d *decode) readZiplistHash(key []byte, expiry int64) error { 478 | ziplist, err := d.readString() 479 | if err != nil { 480 | return err 481 | } 482 | buf := newSliceBuffer(ziplist) 483 | length, err := readZiplistLength(buf) 484 | if err != nil { 485 | return err 486 | } 487 | length /= 2 488 | d.event.StartHash(key, length, expiry) 489 | for i := int64(0); i < length; i++ { 490 | field, err := readZiplistEntry(buf) 491 | if err != nil { 492 | return err 493 | } 494 | value, err := readZiplistEntry(buf) 495 | if err != nil { 496 | return err 497 | } 498 | d.event.Hset(key, field, value) 499 | } 500 | d.event.EndHash(key) 501 | return nil 502 | } 503 | 504 | func readZiplistLength(buf *sliceBuffer) (int64, error) { 505 | buf.Seek(8, 0) // skip the zlbytes and zltail 506 | lenBytes, err := buf.Slice(2) 507 | if err != nil { 508 | return 0, err 509 | } 510 | return int64(binary.LittleEndian.Uint16(lenBytes)), nil 511 | } 512 | 513 | func readZiplistEntry(buf *sliceBuffer) ([]byte, error) { 514 | prevLen, err := buf.ReadByte() 515 | if err != nil { 516 | return nil, err 517 | } 518 | if prevLen == 254 { 519 | buf.Seek(4, 1) // skip the 4-byte prevlen 520 | } 521 | 522 | header, err := buf.ReadByte() 523 | if err != nil { 524 | return nil, err 525 | } 526 | switch { 527 | case header>>6 == rdbZiplist6bitlenString: 528 | return buf.Slice(int(header & 0x3f)) 529 | case header>>6 == rdbZiplist14bitlenString: 530 | b, err := buf.ReadByte() 531 | if err != nil { 532 | return nil, err 533 | } 534 | return buf.Slice((int(header&0x3f) << 8) | int(b)) 535 | case header>>6 == rdbZiplist32bitlenString: 536 | lenBytes, err := buf.Slice(4) 537 | if err != nil { 538 | return nil, err 539 | } 540 | return buf.Slice(int(binary.BigEndian.Uint32(lenBytes))) 541 | case header == rdbZiplistInt16: 542 | intBytes, err := buf.Slice(2) 543 | if err != nil { 544 | return nil, err 545 | } 546 | return []byte(strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10)), nil 547 | case header == rdbZiplistInt32: 548 | intBytes, err := buf.Slice(4) 549 | if err != nil { 550 | return nil, err 551 | } 552 | return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10)), nil 553 | case header == rdbZiplistInt64: 554 | intBytes, err := buf.Slice(8) 555 | if err != nil { 556 | return nil, err 557 | } 558 | return []byte(strconv.FormatInt(int64(binary.LittleEndian.Uint64(intBytes)), 10)), nil 559 | case header == rdbZiplistInt24: 560 | intBytes := make([]byte, 4) 561 | _, err := buf.Read(intBytes[1:]) 562 | if err != nil { 563 | return nil, err 564 | } 565 | return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))>>8), 10)), nil 566 | case header == rdbZiplistInt8: 567 | b, err := buf.ReadByte() 568 | return []byte(strconv.FormatInt(int64(int8(b)), 10)), err 569 | case header>>4 == rdbZiplistInt4: 570 | return []byte(strconv.FormatInt(int64(header&0x0f)-1, 10)), nil 571 | } 572 | 573 | return nil, fmt.Errorf("rdb: unknown ziplist header byte: %d", header) 574 | } 575 | 576 | func (d *decode) readIntset(key []byte, expiry int64) error { 577 | intset, err := d.readString() 578 | if err != nil { 579 | return err 580 | } 581 | buf := newSliceBuffer(intset) 582 | intSizeBytes, err := buf.Slice(4) 583 | if err != nil { 584 | return err 585 | } 586 | intSize := binary.LittleEndian.Uint32(intSizeBytes) 587 | 588 | if intSize != 2 && intSize != 4 && intSize != 8 { 589 | return fmt.Errorf("rdb: unknown intset encoding: %d", intSize) 590 | } 591 | 592 | lenBytes, err := buf.Slice(4) 593 | if err != nil { 594 | return err 595 | } 596 | cardinality := binary.LittleEndian.Uint32(lenBytes) 597 | 598 | d.event.StartSet(key, int64(cardinality), expiry) 599 | for i := uint32(0); i < cardinality; i++ { 600 | intBytes, err := buf.Slice(int(intSize)) 601 | if err != nil { 602 | return err 603 | } 604 | var intString string 605 | switch intSize { 606 | case 2: 607 | intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) 608 | case 4: 609 | intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) 610 | case 8: 611 | intString = strconv.FormatInt(int64(int64(binary.LittleEndian.Uint64(intBytes))), 10) 612 | } 613 | d.event.Sadd(key, []byte(intString)) 614 | } 615 | d.event.EndSet(key) 616 | return nil 617 | } 618 | 619 | func (d *decode) checkHeader() error { 620 | header := make([]byte, 9) 621 | _, err := io.ReadFull(d.r, header) 622 | if err != nil { 623 | return err 624 | } 625 | 626 | if !bytes.Equal(header[:5], []byte("REDIS")) { 627 | return fmt.Errorf("rdb: invalid file format") 628 | } 629 | 630 | version, _ := strconv.ParseInt(string(header[5:]), 10, 64) 631 | if version < 1 || version > 7 { 632 | return fmt.Errorf("rdb: invalid RDB version number %d", version) 633 | } 634 | 635 | return nil 636 | } 637 | 638 | func (d *decode) readString() ([]byte, error) { 639 | length, encoded, err := d.readLength() 640 | if err != nil { 641 | return nil, err 642 | } 643 | if encoded { 644 | switch length { 645 | case rdbEncInt8: 646 | i, err := d.readUint8() 647 | return []byte(strconv.FormatInt(int64(int8(i)), 10)), err 648 | case rdbEncInt16: 649 | i, err := d.readUint16() 650 | return []byte(strconv.FormatInt(int64(int16(i)), 10)), err 651 | case rdbEncInt32: 652 | i, err := d.readUint32() 653 | return []byte(strconv.FormatInt(int64(int32(i)), 10)), err 654 | case rdbEncLZF: 655 | clen, _, err := d.readLength() 656 | if err != nil { 657 | return nil, err 658 | } 659 | ulen, _, err := d.readLength() 660 | if err != nil { 661 | return nil, err 662 | } 663 | compressed := make([]byte, clen) 664 | _, err = io.ReadFull(d.r, compressed) 665 | if err != nil { 666 | return nil, err 667 | } 668 | decompressed := lzfDecompress(compressed, int(ulen)) 669 | if len(decompressed) != int(ulen) { 670 | return nil, fmt.Errorf("decompressed string length %d didn't match expected length %d", len(decompressed), ulen) 671 | } 672 | return decompressed, nil 673 | } 674 | } 675 | 676 | str := make([]byte, length) 677 | _, err = io.ReadFull(d.r, str) 678 | return str, err 679 | } 680 | 681 | func (d *decode) readUint8() (uint8, error) { 682 | b, err := d.r.ReadByte() 683 | return uint8(b), err 684 | } 685 | 686 | func (d *decode) readUint16() (uint16, error) { 687 | _, err := io.ReadFull(d.r, d.intBuf[:2]) 688 | if err != nil { 689 | return 0, err 690 | } 691 | return binary.LittleEndian.Uint16(d.intBuf), nil 692 | } 693 | 694 | func (d *decode) readUint32() (uint32, error) { 695 | _, err := io.ReadFull(d.r, d.intBuf[:4]) 696 | if err != nil { 697 | return 0, err 698 | } 699 | return binary.LittleEndian.Uint32(d.intBuf), nil 700 | } 701 | 702 | func (d *decode) readUint64() (uint64, error) { 703 | _, err := io.ReadFull(d.r, d.intBuf) 704 | if err != nil { 705 | return 0, err 706 | } 707 | return binary.LittleEndian.Uint64(d.intBuf), nil 708 | } 709 | 710 | func (d *decode) readUint32Big() (uint32, error) { 711 | _, err := io.ReadFull(d.r, d.intBuf[:4]) 712 | if err != nil { 713 | return 0, err 714 | } 715 | return binary.BigEndian.Uint32(d.intBuf), nil 716 | } 717 | 718 | // Doubles are saved as strings prefixed by an unsigned 719 | // 8 bit integer specifying the length of the representation. 720 | // This 8 bit integer has special values in order to specify the following 721 | // conditions: 722 | // 253: not a number 723 | // 254: + inf 724 | // 255: - inf 725 | func (d *decode) readFloat64() (float64, error) { 726 | length, err := d.readUint8() 727 | if err != nil { 728 | return 0, err 729 | } 730 | switch length { 731 | case 253: 732 | return math.NaN(), nil 733 | case 254: 734 | return math.Inf(0), nil 735 | case 255: 736 | return math.Inf(-1), nil 737 | default: 738 | floatBytes := make([]byte, length) 739 | _, err := io.ReadFull(d.r, floatBytes) 740 | if err != nil { 741 | return 0, err 742 | } 743 | f, err := strconv.ParseFloat(string(floatBytes), 64) 744 | return f, err 745 | } 746 | 747 | panic("not reached") 748 | } 749 | 750 | func (d *decode) readLength() (uint32, bool, error) { 751 | b, err := d.r.ReadByte() 752 | if err != nil { 753 | return 0, false, err 754 | } 755 | // The first two bits of the first byte are used to indicate the length encoding type 756 | switch (b & 0xc0) >> 6 { 757 | case rdb6bitLen: 758 | // When the first two bits are 00, the next 6 bits are the length. 759 | return uint32(b & 0x3f), false, nil 760 | case rdb14bitLen: 761 | // When the first two bits are 01, the next 14 bits are the length. 762 | bb, err := d.r.ReadByte() 763 | if err != nil { 764 | return 0, false, err 765 | } 766 | return (uint32(b&0x3f) << 8) | uint32(bb), false, nil 767 | case rdbEncVal: 768 | // When the first two bits are 11, the next object is encoded. 769 | // The next 6 bits indicate the encoding type. 770 | return uint32(b & 0x3f), true, nil 771 | default: 772 | // When the first two bits are 10, the next 6 bits are discarded. 773 | // The next 4 bytes are the length. 774 | length, err := d.readUint32Big() 775 | return length, false, err 776 | } 777 | 778 | panic("not reached") 779 | } 780 | 781 | func verifyDump(d []byte) error { 782 | if len(d) < 10 { 783 | return fmt.Errorf("rdb: invalid dump length") 784 | } 785 | version := binary.LittleEndian.Uint16(d[len(d)-10:]) 786 | if version != uint16(Version) { 787 | return fmt.Errorf("rdb: invalid version %d, expecting %d", version, Version) 788 | } 789 | 790 | if binary.LittleEndian.Uint64(d[len(d)-8:]) != crc64.Digest(d[:len(d)-8]) { 791 | return fmt.Errorf("rdb: invalid CRC checksum") 792 | } 793 | 794 | return nil 795 | } 796 | 797 | func lzfDecompress(in []byte, outlen int) []byte { 798 | out := make([]byte, outlen) 799 | for i, o := 0, 0; i < len(in); { 800 | ctrl := int(in[i]) 801 | i++ 802 | if ctrl < 32 { 803 | for x := 0; x <= ctrl; x++ { 804 | out[o] = in[i] 805 | i++ 806 | o++ 807 | } 808 | } else { 809 | length := ctrl >> 5 810 | if length == 7 { 811 | length = length + int(in[i]) 812 | i++ 813 | } 814 | ref := o - ((ctrl & 0x1f) << 8) - int(in[i]) - 1 815 | i++ 816 | for x := 0; x <= length+1; x++ { 817 | out[o] = out[ref] 818 | ref++ 819 | o++ 820 | } 821 | } 822 | } 823 | return out 824 | } 825 | --------------------------------------------------------------------------------