├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── checksum.go ├── doc.go ├── dumps ├── dictionary.rdb ├── easily_compressible_string_key.rdb ├── empty_database.rdb ├── hash_as_ziplist.rdb ├── integer_keys.rdb ├── intset_16.rdb ├── intset_32.rdb ├── intset_64.rdb ├── keys_with_expiry.rdb ├── linkedlist.rdb ├── multiple_databases.rdb ├── parser_filters.rdb ├── rdb_version_5_with_checksum.rdb ├── regular_set.rdb ├── regular_sorted_set.rdb ├── sorted_set_as_ziplist.rdb ├── uncompressible_string_keys.rdb ├── ziplist_that_compresses_easily.rdb ├── ziplist_that_doesnt_compress.rdb ├── ziplist_with_integers.rdb ├── zipmap_that_compresses_easily.rdb ├── zipmap_that_doesnt_compress.rdb └── zipmap_with_big_values.rdb ├── dumps_test.go ├── hashmap.go ├── hashmap_test.go ├── key.go ├── key_test.go ├── list.go ├── list_test.go ├── lzf.go ├── lzf_test.go ├── parser.go ├── parser_test.go ├── set.go ├── set_test.go ├── sortedset.go ├── sortedset_test.go ├── stringobject.go ├── stringobject_test.go ├── utils.go ├── utils_test.go ├── ziplist.go └── ziplist_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | cmd/rdbtools/rdbtools 2 | cmd/genrdb/genrdb 3 | *.rdb 4 | !dumps/* 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.2 5 | - 1.3 6 | - tip 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Vincent Rischmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | rdbtools 2 | ======== 3 | 4 | [![Build Status](https://travis-ci.org/vrischmann/rdbtools.svg?branch=master)](https://travis-ci.org/vrischmann/rdbtools) 5 | 6 | This is a parser for Redis RDB snapshot files. It is loosely based on [redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools). 7 | 8 | It is compatible with Redis RDB files generated since Redis 1.0 and up to the last version. 9 | It comes with an extensive test suite. 10 | 11 | Documentation 12 | ------------- 13 | 14 | The reference is available [here](http://godoc.org/github.com/vrischmann/rdbtools). Right now it's not really useful, but it will be improved. 15 | 16 | License 17 | ------- 18 | 19 | This project is licensed under the MIT license. See the LICENSE file for details. 20 | -------------------------------------------------------------------------------- /checksum.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import "io" 4 | 5 | const ( 6 | crcPolynomial uint64 = 0x95AC9329AC4BC9B5 7 | ) 8 | 9 | type checksumReader struct { 10 | r io.Reader 11 | checksum uint64 12 | crcTable []uint64 13 | update bool 14 | } 15 | 16 | func newChecksumReader(r io.Reader) *checksumReader { 17 | return &checksumReader{ 18 | r: r, 19 | checksum: 0, 20 | crcTable: makeCRCTable(), 21 | update: true, 22 | } 23 | } 24 | 25 | func (r *checksumReader) Read(p []byte) (n int, err error) { 26 | n, err = r.r.Read(p) 27 | r.updateChecksum(p[:n]) 28 | return n, err 29 | } 30 | 31 | func (r *checksumReader) updateChecksum(p []byte) { 32 | for _, e := range p { 33 | lookupIndex := byte(r.checksum) ^ e 34 | r.checksum = (r.checksum >> 8) ^ r.crcTable[lookupIndex] 35 | } 36 | } 37 | 38 | func makeCRCTable() []uint64 { 39 | table := make([]uint64, 256) 40 | var i uint64 41 | var j uint64 42 | 43 | for i = 0; i < 256; i++ { 44 | crc := i 45 | for j = 0; j < 8; j++ { 46 | if (crc & 1) == 1 { 47 | crc = (crc >> 1) ^ crcPolynomial 48 | } else { 49 | crc = (crc >> 1) 50 | } 51 | } 52 | table[i] = crc 53 | } 54 | 55 | return table 56 | } 57 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package rdbtools is a Redis RDB snapshot file parser. 2 | // 3 | // Parsing a file 4 | // 5 | // Example of how to parse a RDB file. 6 | // 7 | // ctx := rdbtools.ParserContext{ 8 | // ListMetadataCh: make(chan rdbtools.ListMetadata), 9 | // ListDataCh: make(chan interface{}), 10 | // } 11 | // p := rdbtools.NewParser(ctx) 12 | // 13 | // go func() { 14 | // stop := false 15 | // for !stop { 16 | // select { 17 | // case md, ok := <-ctx.ListMetadataCh: 18 | // if !ok { 19 | // ctx.ListMetadataCh = nil 20 | // break 21 | // } 22 | // 23 | // // do something with the metadata 24 | // case d, ok := <-ctx.ListDataCh: 25 | // if !ok { 26 | // ctx.ListDataCh = nil 27 | // break 28 | // } 29 | // 30 | // str := rdbtools.DataToString(d) 31 | // // do something with the string 32 | // } 33 | // 34 | // if ctx.Invalid() { 35 | // break 36 | // } 37 | // } 38 | // }() 39 | // 40 | // f, _ := os.Open("/var/lib/redis/dump.rdb") 41 | // if err := p.Parse(f); err != nil { 42 | // log.Fatalln(err) 43 | // } 44 | // 45 | // The context holds the channels you will use to receive data from the parser. 46 | // You only need to provide a channel if you care about it. 47 | // In the example above, we only care about the lists in the RDB file, so we don't 48 | // provide all the other channels. 49 | // 50 | // The parser only has one method Parse(ParserContext) which takes a context. After a call to Parse, 51 | // the parser can't be reused. We plan to change that though. 52 | // 53 | // Why interfaces everywhere 54 | // 55 | // interface{} is used everywhere in rdbtools. The reason is simple: in RDB files, keys and values 56 | // can be encoded as strings or integers or even binary data. 57 | // You might call a key "1" but Redis will happily encode that as an integer. 58 | // 59 | // The majority of the time you will have strings in your keys, and a lot of the times your values 60 | // will be strings as well. 61 | // For this reason, we provided the function DataToString(interface{}) which takes care of casting 62 | // or converting to a string. 63 | package rdbtools 64 | -------------------------------------------------------------------------------- /dumps/dictionary.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/dictionary.rdb -------------------------------------------------------------------------------- /dumps/easily_compressible_string_key.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/easily_compressible_string_key.rdb -------------------------------------------------------------------------------- /dumps/empty_database.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/empty_database.rdb -------------------------------------------------------------------------------- /dumps/hash_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/hash_as_ziplist.rdb -------------------------------------------------------------------------------- /dumps/integer_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/integer_keys.rdb -------------------------------------------------------------------------------- /dumps/intset_16.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/intset_16.rdb -------------------------------------------------------------------------------- /dumps/intset_32.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/intset_32.rdb -------------------------------------------------------------------------------- /dumps/intset_64.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/intset_64.rdb -------------------------------------------------------------------------------- /dumps/keys_with_expiry.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/keys_with_expiry.rdb -------------------------------------------------------------------------------- /dumps/linkedlist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/linkedlist.rdb -------------------------------------------------------------------------------- /dumps/multiple_databases.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/multiple_databases.rdb -------------------------------------------------------------------------------- /dumps/parser_filters.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/parser_filters.rdb -------------------------------------------------------------------------------- /dumps/rdb_version_5_with_checksum.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/rdb_version_5_with_checksum.rdb -------------------------------------------------------------------------------- /dumps/regular_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/regular_set.rdb -------------------------------------------------------------------------------- /dumps/regular_sorted_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/regular_sorted_set.rdb -------------------------------------------------------------------------------- /dumps/sorted_set_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/sorted_set_as_ziplist.rdb -------------------------------------------------------------------------------- /dumps/uncompressible_string_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/uncompressible_string_keys.rdb -------------------------------------------------------------------------------- /dumps/ziplist_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/ziplist_that_compresses_easily.rdb -------------------------------------------------------------------------------- /dumps/ziplist_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/ziplist_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /dumps/ziplist_with_integers.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/ziplist_with_integers.rdb -------------------------------------------------------------------------------- /dumps/zipmap_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/zipmap_that_compresses_easily.rdb -------------------------------------------------------------------------------- /dumps/zipmap_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/zipmap_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /dumps/zipmap_with_big_values.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrischmann/rdbtools/cd9eb17adda8fa11d4415be7e96d8f5fddb37532/dumps/zipmap_with_big_values.rdb -------------------------------------------------------------------------------- /dumps_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | // This file contains tests for all the dumps in the dumps/ directory 10 | // Those dumps are taken from https://github.com/sripathikrishnan/redis-rdb-tools 11 | 12 | func mustOpen(t *testing.T, path string) *os.File { 13 | f, err := os.Open(path) 14 | if err != nil { 15 | t.Fatalf("Error while opening file '%s'; err=%s", path, err) 16 | } 17 | 18 | return f 19 | } 20 | 21 | func doParse(t *testing.T, p Parser, ctx ParserContext, path string) { 22 | err := p.Parse(mustOpen(t, path)) 23 | if err != nil { 24 | ctx.closeChannels() 25 | t.Fatalf("Error while parsing '%s'; err=%s", path, err) 26 | } 27 | } 28 | 29 | func TestDumpDictionary(t *testing.T) { 30 | ctx := ParserContext{ 31 | HashMetadataCh: make(chan HashMetadata), 32 | HashDataCh: make(chan HashEntry), 33 | } 34 | p := NewParser(ctx) 35 | 36 | go doParse(t, p, ctx, "dumps/dictionary.rdb") 37 | 38 | var i, j int 39 | stop := false 40 | for !stop { 41 | select { 42 | case _, ok := <-ctx.HashMetadataCh: 43 | if !ok { 44 | ctx.HashMetadataCh = nil 45 | break 46 | } 47 | i++ 48 | case _, ok := <-ctx.HashDataCh: 49 | if !ok { 50 | ctx.HashDataCh = nil 51 | break 52 | } 53 | j++ 54 | } 55 | 56 | if ctx.Invalid() { 57 | break 58 | } 59 | } 60 | 61 | equals(t, 1, i) 62 | equals(t, 1000, j) 63 | } 64 | 65 | func TestDumpEasilyCompressibleStringKey(t *testing.T) { 66 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 67 | p := NewParser(ctx) 68 | 69 | go doParse(t, p, ctx, "dumps/easily_compressible_string_key.rdb") 70 | 71 | stop := false 72 | for !stop { 73 | select { 74 | case d, ok := <-ctx.StringObjectCh: 75 | if !ok { 76 | ctx.StringObjectCh = nil 77 | break 78 | } 79 | 80 | equals(t, strings.Repeat("a", 200), DataToString(d.Key.Key)) 81 | equals(t, true, d.Key.ExpiryTime.IsZero()) 82 | equals(t, "Key that redis should compress easily", DataToString(d.Value)) 83 | } 84 | 85 | if ctx.Invalid() { 86 | break 87 | } 88 | } 89 | } 90 | 91 | func TestDumpEmptyDatabase(t *testing.T) { 92 | ctx := ParserContext{} 93 | p := NewParser(ctx) 94 | 95 | doParse(t, p, ctx, "dumps/empty_database.rdb") 96 | } 97 | 98 | func TestDumpHashAsZipList(t *testing.T) { 99 | ctx := ParserContext{ 100 | HashMetadataCh: make(chan HashMetadata), 101 | HashDataCh: make(chan HashEntry), 102 | } 103 | p := NewParser(ctx) 104 | 105 | go doParse(t, p, ctx, "dumps/hash_as_ziplist.rdb") 106 | 107 | res := make([]HashEntry, 0) 108 | stop := false 109 | for !stop { 110 | select { 111 | case md, ok := <-ctx.HashMetadataCh: 112 | if !ok { 113 | ctx.HashMetadataCh = nil 114 | break 115 | } 116 | equals(t, int64(3), md.Len) 117 | equals(t, "zipmap_compresses_easily", DataToString(md.Key.Key)) 118 | case d, ok := <-ctx.HashDataCh: 119 | if !ok { 120 | ctx.HashDataCh = nil 121 | break 122 | } 123 | res = append(res, d) 124 | } 125 | 126 | if ctx.Invalid() { 127 | break 128 | } 129 | } 130 | 131 | equals(t, "a", DataToString(res[0].Key)) 132 | equals(t, "aa", DataToString(res[0].Value)) 133 | equals(t, "aa", DataToString(res[1].Key)) 134 | equals(t, "aaaa", DataToString(res[1].Value)) 135 | equals(t, "aaaaa", DataToString(res[2].Key)) 136 | equals(t, "aaaaaaaaaaaaaa", DataToString(res[2].Value)) 137 | } 138 | 139 | func TestDumpIntegerKeys(t *testing.T) { 140 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 141 | p := NewParser(ctx) 142 | 143 | go doParse(t, p, ctx, "dumps/integer_keys.rdb") 144 | 145 | res := make([]StringObject, 0) 146 | stop := false 147 | for !stop { 148 | select { 149 | case d, ok := <-ctx.StringObjectCh: 150 | if !ok { 151 | ctx.StringObjectCh = nil 152 | break 153 | } 154 | 155 | res = append(res, d) 156 | } 157 | 158 | if ctx.Invalid() { 159 | break 160 | } 161 | } 162 | 163 | equals(t, int32(183358245), res[0].Key.Key.(int32)) 164 | equals(t, "Positive 32 bit integer", DataToString(res[0].Value)) 165 | equals(t, int8(125), res[1].Key.Key.(int8)) 166 | equals(t, "Positive 8 bit integer", DataToString(res[1].Value)) 167 | equals(t, int16(-29477), res[2].Key.Key.(int16)) 168 | equals(t, "Negative 16 bit integer", DataToString(res[2].Value)) 169 | equals(t, int8(-123), res[3].Key.Key.(int8)) 170 | equals(t, "Negative 8 bit integer", DataToString(res[3].Value)) 171 | equals(t, int32(43947), res[4].Key.Key.(int32)) 172 | equals(t, "Positive 16 bit integer", DataToString(res[4].Value)) 173 | equals(t, int32(-183358245), res[5].Key.Key.(int32)) 174 | equals(t, "Negative 32 bit integer", DataToString(res[5].Value)) 175 | } 176 | 177 | func TestDumpIntSet16(t *testing.T) { 178 | ctx := ParserContext{ 179 | SetMetadataCh: make(chan SetMetadata), 180 | SetDataCh: make(chan interface{}), 181 | } 182 | p := NewParser(ctx) 183 | 184 | go doParse(t, p, ctx, "dumps/intset_16.rdb") 185 | 186 | res := make([]int16, 0) 187 | stop := false 188 | for !stop { 189 | select { 190 | case md, ok := <-ctx.SetMetadataCh: 191 | if !ok { 192 | ctx.SetMetadataCh = nil 193 | break 194 | } 195 | 196 | equals(t, "intset_16", DataToString(md.Key)) 197 | equals(t, int64(3), md.Len) 198 | case d, ok := <-ctx.SetDataCh: 199 | if !ok { 200 | ctx.SetDataCh = nil 201 | break 202 | } 203 | 204 | res = append(res, d.(int16)) 205 | } 206 | 207 | if ctx.Invalid() { 208 | break 209 | } 210 | } 211 | 212 | equals(t, int16(32764), res[0]) 213 | equals(t, int16(32765), res[1]) 214 | equals(t, int16(32766), res[2]) 215 | } 216 | 217 | func TestDumpIntSet32(t *testing.T) { 218 | ctx := ParserContext{ 219 | SetMetadataCh: make(chan SetMetadata), 220 | SetDataCh: make(chan interface{}), 221 | } 222 | p := NewParser(ctx) 223 | 224 | go doParse(t, p, ctx, "dumps/intset_32.rdb") 225 | 226 | res := make([]int32, 0) 227 | stop := false 228 | for !stop { 229 | select { 230 | case md, ok := <-ctx.SetMetadataCh: 231 | if !ok { 232 | ctx.SetMetadataCh = nil 233 | break 234 | } 235 | 236 | equals(t, "intset_32", DataToString(md.Key)) 237 | equals(t, int64(3), md.Len) 238 | case d, ok := <-ctx.SetDataCh: 239 | if !ok { 240 | ctx.SetDataCh = nil 241 | break 242 | } 243 | 244 | res = append(res, d.(int32)) 245 | } 246 | 247 | if ctx.Invalid() { 248 | break 249 | } 250 | } 251 | 252 | equals(t, int32(2147418108), res[0]) 253 | equals(t, int32(2147418109), res[1]) 254 | equals(t, int32(2147418110), res[2]) 255 | } 256 | 257 | func TestDumpIntSet64(t *testing.T) { 258 | ctx := ParserContext{ 259 | SetMetadataCh: make(chan SetMetadata), 260 | SetDataCh: make(chan interface{}), 261 | } 262 | p := NewParser(ctx) 263 | 264 | go doParse(t, p, ctx, "dumps/intset_64.rdb") 265 | 266 | res := make([]int64, 0) 267 | stop := false 268 | for !stop { 269 | select { 270 | case md, ok := <-ctx.SetMetadataCh: 271 | if !ok { 272 | ctx.SetMetadataCh = nil 273 | break 274 | } 275 | 276 | equals(t, "intset_64", DataToString(md.Key)) 277 | equals(t, int64(3), md.Len) 278 | case d, ok := <-ctx.SetDataCh: 279 | if !ok { 280 | ctx.SetDataCh = nil 281 | break 282 | } 283 | 284 | res = append(res, d.(int64)) 285 | } 286 | 287 | if ctx.Invalid() { 288 | break 289 | } 290 | } 291 | 292 | equals(t, int64(9223090557583032316), res[0]) 293 | equals(t, int64(9223090557583032317), res[1]) 294 | equals(t, int64(9223090557583032318), res[2]) 295 | } 296 | 297 | func TestDumpKeysWithExpiry(t *testing.T) { 298 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 299 | p := NewParser(ctx) 300 | 301 | go doParse(t, p, ctx, "dumps/keys_with_expiry.rdb") 302 | 303 | stop := false 304 | for !stop { 305 | select { 306 | case v, ok := <-ctx.StringObjectCh: 307 | if !ok { 308 | ctx.StringObjectCh = nil 309 | break 310 | } 311 | equals(t, "expires_ms_precision", DataToString(v.Key.Key)) 312 | equals(t, "2022-12-25 10:11:12 +0000 UTC", v.Key.ExpiryTime.UTC().String()) 313 | equals(t, "2022-12-25 10:11:12.573 UTC", DataToString(v.Value)) 314 | } 315 | 316 | if ctx.Invalid() { 317 | break 318 | } 319 | } 320 | } 321 | 322 | func TestDumpLinkedList(t *testing.T) { 323 | ctx := ParserContext{ 324 | ListMetadataCh: make(chan ListMetadata), 325 | ListDataCh: make(chan interface{}), 326 | } 327 | p := NewParser(ctx) 328 | 329 | go doParse(t, p, ctx, "dumps/linkedlist.rdb") 330 | 331 | i := 0 332 | stop := false 333 | for !stop { 334 | select { 335 | case md, ok := <-ctx.ListMetadataCh: 336 | if !ok { 337 | ctx.ListMetadataCh = nil 338 | break 339 | } 340 | 341 | equals(t, "force_linkedlist", DataToString(md.Key)) 342 | equals(t, int64(1000), md.Len) 343 | case d, ok := <-ctx.ListDataCh: 344 | if !ok { 345 | ctx.ListDataCh = nil 346 | break 347 | } 348 | 349 | equals(t, 50, len(DataToString(d))) 350 | i++ 351 | } 352 | 353 | if ctx.Invalid() { 354 | break 355 | } 356 | } 357 | 358 | equals(t, 1000, i) 359 | } 360 | 361 | func TestDumpMultipleDatabases(t *testing.T) { 362 | ctx := ParserContext{ 363 | DbCh: make(chan int), 364 | StringObjectCh: make(chan StringObject), 365 | } 366 | p := NewParser(ctx) 367 | 368 | go doParse(t, p, ctx, "dumps/multiple_databases.rdb") 369 | 370 | data := make(map[int]StringObject) 371 | var db int 372 | stop := false 373 | for !stop { 374 | select { 375 | case d, ok := <-ctx.DbCh: 376 | if !ok { 377 | ctx.DbCh = nil 378 | break 379 | } 380 | 381 | db = d 382 | case d, ok := <-ctx.StringObjectCh: 383 | if !ok { 384 | ctx.StringObjectCh = nil 385 | break 386 | } 387 | 388 | data[db] = d 389 | } 390 | 391 | if ctx.Invalid() { 392 | break 393 | } 394 | } 395 | 396 | equals(t, "key_in_zeroth_database", DataToString(data[0].Key.Key)) 397 | equals(t, "zero", DataToString(data[0].Value)) 398 | equals(t, "key_in_second_database", DataToString(data[2].Key.Key)) 399 | equals(t, "second", DataToString(data[2].Value)) 400 | } 401 | 402 | // Brace yourself for a VERY long test 403 | func TestDumpParserFilters(t *testing.T) { 404 | ctx := ParserContext{ 405 | DbCh: make(chan int), 406 | StringObjectCh: make(chan StringObject), 407 | ListMetadataCh: make(chan ListMetadata), 408 | ListDataCh: make(chan interface{}), 409 | SetMetadataCh: make(chan SetMetadata), 410 | SetDataCh: make(chan interface{}), 411 | HashMetadataCh: make(chan HashMetadata), 412 | HashDataCh: make(chan HashEntry), 413 | SortedSetMetadataCh: make(chan SortedSetMetadata), 414 | SortedSetEntriesCh: make(chan SortedSetEntry), 415 | } 416 | p := NewParser(ctx) 417 | 418 | go doParse(t, p, ctx, "dumps/parser_filters.rdb") 419 | 420 | strings := make([]StringObject, 0) 421 | lists := make(map[string][]interface{}, 0) 422 | var currentList string 423 | sets := make(map[string][]interface{}, 0) 424 | var currentSet string 425 | hashes := make(map[string][]HashEntry, 0) 426 | var currentHash string 427 | sortedSets := make(map[string][]SortedSetEntry, 0) 428 | var currentSortedSet string 429 | 430 | stop := false 431 | for !stop { 432 | select { 433 | case v, ok := <-ctx.DbCh: 434 | if !ok { 435 | ctx.DbCh = nil 436 | break 437 | } 438 | equals(t, int(0), v) 439 | case v, ok := <-ctx.StringObjectCh: 440 | if !ok { 441 | ctx.StringObjectCh = nil 442 | break 443 | } 444 | strings = append(strings, v) 445 | case v, ok := <-ctx.ListMetadataCh: 446 | if !ok { 447 | ctx.ListMetadataCh = nil 448 | break 449 | } 450 | lists[DataToString(v.Key.Key)] = make([]interface{}, 0) 451 | currentList = DataToString(v.Key.Key) 452 | case v, ok := <-ctx.ListDataCh: 453 | if !ok { 454 | ctx.ListDataCh = nil 455 | break 456 | } 457 | lists[currentList] = append(lists[currentList], v) 458 | case v, ok := <-ctx.SetMetadataCh: 459 | if !ok { 460 | ctx.SetMetadataCh = nil 461 | break 462 | } 463 | sets[DataToString(v.Key.Key)] = make([]interface{}, 0) 464 | currentSet = DataToString(v.Key.Key) 465 | case v, ok := <-ctx.SetDataCh: 466 | if !ok { 467 | ctx.SetDataCh = nil 468 | break 469 | } 470 | sets[currentSet] = append(sets[currentSet], v) 471 | case v, ok := <-ctx.SortedSetMetadataCh: 472 | if !ok { 473 | ctx.SortedSetMetadataCh = nil 474 | break 475 | } 476 | sortedSets[DataToString(v.Key.Key)] = make([]SortedSetEntry, 0) 477 | currentSortedSet = DataToString(v.Key.Key) 478 | case v, ok := <-ctx.SortedSetEntriesCh: 479 | if !ok { 480 | ctx.SortedSetEntriesCh = nil 481 | break 482 | } 483 | sortedSets[currentSortedSet] = append(sortedSets[currentSortedSet], v) 484 | case v, ok := <-ctx.HashMetadataCh: 485 | if !ok { 486 | ctx.HashMetadataCh = nil 487 | break 488 | } 489 | hashes[DataToString(v.Key.Key)] = make([]HashEntry, 0) 490 | currentHash = DataToString(v.Key.Key) 491 | case v, ok := <-ctx.HashDataCh: 492 | if !ok { 493 | ctx.HashDataCh = nil 494 | break 495 | } 496 | hashes[currentHash] = append(hashes[currentHash], v) 497 | } 498 | 499 | if ctx.Invalid() { 500 | break 501 | } 502 | } 503 | 504 | // Lists 505 | equals(t, false, lists["l1"] == nil) 506 | equals(t, "yup", DataToString(lists["l1"][0])) 507 | equals(t, "aha", DataToString(lists["l1"][1])) 508 | 509 | equals(t, false, lists["l2"] == nil) 510 | equals(t, "something", DataToString(lists["l2"][0])) 511 | equals(t, "now a bit longer and perhaps more interesting", DataToString(lists["l2"][1])) 512 | 513 | equals(t, false, lists["l3"] == nil) 514 | equals(t, "this one is going to be longer -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------", DataToString(lists["l3"][0])) 515 | equals(t, "a bit more", DataToString(lists["l3"][1])) 516 | 517 | equals(t, false, lists["l4"] == nil) 518 | equals(t, "b", DataToString(lists["l4"][0])) 519 | equals(t, "c", DataToString(lists["l4"][1])) 520 | equals(t, "d", DataToString(lists["l4"][2])) 521 | 522 | equals(t, false, lists["l5"] == nil) 523 | equals(t, "c", DataToString(lists["l5"][0])) 524 | equals(t, "a", DataToString(lists["l5"][1])) 525 | 526 | equals(t, false, lists["l6"] == nil) 527 | equals(t, "b", DataToString(lists["l6"][0])) 528 | 529 | equals(t, false, lists["l7"] == nil) 530 | equals(t, "a", DataToString(lists["l7"][0])) 531 | equals(t, "b", DataToString(lists["l7"][1])) 532 | 533 | equals(t, false, lists["l8"] == nil) 534 | equals(t, "c", DataToString(lists["l8"][0])) 535 | equals(t, int16(1), lists["l8"][1]) 536 | equals(t, int16(2), lists["l8"][2]) 537 | equals(t, int16(3), lists["l8"][3]) 538 | equals(t, int16(4), lists["l8"][4]) 539 | 540 | equals(t, false, lists["l9"] == nil) 541 | equals(t, int16(10001), lists["l9"][0]) 542 | equals(t, int16(10002), lists["l9"][1]) 543 | equals(t, int16(10003), lists["l9"][2]) 544 | equals(t, int16(10004), lists["l9"][3]) 545 | 546 | equals(t, false, lists["l10"] == nil) 547 | equals(t, int32(100001), lists["l10"][0]) 548 | equals(t, int32(100002), lists["l10"][1]) 549 | equals(t, int32(100003), lists["l10"][2]) 550 | equals(t, int32(100004), lists["l10"][3]) 551 | 552 | equals(t, false, lists["l11"] == nil) 553 | equals(t, int64(9999999999), lists["l11"][0]) 554 | equals(t, int64(9999999998), lists["l11"][1]) 555 | equals(t, int64(9999999997), lists["l11"][2]) 556 | 557 | equals(t, false, lists["l12"] == nil) 558 | equals(t, int64(9999999997), lists["l12"][0]) 559 | equals(t, int64(9999999998), lists["l12"][1]) 560 | equals(t, int64(9999999999), lists["l12"][2]) 561 | 562 | // Strings 563 | equals(t, "k1", DataToString(strings[0].Key.Key)) 564 | equals(t, "ssssssss", DataToString(strings[0].Value)) 565 | 566 | equals(t, "k3", DataToString(strings[1].Key.Key)) 567 | equals(t, "wwwwwwww", DataToString(strings[1].Value)) 568 | 569 | equals(t, "s1", DataToString(strings[2].Key.Key)) 570 | equals(t, `.ahaa bit longer and with spaceslonger than 256 characters and trivially compressible --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------`, DataToString(strings[2].Value)) 571 | 572 | equals(t, "s2", DataToString(strings[3].Key.Key)) 573 | equals(t, "now_exists", DataToString(strings[3].Value)) 574 | 575 | equals(t, "n5b", DataToString(strings[4].Key.Key)) 576 | equals(t, int16(1000), strings[4].Value.(int16)) 577 | 578 | equals(t, "b1", DataToString(strings[5].Key.Key)) 579 | equals(t, []byte{0xFF}, strings[5].Value.([]byte)) 580 | 581 | equals(t, "b2", DataToString(strings[6].Key.Key)) 582 | equals(t, []byte{0, 0xFF}, strings[6].Value.([]byte)) 583 | 584 | equals(t, "b3", DataToString(strings[7].Key.Key)) 585 | equals(t, []byte{0, 0, 0xFF}, strings[7].Value.([]byte)) 586 | 587 | equals(t, "b4", DataToString(strings[8].Key.Key)) 588 | equals(t, []byte{0, 0, 0, 0xFF}, strings[8].Value.([]byte)) 589 | 590 | equals(t, "b5", DataToString(strings[9].Key.Key)) 591 | equals(t, []byte{0, 0, 0, 0, 0xFF}, strings[9].Value.([]byte)) 592 | 593 | equals(t, "n1", DataToString(strings[10].Key.Key)) 594 | equals(t, int8(-6), strings[10].Value.(int8)) 595 | 596 | equals(t, "n2", DataToString(strings[11].Key.Key)) 597 | equals(t, int16(501), strings[11].Value.(int16)) 598 | 599 | equals(t, "n3", DataToString(strings[12].Key.Key)) 600 | equals(t, int32(500001), strings[12].Value.(int32)) 601 | 602 | equals(t, "n4", DataToString(strings[13].Key.Key)) 603 | equals(t, int8(1), strings[13].Value.(int8)) 604 | 605 | equals(t, "n5", DataToString(strings[14].Key.Key)) 606 | equals(t, int16(1000), strings[14].Value.(int16)) 607 | 608 | equals(t, "n6", DataToString(strings[15].Key.Key)) 609 | equals(t, int32(1000000), strings[15].Value.(int32)) 610 | 611 | equals(t, "n4b", DataToString(strings[16].Key.Key)) 612 | equals(t, int8(1), strings[16].Value.(int8)) 613 | 614 | equals(t, "n6b", DataToString(strings[17].Key.Key)) 615 | equals(t, int32(1000000), strings[17].Value.(int32)) 616 | 617 | // Sets 618 | equals(t, false, sets["set1"] == nil) 619 | equals(t, []interface{}{[]byte{0x63}, []byte{0x64}, []byte{0x61}, []byte{0x62}}, sets["set1"]) 620 | 621 | equals(t, false, sets["set2"] == nil) 622 | equals(t, []interface{}{[]byte{0x64}, []byte{0x61}}, sets["set2"]) 623 | 624 | equals(t, false, sets["set3"] == nil) 625 | equals(t, []interface{}{[]byte{0x62}}, sets["set3"]) 626 | 627 | equals(t, false, sets["set4"] == nil) 628 | equals(t, []interface{}{int16(1), int16(2), int16(3), int16(4), int16(5), int16(6), int16(7), int16(8), int16(9), int16(10)}, sets["set4"]) 629 | 630 | equals(t, false, sets["set5"] == nil) 631 | equals(t, []interface{}{int32(100000), int32(100001), int32(100002), int32(100003)}, sets["set5"]) 632 | 633 | // Hashes 634 | equals(t, false, hashes["h1"] == nil) 635 | equals(t, HashEntry{Key: []byte("c"), Value: []byte("now this is quite a bit longer, but sort of boring....................................................................................................................................................................................................................................................................................................................................................................")}, hashes["h1"][0]) 636 | equals(t, HashEntry{Key: []byte("a"), Value: []byte("aha")}, hashes["h1"][1]) 637 | equals(t, HashEntry{Key: []byte("b"), Value: []byte("a bit longer, but not very much")}, hashes["h1"][2]) 638 | 639 | equals(t, false, hashes["h2"] == nil) 640 | equals(t, HashEntry{Key: []byte("a"), Value: []byte("101010")}, hashes["h2"][0]) 641 | 642 | equals(t, false, hashes["h3"] == nil) 643 | equals(t, HashEntry{Key: []byte("b"), Value: []byte("b2")}, hashes["h3"][0]) 644 | equals(t, HashEntry{Key: []byte("c"), Value: []byte("c2")}, hashes["h3"][1]) 645 | equals(t, HashEntry{Key: []byte("d"), Value: []byte("d")}, hashes["h3"][2]) 646 | 647 | // Sorted sets 648 | equals(t, false, sortedSets["z1"] == nil) 649 | equals(t, SortedSetEntry{Value: []byte{0x61}, Score: 1.0}, sortedSets["z1"][0]) 650 | equals(t, SortedSetEntry{Value: []byte{0x63}, Score: 13.0}, sortedSets["z1"][1]) 651 | 652 | equals(t, false, sortedSets["z2"] == nil) 653 | equals(t, SortedSetEntry{Value: int16(1), Score: 1.0}, sortedSets["z2"][0]) 654 | equals(t, SortedSetEntry{Value: int16(2), Score: 2.0}, sortedSets["z2"][1]) 655 | equals(t, SortedSetEntry{Value: int16(3), Score: 3.0}, sortedSets["z2"][2]) 656 | 657 | equals(t, false, sortedSets["z3"] == nil) 658 | equals(t, SortedSetEntry{Value: int16(10002), Score: 10001.0}, sortedSets["z3"][0]) 659 | equals(t, SortedSetEntry{Value: int16(10003), Score: 10003.0}, sortedSets["z3"][1]) 660 | 661 | equals(t, false, sortedSets["z4"] == nil) 662 | equals(t, SortedSetEntry{Value: int64(10000000001), Score: 10000000001.0}, sortedSets["z4"][0]) 663 | equals(t, SortedSetEntry{Value: int64(10000000002), Score: 10000000002.0}, sortedSets["z4"][1]) 664 | equals(t, SortedSetEntry{Value: int64(10000000003), Score: 10000000003.0}, sortedSets["z4"][2]) 665 | } 666 | 667 | func TestDumpWithChecksum(t *testing.T) { 668 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 669 | p := NewParser(ctx) 670 | 671 | go doParse(t, p, ctx, "dumps/rdb_version_5_with_checksum.rdb") 672 | 673 | stop := false 674 | res := make([]StringObject, 0) 675 | for !stop { 676 | select { 677 | case v, ok := <-ctx.StringObjectCh: 678 | if !ok { 679 | ctx.StringObjectCh = nil 680 | break 681 | } 682 | 683 | res = append(res, v) 684 | } 685 | 686 | if ctx.Invalid() { 687 | break 688 | } 689 | } 690 | 691 | equals(t, "abcd", DataToString(res[0].Key.Key)) 692 | equals(t, true, res[0].Key.ExpiryTime.IsZero()) 693 | equals(t, "efgh", DataToString(res[0].Value)) 694 | 695 | equals(t, "foo", DataToString(res[1].Key.Key)) 696 | equals(t, true, res[1].Key.ExpiryTime.IsZero()) 697 | equals(t, "bar", DataToString(res[1].Value)) 698 | 699 | equals(t, "bar", DataToString(res[2].Key.Key)) 700 | equals(t, true, res[2].Key.ExpiryTime.IsZero()) 701 | equals(t, "baz", DataToString(res[2].Value)) 702 | 703 | equals(t, "abcdef", DataToString(res[3].Key.Key)) 704 | equals(t, true, res[3].Key.ExpiryTime.IsZero()) 705 | equals(t, "abcdef", DataToString(res[3].Value)) 706 | 707 | equals(t, "longerstring", DataToString(res[4].Key.Key)) 708 | equals(t, true, res[4].Key.ExpiryTime.IsZero()) 709 | equals(t, "thisisalongerstring.idontknowwhatitmeans", DataToString(res[4].Value)) 710 | 711 | equals(t, "abc", DataToString(res[5].Key.Key)) 712 | equals(t, true, res[5].Key.ExpiryTime.IsZero()) 713 | equals(t, "def", DataToString(res[5].Value)) 714 | } 715 | 716 | func TestDumpRegularSet(t *testing.T) { 717 | ctx := ParserContext{ 718 | SetMetadataCh: make(chan SetMetadata), 719 | SetDataCh: make(chan interface{}), 720 | } 721 | p := NewParser(ctx) 722 | 723 | go doParse(t, p, ctx, "dumps/regular_set.rdb") 724 | 725 | res := make([]string, 0) 726 | stop := false 727 | for !stop { 728 | select { 729 | case md, ok := <-ctx.SetMetadataCh: 730 | if !ok { 731 | ctx.SetMetadataCh = nil 732 | break 733 | } 734 | 735 | equals(t, "regular_set", DataToString(md.Key.Key)) 736 | equals(t, int64(6), md.Len) 737 | case d, ok := <-ctx.SetDataCh: 738 | if !ok { 739 | ctx.SetDataCh = nil 740 | break 741 | } 742 | 743 | res = append(res, DataToString(d)) 744 | } 745 | 746 | if ctx.Invalid() { 747 | break 748 | } 749 | } 750 | 751 | equals(t, []string{"beta", "delta", "alpha", "phi", "gamma", "kappa"}, res) 752 | } 753 | 754 | func TestDumpRegularSortedSet(t *testing.T) { 755 | ctx := ParserContext{ 756 | SortedSetMetadataCh: make(chan SortedSetMetadata), 757 | SortedSetEntriesCh: make(chan SortedSetEntry), 758 | } 759 | p := NewParser(ctx) 760 | 761 | go doParse(t, p, ctx, "dumps/regular_sorted_set.rdb") 762 | 763 | stop := false 764 | for !stop { 765 | select { 766 | case md, ok := <-ctx.SortedSetMetadataCh: 767 | if !ok { 768 | ctx.SortedSetMetadataCh = nil 769 | break 770 | } 771 | 772 | equals(t, "force_sorted_set", DataToString(md.Key.Key)) 773 | equals(t, int64(500), md.Len) 774 | case d, ok := <-ctx.SortedSetEntriesCh: 775 | if !ok { 776 | ctx.SortedSetEntriesCh = nil 777 | break 778 | } 779 | 780 | equals(t, 50, len(DataToString(d.Value))) 781 | } 782 | 783 | if ctx.Invalid() { 784 | break 785 | } 786 | } 787 | } 788 | 789 | func TestDumpSortedSetAsZipList(t *testing.T) { 790 | ctx := ParserContext{ 791 | SortedSetMetadataCh: make(chan SortedSetMetadata), 792 | SortedSetEntriesCh: make(chan SortedSetEntry), 793 | } 794 | p := NewParser(ctx) 795 | 796 | go doParse(t, p, ctx, "dumps/sorted_set_as_ziplist.rdb") 797 | 798 | res := make([]SortedSetEntry, 0) 799 | stop := false 800 | for !stop { 801 | select { 802 | case md, ok := <-ctx.SortedSetMetadataCh: 803 | if !ok { 804 | ctx.SortedSetMetadataCh = nil 805 | break 806 | } 807 | 808 | equals(t, "sorted_set_as_ziplist", DataToString(md.Key.Key)) 809 | equals(t, int64(3), md.Len) 810 | case d, ok := <-ctx.SortedSetEntriesCh: 811 | if !ok { 812 | ctx.SortedSetEntriesCh = nil 813 | break 814 | } 815 | 816 | res = append(res, d) 817 | } 818 | 819 | if ctx.Invalid() { 820 | break 821 | } 822 | } 823 | 824 | equals(t, "8b6ba6718a786daefa69438148361901", DataToString(res[0].Value)) 825 | equals(t, 1.0, res[0].Score) 826 | equals(t, "cb7a24bb7528f934b841b34c3a73e0c7", DataToString(res[1].Value)) 827 | equals(t, 2.37, res[1].Score) 828 | equals(t, "523af537946b79c4f8369ed39ba78605", DataToString(res[2].Value)) 829 | equals(t, 3.4230, res[2].Score) 830 | } 831 | 832 | func TestDumpUncompressibleStringKeys(t *testing.T) { 833 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 834 | p := NewParser(ctx) 835 | 836 | go doParse(t, p, ctx, "dumps/uncompressible_string_keys.rdb") 837 | 838 | res := make([]StringObject, 0) 839 | stop := false 840 | for !stop { 841 | select { 842 | case d, ok := <-ctx.StringObjectCh: 843 | if !ok { 844 | ctx.StringObjectCh = nil 845 | break 846 | } 847 | 848 | res = append(res, d) 849 | } 850 | 851 | if ctx.Invalid() { 852 | break 853 | } 854 | } 855 | 856 | equals(t, 16382, len(DataToString(res[0].Key.Key))) 857 | equals(t, "Key length more than 6 bits but less than 14 bits", DataToString(res[0].Value)) 858 | equals(t, 60, len(DataToString(res[1].Key.Key))) 859 | equals(t, "Key length within 6 bits", DataToString(res[1].Value)) 860 | equals(t, 16386, len(DataToString(res[2].Key.Key))) 861 | equals(t, "Key length more than 14 bits but less than 32", DataToString(res[2].Value)) 862 | } 863 | 864 | func TestDumpZipListThatCompressesEasily(t *testing.T) { 865 | ctx := ParserContext{ 866 | ListMetadataCh: make(chan ListMetadata), 867 | ListDataCh: make(chan interface{}), 868 | } 869 | p := NewParser(ctx) 870 | 871 | go doParse(t, p, ctx, "dumps/ziplist_that_compresses_easily.rdb") 872 | 873 | res := make([]string, 0) 874 | stop := false 875 | for !stop { 876 | select { 877 | case md, ok := <-ctx.ListMetadataCh: 878 | if !ok { 879 | ctx.ListMetadataCh = nil 880 | break 881 | } 882 | 883 | equals(t, int64(6), md.Len) 884 | equals(t, "ziplist_compresses_easily", DataToString(md.Key.Key)) 885 | case d, ok := <-ctx.ListDataCh: 886 | if !ok { 887 | ctx.ListDataCh = nil 888 | break 889 | } 890 | 891 | res = append(res, DataToString(d)) 892 | } 893 | 894 | if ctx.Invalid() { 895 | break 896 | } 897 | } 898 | 899 | j := 0 900 | for i := 6; i < 36; i += 6 { 901 | equals(t, strings.Repeat("a", i), res[j]) 902 | j++ 903 | } 904 | } 905 | 906 | func TestDumpZipListThatDoesntCompress(t *testing.T) { 907 | ctx := ParserContext{ 908 | ListMetadataCh: make(chan ListMetadata), 909 | ListDataCh: make(chan interface{}), 910 | } 911 | p := NewParser(ctx) 912 | 913 | go doParse(t, p, ctx, "dumps/ziplist_that_doesnt_compress.rdb") 914 | 915 | res := make([]string, 0) 916 | stop := false 917 | for !stop { 918 | select { 919 | case md, ok := <-ctx.ListMetadataCh: 920 | if !ok { 921 | ctx.ListMetadataCh = nil 922 | break 923 | } 924 | 925 | equals(t, int64(2), md.Len) 926 | equals(t, "ziplist_doesnt_compress", DataToString(md.Key.Key)) 927 | case d, ok := <-ctx.ListDataCh: 928 | if !ok { 929 | ctx.ListDataCh = nil 930 | break 931 | } 932 | 933 | res = append(res, DataToString(d)) 934 | } 935 | 936 | if ctx.Invalid() { 937 | break 938 | } 939 | } 940 | 941 | equals(t, "aj2410", res[0]) 942 | equals(t, "cc953a17a8e096e76a44169ad3f9ac87c5f8248a403274416179aa9fbd852344", res[1]) 943 | } 944 | 945 | func TestDumpZipListWithIntegers(t *testing.T) { 946 | ctx := ParserContext{ 947 | ListMetadataCh: make(chan ListMetadata), 948 | ListDataCh: make(chan interface{}), 949 | } 950 | p := NewParser(ctx) 951 | 952 | go doParse(t, p, ctx, "dumps/ziplist_with_integers.rdb") 953 | 954 | res := make([]interface{}, 0) 955 | stop := false 956 | for !stop { 957 | select { 958 | case md, ok := <-ctx.ListMetadataCh: 959 | if !ok { 960 | ctx.ListMetadataCh = nil 961 | break 962 | } 963 | 964 | equals(t, int64(24), md.Len) 965 | equals(t, "ziplist_with_integers", DataToString(md.Key.Key)) 966 | case d, ok := <-ctx.ListDataCh: 967 | if !ok { 968 | ctx.ListDataCh = nil 969 | break 970 | } 971 | 972 | res = append(res, d) 973 | } 974 | 975 | if ctx.Invalid() { 976 | break 977 | } 978 | } 979 | 980 | equals(t, []interface{}{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res[0:13]) 981 | equals(t, []interface{}{int8(-2), int8(13), int8(25), int8(-61), int8(63)}, res[13:18]) 982 | equals(t, []interface{}{int16(16380), int16(-16000)}, res[18:20]) 983 | equals(t, []interface{}{int32(65535), int32(-65523), int32(4194304)}, res[20:23]) 984 | equals(t, int64(9223372036854775807), res[23]) 985 | } 986 | 987 | func TestDumpZipMapThatCompressesEasily(t *testing.T) { 988 | ctx := ParserContext{ 989 | HashMetadataCh: make(chan HashMetadata), 990 | HashDataCh: make(chan HashEntry), 991 | } 992 | p := NewParser(ctx) 993 | 994 | go doParse(t, p, ctx, "dumps/zipmap_that_compresses_easily.rdb") 995 | 996 | res := make([]HashEntry, 0) 997 | stop := false 998 | for !stop { 999 | select { 1000 | case md, ok := <-ctx.HashMetadataCh: 1001 | if !ok { 1002 | ctx.HashMetadataCh = nil 1003 | break 1004 | } 1005 | 1006 | equals(t, "zipmap_compresses_easily", DataToString(md.Key.Key)) 1007 | equals(t, int64(3), md.Len) 1008 | case d, ok := <-ctx.HashDataCh: 1009 | if !ok { 1010 | ctx.HashDataCh = nil 1011 | break 1012 | } 1013 | 1014 | res = append(res, d) 1015 | } 1016 | 1017 | if ctx.Invalid() { 1018 | break 1019 | } 1020 | } 1021 | 1022 | equals(t, "a", DataToString(res[0].Key)) 1023 | equals(t, "aa", DataToString(res[0].Value)) 1024 | equals(t, "aa", DataToString(res[1].Key)) 1025 | equals(t, "aaaa", DataToString(res[1].Value)) 1026 | equals(t, "aaaaa", DataToString(res[2].Key)) 1027 | equals(t, "aaaaaaaaaaaaaa", DataToString(res[2].Value)) 1028 | } 1029 | 1030 | func TestDumpZipMapThatDoesntCompress(t *testing.T) { 1031 | ctx := ParserContext{ 1032 | HashMetadataCh: make(chan HashMetadata), 1033 | HashDataCh: make(chan HashEntry), 1034 | } 1035 | p := NewParser(ctx) 1036 | 1037 | go doParse(t, p, ctx, "dumps/zipmap_that_doesnt_compress.rdb") 1038 | 1039 | res := make([]HashEntry, 0) 1040 | stop := false 1041 | for !stop { 1042 | select { 1043 | case md, ok := <-ctx.HashMetadataCh: 1044 | if !ok { 1045 | ctx.HashMetadataCh = nil 1046 | break 1047 | } 1048 | 1049 | equals(t, "zimap_doesnt_compress", DataToString(md.Key.Key)) 1050 | equals(t, int64(2), md.Len) 1051 | case d, ok := <-ctx.HashDataCh: 1052 | if !ok { 1053 | ctx.HashDataCh = nil 1054 | break 1055 | } 1056 | 1057 | res = append(res, d) 1058 | } 1059 | 1060 | if ctx.Invalid() { 1061 | break 1062 | } 1063 | } 1064 | 1065 | equals(t, "MKD1G6", DataToString(res[0].Key)) 1066 | equals(t, "YNNXK", DataToString(res[1].Key)) 1067 | equals(t, "2", DataToString(res[0].Value)) 1068 | equals(t, "F7TI", DataToString(res[1].Value)) 1069 | } 1070 | 1071 | func TestDumpZipMapWithBigValues(t *testing.T) { 1072 | ctx := ParserContext{ 1073 | HashMetadataCh: make(chan HashMetadata), 1074 | HashDataCh: make(chan HashEntry), 1075 | } 1076 | p := NewParser(ctx) 1077 | 1078 | go doParse(t, p, ctx, "dumps/zipmap_with_big_values.rdb") 1079 | 1080 | res := make([]HashEntry, 0) 1081 | stop := false 1082 | for !stop { 1083 | select { 1084 | case md, ok := <-ctx.HashMetadataCh: 1085 | if !ok { 1086 | ctx.HashMetadataCh = nil 1087 | break 1088 | } 1089 | 1090 | equals(t, "zipmap_with_big_values", DataToString(md.Key.Key)) 1091 | equals(t, int64(5), md.Len) 1092 | case d, ok := <-ctx.HashDataCh: 1093 | if !ok { 1094 | ctx.HashDataCh = nil 1095 | break 1096 | } 1097 | 1098 | res = append(res, d) 1099 | } 1100 | 1101 | if ctx.Invalid() { 1102 | break 1103 | } 1104 | } 1105 | } 1106 | -------------------------------------------------------------------------------- /hashmap.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | ) 10 | 11 | // Represents the metadata of a hash, which is the key and the hash length 12 | type HashMetadata struct { 13 | Key KeyObject 14 | Len int64 15 | } 16 | 17 | // Returns a visualization of the hash metadata 18 | func (m HashMetadata) String() string { 19 | return fmt.Sprintf("HashMetadata{Key: %s, Len: %d}", DataToString(m.Key), m.Len) 20 | } 21 | 22 | // Represents an entry in a hash 23 | type HashEntry struct { 24 | Key interface{} 25 | Value interface{} 26 | } 27 | 28 | // Returns a string visualization of the entry 29 | func (e HashEntry) String() string { 30 | return fmt.Sprintf("HashEntry{Key: %s, Value: %s}", DataToString(e.Key), DataToString(e.Value)) 31 | } 32 | 33 | func (p *parser) readHashMap(key KeyObject, r io.Reader) error { 34 | l, e, err := p.readLen(r) 35 | if err != nil { 36 | return err 37 | } 38 | if e { 39 | return ErrUnexpectedEncodedLength 40 | } 41 | 42 | if p.ctx.HashMetadataCh != nil { 43 | p.ctx.HashMetadataCh <- HashMetadata{Key: key, Len: l} 44 | } 45 | 46 | for i := int64(0); i < l; i++ { 47 | entryKey, err := p.readString(r) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | entryValue, err := p.readString(r) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | if p.ctx.HashDataCh != nil { 58 | p.ctx.HashDataCh <- HashEntry{Key: entryKey, Value: entryValue} 59 | } 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func (p *parser) readHashMapInZipList(key KeyObject, r io.Reader) error { 66 | data, err := p.readString(r) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | var entryKey interface{} = nil 72 | onLenCallback := func(length int64) error { 73 | if p.ctx.HashMetadataCh != nil { 74 | p.ctx.HashMetadataCh <- HashMetadata{Key: key, Len: length / 2} 75 | } 76 | return nil 77 | } 78 | onElementCallback := func(e interface{}) error { 79 | if entryKey == nil { 80 | entryKey = e 81 | } else { 82 | if p.ctx.HashDataCh != nil { 83 | p.ctx.HashDataCh <- HashEntry{Key: entryKey, Value: e} 84 | } 85 | entryKey = nil 86 | } 87 | return nil 88 | } 89 | dr := bufio.NewReader(bytes.NewReader(data.([]byte))) 90 | 91 | if err := p.readZipList(dr, onLenCallback, onElementCallback); err != nil { 92 | return err 93 | } 94 | 95 | return nil 96 | } 97 | 98 | func readZipMapLength(r io.Reader, b byte) (int64, error) { 99 | var l uint32 100 | switch b { 101 | case 253: 102 | if err := binary.Read(r, binary.LittleEndian, &l); err != nil { 103 | return -1, err 104 | } 105 | default: 106 | l = uint32(b) 107 | } 108 | 109 | return int64(l), nil 110 | } 111 | 112 | // Read a hash map encoded as a zipmap (Redis < 2.6) 113 | func (p *parser) readZipMap(key KeyObject, r io.Reader) error { 114 | data, err := p.readString(r) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | dr := bufio.NewReader(bytes.NewReader(data.([]byte))) 120 | 121 | // Hash map length, valid only when < 254 122 | mapLen, err := dr.ReadByte() 123 | if err != nil { 124 | return err 125 | } 126 | 127 | b, err := dr.ReadByte() 128 | if err != nil { 129 | return err 130 | } 131 | 132 | // This is fugly 133 | // We need the length of the hashmap before we start sending hashmap entries 134 | // so that the metadata we send is before the data, and is also correct. 135 | // Users will rely on this to know when to end processing entries for a given hashmap. 136 | // 137 | // This is why we have to buffer the entries and then sending them once we processed the RDB data. 138 | var results []HashEntry 139 | if mapLen >= 254 { 140 | results = make([]HashEntry, 0) 141 | } else { 142 | if p.ctx.HashMetadataCh != nil { 143 | p.ctx.HashMetadataCh <- HashMetadata{Key: key, Len: int64(mapLen)} 144 | } 145 | } 146 | 147 | for b != 0xFF { 148 | // Entry key data 149 | l, err := readZipMapLength(dr, b) 150 | if err != nil { 151 | return err 152 | } 153 | 154 | entryKey, err := readBytes(dr, l) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | // Entry value data 160 | b, err = dr.ReadByte() 161 | if err != nil { 162 | return err 163 | } 164 | 165 | l, err = readZipMapLength(dr, b) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | // FYI, that free shit is weird 171 | free, err := dr.ReadByte() 172 | if err != nil { 173 | return err 174 | } 175 | 176 | entryValue, err := readBytes(dr, l) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | // skip if necessary 182 | if free > 0 { 183 | if _, err = readBytes(dr, int64(free)); err != nil { 184 | return err 185 | } 186 | } 187 | 188 | if mapLen >= 254 { 189 | results = append(results, HashEntry{Key: entryKey, Value: entryValue}) 190 | } else { 191 | if p.ctx.HashDataCh != nil { 192 | p.ctx.HashDataCh <- HashEntry{Key: entryKey, Value: entryValue} 193 | } 194 | } 195 | 196 | b, err = dr.ReadByte() 197 | if err != nil { 198 | return err 199 | } 200 | } 201 | 202 | if mapLen >= 254 { 203 | if p.ctx.HashMetadataCh != nil { 204 | p.ctx.HashMetadataCh <- HashMetadata{Key: key, Len: int64(len(results))} 205 | } 206 | if p.ctx.HashDataCh != nil { 207 | for _, e := range results { 208 | p.ctx.HashDataCh <- e 209 | } 210 | } 211 | } 212 | 213 | return nil 214 | } 215 | -------------------------------------------------------------------------------- /hashmap_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "io" 8 | "testing" 9 | ) 10 | 11 | func TestHashMetadataString(t *testing.T) { 12 | md := HashMetadata{Key: KeyObject{Key: "foobar"}, Len: 10} 13 | equals(t, "HashMetadata{Key: foobar, Len: 10}", md.String()) 14 | } 15 | 16 | func TestReadHashMap(t *testing.T) { 17 | var buffer bytes.Buffer 18 | br := bufio.NewWriter(&buffer) 19 | 20 | br.WriteByte(1) // hashmap len 21 | br.WriteByte(3) // key len 22 | br.WriteString("foo") // key 23 | br.WriteByte(3) // value len 24 | br.WriteString("bar") // value 25 | br.Flush() 26 | 27 | ctx := ParserContext{ 28 | HashMetadataCh: make(chan HashMetadata), 29 | HashDataCh: make(chan HashEntry), 30 | } 31 | p := &parser{ctx: ctx} 32 | 33 | go readAndNotify(t, &buffer, "hashmap", p.readHashMap) 34 | 35 | stop := false 36 | for !stop { 37 | select { 38 | case md := <-ctx.HashMetadataCh: 39 | equals(t, "hashmap", DataToString(md.Key)) 40 | equals(t, int64(1), md.Len) 41 | case d := <-ctx.HashDataCh: 42 | equals(t, "foo", DataToString(d.Key)) 43 | equals(t, "bar", DataToString(d.Value)) 44 | case <-end: 45 | stop = true 46 | } 47 | } 48 | } 49 | 50 | func TestReadHashMapNoData(t *testing.T) { 51 | var buffer bytes.Buffer 52 | 53 | p := &parser{} 54 | err := p.readHashMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 55 | equals(t, io.EOF, err) 56 | } 57 | 58 | func TestReadHashMapEncodedLen(t *testing.T) { 59 | var buffer bytes.Buffer 60 | 61 | br := bufio.NewWriter(&buffer) 62 | br.WriteByte(0xC0) 63 | br.Flush() 64 | 65 | p := &parser{} 66 | err := p.readHashMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 67 | equals(t, ErrUnexpectedEncodedLength, err) 68 | } 69 | 70 | func TestReadHashMapNoEntryKey(t *testing.T) { 71 | var buffer bytes.Buffer 72 | 73 | br := bufio.NewWriter(&buffer) 74 | br.WriteByte(1) 75 | br.Flush() 76 | 77 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 78 | p := &parser{ctx: ctx} 79 | 80 | go func() { 81 | md := <-ctx.HashMetadataCh 82 | equals(t, "hashmap", DataToString(md.Key)) 83 | equals(t, int64(1), md.Len) 84 | }() 85 | 86 | err := p.readHashMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 87 | equals(t, io.EOF, err) 88 | } 89 | 90 | func TestReadHashMapNoEntryValue(t *testing.T) { 91 | var buffer bytes.Buffer 92 | 93 | br := bufio.NewWriter(&buffer) 94 | br.WriteByte(1) 95 | br.WriteByte(1) 96 | br.WriteString("a") 97 | br.Flush() 98 | 99 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata, 1)} 100 | p := &parser{ctx: ctx} 101 | 102 | go func() { 103 | md := <-ctx.HashMetadataCh 104 | equals(t, "hashmap", DataToString(md.Key)) 105 | equals(t, int64(1), md.Len) 106 | }() 107 | 108 | err := p.readHashMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 109 | equals(t, io.EOF, err) 110 | } 111 | 112 | func TestReadHashMapInZipList(t *testing.T) { 113 | var buffer bytes.Buffer 114 | br := bufio.NewWriter(&buffer) 115 | 116 | br.WriteByte(26) // String length 117 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 118 | br.Write([]byte{0, 0, 0, 0}) // zlTail 119 | br.Write([]byte{2, 0}) // zlLen 120 | 121 | br.WriteByte(0) // len prev entry 122 | br.WriteByte(6) // Special flag 123 | br.WriteString("foobar") 124 | br.WriteByte(0) // len prev entry 125 | br.WriteByte(6) // special flag 126 | br.WriteString("foobar") 127 | 128 | br.Flush() 129 | 130 | ctx := ParserContext{ 131 | HashMetadataCh: make(chan HashMetadata), 132 | HashDataCh: make(chan HashEntry), 133 | } 134 | p := &parser{ctx: ctx} 135 | 136 | go readAndNotify(t, &buffer, "hashmap", p.readHashMapInZipList) 137 | 138 | stop := false 139 | for !stop { 140 | select { 141 | case md := <-ctx.HashMetadataCh: 142 | equals(t, "hashmap", DataToString(md.Key)) 143 | equals(t, int64(1), md.Len) 144 | case d := <-ctx.HashDataCh: 145 | equals(t, "foobar", DataToString(d.Key)) 146 | equals(t, "foobar", DataToString(d.Value)) 147 | case <-end: 148 | stop = true 149 | } 150 | } 151 | } 152 | 153 | func TestReadHashMapInZipListNoData(t *testing.T) { 154 | var buffer bytes.Buffer 155 | 156 | p := &parser{} 157 | err := p.readHashMapInZipList(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 158 | equals(t, io.EOF, err) 159 | } 160 | 161 | func TestReadHashMapInZipListEmptyZipList(t *testing.T) { 162 | var buffer bytes.Buffer 163 | 164 | br := bufio.NewWriter(&buffer) 165 | br.WriteByte(1) 166 | br.WriteByte(0) 167 | br.Flush() 168 | 169 | p := &parser{} 170 | err := p.readHashMapInZipList(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 171 | equals(t, "unexpected EOF", err.Error()) 172 | } 173 | 174 | func TestReadZipMap(t *testing.T) { 175 | var buffer bytes.Buffer 176 | 177 | br := bufio.NewWriter(&buffer) 178 | 179 | br.WriteByte(7) // string length 180 | br.WriteByte(1) // map len 181 | br.WriteByte(1) // key len 182 | br.WriteByte('a') // key 183 | br.WriteByte(1) // value len 184 | br.WriteByte(0) // free byte 185 | br.WriteByte('b') // value 186 | br.WriteByte(0xFF) // end of map 187 | br.Flush() 188 | 189 | ctx := ParserContext{ 190 | HashMetadataCh: make(chan HashMetadata), 191 | HashDataCh: make(chan HashEntry), 192 | } 193 | p := &parser{ctx: ctx} 194 | 195 | go readAndNotify(t, &buffer, "hashmap", p.readZipMap) 196 | 197 | stop := false 198 | for !stop { 199 | select { 200 | case md := <-ctx.HashMetadataCh: 201 | equals(t, "hashmap", DataToString(md.Key)) 202 | equals(t, int64(1), md.Len) 203 | case d := <-ctx.HashDataCh: 204 | equals(t, "a", DataToString(d.Key)) 205 | equals(t, "b", DataToString(d.Value)) 206 | case <-end: 207 | stop = true 208 | } 209 | } 210 | } 211 | 212 | func TestReadZipMapBigKey(t *testing.T) { 213 | var buffer bytes.Buffer 214 | 215 | br := bufio.NewWriter(&buffer) 216 | 217 | br.WriteByte(11) // string length 218 | br.WriteByte(1) // map len 219 | br.WriteByte(253) // key len (special) 220 | binary.Write(br, binary.LittleEndian, int32(1)) // real key len 221 | br.WriteByte('a') // key 222 | br.WriteByte(1) // value len 223 | br.WriteByte(0) // free byte 224 | br.WriteByte('b') 225 | br.WriteByte(0xFF) // end of map 226 | br.Flush() 227 | 228 | ctx := ParserContext{ 229 | HashMetadataCh: make(chan HashMetadata), 230 | HashDataCh: make(chan HashEntry), 231 | } 232 | p := &parser{ctx: ctx} 233 | 234 | go readAndNotify(t, &buffer, "hashmap", p.readZipMap) 235 | 236 | stop := false 237 | for !stop { 238 | select { 239 | case md := <-ctx.HashMetadataCh: 240 | equals(t, "hashmap", DataToString(md.Key)) 241 | equals(t, int64(1), md.Len) 242 | case d := <-ctx.HashDataCh: 243 | equals(t, "a", DataToString(d.Key)) 244 | equals(t, "b", DataToString(d.Value)) 245 | case <-end: 246 | stop = true 247 | } 248 | } 249 | } 250 | 251 | func TestReadZipMapBigMapLen(t *testing.T) { 252 | var buffer bytes.Buffer 253 | 254 | br := bufio.NewWriter(&buffer) 255 | 256 | br.WriteByte(7) // string length 257 | br.WriteByte(254) // map len (special) 258 | br.WriteByte(1) // key len 259 | br.WriteByte('a') // key 260 | br.WriteByte(1) // value len 261 | br.WriteByte(0) // free byte 262 | br.WriteByte('b') // value 263 | br.WriteByte(0xFF) // end of map 264 | br.Flush() 265 | 266 | ctx := ParserContext{ 267 | HashMetadataCh: make(chan HashMetadata), 268 | HashDataCh: make(chan HashEntry), 269 | } 270 | p := &parser{ctx: ctx} 271 | 272 | go readAndNotify(t, &buffer, "hashmap", p.readZipMap) 273 | 274 | stop := false 275 | for !stop { 276 | select { 277 | case md := <-ctx.HashMetadataCh: 278 | equals(t, "hashmap", DataToString(md.Key)) 279 | equals(t, int64(1), md.Len) 280 | case d := <-ctx.HashDataCh: 281 | equals(t, "a", DataToString(d.Key)) 282 | equals(t, "b", DataToString(d.Value)) 283 | case <-end: 284 | stop = true 285 | } 286 | } 287 | } 288 | 289 | func TestReadZipMapSkipFreeBytes(t *testing.T) { 290 | var buffer bytes.Buffer 291 | 292 | br := bufio.NewWriter(&buffer) 293 | 294 | br.WriteByte(11) // string length 295 | br.WriteByte(1) // map len (special) 296 | br.WriteByte(1) // key len 297 | br.WriteByte('a') // key 298 | br.WriteByte(1) // value len 299 | br.WriteByte(4) // free byte 300 | br.WriteByte('b') // value 301 | br.Write([]byte{0, 0, 0, 0}) // free bytes 302 | br.WriteByte(0xFF) // end of map 303 | br.Flush() 304 | 305 | ctx := ParserContext{ 306 | HashMetadataCh: make(chan HashMetadata), 307 | HashDataCh: make(chan HashEntry), 308 | } 309 | p := &parser{ctx: ctx} 310 | 311 | go readAndNotify(t, &buffer, "hashmap", p.readZipMap) 312 | 313 | stop := false 314 | for !stop { 315 | select { 316 | case md := <-ctx.HashMetadataCh: 317 | equals(t, "hashmap", DataToString(md.Key)) 318 | equals(t, int64(1), md.Len) 319 | case d := <-ctx.HashDataCh: 320 | equals(t, "a", DataToString(d.Key)) 321 | equals(t, "b", DataToString(d.Value)) 322 | case <-end: 323 | stop = true 324 | } 325 | } 326 | } 327 | 328 | func TestReadZipMapNoData(t *testing.T) { 329 | var buffer bytes.Buffer 330 | 331 | p := &parser{} 332 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 333 | equals(t, io.EOF, err) 334 | } 335 | 336 | func TestReadZipMapNoMapLen(t *testing.T) { 337 | var buffer bytes.Buffer 338 | 339 | br := bufio.NewWriter(&buffer) 340 | 341 | br.WriteByte(0) 342 | br.Flush() 343 | 344 | p := &parser{} 345 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 346 | equals(t, io.EOF, err) 347 | } 348 | 349 | func TestReadZipMapNoFirstByte(t *testing.T) { 350 | var buffer bytes.Buffer 351 | 352 | br := bufio.NewWriter(&buffer) 353 | 354 | br.WriteByte(1) 355 | br.WriteByte(1) 356 | br.Flush() 357 | 358 | p := &parser{} 359 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 360 | equals(t, io.EOF, err) 361 | } 362 | 363 | func TestReadZipMapFailEntryKeyLength(t *testing.T) { 364 | var buffer bytes.Buffer 365 | 366 | br := bufio.NewWriter(&buffer) 367 | 368 | br.WriteByte(3) 369 | br.WriteByte(1) 370 | br.WriteByte(253) 371 | br.WriteByte(0xFF) 372 | br.Flush() 373 | 374 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 375 | p := &parser{ctx: ctx} 376 | 377 | go func() { 378 | md := <-ctx.HashMetadataCh 379 | equals(t, "hashmap", DataToString(md.Key)) 380 | equals(t, int64(1), md.Len) 381 | }() 382 | 383 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 384 | equals(t, "unexpected EOF", err.Error()) 385 | } 386 | 387 | func TestReadZipMapFailEntryKeyData(t *testing.T) { 388 | var buffer bytes.Buffer 389 | 390 | br := bufio.NewWriter(&buffer) 391 | 392 | br.WriteByte(2) 393 | br.WriteByte(1) 394 | br.WriteByte(1) 395 | br.Flush() 396 | 397 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 398 | p := &parser{ctx: ctx} 399 | 400 | go func() { 401 | md := <-ctx.HashMetadataCh 402 | equals(t, "hashmap", DataToString(md.Key)) 403 | equals(t, int64(1), md.Len) 404 | }() 405 | 406 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 407 | equals(t, io.EOF, err) 408 | } 409 | 410 | func TestReadZipMapFailEntryValByte(t *testing.T) { 411 | var buffer bytes.Buffer 412 | 413 | br := bufio.NewWriter(&buffer) 414 | 415 | br.WriteByte(3) 416 | br.WriteByte(1) 417 | br.WriteByte(1) 418 | br.WriteByte('a') 419 | br.Flush() 420 | 421 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 422 | p := &parser{ctx: ctx} 423 | 424 | go func() { 425 | md := <-ctx.HashMetadataCh 426 | equals(t, "hashmap", DataToString(md.Key)) 427 | equals(t, int64(1), md.Len) 428 | }() 429 | 430 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 431 | equals(t, io.EOF, err) 432 | } 433 | 434 | func TestReadZipMapFailEntryValLength(t *testing.T) { 435 | var buffer bytes.Buffer 436 | 437 | br := bufio.NewWriter(&buffer) 438 | 439 | br.WriteByte(5) 440 | br.WriteByte(1) 441 | br.WriteByte(1) 442 | br.WriteByte('a') 443 | br.WriteByte(253) 444 | br.WriteByte(1) 445 | br.Flush() 446 | 447 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 448 | p := &parser{ctx: ctx} 449 | 450 | go func() { 451 | md := <-ctx.HashMetadataCh 452 | equals(t, "hashmap", DataToString(md.Key)) 453 | equals(t, int64(1), md.Len) 454 | }() 455 | 456 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 457 | equals(t, "unexpected EOF", err.Error()) 458 | } 459 | 460 | func TestReadZipMapFailEntryFreeByte(t *testing.T) { 461 | var buffer bytes.Buffer 462 | 463 | br := bufio.NewWriter(&buffer) 464 | 465 | br.WriteByte(4) 466 | br.WriteByte(1) 467 | br.WriteByte(1) 468 | br.WriteByte('a') 469 | br.WriteByte(1) 470 | br.Flush() 471 | 472 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 473 | p := &parser{ctx: ctx} 474 | 475 | go func() { 476 | md := <-ctx.HashMetadataCh 477 | equals(t, "hashmap", DataToString(md.Key)) 478 | equals(t, int64(1), md.Len) 479 | }() 480 | 481 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 482 | equals(t, io.EOF, err) 483 | } 484 | 485 | func TestReadZipMapFailEntryValData(t *testing.T) { 486 | var buffer bytes.Buffer 487 | 488 | br := bufio.NewWriter(&buffer) 489 | 490 | br.WriteByte(5) 491 | br.WriteByte(1) 492 | br.WriteByte(1) 493 | br.WriteByte('a') 494 | br.WriteByte(1) 495 | br.WriteByte(0) 496 | br.Flush() 497 | 498 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 499 | p := &parser{ctx: ctx} 500 | 501 | go func() { 502 | md := <-ctx.HashMetadataCh 503 | equals(t, "hashmap", DataToString(md.Key)) 504 | equals(t, int64(1), md.Len) 505 | }() 506 | 507 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 508 | equals(t, io.EOF, err) 509 | } 510 | 511 | func TestReadZipMapFailSkipFreeBytes(t *testing.T) { 512 | var buffer bytes.Buffer 513 | 514 | br := bufio.NewWriter(&buffer) 515 | 516 | br.WriteByte(6) 517 | br.WriteByte(1) 518 | br.WriteByte(1) 519 | br.WriteByte('a') 520 | br.WriteByte(1) 521 | br.WriteByte(4) 522 | br.WriteByte('b') 523 | br.Flush() 524 | 525 | ctx := ParserContext{HashMetadataCh: make(chan HashMetadata)} 526 | p := &parser{ctx: ctx} 527 | 528 | go func() { 529 | md := <-ctx.HashMetadataCh 530 | equals(t, "hashmap", DataToString(md.Key)) 531 | equals(t, int64(1), md.Len) 532 | }() 533 | 534 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 535 | equals(t, io.EOF, err) 536 | } 537 | 538 | func TestReadZipMapFailLastReadByte(t *testing.T) { 539 | var buffer bytes.Buffer 540 | 541 | br := bufio.NewWriter(&buffer) 542 | 543 | br.WriteByte(6) 544 | br.WriteByte(1) 545 | br.WriteByte(1) 546 | br.WriteByte('a') 547 | br.WriteByte(1) 548 | br.WriteByte(0) 549 | br.WriteByte('b') 550 | br.Flush() 551 | 552 | ctx := ParserContext{ 553 | HashMetadataCh: make(chan HashMetadata), 554 | HashDataCh: make(chan HashEntry), 555 | } 556 | p := &parser{ctx: ctx} 557 | 558 | go func() { 559 | stop := false 560 | for !stop { 561 | select { 562 | case md := <-ctx.HashMetadataCh: 563 | equals(t, "hashmap", DataToString(md.Key)) 564 | equals(t, int64(1), md.Len) 565 | case d := <-ctx.HashDataCh: 566 | equals(t, "a", DataToString(d.Key)) 567 | equals(t, "b", DataToString(d.Value)) 568 | case <-end: 569 | stop = true 570 | } 571 | } 572 | }() 573 | 574 | err := p.readZipMap(KeyObject{Key: []byte("hashmap")}, bufio.NewReader(&buffer)) 575 | equals(t, io.EOF, err) 576 | end <- true 577 | } 578 | -------------------------------------------------------------------------------- /key.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Represents a Redis key. 9 | type KeyObject struct { 10 | ExpiryTime time.Time // The expiry time of the key. If none, this object IsZero() method will return true 11 | Key interface{} // The key value 12 | } 13 | 14 | // Create a new key. If expiryTime >= 0 it will be used. 15 | func NewKeyObject(key interface{}, expiryTime int64) KeyObject { 16 | k := KeyObject{ 17 | Key: key, 18 | } 19 | if expiryTime >= 0 { 20 | k.ExpiryTime = time.Unix(expiryTime/1000, 0).UTC() 21 | } 22 | 23 | return k 24 | } 25 | 26 | // Returns true if the key is expired (meaning the key's expiry time is before now), false otherwise. 27 | func (k KeyObject) Expired() bool { 28 | return k.ExpiryTime.Before(time.Now()) 29 | } 30 | 31 | // Return a visualization of the key. 32 | func (k KeyObject) String() string { 33 | if !k.ExpiryTime.IsZero() { 34 | return fmt.Sprintf("KeyObject{ExpiryTime: %s, Key: %s}", k.ExpiryTime, DataToString(k.Key)) 35 | } 36 | 37 | return fmt.Sprintf("%s", DataToString(k.Key)) 38 | } 39 | -------------------------------------------------------------------------------- /key_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestNewKeyObject(t *testing.T) { 9 | k := NewKeyObject("test", -1) 10 | 11 | equals(t, "test", k.Key) 12 | equals(t, true, k.ExpiryTime.IsZero()) 13 | equals(t, "test", k.String()) 14 | } 15 | 16 | // With expiry time, not yet expired 17 | func TestNewKeyObjectNotExpired(t *testing.T) { 18 | dt := time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC) 19 | k := NewKeyObject("test", dt.Unix()*1000) 20 | 21 | equals(t, "test", k.Key) 22 | equals(t, false, k.ExpiryTime.IsZero()) 23 | equals(t, false, k.Expired()) 24 | equals(t, "KeyObject{ExpiryTime: 2100-01-01 00:00:00 +0000 UTC, Key: test}", k.String()) 25 | } 26 | 27 | // With expiry time, expired 28 | func TestNewKeyObjectExpired(t *testing.T) { 29 | dt := time.Now().Add(time.Second * -10) 30 | k := NewKeyObject("test", dt.Unix()*1000) 31 | 32 | equals(t, "test", k.Key) 33 | equals(t, false, k.ExpiryTime.IsZero()) 34 | equals(t, true, k.Expired()) 35 | } 36 | -------------------------------------------------------------------------------- /list.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | // Represents the metadata of a list, which is the key and the list length 11 | type ListMetadata struct { 12 | Key KeyObject 13 | Len int64 14 | } 15 | 16 | // Returns a visualization of the list metadata 17 | func (m ListMetadata) String() string { 18 | return fmt.Sprintf("ListMetadata{Key: %s, Len: %d}", DataToString(m.Key), m.Len) 19 | } 20 | 21 | func (p *parser) readList(key KeyObject, r io.Reader) error { 22 | l, e, err := p.readLen(r) 23 | if err != nil { 24 | return err 25 | } 26 | if e { 27 | return ErrUnexpectedEncodedLength 28 | } 29 | 30 | p.ctx.ListMetadataCh <- ListMetadata{Key: key, Len: l} 31 | 32 | for i := int64(0); i < l; i++ { 33 | value, err := p.readString(r) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | p.ctx.ListDataCh <- value 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func (p *parser) readListInZipList(key KeyObject, r io.Reader) error { 45 | data, err := p.readString(r) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | onLenCallback := func(length int64) error { 51 | p.ctx.ListMetadataCh <- ListMetadata{Key: key, Len: length} 52 | return nil 53 | } 54 | onElementCallback := func(e interface{}) error { 55 | p.ctx.ListDataCh <- e 56 | return nil 57 | } 58 | dr := bufio.NewReader(bytes.NewReader(data.([]byte))) 59 | 60 | if err := p.readZipList(dr, onLenCallback, onElementCallback); err != nil { 61 | return err 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /list_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | func TestListMetadataString(t *testing.T) { 11 | md := ListMetadata{Key: KeyObject{Key: "foobar"}, Len: 10} 12 | equals(t, "ListMetadata{Key: foobar, Len: 10}", md.String()) 13 | } 14 | 15 | func TestReadList(t *testing.T) { 16 | var buffer bytes.Buffer 17 | 18 | br := bufio.NewWriter(&buffer) 19 | 20 | br.WriteByte(1) 21 | br.WriteByte(1) 22 | br.WriteByte('a') 23 | br.Flush() 24 | 25 | ctx := ParserContext{ 26 | ListMetadataCh: make(chan ListMetadata), 27 | ListDataCh: make(chan interface{}), 28 | } 29 | p := &parser{ctx: ctx} 30 | 31 | go readAndNotify(t, &buffer, "list", p.readList) 32 | 33 | stop := false 34 | for !stop { 35 | select { 36 | case md := <-p.ctx.ListMetadataCh: 37 | equals(t, "list", DataToString(md.Key)) 38 | equals(t, int64(1), md.Len) 39 | case d := <-p.ctx.ListDataCh: 40 | equals(t, "a", DataToString(d)) 41 | case <-end: 42 | stop = true 43 | } 44 | } 45 | } 46 | 47 | func TestReadListNoData(t *testing.T) { 48 | var buffer bytes.Buffer 49 | 50 | p := &parser{} 51 | err := p.readList(KeyObject{Key: []byte("list")}, bufio.NewReader(&buffer)) 52 | equals(t, io.EOF, err) 53 | } 54 | 55 | func TestReadListEncodedLen(t *testing.T) { 56 | var buffer bytes.Buffer 57 | 58 | br := bufio.NewWriter(&buffer) 59 | 60 | br.WriteByte(0xC0) 61 | br.Flush() 62 | 63 | p := &parser{} 64 | err := p.readList(KeyObject{Key: []byte("list")}, bufio.NewReader(&buffer)) 65 | equals(t, ErrUnexpectedEncodedLength, err) 66 | } 67 | 68 | func TestReadListNoElementData(t *testing.T) { 69 | var buffer bytes.Buffer 70 | 71 | br := bufio.NewWriter(&buffer) 72 | 73 | br.WriteByte(1) 74 | br.Flush() 75 | 76 | ctx := ParserContext{ListMetadataCh: make(chan ListMetadata)} 77 | p := &parser{ctx: ctx} 78 | 79 | go func() { 80 | md := <-ctx.ListMetadataCh 81 | equals(t, "list", DataToString(md.Key)) 82 | equals(t, int64(1), md.Len) 83 | }() 84 | 85 | err := p.readList(KeyObject{Key: []byte("list")}, bufio.NewReader(&buffer)) 86 | equals(t, io.EOF, err) 87 | } 88 | 89 | func TestReadListInZipList(t *testing.T) { 90 | var buffer bytes.Buffer 91 | 92 | br := bufio.NewWriter(&buffer) 93 | 94 | br.WriteByte(18) // String length 95 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 96 | br.Write([]byte{0, 0, 0, 0}) // zlTail 97 | br.Write([]byte{1, 0}) // zlLen 98 | 99 | br.WriteByte(0) // len prev entry 100 | br.WriteByte(6) // Special flag 101 | br.WriteString("foobar") 102 | 103 | br.Flush() 104 | 105 | ctx := ParserContext{ 106 | ListMetadataCh: make(chan ListMetadata), 107 | ListDataCh: make(chan interface{}), 108 | } 109 | p := &parser{ctx: ctx} 110 | 111 | go readAndNotify(t, &buffer, "list", p.readListInZipList) 112 | 113 | stop := false 114 | for !stop { 115 | select { 116 | case md := <-ctx.ListMetadataCh: 117 | equals(t, "list", DataToString(md.Key)) 118 | equals(t, int64(1), md.Len) 119 | case d := <-ctx.ListDataCh: 120 | equals(t, "foobar", DataToString(d)) 121 | case <-end: 122 | stop = true 123 | } 124 | } 125 | } 126 | 127 | func TestReadListInZipListNoData(t *testing.T) { 128 | var buffer bytes.Buffer 129 | 130 | p := &parser{} 131 | err := p.readListInZipList(KeyObject{Key: []byte("list")}, bufio.NewReader(&buffer)) 132 | equals(t, io.EOF, err) 133 | } 134 | 135 | func TestReadListInZipListFail(t *testing.T) { 136 | var buffer bytes.Buffer 137 | 138 | br := bufio.NewWriter(&buffer) 139 | br.WriteByte(0) 140 | br.Flush() 141 | 142 | p := &parser{} 143 | err := p.readListInZipList(KeyObject{Key: []byte("list")}, bufio.NewReader(&buffer)) 144 | equals(t, io.EOF, err) 145 | } 146 | -------------------------------------------------------------------------------- /lzf.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | func lzfDecompress(data []byte, ulen int64) []byte { 4 | output := make([]byte, ulen) 5 | 6 | if len(data) <= 0 { 7 | return output 8 | } 9 | 10 | i := uint32(0) 11 | o := uint32(0) 12 | 13 | for i < uint32(len(data)) { 14 | var ctrl uint32 = uint32(data[i]) 15 | i++ 16 | if ctrl < 32 { 17 | copy(output[o:], data[i:i+ctrl+1]) 18 | i += ctrl + 1 19 | o += ctrl + 1 20 | } else { 21 | var length uint32 = uint32(ctrl >> 5) 22 | if length == 7 { 23 | length += uint32(data[i]) 24 | i++ 25 | } 26 | 27 | var ref uint32 = uint32(o) - uint32(ctrl&0x1F<<8) - uint32(data[i]) - 1 28 | 29 | i++ 30 | for j := uint32(0); j < length+2; j++ { 31 | output[o] = output[ref] 32 | ref++ 33 | o++ 34 | } 35 | } 36 | } 37 | 38 | return output 39 | } 40 | -------------------------------------------------------------------------------- /lzf_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestLzfDecompress(t *testing.T) { 9 | data := []byte{1, 97, 97, 224, 246, 0, 1, 97, 97} 10 | ulen := int64(259) 11 | 12 | output := lzfDecompress(data, ulen) 13 | expected := strings.Repeat("a", int(ulen)) 14 | if string(output) != expected { 15 | t.Errorf("expected %s but got %s", expected, string(output)) 16 | } 17 | } 18 | 19 | func TestLzfDecompressNoData(t *testing.T) { 20 | output := lzfDecompress([]byte{}, 0) 21 | if len(output) != 0 { 22 | t.Errorf("expected empty slice but got %s", string(output)) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "io" 7 | "math" 8 | "strconv" 9 | ) 10 | 11 | type Parser interface { 12 | Parse(r io.Reader) (err error) 13 | } 14 | 15 | // Parser is the main parser for RDB files 16 | type parser struct { 17 | ctx ParserContext 18 | r io.Reader 19 | scratch [4]byte 20 | } 21 | 22 | const ( 23 | // The last version of RDB files 24 | RedisRdbVersion = 6 25 | ) 26 | 27 | var ( 28 | errNoMoreDatabases = errors.New("errNoMoreDatabases") 29 | errNoMoreKeyValuePair = errors.New("errNoMoreKeyValuePair") 30 | ErrInvalidMagicString = errors.New("invalid magic string") 31 | ErrInvalidRDBVersionNumber = errors.New("invalid RDB version number") 32 | ErrInvalidChecksum = errors.New("invalid checksum") 33 | ErrUnexpectedEncodedLength = errors.New("unexpected encoded length") 34 | ErrUnknownValueType = errors.New("unknown value type") 35 | ErrUnknownLengthEncoding = errors.New("unknown length encoding") 36 | ErrUnexpectedPrevLengthEntryByte = errors.New("unexpected prev length entry byte") 37 | ) 38 | 39 | // A ParserContext holds the channels used to receive data from the parser 40 | type ParserContext struct { 41 | DbCh chan int 42 | StringObjectCh chan StringObject 43 | ListMetadataCh chan ListMetadata 44 | ListDataCh chan interface{} 45 | SetMetadataCh chan SetMetadata 46 | SetDataCh chan interface{} 47 | HashMetadataCh chan HashMetadata 48 | HashDataCh chan HashEntry 49 | SortedSetMetadataCh chan SortedSetMetadata 50 | SortedSetEntriesCh chan SortedSetEntry 51 | endOfFileCh chan struct{} 52 | } 53 | 54 | func (c *ParserContext) closeChannels() { 55 | if c.DbCh != nil { 56 | close(c.DbCh) 57 | } 58 | if c.StringObjectCh != nil { 59 | close(c.StringObjectCh) 60 | } 61 | if c.ListMetadataCh != nil { 62 | close(c.ListMetadataCh) 63 | } 64 | if c.ListDataCh != nil { 65 | close(c.ListDataCh) 66 | } 67 | if c.SetMetadataCh != nil { 68 | close(c.SetMetadataCh) 69 | } 70 | if c.SetDataCh != nil { 71 | close(c.SetDataCh) 72 | } 73 | if c.HashMetadataCh != nil { 74 | close(c.HashMetadataCh) 75 | } 76 | if c.HashDataCh != nil { 77 | close(c.HashDataCh) 78 | } 79 | if c.SortedSetMetadataCh != nil { 80 | close(c.SortedSetMetadataCh) 81 | } 82 | if c.SortedSetEntriesCh != nil { 83 | close(c.SortedSetEntriesCh) 84 | } 85 | close(c.endOfFileCh) 86 | } 87 | 88 | // Invalid returns true if the context is invalid (all channels are nil), false otherwise. 89 | // This is needed to actually terminate parsing if you use a for-select loop 90 | func (c *ParserContext) Invalid() bool { 91 | return c.DbCh == nil && c.StringObjectCh == nil && c.ListMetadataCh == nil && c.ListDataCh == nil && c.SetMetadataCh == nil && c.SetDataCh == nil && c.HashMetadataCh == nil && c.HashDataCh == nil && c.SortedSetMetadataCh == nil && c.SortedSetEntriesCh == nil 92 | } 93 | 94 | // Create a new parser using the provided context 95 | func NewParser(ctx ParserContext) Parser { 96 | ctx.endOfFileCh = make(chan struct{}) 97 | return &parser{ctx: ctx} 98 | } 99 | 100 | // Parse a RDB file reading data from the provided reader r 101 | // Any error occurring while parsing will be returned here 102 | func (p *parser) Parse(r io.Reader) (err error) { 103 | cr := newChecksumReader(r) 104 | 105 | if err = readMagicString(cr); err != nil { 106 | return err 107 | } 108 | 109 | var rdbVersion int 110 | if rdbVersion, err = readVersionNumber(cr); err != nil { 111 | return err 112 | } 113 | 114 | for { 115 | if err = p.readDatabase(cr); err != nil && err != errNoMoreDatabases { 116 | return err 117 | } else if err != nil && err == errNoMoreDatabases { 118 | break 119 | } 120 | 121 | for { 122 | if err = p.readKeyValuePair(cr); err != nil && err != errNoMoreKeyValuePair { 123 | return err 124 | } else if err != nil && err == errNoMoreKeyValuePair { 125 | break 126 | } 127 | } 128 | 129 | if p.scratch[0] == 0xFF { 130 | break 131 | } 132 | } 133 | 134 | // Read the CRC64 checksum with RDB version >= 5 135 | if rdbVersion >= 5 { 136 | sum := cr.checksum 137 | 138 | var checksum uint64 139 | if err := binary.Read(cr, binary.LittleEndian, &checksum); err != nil { 140 | return err 141 | } 142 | 143 | if sum != checksum { 144 | return ErrInvalidChecksum 145 | } 146 | } 147 | 148 | p.ctx.closeChannels() 149 | 150 | return nil 151 | } 152 | 153 | func readMagicString(r io.Reader) error { 154 | data := make([]byte, 5) 155 | read, err := r.Read(data) 156 | if err != nil { 157 | return err 158 | } 159 | 160 | if read != 5 { 161 | return ErrInvalidMagicString 162 | } 163 | 164 | if string(data) != "REDIS" { 165 | return ErrInvalidMagicString 166 | } 167 | 168 | return nil 169 | } 170 | 171 | func readVersionNumber(r io.Reader) (int, error) { 172 | data := make([]byte, 4) 173 | read, err := r.Read(data) 174 | if err != nil { 175 | return -1, err 176 | } 177 | 178 | if read != 4 { 179 | return -1, ErrInvalidRDBVersionNumber 180 | } 181 | 182 | val := string(data) 183 | ival, err := strconv.Atoi(val) 184 | if err != nil { 185 | return -1, err 186 | } 187 | 188 | if ival < 1 || ival > RedisRdbVersion { 189 | return -1, ErrInvalidRDBVersionNumber 190 | } 191 | 192 | return ival, nil 193 | } 194 | 195 | func (p *parser) readDatabase(r io.Reader) error { 196 | // Might have read the 0xFE byte already in the last readKeyValuePair call 197 | if p.scratch[0] != 0xFE { 198 | _, err := io.ReadFull(r, p.scratch[0:1]) 199 | if err != nil { 200 | return err 201 | } 202 | 203 | if p.scratch[0] != 0xFE { 204 | return errNoMoreDatabases 205 | } 206 | } 207 | 208 | var dbNumber uint8 209 | if err := binary.Read(r, binary.BigEndian, &dbNumber); err != nil { 210 | return err 211 | } 212 | 213 | if p.ctx.DbCh != nil { 214 | p.ctx.DbCh <- int(dbNumber) 215 | } 216 | 217 | return nil 218 | } 219 | 220 | func (p *parser) readLen(r io.Reader) (int64, bool, error) { 221 | _, err := io.ReadFull(r, p.scratch[0:1]) 222 | if err != nil { 223 | return -1, false, err 224 | } 225 | 226 | b := p.scratch[0] 227 | 228 | bits := (b & 0xC0) >> 6 229 | switch bits { 230 | case 0: 231 | return int64(b) & 0x3f, false, nil 232 | case 1: 233 | _, err := io.ReadFull(r, p.scratch[0:1]) 234 | if err != nil { 235 | return -1, false, err 236 | } 237 | return int64((int64(b)&0x3f)<<8) | int64(p.scratch[0]), false, nil 238 | case 2: 239 | var tmp uint32 240 | if err := binary.Read(r, binary.BigEndian, &tmp); err != nil { 241 | return -1, false, err 242 | } 243 | 244 | return int64(tmp), false, nil 245 | default: 246 | return int64(b) & 0x3f, true, nil 247 | } 248 | } 249 | 250 | func (p *parser) readDoubleValue(r io.Reader) (float64, error) { 251 | _, err := io.ReadFull(r, p.scratch[0:1]) 252 | if err != nil { 253 | return 0, err 254 | } 255 | 256 | l := p.scratch[0] 257 | 258 | switch l { 259 | case 255: 260 | return math.Inf(-1), nil 261 | case 254: 262 | return math.Inf(1), nil 263 | case 253: 264 | return math.NaN(), nil 265 | default: 266 | bytes, err := readBytes(r, int64(l)) 267 | if err != nil { 268 | return 0, err 269 | } 270 | 271 | return strconv.ParseFloat(string(bytes), 64) 272 | } 273 | } 274 | 275 | func readBytes(r io.Reader, length int64) ([]byte, error) { 276 | bytes := make([]byte, length) 277 | _, err := io.ReadFull(r, bytes) 278 | if err != nil { 279 | return nil, err 280 | } 281 | 282 | return bytes, nil 283 | } 284 | 285 | func (p *parser) readLZFString(r io.Reader) ([]byte, error) { 286 | clen, _, err := p.readLen(r) 287 | if err != nil { 288 | return nil, err 289 | } 290 | 291 | ulen, _, err := p.readLen(r) 292 | if err != nil { 293 | return nil, err 294 | } 295 | 296 | cdata, err := readBytes(r, clen) 297 | if err != nil { 298 | return nil, err 299 | } 300 | 301 | return lzfDecompress(cdata, ulen), nil 302 | } 303 | 304 | func (p *parser) readString(r io.Reader) (interface{}, error) { 305 | l, e, err := p.readLen(r) 306 | if err != nil { 307 | return nil, err 308 | } 309 | 310 | var bytes []byte 311 | if e { 312 | // Encoded string 313 | switch l { 314 | case 0: // INT8 315 | var i int8 316 | if err = binary.Read(r, binary.LittleEndian, &i); err != nil { 317 | return nil, err 318 | } 319 | return i, nil 320 | case 1: // INT16 321 | var i int16 322 | if err = binary.Read(r, binary.LittleEndian, &i); err != nil { 323 | return nil, err 324 | } 325 | return i, nil 326 | case 2: // INT32 327 | var i int32 328 | if err = binary.Read(r, binary.LittleEndian, &i); err != nil { 329 | return nil, err 330 | } 331 | return i, nil 332 | case 3: // LZF 333 | bytes, err = p.readLZFString(r) 334 | if err != nil { 335 | return nil, err 336 | } 337 | } 338 | } else { 339 | // Length prefixed string 340 | bytes, err = readBytes(r, l) 341 | if err != nil { 342 | return nil, err 343 | } 344 | } 345 | 346 | return bytes, nil 347 | } 348 | 349 | func (p *parser) readKeyValuePair(r io.Reader) error { 350 | _, err := io.ReadFull(r, p.scratch[0:1]) 351 | if err != nil { 352 | return err 353 | } 354 | 355 | if p.scratch[0] == 0xFE || p.scratch[0] == 0xFF { 356 | return errNoMoreKeyValuePair 357 | } 358 | 359 | b := p.scratch[0] 360 | 361 | // Read expiry time in seconds 362 | var expiryTime int64 = -1 363 | if b == 0xFD { 364 | var tmp uint32 365 | if err := binary.Read(r, binary.LittleEndian, &tmp); err != nil { 366 | return err 367 | } 368 | expiryTime = int64(int64(tmp) * 1000) 369 | } 370 | 371 | // Read expiry time in milliseconds 372 | if b == 0xFC { 373 | if err := binary.Read(r, binary.LittleEndian, &expiryTime); err != nil { 374 | return err 375 | } 376 | } 377 | 378 | // If the byte was a expiry time flag, we need to reread a byte 379 | if b == 0xFD || b == 0xFC { 380 | _, err := io.ReadFull(r, p.scratch[0:1]) 381 | b = p.scratch[0] 382 | if err != nil { 383 | return err 384 | } 385 | } 386 | 387 | keyStr, err := p.readString(r) 388 | if err != nil { 389 | return err 390 | } 391 | 392 | key := NewKeyObject(keyStr, expiryTime) 393 | 394 | switch b { 395 | case 0: // String encoding 396 | value, err := p.readString(r) 397 | if err != nil { 398 | return err 399 | } 400 | 401 | if p.ctx.StringObjectCh != nil { 402 | p.ctx.StringObjectCh <- StringObject{Key: key, Value: value} 403 | } 404 | case 1: // List encoding 405 | if err := p.readList(key, r); err != nil { 406 | return err 407 | } 408 | case 2: // Set encoding 409 | if err := p.readSet(key, r); err != nil { 410 | return err 411 | } 412 | case 3: // Sorted set encoding 413 | if err := p.readSortedSet(key, r); err != nil { 414 | return err 415 | } 416 | case 4: // Hash encoding 417 | if err := p.readHashMap(key, r); err != nil { 418 | return err 419 | } 420 | case 9: // Zipmap encoding 421 | if err := p.readZipMap(key, r); err != nil { 422 | return err 423 | } 424 | case 10: // Zip list encoding 425 | if err := p.readListInZipList(key, r); err != nil { 426 | return err 427 | } 428 | case 11: // int set encoding 429 | if err := p.readIntSet(key, r); err != nil { 430 | return err 431 | } 432 | case 12: // Sorted set in zip list encoding 433 | if err := p.readSortedSetInZipList(key, r); err != nil { 434 | return err 435 | } 436 | case 13: // hash map in zip list encoding 437 | if err := p.readHashMapInZipList(key, r); err != nil { 438 | return err 439 | } 440 | default: 441 | return ErrUnknownValueType 442 | } 443 | 444 | return nil 445 | } 446 | -------------------------------------------------------------------------------- /parser_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "io" 8 | "math" 9 | "strings" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | func mustParse(t *testing.T, p Parser, ctx ParserContext, r io.Reader) { 15 | err := p.Parse(r) 16 | if err != nil { 17 | ctx.closeChannels() 18 | t.Fatalf("Error while parsing; err=%s", err) 19 | } 20 | } 21 | 22 | func TestReadMagicString(t *testing.T) { 23 | var buffer bytes.Buffer 24 | 25 | br := bufio.NewWriter(&buffer) 26 | br.WriteString("REDIS") 27 | br.Flush() 28 | 29 | err := readMagicString(bufio.NewReader(&buffer)) 30 | ok(t, err) 31 | 32 | // No data 33 | buffer.Reset() 34 | 35 | err = readMagicString(bufio.NewReader(&buffer)) 36 | equals(t, io.EOF, err) 37 | 38 | // Not enough data 39 | buffer.Reset() 40 | br.WriteString("FOO") 41 | br.Flush() 42 | 43 | err = readMagicString(bufio.NewReader(&buffer)) 44 | equals(t, ErrInvalidMagicString, err) 45 | 46 | // Invalid data 47 | buffer.Reset() 48 | br.WriteString("FOOBA") 49 | br.Flush() 50 | 51 | err = readMagicString(bufio.NewReader(&buffer)) 52 | equals(t, ErrInvalidMagicString, err) 53 | } 54 | 55 | func TestReadVersionNumber(t *testing.T) { 56 | var buffer bytes.Buffer 57 | 58 | br := bufio.NewWriter(&buffer) 59 | br.WriteString("0006") 60 | br.Flush() 61 | 62 | v, err := readVersionNumber(bufio.NewReader(&buffer)) 63 | ok(t, err) 64 | equals(t, 6, v) 65 | 66 | // No data 67 | buffer.Reset() 68 | 69 | v, err = readVersionNumber(bufio.NewReader(&buffer)) 70 | equals(t, io.EOF, err) 71 | equals(t, -1, v) 72 | 73 | // Not enough data 74 | buffer.Reset() 75 | br.WriteString("FOO") 76 | br.Flush() 77 | 78 | v, err = readVersionNumber(bufio.NewReader(&buffer)) 79 | equals(t, ErrInvalidRDBVersionNumber, err) 80 | equals(t, -1, v) 81 | 82 | // Not a number 83 | buffer.Reset() 84 | br.WriteString("foob") 85 | br.Flush() 86 | 87 | v, err = readVersionNumber(bufio.NewReader(&buffer)) 88 | equals(t, "strconv.ParseInt: parsing \"foob\": invalid syntax", err.Error()) 89 | equals(t, -1, v) 90 | 91 | // Wrong version number 92 | buffer.Reset() 93 | br.WriteString("0010") 94 | br.Flush() 95 | 96 | v, err = readVersionNumber(bufio.NewReader(&buffer)) 97 | equals(t, ErrInvalidRDBVersionNumber, err) 98 | equals(t, -1, v) 99 | } 100 | 101 | func TestReadDatabase(t *testing.T) { 102 | var buffer bytes.Buffer 103 | 104 | br := bufio.NewWriter(&buffer) 105 | br.WriteByte(0xFE) // indicate next database 106 | br.WriteByte(0) 107 | br.Flush() 108 | 109 | ctx := ParserContext{DbCh: make(chan int)} 110 | p := &parser{ctx: ctx} 111 | 112 | go func() { 113 | db := <-ctx.DbCh 114 | equals(t, int(0), db) 115 | }() 116 | 117 | err := p.readDatabase(bufio.NewReader(&buffer)) 118 | ok(t, err) 119 | } 120 | 121 | func TestReadDatabaseNoData(t *testing.T) { 122 | var buffer bytes.Buffer 123 | 124 | ctx := ParserContext{DbCh: make(chan int)} 125 | p := &parser{ctx: ctx} 126 | 127 | err := p.readDatabase(bufio.NewReader(&buffer)) 128 | equals(t, io.EOF, err) 129 | } 130 | 131 | func TestReadDatabaseNoMoreDatabase(t *testing.T) { 132 | var buffer bytes.Buffer 133 | 134 | br := bufio.NewWriter(&buffer) 135 | br.WriteByte(0x01) 136 | br.Flush() 137 | 138 | ctx := ParserContext{DbCh: make(chan int)} 139 | p := &parser{ctx: ctx} 140 | 141 | err := p.readDatabase(bufio.NewReader(&buffer)) 142 | equals(t, errNoMoreDatabases, err) 143 | } 144 | 145 | func TestReadDatabaseNoDbNumber(t *testing.T) { 146 | var buffer bytes.Buffer 147 | 148 | br := bufio.NewWriter(&buffer) 149 | br.WriteByte(0xFE) 150 | br.Flush() 151 | 152 | ctx := ParserContext{DbCh: make(chan int)} 153 | p := &parser{ctx: ctx} 154 | 155 | err := p.readDatabase(bufio.NewReader(&buffer)) 156 | equals(t, io.EOF, err) 157 | } 158 | 159 | func TestReadLen(t *testing.T) { 160 | var buffer bytes.Buffer 161 | 162 | br := bufio.NewWriter(&buffer) 163 | p := &parser{} 164 | 165 | // 6 bits encoding 166 | br.WriteByte(1) 167 | br.Flush() 168 | 169 | l, e, err := p.readLen(bufio.NewReader(&buffer)) 170 | ok(t, err) 171 | equals(t, int64(1), l) 172 | equals(t, false, e) 173 | 174 | // 14 bits encoding 175 | buffer.Reset() 176 | br.WriteByte(0x41) 177 | br.WriteByte(1) 178 | br.Flush() 179 | 180 | l, e, err = p.readLen(bufio.NewReader(&buffer)) 181 | ok(t, err) 182 | equals(t, int64(257), l) 183 | equals(t, false, e) 184 | 185 | // 32 bit encoding 186 | buffer.Reset() 187 | br.WriteByte(0xB0) 188 | binary.Write(br, binary.BigEndian, int32(1)) 189 | br.Flush() 190 | 191 | l, e, err = p.readLen(bufio.NewReader(&buffer)) 192 | ok(t, err) 193 | equals(t, int64(1), l) 194 | equals(t, false, e) 195 | 196 | // special encoding 197 | buffer.Reset() 198 | br.WriteByte(0xD1) 199 | br.Flush() 200 | 201 | l, e, err = p.readLen(bufio.NewReader(&buffer)) 202 | ok(t, err) 203 | equals(t, int64(17), l) 204 | equals(t, true, e) 205 | 206 | // 14 bits encoding - no additional byte 207 | buffer.Reset() 208 | br.WriteByte(0x41) 209 | br.Flush() 210 | 211 | l, e, err = p.readLen(bufio.NewReader(&buffer)) 212 | equals(t, int64(-1), l) 213 | equals(t, false, e) 214 | equals(t, io.EOF, err) 215 | 216 | // 32 bits encoding - no additional data 217 | buffer.Reset() 218 | br.WriteByte(0x80) 219 | br.Flush() 220 | 221 | l, e, err = p.readLen(bufio.NewReader(&buffer)) 222 | equals(t, int64(-1), l) 223 | equals(t, false, e) 224 | equals(t, io.EOF, err) 225 | } 226 | 227 | func TestReadDoubleValue(t *testing.T) { 228 | var buffer bytes.Buffer 229 | 230 | br := bufio.NewWriter(&buffer) 231 | p := &parser{} 232 | 233 | // Negative inf 234 | br.WriteByte(0xFF) 235 | br.Flush() 236 | 237 | v, err := p.readDoubleValue(bufio.NewReader(&buffer)) 238 | ok(t, err) 239 | equals(t, true, math.IsInf(v, -1)) 240 | 241 | // Positive inf 242 | buffer.Reset() 243 | br.WriteByte(0xFE) 244 | br.Flush() 245 | 246 | v, err = p.readDoubleValue(bufio.NewReader(&buffer)) 247 | ok(t, err) 248 | equals(t, true, math.IsInf(v, 1)) 249 | 250 | // NaN 251 | buffer.Reset() 252 | br.WriteByte(0xFD) 253 | br.Flush() 254 | 255 | v, err = p.readDoubleValue(bufio.NewReader(&buffer)) 256 | ok(t, err) 257 | equals(t, true, math.IsNaN(v)) 258 | 259 | // Normal case 260 | buffer.Reset() 261 | br.WriteByte(4) 262 | br.WriteString("20.1") 263 | br.Flush() 264 | 265 | v, err = p.readDoubleValue(bufio.NewReader(&buffer)) 266 | ok(t, err) 267 | equals(t, 20.1, v) 268 | 269 | // No data 270 | buffer.Reset() 271 | 272 | v, err = p.readDoubleValue(bufio.NewReader(&buffer)) 273 | equals(t, io.EOF, err) 274 | 275 | // No additional bytes 276 | buffer.Reset() 277 | br.WriteByte(1) 278 | br.Flush() 279 | 280 | v, err = p.readDoubleValue(bufio.NewReader(&buffer)) 281 | equals(t, io.EOF, err) 282 | 283 | // Not a float value 284 | buffer.Reset() 285 | br.WriteByte(6) 286 | br.WriteString("foobar") 287 | br.Flush() 288 | 289 | v, err = p.readDoubleValue(bufio.NewReader(&buffer)) 290 | equals(t, "strconv.ParseFloat: parsing \"foobar\": invalid syntax", err.Error()) 291 | } 292 | 293 | func TestReadLZFString(t *testing.T) { 294 | var buffer bytes.Buffer 295 | 296 | br := bufio.NewWriter(&buffer) 297 | p := &parser{} 298 | 299 | data := []byte{1, 97, 97, 224, 246, 0, 1, 97, 97} 300 | 301 | br.WriteByte(byte(len(data))) 302 | br.WriteByte(0x41) // ulen - 256 303 | br.WriteByte(3) // +3 304 | br.Write(data) 305 | br.Flush() 306 | 307 | v, err := p.readLZFString(bufio.NewReader(&buffer)) 308 | ok(t, err) 309 | equals(t, strings.Repeat("a", 259), DataToString(v)) 310 | 311 | // No clen data 312 | buffer.Reset() 313 | 314 | v, err = p.readLZFString(bufio.NewReader(&buffer)) 315 | equals(t, io.EOF, err) 316 | 317 | // No ulen data 318 | buffer.Reset() 319 | br.WriteByte(1) 320 | br.Flush() 321 | 322 | v, err = p.readLZFString(bufio.NewReader(&buffer)) 323 | equals(t, io.EOF, err) 324 | 325 | // No cdata 326 | buffer.Reset() 327 | br.WriteByte(1) 328 | br.WriteByte(1) 329 | br.Flush() 330 | 331 | v, err = p.readLZFString(bufio.NewReader(&buffer)) 332 | equals(t, io.EOF, err) 333 | } 334 | 335 | func TestReadString(t *testing.T) { 336 | var buffer bytes.Buffer 337 | 338 | br := bufio.NewWriter(&buffer) 339 | p := &parser{} 340 | 341 | // Length prefixed string 342 | br.WriteByte(1) 343 | br.WriteByte('a') 344 | br.Flush() 345 | 346 | v, err := p.readString(bufio.NewReader(&buffer)) 347 | ok(t, err) 348 | equals(t, "a", DataToString(v)) 349 | 350 | // Int8 encoding 351 | buffer.Reset() 352 | br.WriteByte(0xC0) 353 | br.WriteByte(1) 354 | br.Flush() 355 | 356 | v, err = p.readString(bufio.NewReader(&buffer)) 357 | ok(t, err) 358 | equals(t, int8(1), v) 359 | 360 | // Int16 encoding 361 | buffer.Reset() 362 | br.WriteByte(0xC1) 363 | binary.Write(br, binary.LittleEndian, int16(1)) 364 | br.Flush() 365 | 366 | v, err = p.readString(bufio.NewReader(&buffer)) 367 | ok(t, err) 368 | equals(t, int16(1), v) 369 | 370 | // Int32 encoding 371 | buffer.Reset() 372 | br.WriteByte(0xC2) 373 | binary.Write(br, binary.LittleEndian, int32(1)) 374 | br.Flush() 375 | 376 | v, err = p.readString(bufio.NewReader(&buffer)) 377 | ok(t, err) 378 | equals(t, int32(1), v) 379 | 380 | // LZF string 381 | data := []byte{1, 97, 97, 224, 246, 0, 1, 97, 97} 382 | buffer.Reset() 383 | br.WriteByte(0xC3) 384 | br.WriteByte(byte(len(data))) 385 | br.WriteByte(0x41) // ulen - 256 386 | br.WriteByte(3) // +3 387 | br.Write(data) 388 | br.Flush() 389 | 390 | v, err = p.readString(bufio.NewReader(&buffer)) 391 | ok(t, err) 392 | equals(t, strings.Repeat("a", 259), DataToString(v)) 393 | 394 | // Length prefixed - no data 395 | buffer.Reset() 396 | br.WriteByte(1) 397 | br.Flush() 398 | 399 | v, err = p.readString(bufio.NewReader(&buffer)) 400 | equals(t, nil, v) 401 | equals(t, io.EOF, err) 402 | 403 | // Int8 encoding - no data 404 | buffer.Reset() 405 | br.WriteByte(0xC0) 406 | br.Flush() 407 | 408 | v, err = p.readString(bufio.NewReader(&buffer)) 409 | equals(t, nil, v) 410 | equals(t, io.EOF, err) 411 | 412 | // Int16 encoding - no data 413 | buffer.Reset() 414 | br.WriteByte(0xC1) 415 | br.Flush() 416 | 417 | v, err = p.readString(bufio.NewReader(&buffer)) 418 | equals(t, nil, v) 419 | equals(t, io.EOF, err) 420 | 421 | // Int32 encoding - no data 422 | buffer.Reset() 423 | br.WriteByte(0xC2) 424 | br.Flush() 425 | 426 | v, err = p.readString(bufio.NewReader(&buffer)) 427 | equals(t, nil, v) 428 | equals(t, io.EOF, err) 429 | 430 | // LZF string - no data 431 | buffer.Reset() 432 | br.WriteByte(0xC3) 433 | br.Flush() 434 | 435 | v, err = p.readString(bufio.NewReader(&buffer)) 436 | equals(t, nil, v) 437 | equals(t, io.EOF, err) 438 | } 439 | 440 | func TestReadKeyValuePairErrors(t *testing.T) { 441 | var buffer bytes.Buffer 442 | 443 | br := bufio.NewWriter(&buffer) 444 | 445 | p := &parser{} 446 | 447 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 448 | equals(t, io.EOF, err) 449 | 450 | // No more key value pair 0xFE 451 | buffer.Reset() 452 | br.WriteByte(0xFE) 453 | br.Flush() 454 | 455 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 456 | equals(t, errNoMoreKeyValuePair, err) 457 | 458 | // No more key value pair 0xFF 459 | buffer.Reset() 460 | br.WriteByte(0xFF) 461 | br.Flush() 462 | 463 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 464 | equals(t, errNoMoreKeyValuePair, err) 465 | 466 | // Expiry in seconds byte but no data 467 | buffer.Reset() 468 | br.WriteByte(0xFD) 469 | br.Flush() 470 | 471 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 472 | equals(t, io.EOF, err) 473 | 474 | // Expiry in milliseconds byte but no data 475 | buffer.Reset() 476 | br.WriteByte(0xFC) 477 | br.Flush() 478 | 479 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 480 | equals(t, io.EOF, err) 481 | 482 | // Expiry byte but no value type byte 483 | buffer.Reset() 484 | br.WriteByte(0xFD) 485 | binary.Write(br, binary.LittleEndian, uint32(1)) 486 | br.Flush() 487 | 488 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 489 | equals(t, io.EOF, err) 490 | 491 | // No key data 492 | buffer.Reset() 493 | br.WriteByte(0) 494 | br.Flush() 495 | 496 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 497 | equals(t, io.EOF, err) 498 | } 499 | 500 | func TestReadKeyValuePairStringEncoding(t *testing.T) { 501 | var buffer bytes.Buffer 502 | 503 | br := bufio.NewWriter(&buffer) 504 | 505 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 506 | p := &parser{ctx: ctx} 507 | 508 | go func() { 509 | v := <-ctx.StringObjectCh 510 | equals(t, "a", DataToString(v.Key.Key)) 511 | equals(t, "b", DataToString(v.Value)) 512 | }() 513 | 514 | br.WriteByte(0) // string encoding 515 | br.WriteByte(1) // key length 516 | br.WriteByte('a') // key data 517 | br.WriteByte(1) // string length 518 | br.WriteByte('b') // string data 519 | br.Flush() 520 | 521 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 522 | ok(t, err) 523 | 524 | // No string data 525 | buffer.Reset() 526 | br.WriteByte(0) 527 | br.WriteByte(1) 528 | br.WriteByte('a') 529 | br.WriteByte(1) 530 | br.Flush() 531 | 532 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 533 | equals(t, io.EOF, err) 534 | } 535 | 536 | func TestReadKeyValuePairSecondExpiry(t *testing.T) { 537 | var buffer bytes.Buffer 538 | 539 | br := bufio.NewWriter(&buffer) 540 | 541 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 542 | p := &parser{ctx: ctx} 543 | 544 | etime := time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC) 545 | 546 | df := func() { 547 | v := <-ctx.StringObjectCh 548 | equals(t, "a", DataToString(v.Key.Key)) 549 | equals(t, "2100-01-01 00:00:00 +0000 UTC", v.Key.ExpiryTime.UTC().String()) 550 | equals(t, false, v.Key.Expired()) 551 | equals(t, "foobar", DataToString(v.Value)) 552 | } 553 | 554 | br.WriteByte(0xFD) // expiry in second 555 | binary.Write(br, binary.LittleEndian, uint32(etime.Unix())) 556 | br.WriteByte(0) 557 | br.Write([]byte{1, 'a'}) 558 | br.WriteByte(6) 559 | br.WriteString("foobar") 560 | br.Flush() 561 | 562 | go df() 563 | 564 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 565 | ok(t, err) 566 | } 567 | 568 | func TestReadKeyValuePairMillisecondExpiry(t *testing.T) { 569 | var buffer bytes.Buffer 570 | 571 | br := bufio.NewWriter(&buffer) 572 | 573 | ctx := ParserContext{StringObjectCh: make(chan StringObject)} 574 | p := &parser{ctx: ctx} 575 | 576 | etime := time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC) 577 | 578 | df := func() { 579 | v := <-ctx.StringObjectCh 580 | equals(t, "a", DataToString(v.Key.Key)) 581 | equals(t, "2100-01-01 00:00:00 +0000 UTC", v.Key.ExpiryTime.UTC().String()) 582 | equals(t, false, v.Key.Expired()) 583 | equals(t, "foobar", DataToString(v.Value)) 584 | } 585 | 586 | br.WriteByte(0xFC) // expiry in second 587 | binary.Write(br, binary.LittleEndian, uint64(etime.Unix()*1000)) 588 | br.WriteByte(0) 589 | br.Write([]byte{1, 'a'}) 590 | br.WriteByte(6) 591 | br.WriteString("foobar") 592 | br.Flush() 593 | 594 | go df() 595 | 596 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 597 | ok(t, err) 598 | } 599 | 600 | func TestReadKeyValuePairListEncoding(t *testing.T) { 601 | var buffer bytes.Buffer 602 | 603 | br := bufio.NewWriter(&buffer) 604 | 605 | ctx := ParserContext{ 606 | ListMetadataCh: make(chan ListMetadata), 607 | ListDataCh: make(chan interface{}), 608 | } 609 | p := &parser{ctx: ctx} 610 | 611 | mf := func() { 612 | l := <-ctx.ListMetadataCh 613 | equals(t, "a", DataToString(l.Key)) 614 | equals(t, int64(1), l.Len) 615 | } 616 | 617 | df := func() { 618 | v := <-ctx.ListDataCh 619 | equals(t, "v", DataToString(v)) 620 | } 621 | 622 | br.WriteByte(1) // list encoding 623 | br.WriteByte(1) // key length 624 | br.WriteByte('a') // key data 625 | br.WriteByte(1) // list length 626 | br.WriteByte(1) // element length 627 | br.WriteByte('v') // element data 628 | br.Flush() 629 | 630 | go mf() 631 | go df() 632 | 633 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 634 | ok(t, err) 635 | 636 | // No list data 637 | buffer.Reset() 638 | br.WriteByte(1) 639 | br.WriteByte(1) 640 | br.WriteByte('a') 641 | br.WriteByte(1) 642 | br.WriteByte(1) 643 | br.Flush() 644 | 645 | go mf() 646 | go df() 647 | 648 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 649 | equals(t, io.EOF, err) 650 | } 651 | 652 | func TestReadKeyValuePairSetEncoding(t *testing.T) { 653 | var buffer bytes.Buffer 654 | 655 | br := bufio.NewWriter(&buffer) 656 | 657 | ctx := ParserContext{ 658 | SetMetadataCh: make(chan SetMetadata), 659 | SetDataCh: make(chan interface{}), 660 | } 661 | p := &parser{ctx: ctx} 662 | 663 | mf := func() { 664 | l := <-ctx.SetMetadataCh 665 | equals(t, "a", DataToString(l.Key)) 666 | equals(t, int64(1), l.Len) 667 | } 668 | 669 | df := func() { 670 | v := <-ctx.SetDataCh 671 | equals(t, "v", DataToString(v)) 672 | } 673 | 674 | br.WriteByte(2) // set encoding 675 | br.WriteByte(1) // key length 676 | br.WriteByte('a') // key data 677 | br.WriteByte(1) // set length 678 | br.WriteByte(1) // element length 679 | br.WriteByte('v') // element data 680 | br.Flush() 681 | 682 | go mf() 683 | go df() 684 | 685 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 686 | ok(t, err) 687 | 688 | // No set data 689 | buffer.Reset() 690 | br.WriteByte(2) 691 | br.WriteByte(1) 692 | br.WriteByte('a') 693 | br.WriteByte(1) 694 | br.WriteByte(1) 695 | br.Flush() 696 | 697 | go mf() 698 | go df() 699 | 700 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 701 | equals(t, io.EOF, err) 702 | } 703 | 704 | func TestReadKeyValuePairSortedSetEncoding(t *testing.T) { 705 | var buffer bytes.Buffer 706 | 707 | br := bufio.NewWriter(&buffer) 708 | 709 | ctx := ParserContext{ 710 | SortedSetMetadataCh: make(chan SortedSetMetadata), 711 | SortedSetEntriesCh: make(chan SortedSetEntry), 712 | } 713 | p := &parser{ctx: ctx} 714 | 715 | mf := func() { 716 | l := <-ctx.SortedSetMetadataCh 717 | equals(t, "a", DataToString(l.Key)) 718 | equals(t, int64(1), l.Len) 719 | } 720 | 721 | df := func() { 722 | v := <-ctx.SortedSetEntriesCh 723 | equals(t, "v", DataToString(v.Value)) 724 | equals(t, 20.1, v.Score) 725 | } 726 | 727 | br.WriteByte(3) // sorted set encoding 728 | br.WriteByte(1) // key length 729 | br.WriteByte('a') // key data 730 | br.WriteByte(1) // sorted set length 731 | br.WriteByte(1) // entry val length 732 | br.WriteByte('v') // entry val data 733 | br.WriteByte(4) // entry score length 734 | br.WriteString("20.1") // entry score data 735 | br.Flush() 736 | 737 | go mf() 738 | go df() 739 | 740 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 741 | ok(t, err) 742 | 743 | // No sorted set data 744 | buffer.Reset() 745 | br.WriteByte(3) 746 | br.WriteByte(1) 747 | br.WriteByte('a') 748 | br.WriteByte(1) 749 | br.WriteByte(1) 750 | br.WriteByte('v') 751 | br.WriteByte(4) 752 | br.Flush() 753 | 754 | go mf() 755 | go df() 756 | 757 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 758 | equals(t, io.EOF, err) 759 | } 760 | 761 | func TestReadKeyValuePairHashMapEncoding(t *testing.T) { 762 | var buffer bytes.Buffer 763 | 764 | br := bufio.NewWriter(&buffer) 765 | 766 | ctx := ParserContext{ 767 | HashMetadataCh: make(chan HashMetadata), 768 | HashDataCh: make(chan HashEntry), 769 | } 770 | p := &parser{ctx: ctx} 771 | 772 | mf := func() { 773 | l := <-ctx.HashMetadataCh 774 | equals(t, "a", DataToString(l.Key)) 775 | equals(t, int64(1), l.Len) 776 | } 777 | 778 | df := func() { 779 | v := <-ctx.HashDataCh 780 | equals(t, "a", DataToString(v.Key)) 781 | equals(t, "b", DataToString(v.Value)) 782 | } 783 | 784 | br.WriteByte(4) // hash map encoding 785 | br.WriteByte(1) // key length 786 | br.WriteByte('a') // key data 787 | br.WriteByte(1) // hash map length 788 | br.WriteByte(1) // entry key length 789 | br.WriteByte('a') // entry key data 790 | br.WriteByte(1) // entry val length 791 | br.WriteByte('b') // entry val data 792 | br.Flush() 793 | 794 | go mf() 795 | go df() 796 | 797 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 798 | ok(t, err) 799 | 800 | // No hash map data 801 | buffer.Reset() 802 | br.WriteByte(4) 803 | br.WriteByte(1) 804 | br.WriteByte('a') 805 | br.WriteByte(1) 806 | br.WriteByte(1) 807 | br.WriteByte('a') 808 | br.Flush() 809 | 810 | go mf() 811 | go df() 812 | 813 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 814 | equals(t, io.EOF, err) 815 | } 816 | 817 | func TestReadKeyValuePairZipMapEncoding(t *testing.T) { 818 | var buffer bytes.Buffer 819 | 820 | br := bufio.NewWriter(&buffer) 821 | 822 | ctx := ParserContext{ 823 | HashMetadataCh: make(chan HashMetadata), 824 | HashDataCh: make(chan HashEntry), 825 | } 826 | p := &parser{ctx: ctx} 827 | 828 | mf := func() { 829 | l := <-ctx.HashMetadataCh 830 | equals(t, "a", DataToString(l.Key)) 831 | equals(t, int64(1), l.Len) 832 | } 833 | 834 | df := func() { 835 | v := <-ctx.HashDataCh 836 | equals(t, "a", DataToString(v.Key)) 837 | equals(t, "b", DataToString(v.Value)) 838 | } 839 | 840 | br.WriteByte(9) // hash map encoding 841 | br.WriteByte(1) // key length 842 | br.WriteByte('a') // key data 843 | br.WriteRune(7) // string length 844 | br.WriteByte(1) // hash map length 845 | br.WriteByte(1) // entry key length 846 | br.WriteByte('a') // entry key data 847 | br.WriteByte(1) // entry val length 848 | br.WriteRune(0) // free bytes 849 | br.WriteByte('b') // entry val data 850 | br.WriteByte(0xFF) // end 851 | br.Flush() 852 | 853 | go mf() 854 | go df() 855 | 856 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 857 | ok(t, err) 858 | 859 | // No hash map data 860 | buffer.Reset() 861 | br.WriteByte(9) // hash map encoding 862 | br.WriteByte(1) // key length 863 | br.WriteByte('a') // key data 864 | br.WriteByte(6) // string length 865 | br.WriteByte(1) // hash map length 866 | br.WriteByte(1) // entry key length 867 | br.WriteByte('a') // entry key data 868 | br.WriteByte(1) // entry val length 869 | br.WriteRune(0) // free bytes 870 | br.WriteByte('b') // entry val data 871 | br.Flush() 872 | 873 | go mf() 874 | go df() 875 | 876 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 877 | equals(t, io.EOF, err) 878 | } 879 | 880 | func TestReadKeyValuePairZipListEncoding(t *testing.T) { 881 | var buffer bytes.Buffer 882 | 883 | br := bufio.NewWriter(&buffer) 884 | 885 | ctx := ParserContext{ 886 | ListMetadataCh: make(chan ListMetadata), 887 | ListDataCh: make(chan interface{}), 888 | } 889 | p := &parser{ctx: ctx} 890 | 891 | mf := func() { 892 | l := <-ctx.ListMetadataCh 893 | equals(t, "a", DataToString(l.Key)) 894 | equals(t, int64(1), l.Len) 895 | } 896 | 897 | df := func() { 898 | v := <-ctx.ListDataCh 899 | equals(t, "a", DataToString(v)) 900 | } 901 | 902 | br.WriteByte(10) // zip list encoding 903 | br.WriteByte(1) 904 | br.WriteByte('a') 905 | br.WriteByte(13) 906 | binary.Write(br, binary.LittleEndian, int32(0)) 907 | binary.Write(br, binary.LittleEndian, int32(0)) 908 | binary.Write(br, binary.LittleEndian, int16(1)) 909 | br.WriteByte(0) 910 | br.WriteByte(1) 911 | br.WriteByte('a') 912 | br.Flush() 913 | 914 | go mf() 915 | go df() 916 | 917 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 918 | ok(t, err) 919 | 920 | // No zip list data 921 | buffer.Reset() 922 | br.WriteByte(10) // zip list encoding 923 | br.WriteByte(1) 924 | br.WriteByte('a') 925 | br.WriteByte(12) 926 | binary.Write(br, binary.LittleEndian, int32(0)) 927 | binary.Write(br, binary.LittleEndian, int32(0)) 928 | binary.Write(br, binary.LittleEndian, int16(1)) 929 | br.WriteByte(0) 930 | br.WriteByte(1) 931 | br.Flush() 932 | 933 | go mf() 934 | go df() 935 | 936 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 937 | equals(t, io.EOF, err) 938 | } 939 | 940 | func TestReadKeyValuePairIntSetEncoding(t *testing.T) { 941 | var buffer bytes.Buffer 942 | 943 | br := bufio.NewWriter(&buffer) 944 | 945 | ctx := ParserContext{ 946 | SetMetadataCh: make(chan SetMetadata), 947 | SetDataCh: make(chan interface{}), 948 | } 949 | p := &parser{ctx: ctx} 950 | 951 | mf := func() { 952 | l := <-ctx.SetMetadataCh 953 | equals(t, "a", DataToString(l.Key)) 954 | equals(t, int64(1), l.Len) 955 | } 956 | 957 | df := func() { 958 | v := <-ctx.SetDataCh 959 | equals(t, int16(1), v) 960 | } 961 | 962 | br.WriteByte(11) // intset encoding 963 | br.WriteByte(1) 964 | br.WriteByte('a') 965 | br.WriteByte(10) 966 | binary.Write(br, binary.LittleEndian, uint32(2)) 967 | binary.Write(br, binary.LittleEndian, uint32(1)) 968 | binary.Write(br, binary.LittleEndian, int16(1)) 969 | br.Flush() 970 | 971 | go mf() 972 | go df() 973 | 974 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 975 | ok(t, err) 976 | 977 | // No zip list data 978 | buffer.Reset() 979 | br.WriteByte(11) // zip list encoding 980 | br.WriteByte(1) 981 | br.WriteByte('a') 982 | br.WriteByte(8) 983 | binary.Write(br, binary.LittleEndian, uint32(2)) 984 | binary.Write(br, binary.LittleEndian, uint32(1)) 985 | br.Flush() 986 | 987 | go mf() 988 | go df() 989 | 990 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 991 | equals(t, io.EOF, err) 992 | } 993 | 994 | func TestReadKeyValuePairSortedSetInZipListEncoding(t *testing.T) { 995 | var buffer bytes.Buffer 996 | 997 | br := bufio.NewWriter(&buffer) 998 | 999 | ctx := ParserContext{ 1000 | SortedSetMetadataCh: make(chan SortedSetMetadata), 1001 | SortedSetEntriesCh: make(chan SortedSetEntry), 1002 | } 1003 | p := &parser{ctx: ctx} 1004 | 1005 | mf := func() { 1006 | l := <-ctx.SortedSetMetadataCh 1007 | equals(t, "a", DataToString(l.Key)) 1008 | equals(t, int64(1), l.Len) 1009 | } 1010 | 1011 | df := func() { 1012 | v := <-ctx.SortedSetEntriesCh 1013 | equals(t, "a", DataToString(v.Value)) 1014 | equals(t, 1.2, v.Score) 1015 | } 1016 | 1017 | br.WriteByte(12) // sorted set in ziplist encoding 1018 | br.WriteByte(1) 1019 | br.WriteByte('a') 1020 | br.WriteByte(18) 1021 | binary.Write(br, binary.LittleEndian, int32(0)) 1022 | binary.Write(br, binary.LittleEndian, int32(0)) 1023 | binary.Write(br, binary.LittleEndian, int16(2)) 1024 | br.WriteByte(0) 1025 | br.WriteByte(1) 1026 | br.WriteByte('a') 1027 | br.WriteByte(0) 1028 | br.WriteByte(3) 1029 | br.WriteString("1.2") 1030 | br.Flush() 1031 | 1032 | go mf() 1033 | go df() 1034 | 1035 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 1036 | ok(t, err) 1037 | 1038 | // No zip list data 1039 | buffer.Reset() 1040 | br.WriteByte(12) 1041 | br.WriteByte(1) 1042 | br.WriteByte('a') 1043 | br.WriteByte(15) 1044 | binary.Write(br, binary.LittleEndian, int32(0)) 1045 | binary.Write(br, binary.LittleEndian, int32(0)) 1046 | binary.Write(br, binary.LittleEndian, int16(2)) 1047 | br.WriteByte(0) 1048 | br.WriteByte(1) 1049 | br.WriteByte('a') 1050 | br.WriteByte(0) 1051 | br.WriteByte(3) 1052 | br.Flush() 1053 | 1054 | go mf() 1055 | go df() 1056 | 1057 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 1058 | equals(t, io.EOF, err) 1059 | } 1060 | 1061 | func TestReadKeyValuePairHashMapInZipListEncoding(t *testing.T) { 1062 | var buffer bytes.Buffer 1063 | 1064 | br := bufio.NewWriter(&buffer) 1065 | 1066 | ctx := ParserContext{ 1067 | HashMetadataCh: make(chan HashMetadata), 1068 | HashDataCh: make(chan HashEntry), 1069 | } 1070 | p := &parser{ctx: ctx} 1071 | 1072 | mf := func() { 1073 | l := <-ctx.HashMetadataCh 1074 | equals(t, "a", DataToString(l.Key)) 1075 | equals(t, int64(1), l.Len) 1076 | } 1077 | 1078 | df := func() { 1079 | v := <-ctx.HashDataCh 1080 | equals(t, "a", DataToString(v.Key)) 1081 | equals(t, "b", DataToString(v.Value)) 1082 | } 1083 | 1084 | br.WriteByte(13) // hashmap in ziplist encoding 1085 | br.WriteByte(1) 1086 | br.WriteByte('a') 1087 | br.WriteByte(16) 1088 | binary.Write(br, binary.LittleEndian, int32(0)) 1089 | binary.Write(br, binary.LittleEndian, int32(0)) 1090 | binary.Write(br, binary.LittleEndian, int16(2)) 1091 | br.WriteByte(0) 1092 | br.WriteByte(1) 1093 | br.WriteByte('a') 1094 | br.WriteByte(0) 1095 | br.WriteByte(1) 1096 | br.WriteByte('b') 1097 | br.Flush() 1098 | 1099 | go mf() 1100 | go df() 1101 | 1102 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 1103 | ok(t, err) 1104 | 1105 | // No zip list data 1106 | buffer.Reset() 1107 | br.WriteByte(13) // hashmap in ziplist encoding 1108 | br.WriteByte(1) 1109 | br.WriteByte('a') 1110 | br.WriteByte(15) 1111 | binary.Write(br, binary.LittleEndian, int32(0)) 1112 | binary.Write(br, binary.LittleEndian, int32(0)) 1113 | binary.Write(br, binary.LittleEndian, int16(2)) 1114 | br.WriteByte(0) 1115 | br.WriteByte(1) 1116 | br.WriteByte('a') 1117 | br.WriteByte(0) 1118 | br.WriteByte(1) 1119 | br.Flush() 1120 | 1121 | go mf() 1122 | go df() 1123 | 1124 | err = p.readKeyValuePair(bufio.NewReader(&buffer)) 1125 | equals(t, io.EOF, err) 1126 | } 1127 | 1128 | func TestReadKeyValuePairUnknownValueType(t *testing.T) { 1129 | var buffer bytes.Buffer 1130 | 1131 | br := bufio.NewWriter(&buffer) 1132 | 1133 | br.WriteByte(0xF0) 1134 | br.WriteByte(1) 1135 | br.WriteByte('a') 1136 | br.Flush() 1137 | 1138 | p := &parser{} 1139 | err := p.readKeyValuePair(bufio.NewReader(&buffer)) 1140 | equals(t, ErrUnknownValueType, err) 1141 | } 1142 | 1143 | func TestParse(t *testing.T) { 1144 | var buffer bytes.Buffer 1145 | 1146 | ctx := ParserContext{ 1147 | DbCh: make(chan int), 1148 | StringObjectCh: make(chan StringObject), 1149 | endOfFileCh: make(chan struct{}), 1150 | } 1151 | p := &parser{ctx: ctx} 1152 | 1153 | br := bufio.NewWriter(&buffer) 1154 | br.WriteString("REDIS") // magic string 1155 | br.WriteString("0004") // RDB version TODO use version >= 5 and handle checksum 1156 | br.WriteByte(0xFE) // next database byte 1157 | br.WriteByte(0) // database number 1158 | br.WriteByte(0) // string 1159 | br.WriteByte(1) // key len 1160 | br.WriteByte('a') // key data 1161 | br.WriteByte(6) // string len 1162 | br.WriteString("foobar") // string data 1163 | br.WriteByte(0xFF) // end of file 1164 | br.Flush() 1165 | 1166 | go mustParse(t, p, ctx, bufio.NewReader(&buffer)) 1167 | 1168 | stop := false 1169 | for !stop { 1170 | select { 1171 | case v, ok := <-ctx.StringObjectCh: 1172 | if !ok { 1173 | ctx.StringObjectCh = nil 1174 | break 1175 | } 1176 | equals(t, "a", DataToString(v.Key.Key)) 1177 | equals(t, "foobar", DataToString(v.Value)) 1178 | case v, ok := <-ctx.DbCh: 1179 | if !ok { 1180 | ctx.DbCh = nil 1181 | break 1182 | } 1183 | equals(t, int(0), v) 1184 | } 1185 | 1186 | if ctx.Invalid() { 1187 | break 1188 | } 1189 | } 1190 | } 1191 | 1192 | func TestParseAllTypes(t *testing.T) { 1193 | var buffer bytes.Buffer 1194 | 1195 | ctx := ParserContext{ 1196 | DbCh: make(chan int), 1197 | StringObjectCh: make(chan StringObject), 1198 | ListMetadataCh: make(chan ListMetadata), 1199 | ListDataCh: make(chan interface{}), 1200 | SetMetadataCh: make(chan SetMetadata), 1201 | SetDataCh: make(chan interface{}), 1202 | HashMetadataCh: make(chan HashMetadata), 1203 | HashDataCh: make(chan HashEntry), 1204 | SortedSetMetadataCh: make(chan SortedSetMetadata), 1205 | SortedSetEntriesCh: make(chan SortedSetEntry), 1206 | endOfFileCh: make(chan struct{}), 1207 | } 1208 | p := &parser{ctx: ctx} 1209 | 1210 | br := bufio.NewWriter(&buffer) 1211 | br.WriteString("REDIS") // magic string 1212 | br.WriteString("0004") // RDB version TODO use version >= 5 and handling checksum 1213 | br.WriteByte(0xFE) // next database byte 1214 | br.WriteByte(0) // database number 1215 | 1216 | br.WriteByte(0) // string 1217 | br.Write([]byte{1, 'a'}) // key 1218 | br.WriteByte(6) // string len 1219 | br.WriteString("foobar") // string data 1220 | 1221 | br.WriteByte(1) // list 1222 | br.Write([]byte{1, 'b'}) // key 1223 | br.WriteByte(1) // list len 1224 | br.Write([]byte{1, 'Z'}) // list element 1225 | 1226 | br.WriteByte(2) // set 1227 | br.Write([]byte{1, 'c'}) // key 1228 | br.WriteByte(1) // set len 1229 | br.Write([]byte{1, 'Z'}) // set element 1230 | 1231 | br.WriteByte(3) // sorted set 1232 | br.Write([]byte{1, 'd'}) // key 1233 | br.WriteByte(1) // sorted set len 1234 | br.Write([]byte{1, 'Z'}) // entry member 1235 | br.Write([]byte{3, '0', '.', '1'}) // entry score 1236 | 1237 | br.WriteByte(4) // hash 1238 | br.Write([]byte{1, 'e'}) // key 1239 | br.WriteByte(1) // hash len 1240 | br.Write([]byte{1, 'Z'}) // entry key 1241 | br.Write([]byte{2, 'Z', '1'}) // entry value 1242 | br.WriteByte(0xFF) // end of file 1243 | br.Flush() 1244 | 1245 | go mustParse(t, p, ctx, bufio.NewReader(&buffer)) 1246 | 1247 | stop := false 1248 | for !stop { 1249 | select { 1250 | case v, ok := <-ctx.DbCh: 1251 | if !ok { 1252 | ctx.DbCh = nil 1253 | break 1254 | } 1255 | equals(t, int(0), v) 1256 | case v, ok := <-ctx.StringObjectCh: 1257 | if !ok { 1258 | ctx.StringObjectCh = nil 1259 | break 1260 | } 1261 | equals(t, "a", DataToString(v.Key.Key)) 1262 | equals(t, "foobar", DataToString(v.Value)) 1263 | case v, ok := <-ctx.ListMetadataCh: 1264 | if !ok { 1265 | ctx.ListMetadataCh = nil 1266 | break 1267 | } 1268 | equals(t, int64(1), v.Len) 1269 | equals(t, "b", DataToString(v.Key.Key)) 1270 | case v, ok := <-ctx.ListDataCh: 1271 | if !ok { 1272 | ctx.ListDataCh = nil 1273 | break 1274 | } 1275 | equals(t, "Z", DataToString(v)) 1276 | case v, ok := <-ctx.SetMetadataCh: 1277 | if !ok { 1278 | ctx.SetMetadataCh = nil 1279 | break 1280 | } 1281 | equals(t, int64(1), v.Len) 1282 | equals(t, "c", DataToString(v.Key.Key)) 1283 | case v, ok := <-ctx.SetDataCh: 1284 | if !ok { 1285 | ctx.SetDataCh = nil 1286 | break 1287 | } 1288 | equals(t, "Z", DataToString(v)) 1289 | case v, ok := <-ctx.SortedSetMetadataCh: 1290 | if !ok { 1291 | ctx.SortedSetMetadataCh = nil 1292 | break 1293 | } 1294 | equals(t, int64(1), v.Len) 1295 | equals(t, "d", DataToString(v.Key.Key)) 1296 | case v, ok := <-ctx.SortedSetEntriesCh: 1297 | if !ok { 1298 | ctx.SortedSetEntriesCh = nil 1299 | break 1300 | } 1301 | equals(t, "Z", DataToString(v.Value)) 1302 | equals(t, 0.1, v.Score) 1303 | case v, ok := <-ctx.HashMetadataCh: 1304 | if !ok { 1305 | ctx.HashMetadataCh = nil 1306 | break 1307 | } 1308 | equals(t, int64(1), v.Len) 1309 | equals(t, "e", DataToString(v.Key.Key)) 1310 | case v, ok := <-ctx.HashDataCh: 1311 | if !ok { 1312 | ctx.HashDataCh = nil 1313 | break 1314 | } 1315 | equals(t, "Z", DataToString(v.Key)) 1316 | equals(t, "Z1", DataToString(v.Value)) 1317 | } 1318 | 1319 | if ctx.Invalid() { 1320 | break 1321 | } 1322 | } 1323 | } 1324 | 1325 | func TestParseNoMagicString(t *testing.T) { 1326 | var buffer bytes.Buffer 1327 | 1328 | p := NewParser(ParserContext{}) 1329 | 1330 | err := p.Parse(bufio.NewReader(&buffer)) 1331 | equals(t, io.EOF, err) 1332 | } 1333 | 1334 | func TestParseNoVersionNumber(t *testing.T) { 1335 | var buffer bytes.Buffer 1336 | 1337 | p := NewParser(ParserContext{}) 1338 | 1339 | br := bufio.NewWriter(&buffer) 1340 | 1341 | br.WriteString("REDIS") 1342 | br.Flush() 1343 | 1344 | err := p.Parse(bufio.NewReader(&buffer)) 1345 | equals(t, io.EOF, err) 1346 | } 1347 | 1348 | func TestParseNoDatabaseNumber(t *testing.T) { 1349 | var buffer bytes.Buffer 1350 | 1351 | p := NewParser(ParserContext{}) 1352 | 1353 | br := bufio.NewWriter(&buffer) 1354 | 1355 | br.WriteString("REDIS") 1356 | br.WriteString("0006") 1357 | br.Flush() 1358 | 1359 | err := p.Parse(bufio.NewReader(&buffer)) 1360 | equals(t, io.EOF, err) 1361 | } 1362 | 1363 | func TestParseNoKeyValuePair(t *testing.T) { 1364 | var buffer bytes.Buffer 1365 | 1366 | ctx := ParserContext{DbCh: make(chan int)} 1367 | p := &parser{ctx: ctx} 1368 | 1369 | go func() { 1370 | v := <-ctx.DbCh 1371 | equals(t, int(0), v) 1372 | }() 1373 | 1374 | br := bufio.NewWriter(&buffer) 1375 | 1376 | br.WriteString("REDIS") 1377 | br.WriteString("0006") 1378 | br.WriteByte(0xFE) 1379 | br.WriteByte(0) 1380 | br.Flush() 1381 | 1382 | err := p.Parse(bufio.NewReader(&buffer)) 1383 | equals(t, io.EOF, err) 1384 | } 1385 | -------------------------------------------------------------------------------- /set.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | ) 10 | 11 | // Represents the metadata of a set, which is the key and the set length 12 | type SetMetadata struct { 13 | Key KeyObject 14 | Len int64 15 | } 16 | 17 | // Returns a visualization of the set metadata 18 | func (m SetMetadata) String() string { 19 | return fmt.Sprintf("SetMetadata{Key: %s, Len: %d}", DataToString(m.Key), m.Len) 20 | } 21 | 22 | func (p *parser) readSet(key KeyObject, r io.Reader) error { 23 | l, e, err := p.readLen(r) 24 | if err != nil { 25 | return err 26 | } 27 | if e { 28 | return ErrUnexpectedEncodedLength 29 | } 30 | 31 | if p.ctx.SetMetadataCh != nil { 32 | p.ctx.SetMetadataCh <- SetMetadata{Key: key, Len: l} 33 | } 34 | 35 | for i := int64(0); i < l; i++ { 36 | value, err := p.readString(r) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | if p.ctx.SetDataCh != nil { 42 | p.ctx.SetDataCh <- value 43 | } 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func (p *parser) readIntSet(key KeyObject, r io.Reader) error { 50 | data, err := p.readString(r) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | dr := bufio.NewReader(bytes.NewReader(data.([]byte))) 56 | 57 | // read encoding (2, 4, 8 bytes per int) 58 | var encoding uint32 59 | if err := binary.Read(dr, binary.LittleEndian, &encoding); err != nil { 60 | return err 61 | } 62 | 63 | // read length of contents 64 | var length uint32 65 | if err := binary.Read(dr, binary.LittleEndian, &length); err != nil { 66 | return err 67 | } 68 | 69 | if p.ctx.SetMetadataCh != nil { 70 | p.ctx.SetMetadataCh <- SetMetadata{Key: key, Len: int64(length)} 71 | } 72 | 73 | // decode contents 74 | for i := uint32(0); i < length; i++ { 75 | var e interface{} 76 | switch encoding { 77 | case 2: 78 | var i int16 79 | if err := binary.Read(dr, binary.LittleEndian, &i); err != nil { 80 | return err 81 | } 82 | e = i 83 | case 4: 84 | var i int32 85 | if err := binary.Read(dr, binary.LittleEndian, &i); err != nil { 86 | return err 87 | } 88 | e = i 89 | case 8: 90 | var i int64 91 | if err := binary.Read(dr, binary.LittleEndian, &i); err != nil { 92 | return err 93 | } 94 | e = i 95 | } 96 | 97 | if p.ctx.SetDataCh != nil { 98 | p.ctx.SetDataCh <- e 99 | } 100 | } 101 | 102 | return nil 103 | } 104 | -------------------------------------------------------------------------------- /set_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | func TestSetMetadataString(t *testing.T) { 11 | md := SetMetadata{Key: KeyObject{Key: "foobar"}, Len: 10} 12 | equals(t, "SetMetadata{Key: foobar, Len: 10}", md.String()) 13 | } 14 | 15 | func TestReadSet(t *testing.T) { 16 | var buffer bytes.Buffer 17 | 18 | br := bufio.NewWriter(&buffer) 19 | 20 | br.WriteByte(1) 21 | br.WriteByte(1) 22 | br.WriteByte('a') 23 | br.Flush() 24 | 25 | ctx := ParserContext{ 26 | SetMetadataCh: make(chan SetMetadata), 27 | SetDataCh: make(chan interface{}), 28 | } 29 | p := &parser{ctx: ctx} 30 | 31 | go readAndNotify(t, &buffer, "set", p.readSet) 32 | 33 | stop := false 34 | for !stop { 35 | select { 36 | case md := <-ctx.SetMetadataCh: 37 | equals(t, "set", DataToString(md.Key)) 38 | equals(t, int64(1), md.Len) 39 | case d := <-ctx.SetDataCh: 40 | equals(t, "a", DataToString(d)) 41 | case <-end: 42 | stop = true 43 | } 44 | } 45 | } 46 | 47 | func TestReadSetNoData(t *testing.T) { 48 | var buffer bytes.Buffer 49 | 50 | p := &parser{} 51 | err := p.readSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 52 | equals(t, io.EOF, err) 53 | } 54 | 55 | func TestReadSetEncodedLen(t *testing.T) { 56 | var buffer bytes.Buffer 57 | 58 | br := bufio.NewWriter(&buffer) 59 | br.WriteByte(0xC0) 60 | br.Flush() 61 | 62 | p := &parser{} 63 | err := p.readSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 64 | equals(t, ErrUnexpectedEncodedLength, err) 65 | } 66 | 67 | func TestReadSetNoEntry(t *testing.T) { 68 | var buffer bytes.Buffer 69 | 70 | br := bufio.NewWriter(&buffer) 71 | br.WriteByte(1) 72 | br.Flush() 73 | 74 | ctx := ParserContext{ 75 | SetMetadataCh: make(chan SetMetadata, 1), 76 | SetDataCh: make(chan interface{}, 1), 77 | } 78 | p := &parser{ctx: ctx} 79 | 80 | go func() { 81 | md := <-p.ctx.SetMetadataCh 82 | equals(t, "set", DataToString(md.Key)) 83 | equals(t, int64(1), md.Len) 84 | }() 85 | 86 | err := p.readSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 87 | equals(t, io.EOF, err) 88 | } 89 | 90 | func TestReadInt16Set(t *testing.T) { 91 | var buffer bytes.Buffer 92 | 93 | br := bufio.NewWriter(&buffer) 94 | 95 | br.WriteByte(10) // string length 96 | br.Write([]byte{2, 0, 0, 0}) // encoding 97 | br.Write([]byte{1, 0, 0, 0}) // len 98 | br.Write([]byte{1, 0}) // value 99 | br.Flush() 100 | 101 | ctx := ParserContext{ 102 | SetMetadataCh: make(chan SetMetadata), 103 | SetDataCh: make(chan interface{}), 104 | } 105 | p := parser{ctx: ctx} 106 | 107 | go readAndNotify(t, &buffer, "set", p.readIntSet) 108 | 109 | stop := false 110 | for !stop { 111 | select { 112 | case md := <-ctx.SetMetadataCh: 113 | equals(t, "set", DataToString(md.Key)) 114 | equals(t, int64(1), md.Len) 115 | case d := <-ctx.SetDataCh: 116 | equals(t, int16(1), d) 117 | case <-end: 118 | stop = true 119 | } 120 | } 121 | } 122 | 123 | func TestReadInt32Set(t *testing.T) { 124 | var buffer bytes.Buffer 125 | 126 | br := bufio.NewWriter(&buffer) 127 | 128 | br.WriteByte(12) // string length 129 | br.Write([]byte{4, 0, 0, 0}) // encoding 130 | br.Write([]byte{1, 0, 0, 0}) // len 131 | br.Write([]byte{1, 0, 0, 0}) // value 132 | br.Flush() 133 | 134 | ctx := ParserContext{ 135 | SetMetadataCh: make(chan SetMetadata), 136 | SetDataCh: make(chan interface{}), 137 | } 138 | p := &parser{ctx: ctx} 139 | 140 | go readAndNotify(t, &buffer, "set", p.readIntSet) 141 | 142 | stop := false 143 | for !stop { 144 | select { 145 | case md := <-ctx.SetMetadataCh: 146 | equals(t, "set", DataToString(md.Key)) 147 | equals(t, int64(1), md.Len) 148 | case d := <-ctx.SetDataCh: 149 | equals(t, int32(1), d) 150 | case <-end: 151 | stop = true 152 | } 153 | } 154 | } 155 | 156 | func TestReadInt64Set(t *testing.T) { 157 | var buffer bytes.Buffer 158 | 159 | br := bufio.NewWriter(&buffer) 160 | 161 | br.WriteByte(16) // string length 162 | br.Write([]byte{8, 0, 0, 0}) // encoding 163 | br.Write([]byte{1, 0, 0, 0}) // len 164 | br.Write([]byte{1, 0, 0, 0, 0, 0, 0, 0}) // value 165 | br.Flush() 166 | 167 | ctx := ParserContext{ 168 | SetMetadataCh: make(chan SetMetadata), 169 | SetDataCh: make(chan interface{}), 170 | } 171 | p := &parser{ctx: ctx} 172 | 173 | go readAndNotify(t, &buffer, "set", p.readIntSet) 174 | 175 | stop := false 176 | for !stop { 177 | select { 178 | case md := <-ctx.SetMetadataCh: 179 | equals(t, "set", DataToString(md.Key)) 180 | equals(t, int64(1), md.Len) 181 | case d := <-ctx.SetDataCh: 182 | equals(t, int64(1), d) 183 | case <-end: 184 | stop = true 185 | } 186 | } 187 | } 188 | 189 | func TestReadIntSetNoData(t *testing.T) { 190 | var buffer bytes.Buffer 191 | 192 | p := &parser{} 193 | err := p.readIntSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 194 | equals(t, io.EOF, err) 195 | } 196 | 197 | func TestReadIntSetNoEncoding(t *testing.T) { 198 | var buffer bytes.Buffer 199 | 200 | br := bufio.NewWriter(&buffer) 201 | 202 | br.WriteByte(0) 203 | br.Flush() 204 | 205 | p := &parser{} 206 | err := p.readIntSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 207 | equals(t, io.EOF, err) 208 | } 209 | 210 | func TestReadIntSetNoLength(t *testing.T) { 211 | var buffer bytes.Buffer 212 | 213 | br := bufio.NewWriter(&buffer) 214 | 215 | br.WriteByte(4) 216 | br.Write([]byte{0, 0, 0, 0}) 217 | br.Flush() 218 | 219 | p := &parser{} 220 | err := p.readIntSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 221 | equals(t, io.EOF, err) 222 | } 223 | 224 | func TestReadIntSetNoInt16Value(t *testing.T) { 225 | var buffer bytes.Buffer 226 | 227 | br := bufio.NewWriter(&buffer) 228 | 229 | br.WriteByte(8) 230 | br.Write([]byte{2, 0, 0, 0}) 231 | br.Write([]byte{1, 0, 0, 0}) 232 | br.Flush() 233 | 234 | ctx := ParserContext{ 235 | SetMetadataCh: make(chan SetMetadata), 236 | SetDataCh: make(chan interface{}), 237 | } 238 | p := &parser{ctx: ctx} 239 | 240 | go func() { 241 | md := <-ctx.SetMetadataCh 242 | equals(t, "set", DataToString(md.Key)) 243 | equals(t, int64(1), md.Len) 244 | }() 245 | 246 | err := p.readIntSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 247 | equals(t, io.EOF, err) 248 | } 249 | 250 | func TestReadIntSetNoInt32Value(t *testing.T) { 251 | var buffer bytes.Buffer 252 | 253 | br := bufio.NewWriter(&buffer) 254 | 255 | br.WriteByte(8) 256 | br.Write([]byte{4, 0, 0, 0}) 257 | br.Write([]byte{1, 0, 0, 0}) 258 | br.Flush() 259 | 260 | ctx := ParserContext{ 261 | SetMetadataCh: make(chan SetMetadata), 262 | SetDataCh: make(chan interface{}), 263 | } 264 | p := &parser{ctx: ctx} 265 | 266 | go func() { 267 | md := <-ctx.SetMetadataCh 268 | equals(t, "set", DataToString(md.Key)) 269 | equals(t, int64(1), md.Len) 270 | }() 271 | 272 | err := p.readIntSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 273 | equals(t, io.EOF, err) 274 | } 275 | 276 | func TestReadIntSetNoInt64Value(t *testing.T) { 277 | var buffer bytes.Buffer 278 | 279 | br := bufio.NewWriter(&buffer) 280 | 281 | br.WriteByte(8) 282 | br.Write([]byte{8, 0, 0, 0}) 283 | br.Write([]byte{1, 0, 0, 0}) 284 | br.Flush() 285 | 286 | ctx := ParserContext{ 287 | SetMetadataCh: make(chan SetMetadata), 288 | SetDataCh: make(chan interface{}), 289 | } 290 | p := &parser{ctx: ctx} 291 | 292 | go func() { 293 | md := <-ctx.SetMetadataCh 294 | equals(t, "set", DataToString(md.Key)) 295 | equals(t, int64(1), md.Len) 296 | }() 297 | 298 | err := p.readIntSet(KeyObject{Key: []byte("set")}, bufio.NewReader(&buffer)) 299 | equals(t, io.EOF, err) 300 | } 301 | -------------------------------------------------------------------------------- /sortedset.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "strconv" 9 | ) 10 | 11 | // Represents the metadata of a sorted set, which is the key and the sorted set length 12 | type SortedSetMetadata struct { 13 | Key KeyObject 14 | Len int64 15 | } 16 | 17 | // Returns a visualization of the sorted set metadata 18 | func (m SortedSetMetadata) String() string { 19 | return fmt.Sprintf("SortedSetMetadata{Key: %s, Len: %d}", DataToString(m.Key), m.Len) 20 | } 21 | 22 | // Represents an entry in a sorted set. 23 | type SortedSetEntry struct { 24 | Value interface{} 25 | Score float64 26 | } 27 | 28 | // Returns a visualization of a sorted set entry 29 | func (e SortedSetEntry) String() string { 30 | return fmt.Sprintf("SortedSetEntry{Value: %s, Score: %0.4f}", DataToString(e.Value), e.Score) 31 | } 32 | 33 | func (p *parser) readSortedSet(key KeyObject, r io.Reader) error { 34 | l, e, err := p.readLen(r) 35 | if err != nil { 36 | return err 37 | } 38 | if e { 39 | return ErrUnexpectedEncodedLength 40 | } 41 | 42 | p.ctx.SortedSetMetadataCh <- SortedSetMetadata{Key: key, Len: l} 43 | 44 | for i := int64(0); i < l; i++ { 45 | value, err := p.readString(r) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | score, err := p.readDoubleValue(r) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | e := SortedSetEntry{Value: value, Score: score} 56 | p.ctx.SortedSetEntriesCh <- e 57 | } 58 | 59 | return nil 60 | } 61 | 62 | func (p *parser) readSortedSetInZipList(key KeyObject, r io.Reader) error { 63 | data, err := p.readString(r) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | var el interface{} = nil 69 | onLenCallback := func(length int64) error { 70 | p.ctx.SortedSetMetadataCh <- SortedSetMetadata{Key: key, Len: length / 2} 71 | return nil 72 | } 73 | onElementCallback := func(e interface{}) error { 74 | if el == nil { 75 | el = e 76 | } else { 77 | var score float64 78 | switch v := e.(type) { 79 | case []byte: 80 | score, err = strconv.ParseFloat(string(v), 64) 81 | if err != nil { 82 | return err 83 | } 84 | case int8: 85 | score = float64(v) 86 | case int: 87 | score = float64(v) 88 | case int16: 89 | score = float64(v) 90 | case int32: 91 | score = float64(v) 92 | case int64: 93 | score = float64(v) 94 | } 95 | 96 | p.ctx.SortedSetEntriesCh <- SortedSetEntry{Value: el, Score: score} 97 | el = nil 98 | } 99 | 100 | return nil 101 | } 102 | dr := bufio.NewReader(bytes.NewReader(data.([]byte))) 103 | 104 | if err := p.readZipList(dr, onLenCallback, onElementCallback); err != nil { 105 | return err 106 | } 107 | 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /sortedset_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | func TestSortedSetMetadataString(t *testing.T) { 11 | md := SortedSetMetadata{Key: KeyObject{Key: "foobar"}, Len: 10} 12 | equals(t, "SortedSetMetadata{Key: foobar, Len: 10}", md.String()) 13 | } 14 | 15 | func TestReadSortedSet(t *testing.T) { 16 | var buffer bytes.Buffer 17 | br := bufio.NewWriter(&buffer) 18 | 19 | br.WriteByte(2) // Sorted set len 20 | br.WriteByte(3) // Entry key len 21 | br.WriteString("bar") 22 | br.WriteByte(4) // Entry score length 23 | br.WriteString("20.1") 24 | br.WriteByte(6) // Entry key len 25 | br.WriteString("foobar") 26 | br.WriteByte(2) // Entry score length 27 | br.WriteString("62") 28 | br.Flush() 29 | 30 | ctx := ParserContext{ 31 | SortedSetMetadataCh: make(chan SortedSetMetadata), 32 | SortedSetEntriesCh: make(chan SortedSetEntry), 33 | } 34 | p := &parser{ctx: ctx} 35 | 36 | go readAndNotify(t, &buffer, "zset", p.readSortedSet) 37 | 38 | stop := false 39 | i := 0 40 | for !stop { 41 | select { 42 | case md := <-ctx.SortedSetMetadataCh: 43 | equals(t, "zset", DataToString(md.Key)) 44 | equals(t, int64(2), md.Len) 45 | case d := <-ctx.SortedSetEntriesCh: 46 | v := DataToString(d.Value) 47 | switch i { 48 | case 0: 49 | equals(t, "bar", v) 50 | equals(t, 20.1, d.Score) 51 | equals(t, "SortedSetEntry{Value: bar, Score: 20.1000}", d.String()) 52 | case 1: 53 | equals(t, "foobar", v) 54 | equals(t, 62.0, d.Score) 55 | equals(t, "SortedSetEntry{Value: foobar, Score: 62.0000}", d.String()) 56 | } 57 | i++ 58 | case <-end: 59 | stop = true 60 | } 61 | } 62 | } 63 | 64 | func TestReadSortedSetNoData(t *testing.T) { 65 | var buffer bytes.Buffer 66 | 67 | p := &parser{} 68 | err := p.readSortedSet(KeyObject{Key: []byte("zset")}, bufio.NewReader(&buffer)) 69 | equals(t, io.EOF, err) 70 | } 71 | 72 | func TestReadSortedSetEncodedLen(t *testing.T) { 73 | var buffer bytes.Buffer 74 | 75 | br := bufio.NewWriter(&buffer) 76 | br.WriteByte(0xC0) 77 | br.Flush() 78 | 79 | p := &parser{} 80 | err := p.readSortedSet(KeyObject{Key: []byte("zset")}, bufio.NewReader(&buffer)) 81 | equals(t, ErrUnexpectedEncodedLength, err) 82 | } 83 | 84 | func TestReadSortedSetNoEntryKey(t *testing.T) { 85 | var buffer bytes.Buffer 86 | 87 | br := bufio.NewWriter(&buffer) 88 | br.WriteByte(1) 89 | br.Flush() 90 | 91 | ctx := ParserContext{ 92 | SortedSetMetadataCh: make(chan SortedSetMetadata), 93 | SortedSetEntriesCh: make(chan SortedSetEntry), 94 | } 95 | p := &parser{ctx: ctx} 96 | 97 | go func() { 98 | md := <-ctx.SortedSetMetadataCh 99 | equals(t, "zset", DataToString(md.Key)) 100 | equals(t, int64(1), md.Len) 101 | }() 102 | 103 | err := p.readSortedSet(KeyObject{Key: []byte("zset")}, bufio.NewReader(&buffer)) 104 | equals(t, io.EOF, err) 105 | } 106 | 107 | func TestReadSortedSetNoEntryScore(t *testing.T) { 108 | var buffer bytes.Buffer 109 | 110 | br := bufio.NewWriter(&buffer) 111 | br.WriteByte(1) 112 | br.WriteByte(1) 113 | br.WriteString("a") 114 | br.Flush() 115 | 116 | ctx := ParserContext{ 117 | SortedSetMetadataCh: make(chan SortedSetMetadata), 118 | SortedSetEntriesCh: make(chan SortedSetEntry), 119 | } 120 | p := &parser{ctx: ctx} 121 | 122 | go func() { 123 | md := <-ctx.SortedSetMetadataCh 124 | equals(t, "zset", DataToString(md.Key)) 125 | equals(t, int64(1), md.Len) 126 | }() 127 | 128 | err := p.readSortedSet(KeyObject{Key: []byte("zset")}, bufio.NewReader(&buffer)) 129 | equals(t, io.EOF, err) 130 | } 131 | 132 | func TestReadSortedSetInZipList(t *testing.T) { 133 | var buffer bytes.Buffer 134 | br := bufio.NewWriter(&buffer) 135 | 136 | br.WriteByte(24) // String length 137 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 138 | br.Write([]byte{0, 0, 0, 0}) // zlTail 139 | br.Write([]byte{2, 0}) // zlLen 140 | br.WriteByte(0) // len prev entry 141 | br.WriteByte(6) // Special flag 142 | br.WriteString("foobar") 143 | br.WriteByte(0) // len prev entry 144 | br.WriteByte(4) // special flag 145 | br.WriteString("43.2") 146 | br.Flush() 147 | 148 | ctx := ParserContext{ 149 | SortedSetMetadataCh: make(chan SortedSetMetadata), 150 | SortedSetEntriesCh: make(chan SortedSetEntry), 151 | } 152 | p := &parser{ctx: ctx} 153 | 154 | go readAndNotify(t, &buffer, "zset", p.readSortedSetInZipList) 155 | 156 | stop := false 157 | for !stop { 158 | select { 159 | case md := <-ctx.SortedSetMetadataCh: 160 | equals(t, "zset", DataToString(md.Key)) 161 | equals(t, int64(1), md.Len) 162 | case d := <-ctx.SortedSetEntriesCh: 163 | v := DataToString(d.Value) 164 | equals(t, "foobar", v) 165 | equals(t, 43.2, d.Score) 166 | case <-end: 167 | stop = true 168 | } 169 | } 170 | } 171 | 172 | func TestReadSortedSetInZipListIntScore(t *testing.T) { 173 | var buffer bytes.Buffer 174 | br := bufio.NewWriter(&buffer) 175 | 176 | writeVal := func() { 177 | br.WriteByte(0) 178 | br.WriteByte(2) 179 | br.WriteString("fo") 180 | } 181 | 182 | br.WriteByte(64) 183 | br.WriteByte(64) 184 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 185 | br.Write([]byte{0, 0, 0, 0}) // zlTail 186 | br.Write([]byte{12, 0}) // zlLen 187 | writeVal() 188 | br.WriteByte(0) 189 | br.WriteByte(242) 190 | writeVal() 191 | br.WriteByte(0) 192 | br.WriteByte(0xFE) 193 | br.WriteByte(10) 194 | writeVal() 195 | br.WriteByte(0) 196 | br.WriteByte(0xF0) 197 | br.Write([]byte{1, 1, 1}) 198 | writeVal() 199 | br.WriteByte(0) 200 | br.WriteByte(0xE0) 201 | br.Write([]byte{1, 1, 0, 0, 0, 0, 0, 0}) 202 | writeVal() 203 | br.WriteByte(0) 204 | br.WriteByte(0xD0) 205 | br.Write([]byte{1, 1, 0, 0}) 206 | writeVal() 207 | br.WriteByte(0) 208 | br.WriteByte(0xC0) 209 | br.Write([]byte{1, 1}) 210 | br.Flush() 211 | 212 | ctx := ParserContext{ 213 | SortedSetMetadataCh: make(chan SortedSetMetadata), 214 | SortedSetEntriesCh: make(chan SortedSetEntry), 215 | } 216 | p := &parser{ctx: ctx} 217 | 218 | go readAndNotify(t, &buffer, "zset", p.readSortedSetInZipList) 219 | 220 | stop := false 221 | i := 0 222 | for !stop { 223 | select { 224 | case md := <-ctx.SortedSetMetadataCh: 225 | equals(t, "zset", DataToString(md.Key)) 226 | equals(t, int64(6), md.Len) 227 | case d := <-ctx.SortedSetEntriesCh: 228 | v := DataToString(d.Value) 229 | switch i { 230 | case 0: 231 | equals(t, "fo", v) 232 | equals(t, float64(1), d.Score) 233 | case 1: 234 | equals(t, "fo", v) 235 | equals(t, float64(10), d.Score) 236 | case 2: 237 | equals(t, "fo", v) 238 | equals(t, float64(65793), d.Score) 239 | case 3: 240 | equals(t, "fo", v) 241 | equals(t, float64(257), d.Score) 242 | case 4: 243 | equals(t, "fo", v) 244 | equals(t, float64(257), d.Score) 245 | case 5: 246 | equals(t, "fo", v) 247 | equals(t, float64(257), d.Score) 248 | } 249 | i++ 250 | case <-end: 251 | stop = true 252 | } 253 | } 254 | } 255 | 256 | func TestReadSortedSetInZipListNoData(t *testing.T) { 257 | var buffer bytes.Buffer 258 | 259 | p := &parser{} 260 | err := p.readSortedSetInZipList(KeyObject{Key: []byte("zset")}, bufio.NewReader(&buffer)) 261 | equals(t, io.EOF, err) 262 | } 263 | 264 | func TestReadSortedSetInZipListWrongScore(t *testing.T) { 265 | var buffer bytes.Buffer 266 | 267 | br := bufio.NewWriter(&buffer) 268 | br.WriteByte(26) // String length 269 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 270 | br.Write([]byte{0, 0, 0, 0}) // zlTail 271 | br.Write([]byte{2, 0}) // zlLen 272 | br.WriteByte(0) // len prev entry 273 | br.WriteByte(6) // Special flag 274 | br.WriteString("foobar") 275 | br.WriteByte(0) // len prev entry 276 | br.WriteByte(6) // special flag 277 | br.WriteString("foobar") 278 | br.Flush() 279 | 280 | ctx := ParserContext{ 281 | SortedSetMetadataCh: make(chan SortedSetMetadata), 282 | SortedSetEntriesCh: make(chan SortedSetEntry), 283 | } 284 | p := &parser{ctx: ctx} 285 | 286 | go func() { 287 | md := <-ctx.SortedSetMetadataCh 288 | equals(t, "zset", DataToString(md.Key)) 289 | equals(t, int64(1), md.Len) 290 | }() 291 | 292 | err := p.readSortedSetInZipList(KeyObject{Key: []byte("zset")}, bufio.NewReader(&buffer)) 293 | equals(t, "strconv.ParseFloat: parsing \"foobar\": invalid syntax", err.Error()) 294 | } 295 | -------------------------------------------------------------------------------- /stringobject.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import "fmt" 4 | 5 | // Represents a Redis string (which you get/set with SET, GET, MSET, MGET, etc). 6 | type StringObject struct { 7 | Key KeyObject 8 | Value interface{} 9 | } 10 | 11 | // Returns a visualization of the string. 12 | func (s StringObject) String() string { 13 | return fmt.Sprintf("StringObject{Key: %s, Value: '%s'}", DataToString(s.Key), DataToString(s.Value)) 14 | } 15 | -------------------------------------------------------------------------------- /stringobject_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import "testing" 4 | 5 | func TestStringObjectString(t *testing.T) { 6 | s := StringObject{Key: KeyObject{Key: "foo"}, Value: "bar"} 7 | equals(t, "StringObject{Key: foo, Value: 'bar'}", s.String()) 8 | } 9 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import "fmt" 4 | 5 | // Cast or convert an interface{} object to a string 6 | // If the type is not string, fmt.Stringer, []byte or an integer, it panics 7 | func DataToString(i interface{}) string { 8 | switch v := i.(type) { 9 | case string: 10 | return v 11 | case fmt.Stringer: 12 | return v.String() 13 | case uint8, int8, uint16, int16, uint32, int32, uint64, int64, int, uint: 14 | return fmt.Sprintf("%d", v) 15 | case []byte: 16 | return string(v) 17 | default: 18 | panic("unknown type") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "path/filepath" 8 | "reflect" 9 | "runtime" 10 | "testing" 11 | ) 12 | 13 | var ( 14 | end = make(chan bool, 1) 15 | ) 16 | 17 | type myCustomStringer struct{} 18 | 19 | func (s myCustomStringer) String() string { 20 | return "foobar" 21 | } 22 | 23 | func TestDataToString(t *testing.T) { 24 | equals(t, "foobar", "foobar") 25 | equals(t, "foobar", DataToString(myCustomStringer{})) 26 | equals(t, "1", DataToString(uint8(1))) 27 | equals(t, "1", DataToString(int8(1))) 28 | equals(t, "1", DataToString(uint16(1))) 29 | equals(t, "1", DataToString(int16(1))) 30 | equals(t, "1", DataToString(uint32(1))) 31 | equals(t, "1", DataToString(int32(1))) 32 | equals(t, "1", DataToString(uint64(1))) 33 | equals(t, "1", DataToString(int64(1))) 34 | equals(t, "1", DataToString(int(1))) 35 | equals(t, "1", DataToString(uint(1))) 36 | 37 | defer func() { 38 | e := recover() 39 | equals(t, "unknown type", e) 40 | }() 41 | DataToString(io.EOF) 42 | } 43 | 44 | // Call the read function f and report errors if there are any 45 | func readAndNotify(t *testing.T, r io.Reader, key string, f func(KeyObject, io.Reader) error) { 46 | err := f(KeyObject{Key: []byte(key)}, bufio.NewReader(r)) 47 | if err != nil { 48 | t.Error(err) 49 | } 50 | end <- true 51 | } 52 | 53 | // assert fails the test if the condition is false. 54 | func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 55 | if !condition { 56 | _, file, line, _ := runtime.Caller(1) 57 | fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 58 | tb.FailNow() 59 | } 60 | } 61 | 62 | // ok fails the test if an err is not nil. 63 | func ok(tb testing.TB, err error) { 64 | if err != nil { 65 | _, file, line, _ := runtime.Caller(1) 66 | fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) 67 | tb.FailNow() 68 | } 69 | } 70 | 71 | // equals fails the test if exp is not equal to act. 72 | func equals(tb testing.TB, exp, act interface{}) { 73 | if !reflect.DeepEqual(exp, act) { 74 | _, file, line, _ := runtime.Caller(1) 75 | fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) 76 | tb.FailNow() 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /ziplist.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | ) 7 | 8 | type zipListOnLenCallback func(length int64) error 9 | type zipListOnElementCallback func(element interface{}) error 10 | 11 | func (p *parser) readZipList(r io.Reader, onLenCallback zipListOnLenCallback, onElementCallback zipListOnElementCallback) error { 12 | var zlBytes int32 13 | var zlTail int32 14 | var zlLen int16 15 | var err error 16 | 17 | if err = binary.Read(r, binary.LittleEndian, &zlBytes); err != nil { 18 | return err 19 | } 20 | 21 | if err = binary.Read(r, binary.LittleEndian, &zlTail); err != nil { 22 | return err 23 | } 24 | 25 | if err = binary.Read(r, binary.LittleEndian, &zlLen); err != nil { 26 | return err 27 | } 28 | 29 | if err := onLenCallback(int64(zlLen)); err != nil { 30 | return err 31 | } 32 | 33 | for i := 0; i < int(zlLen); i++ { 34 | _, err := io.ReadFull(r, p.scratch[0:1]) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | b := p.scratch[0] 40 | 41 | // Read length of the previous entry 42 | // We don't use it though 43 | if b <= 0xFD { // 253 44 | // Do nothing 45 | } else if b == 0xFE { // 254 46 | var tmp int32 47 | if err = binary.Read(r, binary.LittleEndian, &tmp); err != nil { 48 | return err 49 | } 50 | // Do nothing 51 | } else { 52 | return ErrUnexpectedPrevLengthEntryByte 53 | } 54 | 55 | _, err = io.ReadFull(r, p.scratch[0:1]) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | flag := p.scratch[0] 61 | var data interface{} 62 | 63 | if (flag & 0xC0) == 0 { 64 | // String with length <= 63 bytes 65 | length := int64(flag & 0x3F) 66 | data, err = readBytes(r, length) 67 | if err != nil { 68 | return err 69 | } 70 | } else if (flag & 0xC0) == 0x40 { 71 | // String with length <= 16383 bytes 72 | _, err = io.ReadFull(r, p.scratch[0:1]) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | length := (int64(flag&0x3F) << 8) | int64(p.scratch[0]) 78 | data, err = readBytes(r, length) 79 | if err != nil { 80 | return err 81 | } 82 | } else if (flag & 0xC0) == 0x80 { 83 | // String with length >= 16384 bytes 84 | var tmp int32 85 | if err := binary.Read(r, binary.BigEndian, &tmp); err != nil { 86 | return err 87 | } 88 | 89 | length := int64(tmp) 90 | data, err = readBytes(r, length) 91 | if err != nil { 92 | return err 93 | } 94 | } else if (flag & 0xF0) == 0xC0 { 95 | // int16 96 | var tmp int16 97 | if err := binary.Read(r, binary.LittleEndian, &tmp); err != nil { 98 | return err 99 | } 100 | 101 | data = tmp 102 | } else if (flag & 0xF0) == 0xD0 { 103 | // int32 104 | var tmp int32 105 | if err := binary.Read(r, binary.LittleEndian, &tmp); err != nil { 106 | return err 107 | } 108 | 109 | data = tmp 110 | } else if (flag & 0xF0) == 0xE0 { 111 | // int64 112 | var tmp int64 113 | if err := binary.Read(r, binary.LittleEndian, &tmp); err != nil { 114 | return err 115 | } 116 | 117 | data = tmp 118 | } else if flag == 0xF0 { 119 | // int24 120 | ab, err := readBytes(r, 3) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | tmp := uint32(ab[0])<<8 | uint32(ab[1])<<16 | uint32(ab[2])<<24 126 | data = int32(tmp) >> 8 127 | } else if flag == 0xFE { 128 | // int8 129 | _, err := io.ReadFull(r, p.scratch[0:1]) 130 | if err != nil { 131 | return err 132 | } 133 | 134 | data = int8(p.scratch[0]) 135 | } else if (flag & 0xF0) == 0xF0 { 136 | // int4 137 | data = (int(int(flag) & 0x0F)) - 1 138 | } 139 | 140 | if err := onElementCallback(data); err != nil { 141 | return err 142 | } 143 | } 144 | 145 | return nil 146 | } 147 | -------------------------------------------------------------------------------- /ziplist_test.go: -------------------------------------------------------------------------------- 1 | package rdbtools 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "errors" 8 | "io" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func TestReadZipListStringLengthLte63(t *testing.T) { 14 | var buffer bytes.Buffer 15 | 16 | br := bufio.NewWriter(&buffer) 17 | p := &parser{} 18 | 19 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 20 | br.Write([]byte{0, 0, 0, 0}) // zlTail 21 | br.Write([]byte{1, 0}) // zlLen 22 | 23 | br.WriteByte(0) // len prev entry 24 | br.WriteByte(6) // Special flag 25 | br.WriteString("foobar") 26 | 27 | br.Flush() 28 | 29 | onLenCallback := func(length int64) error { 30 | equals(t, int64(1), length) 31 | return nil 32 | } 33 | 34 | onElementCallback := func(e interface{}) error { 35 | equals(t, "foobar", DataToString(e)) 36 | return nil 37 | } 38 | 39 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 40 | ok(t, err) 41 | } 42 | 43 | func TestReadZipListStringLengthLte16383(t *testing.T) { 44 | var buffer bytes.Buffer 45 | 46 | br := bufio.NewWriter(&buffer) 47 | p := &parser{} 48 | 49 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 50 | br.Write([]byte{0, 0, 0, 0}) // zlTail 51 | br.Write([]byte{1, 0}) // zlLen 52 | 53 | br.WriteByte(0) // len prev entry 54 | br.WriteByte(0x57) // Special flag 55 | br.WriteByte(0x70) // additional length byte 56 | for i := 0; i < 1000; i++ { 57 | br.WriteString("foobar") 58 | } 59 | 60 | br.Flush() 61 | 62 | onLenCallback := func(length int64) error { 63 | equals(t, int64(1), length) 64 | return nil 65 | } 66 | 67 | onElementCallback := func(e interface{}) error { 68 | equals(t, strings.Repeat("foobar", 1000), DataToString(e)) 69 | return nil 70 | } 71 | 72 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 73 | ok(t, err) 74 | } 75 | 76 | func TestReadZipListStringLengthGte16384(t *testing.T) { 77 | var buffer bytes.Buffer 78 | 79 | br := bufio.NewWriter(&buffer) 80 | p := &parser{} 81 | 82 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 83 | br.Write([]byte{0, 0, 0, 0}) // zlTail 84 | br.Write([]byte{1, 0}) // zlLen 85 | 86 | br.WriteByte(0) // len prev entry 87 | br.WriteByte(0x80) // Special flag 88 | binary.Write(br, binary.BigEndian, int32(30000)) 89 | for i := 0; i < 5000; i++ { 90 | br.WriteString("foobar") 91 | } 92 | 93 | br.Flush() 94 | 95 | onLenCallback := func(length int64) error { 96 | equals(t, int64(1), length) 97 | return nil 98 | } 99 | 100 | onElementCallback := func(e interface{}) error { 101 | equals(t, strings.Repeat("foobar", 5000), DataToString(e)) 102 | return nil 103 | } 104 | 105 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 106 | ok(t, err) 107 | } 108 | 109 | func TestReadZipListInt16(t *testing.T) { 110 | var buffer bytes.Buffer 111 | 112 | br := bufio.NewWriter(&buffer) 113 | p := &parser{} 114 | 115 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 116 | br.Write([]byte{0, 0, 0, 0}) // zlTail 117 | br.Write([]byte{1, 0}) // zlLen 118 | 119 | br.WriteByte(0) // len prev entry 120 | br.WriteByte(0xC0) // Special flag 121 | binary.Write(br, binary.LittleEndian, int16(1)) 122 | br.Flush() 123 | 124 | onLenCallback := func(length int64) error { 125 | equals(t, int64(1), length) 126 | return nil 127 | } 128 | 129 | onElementCallback := func(e interface{}) error { 130 | equals(t, int16(1), e) 131 | return nil 132 | } 133 | 134 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 135 | ok(t, err) 136 | } 137 | 138 | func TestReadZipListInt32(t *testing.T) { 139 | var buffer bytes.Buffer 140 | 141 | br := bufio.NewWriter(&buffer) 142 | p := &parser{} 143 | 144 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 145 | br.Write([]byte{0, 0, 0, 0}) // zlTail 146 | br.Write([]byte{1, 0}) // zlLen 147 | 148 | br.WriteByte(0) // len prev entry 149 | br.WriteByte(0xD0) // Special flag 150 | binary.Write(br, binary.LittleEndian, int32(1)) 151 | br.Flush() 152 | 153 | onLenCallback := func(length int64) error { 154 | equals(t, int64(1), length) 155 | return nil 156 | } 157 | 158 | onElementCallback := func(e interface{}) error { 159 | equals(t, int32(1), e) 160 | return nil 161 | } 162 | 163 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 164 | ok(t, err) 165 | } 166 | 167 | func TestReadZipListInt64(t *testing.T) { 168 | var buffer bytes.Buffer 169 | 170 | br := bufio.NewWriter(&buffer) 171 | p := &parser{} 172 | 173 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 174 | br.Write([]byte{0, 0, 0, 0}) // zlTail 175 | br.Write([]byte{1, 0}) // zlLen 176 | 177 | br.WriteByte(0) // len prev entry 178 | br.WriteByte(0xE0) // Special flag 179 | binary.Write(br, binary.LittleEndian, int64(1)) 180 | br.Flush() 181 | 182 | onLenCallback := func(length int64) error { 183 | equals(t, int64(1), length) 184 | return nil 185 | } 186 | 187 | onElementCallback := func(e interface{}) error { 188 | equals(t, int64(1), e) 189 | return nil 190 | } 191 | 192 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 193 | ok(t, err) 194 | } 195 | 196 | func TestReadZipListInt24(t *testing.T) { 197 | var buffer bytes.Buffer 198 | 199 | br := bufio.NewWriter(&buffer) 200 | p := &parser{} 201 | 202 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 203 | br.Write([]byte{0, 0, 0, 0}) // zlTail 204 | br.Write([]byte{1, 0}) // zlLen 205 | 206 | br.WriteByte(0) // len prev entry 207 | br.WriteByte(0xF0) // Special flag 208 | br.Write([]byte{13, 0, 0xFF}) 209 | br.Flush() 210 | 211 | onLenCallback := func(length int64) error { 212 | equals(t, int64(1), length) 213 | return nil 214 | } 215 | 216 | onElementCallback := func(e interface{}) error { 217 | equals(t, int32(-65523), e) 218 | return nil 219 | } 220 | 221 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 222 | ok(t, err) 223 | } 224 | 225 | func TestReadZipListInt8(t *testing.T) { 226 | var buffer bytes.Buffer 227 | 228 | br := bufio.NewWriter(&buffer) 229 | p := &parser{} 230 | 231 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 232 | br.Write([]byte{0, 0, 0, 0}) // zlTail 233 | br.Write([]byte{1, 0}) // zlLen 234 | 235 | br.WriteByte(0) // len prev entry 236 | br.WriteByte(0xFE) // Special flag 237 | br.WriteByte(1) 238 | br.Flush() 239 | 240 | onLenCallback := func(length int64) error { 241 | equals(t, int64(1), length) 242 | return nil 243 | } 244 | 245 | onElementCallback := func(e interface{}) error { 246 | equals(t, int8(1), e) 247 | return nil 248 | } 249 | 250 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 251 | ok(t, err) 252 | } 253 | 254 | func TestReadZipListInt4(t *testing.T) { 255 | var buffer bytes.Buffer 256 | 257 | br := bufio.NewWriter(&buffer) 258 | p := &parser{} 259 | 260 | br.Write([]byte{0, 0, 0, 0}) // zlBytes 261 | br.Write([]byte{0, 0, 0, 0}) // zlTail 262 | br.Write([]byte{1, 0}) // zlLen 263 | 264 | br.WriteByte(0) // len prev entry 265 | br.WriteByte(0xF2) // Special flag 266 | br.Flush() 267 | 268 | onLenCallback := func(length int64) error { 269 | equals(t, int64(1), length) 270 | return nil 271 | } 272 | 273 | onElementCallback := func(e interface{}) error { 274 | equals(t, int(1), e) 275 | return nil 276 | } 277 | 278 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 279 | ok(t, err) 280 | } 281 | 282 | func TestReadZipListNoZlBytes(t *testing.T) { 283 | var buffer bytes.Buffer 284 | p := &parser{} 285 | err := p.readZipList(bufio.NewReader(&buffer), nil, nil) 286 | equals(t, io.EOF, err) 287 | } 288 | 289 | func TestReadZipListNoZlTail(t *testing.T) { 290 | var buffer bytes.Buffer 291 | 292 | br := bufio.NewWriter(&buffer) 293 | p := &parser{} 294 | 295 | binary.Write(br, binary.LittleEndian, int32(1)) 296 | br.Flush() 297 | 298 | err := p.readZipList(bufio.NewReader(&buffer), nil, nil) 299 | equals(t, io.EOF, err) 300 | } 301 | 302 | func TestReadZipListNoZlLen(t *testing.T) { 303 | var buffer bytes.Buffer 304 | 305 | br := bufio.NewWriter(&buffer) 306 | p := &parser{} 307 | 308 | binary.Write(br, binary.LittleEndian, int32(1)) 309 | binary.Write(br, binary.LittleEndian, int32(1)) 310 | br.Flush() 311 | 312 | err := p.readZipList(bufio.NewReader(&buffer), nil, nil) 313 | equals(t, io.EOF, err) 314 | } 315 | 316 | func TestReadZipListNoPrevEntryLength(t *testing.T) { 317 | var buffer bytes.Buffer 318 | 319 | br := bufio.NewWriter(&buffer) 320 | p := &parser{} 321 | 322 | binary.Write(br, binary.LittleEndian, int32(1)) 323 | binary.Write(br, binary.LittleEndian, int32(1)) 324 | binary.Write(br, binary.LittleEndian, int16(1)) 325 | br.Flush() 326 | 327 | onLenCallback := func(l int64) error { 328 | equals(t, int64(1), l) 329 | return nil 330 | } 331 | 332 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 333 | equals(t, io.EOF, err) 334 | } 335 | 336 | // error on flag reading 337 | func TestReadZipListPrevEntryLengthLte253(t *testing.T) { 338 | var buffer bytes.Buffer 339 | 340 | br := bufio.NewWriter(&buffer) 341 | p := &parser{} 342 | 343 | binary.Write(br, binary.LittleEndian, int32(1)) 344 | binary.Write(br, binary.LittleEndian, int32(1)) 345 | binary.Write(br, binary.LittleEndian, int16(1)) 346 | br.WriteByte(0) // prev entry len 347 | br.Flush() 348 | 349 | onLenCallback := func(l int64) error { 350 | equals(t, int64(1), l) 351 | return nil 352 | } 353 | 354 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 355 | equals(t, io.EOF, err) 356 | } 357 | 358 | // error on flag reading 359 | func TestReadZipListPrevEntryLengthEq254(t *testing.T) { 360 | var buffer bytes.Buffer 361 | 362 | br := bufio.NewWriter(&buffer) 363 | p := &parser{} 364 | 365 | binary.Write(br, binary.LittleEndian, int32(1)) 366 | binary.Write(br, binary.LittleEndian, int32(1)) 367 | binary.Write(br, binary.LittleEndian, int16(1)) 368 | br.WriteByte(0xFE) // prev entry len 369 | binary.Write(br, binary.LittleEndian, int32(1)) 370 | br.Flush() 371 | 372 | onLenCallback := func(l int64) error { 373 | equals(t, int64(1), l) 374 | return nil 375 | } 376 | 377 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 378 | equals(t, io.EOF, err) 379 | } 380 | 381 | func TestReadZipListPrevEntryLengthEq254NoData(t *testing.T) { 382 | var buffer bytes.Buffer 383 | 384 | br := bufio.NewWriter(&buffer) 385 | p := &parser{} 386 | 387 | binary.Write(br, binary.LittleEndian, int32(1)) 388 | binary.Write(br, binary.LittleEndian, int32(1)) 389 | binary.Write(br, binary.LittleEndian, int16(1)) 390 | br.WriteByte(0xFE) // prev entry len 391 | br.Flush() 392 | 393 | onLenCallback := func(l int64) error { 394 | equals(t, int64(1), l) 395 | return nil 396 | } 397 | 398 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 399 | equals(t, io.EOF, err) 400 | } 401 | 402 | func TestReadZipListPrevEntryLengthUnexpected(t *testing.T) { 403 | var buffer bytes.Buffer 404 | 405 | br := bufio.NewWriter(&buffer) 406 | p := &parser{} 407 | 408 | binary.Write(br, binary.LittleEndian, int32(1)) 409 | binary.Write(br, binary.LittleEndian, int32(1)) 410 | binary.Write(br, binary.LittleEndian, int16(1)) 411 | br.WriteByte(0xFF) // prev entry len 412 | br.Flush() 413 | 414 | onLenCallback := func(l int64) error { 415 | equals(t, int64(1), l) 416 | return nil 417 | } 418 | 419 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 420 | equals(t, ErrUnexpectedPrevLengthEntryByte, err) 421 | } 422 | 423 | func TestReadZipListStringLengthLte63NoData(t *testing.T) { 424 | var buffer bytes.Buffer 425 | 426 | br := bufio.NewWriter(&buffer) 427 | p := &parser{} 428 | 429 | binary.Write(br, binary.LittleEndian, int32(1)) 430 | binary.Write(br, binary.LittleEndian, int32(1)) 431 | binary.Write(br, binary.LittleEndian, int16(1)) 432 | br.WriteByte(0) // prev entry len 433 | br.WriteByte(1) // flag 434 | br.Flush() 435 | 436 | onLenCallback := func(l int64) error { 437 | equals(t, int64(1), l) 438 | return nil 439 | } 440 | 441 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 442 | equals(t, io.EOF, err) 443 | } 444 | 445 | func TestReadZipListStringLengthLte16383NoAdditionalByte(t *testing.T) { 446 | var buffer bytes.Buffer 447 | 448 | br := bufio.NewWriter(&buffer) 449 | p := &parser{} 450 | 451 | binary.Write(br, binary.LittleEndian, int32(1)) 452 | binary.Write(br, binary.LittleEndian, int32(1)) 453 | binary.Write(br, binary.LittleEndian, int16(1)) 454 | br.WriteByte(0) // prev entry len 455 | br.WriteByte(0x40) // flag 456 | br.Flush() 457 | 458 | onLenCallback := func(l int64) error { 459 | equals(t, int64(1), l) 460 | return nil 461 | } 462 | 463 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 464 | equals(t, io.EOF, err) 465 | } 466 | 467 | func TestReadZipListStringLengthLte16383NoData(t *testing.T) { 468 | var buffer bytes.Buffer 469 | 470 | br := bufio.NewWriter(&buffer) 471 | p := &parser{} 472 | 473 | binary.Write(br, binary.LittleEndian, int32(1)) 474 | binary.Write(br, binary.LittleEndian, int32(1)) 475 | binary.Write(br, binary.LittleEndian, int16(1)) 476 | br.WriteByte(0) // prev entry len 477 | br.WriteByte(0x41) // flag 478 | br.WriteByte(0) 479 | br.Flush() 480 | 481 | onLenCallback := func(l int64) error { 482 | equals(t, int64(1), l) 483 | return nil 484 | } 485 | 486 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 487 | equals(t, io.EOF, err) 488 | } 489 | 490 | func TestReadZipListStringLengthGte16384NoAdditionalBytes(t *testing.T) { 491 | var buffer bytes.Buffer 492 | 493 | br := bufio.NewWriter(&buffer) 494 | p := &parser{} 495 | 496 | binary.Write(br, binary.LittleEndian, int32(1)) 497 | binary.Write(br, binary.LittleEndian, int32(1)) 498 | binary.Write(br, binary.LittleEndian, int16(1)) 499 | br.WriteByte(0) // prev entry len 500 | br.WriteByte(0x80) // flag 501 | br.Flush() 502 | 503 | onLenCallback := func(l int64) error { 504 | equals(t, int64(1), l) 505 | return nil 506 | } 507 | 508 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 509 | equals(t, io.EOF, err) 510 | } 511 | 512 | func TestReadZipListStringLengthGte16384NoData(t *testing.T) { 513 | var buffer bytes.Buffer 514 | 515 | br := bufio.NewWriter(&buffer) 516 | p := &parser{} 517 | 518 | binary.Write(br, binary.LittleEndian, int32(1)) 519 | binary.Write(br, binary.LittleEndian, int32(1)) 520 | binary.Write(br, binary.LittleEndian, int16(1)) 521 | br.WriteByte(0) // prev entry len 522 | br.WriteByte(0x80) // flag 523 | binary.Write(br, binary.LittleEndian, int32(1)) 524 | br.Flush() 525 | 526 | onLenCallback := func(l int64) error { 527 | equals(t, int64(1), l) 528 | return nil 529 | } 530 | 531 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 532 | equals(t, io.EOF, err) 533 | } 534 | 535 | func TestReadZipListInt16NoData(t *testing.T) { 536 | var buffer bytes.Buffer 537 | 538 | br := bufio.NewWriter(&buffer) 539 | p := &parser{} 540 | 541 | binary.Write(br, binary.LittleEndian, int32(1)) 542 | binary.Write(br, binary.LittleEndian, int32(1)) 543 | binary.Write(br, binary.LittleEndian, int16(1)) 544 | br.WriteByte(0) // prev entry len 545 | br.WriteByte(0xC0) // flag 546 | br.Flush() 547 | 548 | onLenCallback := func(l int64) error { 549 | equals(t, int64(1), l) 550 | return nil 551 | } 552 | 553 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 554 | equals(t, io.EOF, err) 555 | } 556 | 557 | func TestReadZipListInt32NoData(t *testing.T) { 558 | var buffer bytes.Buffer 559 | 560 | br := bufio.NewWriter(&buffer) 561 | p := &parser{} 562 | 563 | binary.Write(br, binary.LittleEndian, int32(1)) 564 | binary.Write(br, binary.LittleEndian, int32(1)) 565 | binary.Write(br, binary.LittleEndian, int16(1)) 566 | br.WriteByte(0) // prev entry len 567 | br.WriteByte(0xD0) // flag 568 | br.Flush() 569 | 570 | onLenCallback := func(l int64) error { 571 | equals(t, int64(1), l) 572 | return nil 573 | } 574 | 575 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 576 | equals(t, io.EOF, err) 577 | } 578 | 579 | func TestReadZipListInt64NoData(t *testing.T) { 580 | var buffer bytes.Buffer 581 | 582 | br := bufio.NewWriter(&buffer) 583 | p := &parser{} 584 | 585 | binary.Write(br, binary.LittleEndian, int32(1)) 586 | binary.Write(br, binary.LittleEndian, int32(1)) 587 | binary.Write(br, binary.LittleEndian, int16(1)) 588 | br.WriteByte(0) // prev entry len 589 | br.WriteByte(0xE0) // flag 590 | br.Flush() 591 | 592 | onLenCallback := func(l int64) error { 593 | equals(t, int64(1), l) 594 | return nil 595 | } 596 | 597 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 598 | equals(t, io.EOF, err) 599 | } 600 | 601 | func TestReadZipListInt24NoAdditionalBytes(t *testing.T) { 602 | var buffer bytes.Buffer 603 | 604 | br := bufio.NewWriter(&buffer) 605 | p := &parser{} 606 | 607 | binary.Write(br, binary.LittleEndian, int32(1)) 608 | binary.Write(br, binary.LittleEndian, int32(1)) 609 | binary.Write(br, binary.LittleEndian, int16(1)) 610 | br.WriteByte(0) // prev entry len 611 | br.WriteByte(0xF0) // flag 612 | br.Flush() 613 | 614 | onLenCallback := func(l int64) error { 615 | equals(t, int64(1), l) 616 | return nil 617 | } 618 | 619 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 620 | equals(t, io.EOF, err) 621 | } 622 | 623 | func TestReadZipListInt8NoAdditionalByte(t *testing.T) { 624 | var buffer bytes.Buffer 625 | 626 | br := bufio.NewWriter(&buffer) 627 | p := &parser{} 628 | 629 | binary.Write(br, binary.LittleEndian, int32(1)) 630 | binary.Write(br, binary.LittleEndian, int32(1)) 631 | binary.Write(br, binary.LittleEndian, int16(1)) 632 | br.WriteByte(0) // prev entry len 633 | br.WriteByte(0xFE) // flag 634 | br.Flush() 635 | 636 | onLenCallback := func(l int64) error { 637 | equals(t, int64(1), l) 638 | return nil 639 | } 640 | 641 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 642 | equals(t, io.EOF, err) 643 | } 644 | 645 | func TestReadZipListOnLenCallbackError(t *testing.T) { 646 | var buffer bytes.Buffer 647 | 648 | br := bufio.NewWriter(&buffer) 649 | p := &parser{} 650 | 651 | binary.Write(br, binary.LittleEndian, int32(1)) 652 | binary.Write(br, binary.LittleEndian, int32(1)) 653 | binary.Write(br, binary.LittleEndian, int16(1)) 654 | br.Flush() 655 | 656 | myErr := errors.New("myErr") 657 | 658 | onLenCallback := func(l int64) error { 659 | return myErr 660 | } 661 | 662 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, nil) 663 | equals(t, myErr, err) 664 | } 665 | 666 | func TestReadZipListOnElementCallbackError(t *testing.T) { 667 | var buffer bytes.Buffer 668 | 669 | br := bufio.NewWriter(&buffer) 670 | p := &parser{} 671 | 672 | binary.Write(br, binary.LittleEndian, int32(1)) 673 | binary.Write(br, binary.LittleEndian, int32(1)) 674 | binary.Write(br, binary.LittleEndian, int16(1)) 675 | br.WriteByte(0) // len prev entry 676 | br.WriteByte(6) // Special flag 677 | br.WriteString("foobar") 678 | br.Flush() 679 | 680 | myErr := errors.New("myErr") 681 | 682 | onLenCallback := func(l int64) error { 683 | return nil 684 | } 685 | onElementCallback := func(e interface{}) error { 686 | return myErr 687 | } 688 | 689 | err := p.readZipList(bufio.NewReader(&buffer), onLenCallback, onElementCallback) 690 | equals(t, myErr, err) 691 | } 692 | --------------------------------------------------------------------------------