├── cmd └── rmr │ ├── .gitignore │ ├── README.md │ └── main.go ├── .travis.yml ├── vendor ├── modules.txt └── github.com │ └── pkg │ └── errors │ ├── .travis.yml │ ├── .gitignore │ ├── appveyor.yml │ ├── LICENSE │ ├── README.md │ ├── stack.go │ └── errors.go ├── go.mod ├── testdata ├── dumps │ ├── big_string.rdb │ ├── dictionary.rdb │ ├── intset_16.rdb │ ├── intset_32.rdb │ ├── intset_64.rdb │ ├── linkedlist.rdb │ ├── quicklist.rdb │ ├── regular_set.rdb │ ├── integer_keys.rdb │ ├── empty_database.rdb │ ├── hash_as_ziplist.rdb │ ├── keys_with_expiry.rdb │ ├── non_ascii_values.rdb │ ├── parser_filters.rdb │ ├── multiple_databases.rdb │ ├── regular_sorted_set.rdb │ ├── sorted_set_as_ziplist.rdb │ ├── ziplist_with_integers.rdb │ ├── rdb_version_9_with_freq.rdb │ ├── rdb_version_9_with_idle.rdb │ ├── zipmap_with_big_values.rdb │ ├── uncompressible_string_keys.rdb │ ├── rdb_version_5_with_checksum.rdb │ ├── ziplist_that_doesnt_compress.rdb │ ├── zipmap_that_doesnt_compress.rdb │ ├── easily_compressible_string_key.rdb │ ├── ziplist_that_compresses_easily.rdb │ ├── zipmap_that_compresses_easily.rdb │ └── rdb_version_8_with_64b_length_and_scores.rdb └── .20kbytes ├── manifest.json ├── go.sum ├── mmap_other.go ├── lock.json ├── mmap_test.go ├── mmap_linux.go ├── LICENSE ├── doc.go ├── README.md ├── reader.go ├── util.go ├── type.go ├── rdb.go └── rdb_test.go /cmd/rmr/.gitignore: -------------------------------------------------------------------------------- 1 | rmr 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.7 4 | - 1.8 5 | - tip 6 | -------------------------------------------------------------------------------- /vendor/modules.txt: -------------------------------------------------------------------------------- 1 | # github.com/pkg/errors v0.8.0 2 | ## explicit 3 | github.com/pkg/errors 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/replit/rdb 2 | 3 | go 1.16 4 | 5 | require github.com/pkg/errors v0.8.0 6 | -------------------------------------------------------------------------------- /testdata/dumps/big_string.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/big_string.rdb -------------------------------------------------------------------------------- /testdata/dumps/dictionary.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/dictionary.rdb -------------------------------------------------------------------------------- /testdata/dumps/intset_16.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/intset_16.rdb -------------------------------------------------------------------------------- /testdata/dumps/intset_32.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/intset_32.rdb -------------------------------------------------------------------------------- /testdata/dumps/intset_64.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/intset_64.rdb -------------------------------------------------------------------------------- /testdata/dumps/linkedlist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/linkedlist.rdb -------------------------------------------------------------------------------- /testdata/dumps/quicklist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/quicklist.rdb -------------------------------------------------------------------------------- /testdata/dumps/regular_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/regular_set.rdb -------------------------------------------------------------------------------- /testdata/dumps/integer_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/integer_keys.rdb -------------------------------------------------------------------------------- /testdata/dumps/empty_database.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/empty_database.rdb -------------------------------------------------------------------------------- /testdata/dumps/hash_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/hash_as_ziplist.rdb -------------------------------------------------------------------------------- /testdata/dumps/keys_with_expiry.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/keys_with_expiry.rdb -------------------------------------------------------------------------------- /testdata/dumps/non_ascii_values.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/non_ascii_values.rdb -------------------------------------------------------------------------------- /testdata/dumps/parser_filters.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/parser_filters.rdb -------------------------------------------------------------------------------- /testdata/dumps/multiple_databases.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/multiple_databases.rdb -------------------------------------------------------------------------------- /testdata/dumps/regular_sorted_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/regular_sorted_set.rdb -------------------------------------------------------------------------------- /testdata/dumps/sorted_set_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/sorted_set_as_ziplist.rdb -------------------------------------------------------------------------------- /testdata/dumps/ziplist_with_integers.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/ziplist_with_integers.rdb -------------------------------------------------------------------------------- /testdata/dumps/rdb_version_9_with_freq.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/rdb_version_9_with_freq.rdb -------------------------------------------------------------------------------- /testdata/dumps/rdb_version_9_with_idle.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/rdb_version_9_with_idle.rdb -------------------------------------------------------------------------------- /testdata/dumps/zipmap_with_big_values.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/zipmap_with_big_values.rdb -------------------------------------------------------------------------------- /testdata/dumps/uncompressible_string_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/uncompressible_string_keys.rdb -------------------------------------------------------------------------------- /testdata/dumps/rdb_version_5_with_checksum.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/rdb_version_5_with_checksum.rdb -------------------------------------------------------------------------------- /testdata/dumps/ziplist_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/ziplist_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /testdata/dumps/zipmap_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/zipmap_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /testdata/dumps/easily_compressible_string_key.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/easily_compressible_string_key.rdb -------------------------------------------------------------------------------- /testdata/dumps/ziplist_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/ziplist_that_compresses_easily.rdb -------------------------------------------------------------------------------- /testdata/dumps/zipmap_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/zipmap_that_compresses_easily.rdb -------------------------------------------------------------------------------- /manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "github.com/pkg/errors": { 4 | "version": "v0.8.0" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testdata/dumps/rdb_version_8_with_64b_length_and_scores.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/rdb/master/testdata/dumps/rdb_version_8_with_64b_length_and_scores.rdb -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 2 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 3 | -------------------------------------------------------------------------------- /mmap_other.go: -------------------------------------------------------------------------------- 1 | // +build !linux 2 | 3 | package rdb 4 | 5 | import "io/ioutil" 6 | 7 | func mmap(filename string) ([]byte, error) { 8 | return ioutil.ReadFile(filename) 9 | } 10 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go_import_path: github.com/pkg/errors 3 | go: 4 | - 1.4.3 5 | - 1.5.4 6 | - 1.6.2 7 | - 1.7.1 8 | - tip 9 | 10 | script: 11 | - go test -v ./... 12 | -------------------------------------------------------------------------------- /lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "memo": "fe3d778b7939daa14c4c85b19faf1496b476710b9aa988448787cd10d6a69b26", 3 | "projects": [ 4 | { 5 | "name": "github.com/pkg/errors", 6 | "version": "v0.8.0", 7 | "revision": "3866ebc348c54054262feae422da428fe6cf147d", 8 | "packages": [ 9 | "." 10 | ] 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /mmap_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "testing" 7 | ) 8 | 9 | func TestMmap(t *testing.T) { 10 | const filename = "mmap_test.go" 11 | got, err := mmap(filename) 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | want, err := ioutil.ReadFile(filename) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | if !bytes.Equal(got, want) { 20 | t.Fatalf("got: %v, want: %v", got, want) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /mmap_linux.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | ) 7 | 8 | func mmap(filename string) ([]byte, error) { 9 | f, err := os.Open(filename) 10 | if err != nil { 11 | return nil, err 12 | } 13 | defer f.Close() 14 | 15 | fi, err := f.Stat() 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | size := fi.Size() 21 | if size == 0 { 22 | return nil, nil 23 | } 24 | return syscall.Mmap(int(f.Fd()), 0, int(size), syscall.PROT_READ, syscall.MAP_SHARED) 25 | } 26 | -------------------------------------------------------------------------------- /cmd/rmr/README.md: -------------------------------------------------------------------------------- 1 | # RMR 2 | 3 | RMR is a command line util to generate Redis Memory Report. 4 | 5 | ## Usage 6 | 7 | ``` 8 | Usage: ./rmr [options] -f /path/to/dump.rdb 9 | 10 | Options: 11 | 12 | -b int 13 | Read buffer size. 14 | -d Enable debug output. 15 | -db value 16 | Databases to inspect. Multiple databases can provided. 17 | -f string 18 | Redis RDB file path. 19 | -k value 20 | Keys to inspect. Multiple keys can provided. 21 | -m int 22 | Maximum memory mapping size. (default 1073741824) 23 | -o string 24 | Output file. 25 | -p value 26 | Key match patterns. Multiple patterns can provided. 27 | -t value 28 | Types to inspect. Multiple types can provided. 29 | ``` 30 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/appveyor.yml: -------------------------------------------------------------------------------- 1 | version: build-{build}.{branch} 2 | 3 | clone_folder: C:\gopath\src\github.com\pkg\errors 4 | shallow_clone: true # for startup speed 5 | 6 | environment: 7 | GOPATH: C:\gopath 8 | 9 | platform: 10 | - x64 11 | 12 | # http://www.appveyor.com/docs/installed-software 13 | install: 14 | # some helpful output for debugging builds 15 | - go version 16 | - go env 17 | # pre-installed MinGW at C:\MinGW is 32bit only 18 | # but MSYS2 at C:\msys64 has mingw64 19 | - set PATH=C:\msys64\mingw64\bin;%PATH% 20 | - gcc --version 21 | - g++ --version 22 | 23 | build_script: 24 | - go install -v ./... 25 | 26 | test_script: 27 | - set PATH=C:\gopath\bin;%PATH% 28 | - go test -v ./... 29 | 30 | #artifacts: 31 | # - path: '%GOPATH%\bin\*.exe' 32 | deploy: off 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Matthewjhe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Dave Cheney 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/README.md: -------------------------------------------------------------------------------- 1 | # errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) 2 | 3 | Package errors provides simple error handling primitives. 4 | 5 | `go get github.com/pkg/errors` 6 | 7 | The traditional error handling idiom in Go is roughly akin to 8 | ```go 9 | if err != nil { 10 | return err 11 | } 12 | ``` 13 | which applied recursively up the call stack results in error reports without context or debugging information. The errors package allows programmers to add context to the failure path in their code in a way that does not destroy the original value of the error. 14 | 15 | ## Adding context to an error 16 | 17 | The errors.Wrap function returns a new error that adds context to the original error. For example 18 | ```go 19 | _, err := ioutil.ReadAll(r) 20 | if err != nil { 21 | return errors.Wrap(err, "read failed") 22 | } 23 | ``` 24 | ## Retrieving the cause of an error 25 | 26 | Using `errors.Wrap` constructs a stack of errors, adding context to the preceding error. Depending on the nature of the error it may be necessary to reverse the operation of errors.Wrap to retrieve the original error for inspection. Any error value which implements this interface can be inspected by `errors.Cause`. 27 | ```go 28 | type causer interface { 29 | Cause() error 30 | } 31 | ``` 32 | `errors.Cause` will recursively retrieve the topmost error which does not implement `causer`, which is assumed to be the original cause. For example: 33 | ```go 34 | switch err := errors.Cause(err).(type) { 35 | case *MyError: 36 | // handle specifically 37 | default: 38 | // unknown error 39 | } 40 | ``` 41 | 42 | [Read the package documentation for more information](https://godoc.org/github.com/pkg/errors). 43 | 44 | ## Contributing 45 | 46 | We welcome pull requests, bug fixes and issue reports. With that said, the bar for adding new symbols to this package is intentionally set high. 47 | 48 | Before proposing a change, please discuss your change by raising an issue. 49 | 50 | ## Licence 51 | 52 | BSD-2-Clause 53 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package rdb implements a Redis RDB File parser. 3 | 4 | Basic Example 5 | 6 | The basic use case is very simple, just implements Filter interface and pass it to Parse: 7 | 8 | package main 9 | 10 | import ( 11 | "github.com/replit/rdb" 12 | ) 13 | 14 | func (f filter) Key(k rdb.Key) bool { return false } 15 | func (f filter) Type(t rdb.Type) bool { return false } 16 | func (f filter) Database(db rdb.DB) bool { return false } 17 | func (f filter) Set(v *rdb.Set) { } 18 | func (f filter) List(v *rdb.List) { } 19 | func (f filter) Hash(v *rdb.Hash) { } 20 | func (f filter) String(v *rdb.String) { } 21 | func (f filter) SortedSet(v *rdb.SortedSet) { } 22 | 23 | func main() { 24 | const file = "/tmp/dump.rdb" 25 | reader, err := rdb.NewBufferReader(file, 0) 26 | if err != nil { 27 | panic(err) 28 | } 29 | 30 | if err := rdb.Parse(reader, rdb.WithFilter(filter{})); err != nil { 31 | panic(err) 32 | } 33 | } 34 | 35 | Skipping 36 | 37 | NOTE: RDB file is read sequentially, when we say skips a database, we also need to parse this database's every single key, 38 | we just simply skips actions like decompress, unzip ziplist, etc. It is impossible to skips a key without reading its metadata. 39 | 40 | Global skip strategy applies to the whole parsing lifetime, it can be overwritten by Filter's Database method, 41 | it will be restored when parsing a database completely. 42 | 43 | Global skip strategy is set by a ParseOption when rdb.Parse is called: 44 | 45 | // reads memory report only 46 | strategy := rdb.WithStrategy(rdb.SkipExpiry | rdb.SkipMeta | rdb.SkipValue) 47 | err := rdb.Parse(reader, strategy, rdb.WithFilter(filter{})) 48 | 49 | Database skip strategy applies to a database lifetime, it is set by Filter's Database method, and it can be overwritten 50 | by Filter's Key or Type method. Database skip strategy is set to global skip strategy by default. 51 | 52 | func (f filter) Database(db rdb.DB) bool { 53 | // skips this database 54 | db.Skip(rdb.SkipAll) 55 | 56 | // skips database's metadata and key's expiry 57 | // db.Skip(rdb.SkipMeta | rdb.SkipExpiry) 58 | return false 59 | } 60 | 61 | Type and key skip strategy overwrite database skip strategy. It only applies the key and value which will read next. 62 | 63 | func (f filter) Type(t rdb.Type) bool { 64 | // skips this type 65 | t.Skip(rdb.SkipAll) 66 | return false 67 | } 68 | 69 | func (f filter) Key(key rdb.Key) bool { 70 | // skips this key 71 | // key.Skip(rdb.SkipAll) 72 | 73 | // skips value 74 | key.Skip(rdb.SkipValue) 75 | return false 76 | } 77 | 78 | Syncing 79 | 80 | By default, rdb uses multiple goroutines to parse keys which would cause keys disorder. 81 | If the order of keys is important, use EnableSync ParseOption. 82 | 83 | rdb.Parse(reader, rdb.WithFilter(filter{}), rdb.EnableSync()) 84 | 85 | Aborting 86 | 87 | Sometimes we want to abort the parsing process, this can be done by return true when one of the following methods is called: 88 | Note that the abort action is taking effect immediately when the function returns. 89 | 90 | func (f filter) Key(k rdb.Key) bool { 91 | return true 92 | } 93 | func (f filter) Type(t rdb.Type) bool { 94 | return true 95 | } 96 | func (f filter) Database(db rdb.DB) bool { 97 | return true 98 | } 99 | */ 100 | package rdb // import "github.com/replit/rdb" 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rdb [![Build Status](https://travis-ci.org/matthewjhe/rdb.svg?branch=master)](https://travis-ci.org/matthewjhe/rdb) [![GoDoc](https://godoc.org/github.com/matthewjhe/rdb?status.svg)](http://godoc.org/github.com/matthewjhe/rdb) 2 | 3 | **WARNING**: This project is not production ready. 4 | 5 | Package rdb is a Redis RDB File parser written in [Go](https://golang.org/). 6 | 7 | ## Feature 8 | 9 | - Concurrent parsing. 10 | - Built in memory report. 11 | - Simple filter support. 12 | - Layered skipping strategies. 13 | 14 | ## Install 15 | 16 | `go get -u github.com/matthewjhe/rdb` 17 | 18 | ## Usage 19 | 20 | ### Basic Example 21 | 22 | The basic use case is very simple, just implements Filter interface and pass it to Parse: 23 | 24 | ```go 25 | package main 26 | 27 | import ( 28 | "github.com/matthewjhe/rdb" 29 | ) 30 | 31 | func (f filter) Key(k rdb.Key) bool { return false } 32 | func (f filter) Type(t rdb.Type) bool { return false } 33 | func (f filter) Database(db rdb.DB) bool { return false } 34 | func (f filter) Set(v *rdb.Set) { } 35 | func (f filter) List(v *rdb.List) { } 36 | func (f filter) Hash(v *rdb.Hash) { } 37 | func (f filter) String(v *rdb.String) { } 38 | func (f filter) SortedSet(v *rdb.SortedSet) { } 39 | 40 | func main() { 41 | const file = "/tmp/dump.rdb" 42 | reader, err := rdb.NewBufferReader(file, 0) 43 | if err != nil { 44 | panic(err) 45 | } 46 | 47 | if err := rdb.Parse(reader, rdb.WithFilter(filter{})); err != nil { 48 | panic(err) 49 | } 50 | } 51 | ``` 52 | 53 | ### Skipping 54 | 55 | **NOTE**: RDB file is read sequentially, when we say skips a database, we also need to parse this database's every single key, 56 | we just simply skips actions like `decompress`, `unzip ziplist`, etc. It is impossible to skips a `key` without reading its metadata. 57 | 58 | - Global Skip Strategy 59 | 60 | Global skip strategy applies to the whole parsing lifetime, it can be overwritten by Filter's `Database` method, 61 | it will be restored when parsing a database completely. 62 | 63 | Global skip strategy is set by a ParseOption when rdb.Parse is called: 64 | 65 | ```go 66 | // reads memory report only 67 | strategy := rdb.WithStrategy(rdb.SkipExpiry | rdb.SkipMeta | rdb.SkipValue) 68 | err := rdb.Parse(reader, strategy, rdb.WithFilter(filter{})) 69 | // ... 70 | ``` 71 | 72 | - Database Skip Strategy 73 | 74 | Database skip strategy applies to a database lifetime, it is set by Filter's `Database` method, and it can be overwritten 75 | by Filter's `Key` or `Type` method. Database skip strategy is set to global skip strategy by default. 76 | 77 | ```go 78 | func (f filter) Database(db rdb.DB) bool { 79 | // skips this database 80 | db.Skip(rdb.SkipAll) 81 | 82 | // skips database's metadata and key's expiry 83 | // db.Skip(rdb.SkipMeta | rdb.SkipExpiry) 84 | return false 85 | } 86 | ``` 87 | 88 | - Type and Key Skip Strategy 89 | 90 | Type and key skip strategy overwrite database skip strategy. It only applies the key and value which will read next. 91 | 92 | ```go 93 | func (f filter) Type(t rdb.Type) bool { 94 | // skips this type 95 | t.Skip(rdb.SkipAll) 96 | return false 97 | } 98 | 99 | func (f filter) Key(key rdb.Key) bool { 100 | // skips this key 101 | // key.Skip(rdb.SkipAll) 102 | 103 | // skips value 104 | key.Skip(rdb.SkipValue) 105 | return false 106 | } 107 | ``` 108 | 109 | ### Syncing 110 | 111 | By default, rdb uses multiple goroutines to parse keys which would cause keys disorder. 112 | If the order of keys is important, use `EnableSync` ParseOption. 113 | 114 | ```go 115 | rdb.Parse(reader, rdb.WithFilter(filter{}), rdb.EnableSync()) 116 | ``` 117 | 118 | ### Aborting 119 | 120 | Sometimes we want to abort the parsing process, this can be done by return `true` when one of the following methods is called: 121 | Note that the abort action is taking effect immediately when the function returns. 122 | 123 | ```go 124 | func (f filter) Key(k rdb.Key) bool { 125 | return true 126 | } 127 | func (f filter) Type(t rdb.Type) bool { 128 | return true 129 | } 130 | func (f filter) Database(db rdb.DB) bool { 131 | return true 132 | } 133 | ``` 134 | 135 | ## References 136 | 137 | - [redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools) for Redis RDB format and memory profile. 138 | 139 | ## License 140 | 141 | [MIT](https://github.com/matthewjhe/rdb/blob/master/LICENSE) 142 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/stack.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "path" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | // Frame represents a program counter inside a stack frame. 12 | type Frame uintptr 13 | 14 | // pc returns the program counter for this frame; 15 | // multiple frames may have the same PC value. 16 | func (f Frame) pc() uintptr { return uintptr(f) - 1 } 17 | 18 | // file returns the full path to the file that contains the 19 | // function for this Frame's pc. 20 | func (f Frame) file() string { 21 | fn := runtime.FuncForPC(f.pc()) 22 | if fn == nil { 23 | return "unknown" 24 | } 25 | file, _ := fn.FileLine(f.pc()) 26 | return file 27 | } 28 | 29 | // line returns the line number of source code of the 30 | // function for this Frame's pc. 31 | func (f Frame) line() int { 32 | fn := runtime.FuncForPC(f.pc()) 33 | if fn == nil { 34 | return 0 35 | } 36 | _, line := fn.FileLine(f.pc()) 37 | return line 38 | } 39 | 40 | // Format formats the frame according to the fmt.Formatter interface. 41 | // 42 | // %s source file 43 | // %d source line 44 | // %n function name 45 | // %v equivalent to %s:%d 46 | // 47 | // Format accepts flags that alter the printing of some verbs, as follows: 48 | // 49 | // %+s path of source file relative to the compile time GOPATH 50 | // %+v equivalent to %+s:%d 51 | func (f Frame) Format(s fmt.State, verb rune) { 52 | switch verb { 53 | case 's': 54 | switch { 55 | case s.Flag('+'): 56 | pc := f.pc() 57 | fn := runtime.FuncForPC(pc) 58 | if fn == nil { 59 | io.WriteString(s, "unknown") 60 | } else { 61 | file, _ := fn.FileLine(pc) 62 | fmt.Fprintf(s, "%s\n\t%s", fn.Name(), file) 63 | } 64 | default: 65 | io.WriteString(s, path.Base(f.file())) 66 | } 67 | case 'd': 68 | fmt.Fprintf(s, "%d", f.line()) 69 | case 'n': 70 | name := runtime.FuncForPC(f.pc()).Name() 71 | io.WriteString(s, funcname(name)) 72 | case 'v': 73 | f.Format(s, 's') 74 | io.WriteString(s, ":") 75 | f.Format(s, 'd') 76 | } 77 | } 78 | 79 | // StackTrace is stack of Frames from innermost (newest) to outermost (oldest). 80 | type StackTrace []Frame 81 | 82 | func (st StackTrace) Format(s fmt.State, verb rune) { 83 | switch verb { 84 | case 'v': 85 | switch { 86 | case s.Flag('+'): 87 | for _, f := range st { 88 | fmt.Fprintf(s, "\n%+v", f) 89 | } 90 | case s.Flag('#'): 91 | fmt.Fprintf(s, "%#v", []Frame(st)) 92 | default: 93 | fmt.Fprintf(s, "%v", []Frame(st)) 94 | } 95 | case 's': 96 | fmt.Fprintf(s, "%s", []Frame(st)) 97 | } 98 | } 99 | 100 | // stack represents a stack of program counters. 101 | type stack []uintptr 102 | 103 | func (s *stack) Format(st fmt.State, verb rune) { 104 | switch verb { 105 | case 'v': 106 | switch { 107 | case st.Flag('+'): 108 | for _, pc := range *s { 109 | f := Frame(pc) 110 | fmt.Fprintf(st, "\n%+v", f) 111 | } 112 | } 113 | } 114 | } 115 | 116 | func (s *stack) StackTrace() StackTrace { 117 | f := make([]Frame, len(*s)) 118 | for i := 0; i < len(f); i++ { 119 | f[i] = Frame((*s)[i]) 120 | } 121 | return f 122 | } 123 | 124 | func callers() *stack { 125 | const depth = 32 126 | var pcs [depth]uintptr 127 | n := runtime.Callers(3, pcs[:]) 128 | var st stack = pcs[0:n] 129 | return &st 130 | } 131 | 132 | // funcname removes the path prefix component of a function's name reported by func.Name(). 133 | func funcname(name string) string { 134 | i := strings.LastIndex(name, "/") 135 | name = name[i+1:] 136 | i = strings.Index(name, ".") 137 | return name[i+1:] 138 | } 139 | 140 | func trimGOPATH(name, file string) string { 141 | // Here we want to get the source file path relative to the compile time 142 | // GOPATH. As of Go 1.6.x there is no direct way to know the compiled 143 | // GOPATH at runtime, but we can infer the number of path segments in the 144 | // GOPATH. We note that fn.Name() returns the function name qualified by 145 | // the import path, which does not include the GOPATH. Thus we can trim 146 | // segments from the beginning of the file path until the number of path 147 | // separators remaining is one more than the number of path separators in 148 | // the function name. For example, given: 149 | // 150 | // GOPATH /home/user 151 | // file /home/user/src/pkg/sub/file.go 152 | // fn.Name() pkg/sub.Type.Method 153 | // 154 | // We want to produce: 155 | // 156 | // pkg/sub/file.go 157 | // 158 | // From this we can easily see that fn.Name() has one less path separator 159 | // than our desired output. We count separators from the end of the file 160 | // path until it finds two more than in the function name and then move 161 | // one character forward to preserve the initial path segment without a 162 | // leading separator. 163 | const sep = "/" 164 | goal := strings.Count(name, sep) + 2 165 | i := len(file) 166 | for n := 0; n < goal; n++ { 167 | i = strings.LastIndex(file[:i], sep) 168 | if i == -1 { 169 | // not enough separators found, set i so that the slice expression 170 | // below leaves file unmodified 171 | i = -len(sep) 172 | break 173 | } 174 | } 175 | // get back to 0 or trim the leading separator 176 | file = file[i+len(sep):] 177 | return file 178 | } 179 | -------------------------------------------------------------------------------- /reader.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "io" 7 | "os" 8 | ) 9 | 10 | // Reader is the interface that wraps the operations against rdb data. 11 | // 12 | // Discard skips the next n bytes. 13 | // ReadByte reads and returns a single byte. 14 | // ReadBytes reads and returns exactly n bytes. 15 | type Reader interface { 16 | numberReader 17 | 18 | Discard(n int) 19 | ReadByte() (byte, error) 20 | ReadBytes(n int) ([]byte, error) 21 | } 22 | 23 | // numberReader is the interface that converts byte sequences into number. 24 | type numberReader interface { 25 | big32() (int, error) 26 | big64() (int, error) 27 | little16() (int, error) 28 | little32() (int, error) 29 | little64() (int, error) 30 | } 31 | 32 | // MemReader is a Reader that reads rdb data from memory. 33 | type MemReader struct { 34 | i int 35 | b []byte 36 | } 37 | 38 | // NewMemReader memory-maps the named file and returns a MemReader that reads from it. 39 | func NewMemReader(file string) (Reader, error) { 40 | b, err := mmap(file) 41 | if err != nil { 42 | return nil, err 43 | } 44 | return &MemReader{b: b}, nil 45 | } 46 | 47 | // Discard skips the next n bytes. 48 | func (r *MemReader) Discard(n int) { 49 | r.i += n 50 | } 51 | 52 | // ReadByte reads and returns a single byte. 53 | // If no byte is available, returns an error. 54 | func (r *MemReader) ReadByte() (byte, error) { 55 | if 1 > len(r.b[r.i:]) { 56 | return 0, io.ErrUnexpectedEOF 57 | } 58 | r.i++ 59 | return r.b[r.i-1], nil 60 | } 61 | 62 | // ReadBytes reads and returns exactly n bytes. 63 | // If fewer than n bytes is available, returns an error. 64 | // 65 | // NOTE: It's not safe to modify the returned slice. 66 | func (r *MemReader) ReadBytes(n int) ([]byte, error) { 67 | if n > len(r.b[r.i:]) { 68 | return nil, io.ErrUnexpectedEOF 69 | } 70 | r.i += n 71 | return r.b[r.i-n : r.i], nil 72 | } 73 | 74 | func (r *MemReader) readString(n int) (string, error) { 75 | b, err := r.ReadBytes(n) 76 | if err != nil { 77 | return "", err 78 | } 79 | return bytes2string(b), nil 80 | } 81 | 82 | // helper funcs that converts byte sequences into number. 83 | 84 | func (r *MemReader) little16() (int, error) { 85 | b, err := r.ReadBytes(2) 86 | if err != nil { 87 | return 0, err 88 | } 89 | return int(int16(binary.LittleEndian.Uint16(b))), nil 90 | } 91 | 92 | func (r *MemReader) little32() (int, error) { 93 | b, err := r.ReadBytes(4) 94 | if err != nil { 95 | return 0, err 96 | } 97 | return int(int32(binary.LittleEndian.Uint32(b))), nil 98 | } 99 | 100 | func (r *MemReader) little64() (int, error) { 101 | b, err := r.ReadBytes(8) 102 | if err != nil { 103 | return 0, err 104 | } 105 | return int(int64(binary.LittleEndian.Uint64(b))), nil 106 | } 107 | 108 | func (r *MemReader) big32() (int, error) { 109 | b, err := r.ReadBytes(4) 110 | if err != nil { 111 | return 0, err 112 | } 113 | return int(int32(binary.BigEndian.Uint32(b))), nil 114 | } 115 | 116 | func (r *MemReader) big64() (int, error) { 117 | b, err := r.ReadBytes(8) 118 | if err != nil { 119 | return 0, err 120 | } 121 | return int(int64(binary.BigEndian.Uint64(b))), nil 122 | } 123 | 124 | // BufferReader is a Reader that reads from a *bufio.Reader. 125 | type BufferReader struct { 126 | *bufio.Reader 127 | 128 | buf [8]byte 129 | file *os.File 130 | } 131 | 132 | // NewBufferReader returns a new BufferReader reading from file. 133 | // It's buffer has at least the specified size. If size == 0, use default size. 134 | func NewBufferReader(file string, size int) (Reader, error) { 135 | if size == 0 { 136 | size = 4096 137 | } 138 | f, err := os.Open(file) 139 | if err != nil { 140 | return nil, err 141 | } 142 | return &BufferReader{ 143 | file: f, 144 | Reader: bufio.NewReaderSize(f, size), 145 | }, nil 146 | } 147 | 148 | // Close closes the file. 149 | func (r *BufferReader) Close() error { 150 | if r.file != nil { 151 | f := r.file 152 | r.file = nil 153 | return f.Close() 154 | } 155 | return nil 156 | } 157 | 158 | // Discard skips the next n bytes. 159 | func (r *BufferReader) Discard(n int) { 160 | r.Reader.Discard(n) 161 | } 162 | 163 | // ReadBytes reads and returns exactly n bytes. 164 | // If ReadBytes reads fewer than n bytes, it also returns an error. 165 | func (r *BufferReader) ReadBytes(n int) ([]byte, error) { 166 | buf := make([]byte, n) 167 | _, err := io.ReadFull(r, buf) 168 | if err != nil { 169 | return nil, err 170 | } 171 | return buf, nil 172 | } 173 | 174 | // helper funcs that converts byte sequences into number. 175 | 176 | func (r *BufferReader) readBytes(n int) ([]byte, error) { 177 | _, err := io.ReadFull(r, r.buf[:n]) 178 | if err != nil { 179 | return nil, err 180 | } 181 | return r.buf[:n], nil 182 | } 183 | 184 | func (r *BufferReader) little16() (int, error) { 185 | b, err := r.readBytes(2) 186 | if err != nil { 187 | return 0, err 188 | } 189 | return int(int16(binary.LittleEndian.Uint16(b))), nil 190 | } 191 | 192 | func (r *BufferReader) little32() (int, error) { 193 | b, err := r.readBytes(4) 194 | if err != nil { 195 | return 0, err 196 | } 197 | return int(int32(binary.LittleEndian.Uint32(b))), nil 198 | } 199 | 200 | func (r *BufferReader) little64() (int, error) { 201 | b, err := r.readBytes(8) 202 | if err != nil { 203 | return 0, err 204 | } 205 | return int(int64(binary.LittleEndian.Uint64(b))), nil 206 | } 207 | 208 | func (r *BufferReader) big32() (int, error) { 209 | b, err := r.readBytes(4) 210 | if err != nil { 211 | return 0, err 212 | } 213 | return int(int32(binary.BigEndian.Uint32(b))), nil 214 | } 215 | 216 | func (r *BufferReader) big64() (int, error) { 217 | b, err := r.readBytes(8) 218 | if err != nil { 219 | return 0, err 220 | } 221 | return int(int64(binary.BigEndian.Uint64(b))), nil 222 | } 223 | -------------------------------------------------------------------------------- /cmd/rmr/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "os" 8 | "regexp" 9 | "runtime" 10 | "strconv" 11 | "time" 12 | 13 | "github.com/replit/rdb" 14 | ) 15 | 16 | var ( 17 | f filter 18 | patterns strings 19 | 20 | b = flag.Int("b", 0, "Read buffer size.") 21 | m = flag.Int64("m", 1<<30, "Maximum memory mapping size.") 22 | o = flag.String("o", "", "Output file.") 23 | ) 24 | 25 | type strings []string 26 | 27 | func (s *strings) Set(v string) error { 28 | *s = append(*s, v) 29 | return nil 30 | } 31 | 32 | func (s *strings) String() string { 33 | return "" 34 | } 35 | 36 | type ints []int 37 | 38 | func (i *ints) Set(v string) error { 39 | i32, err := strconv.ParseInt(v, 10, 32) 40 | if err != nil { 41 | return err 42 | } 43 | *i = append(*i, int(i32)) 44 | return nil 45 | } 46 | 47 | func (i *ints) String() string { 48 | return "" 49 | } 50 | 51 | type filter struct { 52 | dbs ints 53 | keys strings 54 | types strings 55 | patterns []*regexp.Regexp 56 | 57 | debug bool 58 | file string 59 | out io.Writer 60 | writeCh chan string 61 | 62 | dbAbort bool 63 | keyAbort bool 64 | } 65 | 66 | func (f *filter) error(err error) { 67 | if f.debug { 68 | fmt.Fprintf(os.Stderr, "%+v\n", err) 69 | os.Exit(2) 70 | } 71 | fmt.Fprintln(os.Stderr, err) 72 | os.Exit(2) 73 | } 74 | 75 | func (f *filter) Key(key rdb.Key) bool { 76 | if len(f.keys) == 0 && len(f.patterns) == 0 { 77 | return f.keyAbort 78 | } 79 | for i, k := range f.keys { 80 | if k == key.Key { 81 | if len(f.keys) == 1 { 82 | f.keyAbort = true 83 | } 84 | f.keys = append(f.keys[0:i], f.keys[i+1:]...) 85 | return false 86 | } 87 | } 88 | for _, p := range f.patterns { 89 | if p.MatchString(key.Key) { 90 | return false 91 | } 92 | } 93 | key.Skip(rdb.SkipAll) 94 | return false 95 | } 96 | 97 | func (f *filter) Type(typ rdb.Type) bool { 98 | if len(f.types) == 0 { 99 | return false 100 | } 101 | for _, t := range f.types { 102 | if rdb.Encoding2Type(typ.Encoding) == t { 103 | return false 104 | } 105 | } 106 | typ.Skip(rdb.SkipAll) 107 | return false 108 | } 109 | 110 | func (f *filter) Database(db rdb.DB) bool { 111 | if len(f.dbs) == 0 { 112 | return f.dbAbort 113 | } 114 | for i, d := range f.dbs { 115 | if db.Num == d { 116 | if len(f.dbs) == 1 { 117 | f.dbAbort = true 118 | } 119 | f.dbs = append(f.dbs[0:i], f.dbs[i+1:]...) 120 | return false 121 | } 122 | } 123 | db.Skip(rdb.SkipAll) 124 | return false 125 | } 126 | 127 | func (f *filter) Set(v *rdb.Set) { 128 | f.writeCh <- fmt.Sprintf( 129 | "%v,%v,%v,%v,%v", 130 | v.Key.DB, 131 | rdb.Encoding2Type(v.Key.Encoding), 132 | rdb.Encoding2String(v.Key.Encoding), 133 | strconv.Quote(v.Key.Key), 134 | v.Memory(), 135 | ) 136 | } 137 | 138 | func (f *filter) List(v *rdb.List) { 139 | f.writeCh <- fmt.Sprintf( 140 | "%v,%v,%v,%v,%v", 141 | v.Key.DB, 142 | rdb.Encoding2Type(v.Key.Encoding), 143 | rdb.Encoding2String(v.Key.Encoding), 144 | strconv.Quote(v.Key.Key), 145 | v.Memory(), 146 | ) 147 | } 148 | 149 | func (f *filter) Hash(v *rdb.Hash) { 150 | f.writeCh <- fmt.Sprintf( 151 | "%v,%v,%v,%v,%v", 152 | v.Key.DB, 153 | rdb.Encoding2Type(v.Key.Encoding), 154 | rdb.Encoding2String(v.Key.Encoding), 155 | strconv.Quote(v.Key.Key), 156 | v.Memory(), 157 | ) 158 | } 159 | 160 | func (f *filter) String(v *rdb.String) { 161 | f.writeCh <- fmt.Sprintf( 162 | "%v,%v,%v,%v,%v", 163 | v.Key.DB, 164 | rdb.Encoding2Type(v.Key.Encoding), 165 | rdb.Encoding2String(v.Key.Encoding), 166 | strconv.Quote(v.Key.Key), 167 | v.Memory(), 168 | ) 169 | } 170 | 171 | func (f *filter) SortedSet(v *rdb.SortedSet) { 172 | f.writeCh <- fmt.Sprintf( 173 | "%v,%v,%v,%v,%v", 174 | v.Key.DB, 175 | rdb.Encoding2Type(v.Key.Encoding), 176 | rdb.Encoding2String(v.Key.Encoding), 177 | strconv.Quote(v.Key.Key), 178 | v.Memory(), 179 | ) 180 | } 181 | 182 | func (f *filter) batchWrite() <-chan struct{} { 183 | wait := make(chan struct{}) 184 | f.writeCh = make(chan string, 512) 185 | f.writeCh <- "db,type,encoding,key,mem" 186 | go func() { 187 | timer := time.NewTimer(200 * time.Millisecond) 188 | defer timer.Stop() 189 | 190 | var ok bool 191 | var str, next string 192 | for { 193 | if !timer.Stop() { 194 | select { 195 | case <-timer.C: 196 | default: 197 | } 198 | } 199 | timer.Reset(200 * time.Millisecond) 200 | 201 | BATCH: 202 | for i := 0; i < 100; i++ { 203 | select { 204 | case next, ok = <-f.writeCh: 205 | if ok { 206 | str += next + "\n" 207 | } else { 208 | break BATCH 209 | } 210 | default: 211 | select { 212 | case next, ok = <-f.writeCh: 213 | if ok { 214 | str += next + "\n" 215 | } else { 216 | break BATCH 217 | } 218 | case <-timer.C: 219 | break BATCH 220 | } 221 | } 222 | } 223 | 224 | io.WriteString(f.out, str) 225 | if !ok { 226 | close(wait) 227 | return 228 | } 229 | str, next = "", "" 230 | } 231 | }() 232 | return wait 233 | } 234 | 235 | func main() { 236 | flag.Parse() 237 | if f.file == "" { 238 | fmt.Fprintf(os.Stderr, "Usage: %s [options] -f /path/to/dump.rdb\n", os.Args[0]) 239 | fmt.Fprintln(os.Stderr) 240 | fmt.Fprintf(os.Stderr, "Options:\n\n") 241 | flag.PrintDefaults() 242 | os.Exit(1) 243 | } 244 | if len(patterns) != 0 { 245 | for _, p := range patterns { 246 | f.patterns = append(f.patterns, regexp.MustCompile(p)) 247 | } 248 | } 249 | 250 | var r rdb.Reader 251 | fi, err := os.Stat(f.file) 252 | if err != nil { 253 | f.error(err) 254 | } 255 | if fi.Size() > *m { 256 | r, err = rdb.NewBufferReader(f.file, *b) 257 | } else { 258 | r, err = rdb.NewMemReader(f.file) 259 | } 260 | if err != nil { 261 | f.error(err) 262 | } 263 | 264 | f.out = os.Stdout 265 | if *o != "" { 266 | of, err := os.OpenFile(*o, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) 267 | if err != nil { 268 | f.error(err) 269 | } 270 | defer of.Close() 271 | 272 | f.out = of 273 | } 274 | 275 | wait := f.batchWrite() 276 | strategy := rdb.WithStrategy(rdb.SkipExpiry | rdb.SkipMeta | rdb.SkipValue) 277 | if err := rdb.Parse(r, rdb.WithFilter(&f), strategy); err != nil { 278 | f.error(err) 279 | } 280 | close(f.writeCh) 281 | <-wait 282 | } 283 | 284 | func init() { 285 | flag.StringVar(&f.file, "f", "", "Redis RDB file path.") 286 | flag.BoolVar(&f.debug, "d", false, "Enable debug output.") 287 | flag.Var(&f.keys, "k", "Keys to inspect. Multiple keys can provided.") 288 | flag.Var(&f.types, "t", "Types to inspect. Multiple types can provided.") 289 | flag.Var(&f.dbs, "db", "Databases to inspect. Multiple databases can provided.") 290 | flag.Var(&patterns, "p", "Key match patterns. Multiple patterns can provided.") 291 | 292 | if cpu := runtime.NumCPU(); cpu == 1 { 293 | runtime.GOMAXPROCS(3) 294 | } else { 295 | runtime.GOMAXPROCS(cpu + 1) 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /vendor/github.com/pkg/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Package errors provides simple error handling primitives. 2 | // 3 | // The traditional error handling idiom in Go is roughly akin to 4 | // 5 | // if err != nil { 6 | // return err 7 | // } 8 | // 9 | // which applied recursively up the call stack results in error reports 10 | // without context or debugging information. The errors package allows 11 | // programmers to add context to the failure path in their code in a way 12 | // that does not destroy the original value of the error. 13 | // 14 | // Adding context to an error 15 | // 16 | // The errors.Wrap function returns a new error that adds context to the 17 | // original error by recording a stack trace at the point Wrap is called, 18 | // and the supplied message. For example 19 | // 20 | // _, err := ioutil.ReadAll(r) 21 | // if err != nil { 22 | // return errors.Wrap(err, "read failed") 23 | // } 24 | // 25 | // If additional control is required the errors.WithStack and errors.WithMessage 26 | // functions destructure errors.Wrap into its component operations of annotating 27 | // an error with a stack trace and an a message, respectively. 28 | // 29 | // Retrieving the cause of an error 30 | // 31 | // Using errors.Wrap constructs a stack of errors, adding context to the 32 | // preceding error. Depending on the nature of the error it may be necessary 33 | // to reverse the operation of errors.Wrap to retrieve the original error 34 | // for inspection. Any error value which implements this interface 35 | // 36 | // type causer interface { 37 | // Cause() error 38 | // } 39 | // 40 | // can be inspected by errors.Cause. errors.Cause will recursively retrieve 41 | // the topmost error which does not implement causer, which is assumed to be 42 | // the original cause. For example: 43 | // 44 | // switch err := errors.Cause(err).(type) { 45 | // case *MyError: 46 | // // handle specifically 47 | // default: 48 | // // unknown error 49 | // } 50 | // 51 | // causer interface is not exported by this package, but is considered a part 52 | // of stable public API. 53 | // 54 | // Formatted printing of errors 55 | // 56 | // All error values returned from this package implement fmt.Formatter and can 57 | // be formatted by the fmt package. The following verbs are supported 58 | // 59 | // %s print the error. If the error has a Cause it will be 60 | // printed recursively 61 | // %v see %s 62 | // %+v extended format. Each Frame of the error's StackTrace will 63 | // be printed in detail. 64 | // 65 | // Retrieving the stack trace of an error or wrapper 66 | // 67 | // New, Errorf, Wrap, and Wrapf record a stack trace at the point they are 68 | // invoked. This information can be retrieved with the following interface. 69 | // 70 | // type stackTracer interface { 71 | // StackTrace() errors.StackTrace 72 | // } 73 | // 74 | // Where errors.StackTrace is defined as 75 | // 76 | // type StackTrace []Frame 77 | // 78 | // The Frame type represents a call site in the stack trace. Frame supports 79 | // the fmt.Formatter interface that can be used for printing information about 80 | // the stack trace of this error. For example: 81 | // 82 | // if err, ok := err.(stackTracer); ok { 83 | // for _, f := range err.StackTrace() { 84 | // fmt.Printf("%+s:%d", f) 85 | // } 86 | // } 87 | // 88 | // stackTracer interface is not exported by this package, but is considered a part 89 | // of stable public API. 90 | // 91 | // See the documentation for Frame.Format for more details. 92 | package errors 93 | 94 | import ( 95 | "fmt" 96 | "io" 97 | ) 98 | 99 | // New returns an error with the supplied message. 100 | // New also records the stack trace at the point it was called. 101 | func New(message string) error { 102 | return &fundamental{ 103 | msg: message, 104 | stack: callers(), 105 | } 106 | } 107 | 108 | // Errorf formats according to a format specifier and returns the string 109 | // as a value that satisfies error. 110 | // Errorf also records the stack trace at the point it was called. 111 | func Errorf(format string, args ...interface{}) error { 112 | return &fundamental{ 113 | msg: fmt.Sprintf(format, args...), 114 | stack: callers(), 115 | } 116 | } 117 | 118 | // fundamental is an error that has a message and a stack, but no caller. 119 | type fundamental struct { 120 | msg string 121 | *stack 122 | } 123 | 124 | func (f *fundamental) Error() string { return f.msg } 125 | 126 | func (f *fundamental) Format(s fmt.State, verb rune) { 127 | switch verb { 128 | case 'v': 129 | if s.Flag('+') { 130 | io.WriteString(s, f.msg) 131 | f.stack.Format(s, verb) 132 | return 133 | } 134 | fallthrough 135 | case 's': 136 | io.WriteString(s, f.msg) 137 | case 'q': 138 | fmt.Fprintf(s, "%q", f.msg) 139 | } 140 | } 141 | 142 | // WithStack annotates err with a stack trace at the point WithStack was called. 143 | // If err is nil, WithStack returns nil. 144 | func WithStack(err error) error { 145 | if err == nil { 146 | return nil 147 | } 148 | return &withStack{ 149 | err, 150 | callers(), 151 | } 152 | } 153 | 154 | type withStack struct { 155 | error 156 | *stack 157 | } 158 | 159 | func (w *withStack) Cause() error { return w.error } 160 | 161 | func (w *withStack) Format(s fmt.State, verb rune) { 162 | switch verb { 163 | case 'v': 164 | if s.Flag('+') { 165 | fmt.Fprintf(s, "%+v", w.Cause()) 166 | w.stack.Format(s, verb) 167 | return 168 | } 169 | fallthrough 170 | case 's': 171 | io.WriteString(s, w.Error()) 172 | case 'q': 173 | fmt.Fprintf(s, "%q", w.Error()) 174 | } 175 | } 176 | 177 | // Wrap returns an error annotating err with a stack trace 178 | // at the point Wrap is called, and the supplied message. 179 | // If err is nil, Wrap returns nil. 180 | func Wrap(err error, message string) error { 181 | if err == nil { 182 | return nil 183 | } 184 | err = &withMessage{ 185 | cause: err, 186 | msg: message, 187 | } 188 | return &withStack{ 189 | err, 190 | callers(), 191 | } 192 | } 193 | 194 | // Wrapf returns an error annotating err with a stack trace 195 | // at the point Wrapf is call, and the format specifier. 196 | // If err is nil, Wrapf returns nil. 197 | func Wrapf(err error, format string, args ...interface{}) error { 198 | if err == nil { 199 | return nil 200 | } 201 | err = &withMessage{ 202 | cause: err, 203 | msg: fmt.Sprintf(format, args...), 204 | } 205 | return &withStack{ 206 | err, 207 | callers(), 208 | } 209 | } 210 | 211 | // WithMessage annotates err with a new message. 212 | // If err is nil, WithMessage returns nil. 213 | func WithMessage(err error, message string) error { 214 | if err == nil { 215 | return nil 216 | } 217 | return &withMessage{ 218 | cause: err, 219 | msg: message, 220 | } 221 | } 222 | 223 | type withMessage struct { 224 | cause error 225 | msg string 226 | } 227 | 228 | func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() } 229 | func (w *withMessage) Cause() error { return w.cause } 230 | 231 | func (w *withMessage) Format(s fmt.State, verb rune) { 232 | switch verb { 233 | case 'v': 234 | if s.Flag('+') { 235 | fmt.Fprintf(s, "%+v\n", w.Cause()) 236 | io.WriteString(s, w.msg) 237 | return 238 | } 239 | fallthrough 240 | case 's', 'q': 241 | io.WriteString(s, w.Error()) 242 | } 243 | } 244 | 245 | // Cause returns the underlying cause of the error, if possible. 246 | // An error value has a cause if it implements the following 247 | // interface: 248 | // 249 | // type causer interface { 250 | // Cause() error 251 | // } 252 | // 253 | // If the error does not implement Cause, the original error will 254 | // be returned. If the error is nil, nil will be returned without further 255 | // investigation. 256 | func Cause(err error) error { 257 | type causer interface { 258 | Cause() error 259 | } 260 | 261 | for err != nil { 262 | cause, ok := err.(causer) 263 | if !ok { 264 | break 265 | } 266 | err = cause.Cause() 267 | } 268 | return err 269 | } 270 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "math/rand" 5 | "sort" 6 | "time" 7 | "unsafe" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // Encoding2Type returns the actual redis type of encoding. 13 | func Encoding2Type(encoding byte) string { 14 | switch encoding { 15 | case EncodingString: 16 | return TypeString 17 | case EncodingList, EncodingZiplist, EncodingQuicklist: 18 | return TypeList 19 | case EncodingSet, EncodingIntset: 20 | return TypeSet 21 | case EncodingSortedSet, EncodingSortedSet2, EncodingSortedSetZip: 22 | return TypeSortedSet 23 | case EncodingHash, EncodingZipmap, EncodingHashZip: 24 | return TypeHash 25 | } 26 | return "unknown" 27 | } 28 | 29 | // Encoding2String returns the string format of encoding. 30 | func Encoding2String(encoding byte) string { 31 | switch encoding { 32 | case EncodingString: 33 | return "string" 34 | case EncodingList: 35 | return "linkedlist" 36 | case EncodingZiplist: 37 | return "ziplist" 38 | case EncodingQuicklist: 39 | return "quicklist" 40 | case EncodingSet: 41 | return "hashtable" 42 | case EncodingIntset: 43 | return "intset" 44 | case EncodingSortedSet: 45 | return "skiplist" 46 | case EncodingSortedSet2: 47 | return "skiplist" 48 | case EncodingSortedSetZip: 49 | return "ziplist" 50 | case EncodingHash: 51 | return "hashtable" 52 | case EncodingZipmap: 53 | return "zipmap" 54 | case EncodingHashZip: 55 | return "ziplist" 56 | } 57 | return "unknown" 58 | } 59 | 60 | func bytes2string(b []byte) string { 61 | return *(*string)(unsafe.Pointer(&b)) 62 | } 63 | 64 | func readLZF(buf []byte, clen, ulen int) ([]byte, error) { 65 | if clen > len(buf) { 66 | return nil, nil 67 | } 68 | 69 | ip, op := 0, 0 70 | out := make([]byte, ulen) 71 | for ip < clen { 72 | ctrl := int(buf[ip]) 73 | ip++ 74 | 75 | if ctrl < (1 << 5) { 76 | ctrl++ 77 | if ctrl+ip > clen || ctrl+op > ulen { 78 | return nil, errors.WithStack(ErrInvalidCompressedData) 79 | } 80 | 81 | copy(out[op:op+ctrl], buf[ip:ip+ctrl]) 82 | ip += ctrl 83 | op += ctrl 84 | continue 85 | } 86 | 87 | length := ctrl >> 5 88 | ref := op - ((ctrl & 0x1f) << 8) - 1 89 | 90 | if length == 7 { 91 | length += int(buf[ip]) 92 | ip++ 93 | 94 | if ip >= clen { 95 | return nil, errors.WithStack(ErrInvalidCompressedData) 96 | } 97 | } 98 | 99 | ref -= int(buf[ip]) 100 | ip++ 101 | length += 2 102 | if op+length > ulen || ref < 0 { 103 | return nil, errors.WithStack(ErrInvalidCompressedData) 104 | } 105 | 106 | for i := 0; i < length; i++ { 107 | out[op+i] = out[ref+i] 108 | } 109 | op += length 110 | } 111 | 112 | return out[:op], nil 113 | } 114 | 115 | func readZipmapEntry(r *MemReader, value bool) (string, error) { 116 | b, err := r.ReadByte() 117 | if err != nil { 118 | return "", err 119 | } 120 | 121 | switch { 122 | case b < 254: 123 | if value { 124 | n, err := r.ReadByte() 125 | if err != nil { 126 | return "", err 127 | } 128 | if n > 0 { 129 | r.Discard(int(n)) 130 | } 131 | } 132 | return r.readString(int(b)) 133 | case b == 254: 134 | l, err := r.little32() 135 | if err != nil { 136 | return "", err 137 | } 138 | if value { 139 | n, err := r.ReadByte() 140 | if err != nil { 141 | return "", err 142 | } 143 | if n > 0 { 144 | r.Discard(int(n)) 145 | } 146 | } 147 | return r.readString(l) 148 | default: 149 | return "", errors.WithStack(ErrInvalidZipmapEntry) 150 | } 151 | } 152 | 153 | type overhead struct{} 154 | 155 | func (o overhead) alloc(l int) uint64 { 156 | switch { 157 | case l < 1<<5: 158 | return o.jemalloc(l + 2) 159 | case l < 1<<8: 160 | return o.jemalloc(l + 2 + 2) 161 | case l < 1<<16: 162 | return o.jemalloc(l + 2 + 4) 163 | case l < 1<<32: 164 | return o.jemalloc(l + 2 + 8) 165 | default: 166 | return o.jemalloc(l + 2 + 16) 167 | } 168 | } 169 | 170 | func (o overhead) top(expiry int) uint64 { 171 | // Each top level object is an entry in a dictionary, and so we have to include 172 | // the overhead of a dictionary entry 173 | return o.hashEntry() + o.root() + o.expiry(expiry) 174 | } 175 | 176 | func (o overhead) hashEntry() uint64 { 177 | // See https://github.com/antirez/redis/blob/unstable/src/dict.h 178 | // Each dictEntry has 2 pointers + int64 179 | return 2*o.arch() + 8 180 | } 181 | 182 | func (o overhead) root() uint64 { 183 | return o.arch() + 8 184 | } 185 | 186 | func (o overhead) expiry(expiry int) uint64 { 187 | if expiry < 0 { 188 | return 0 189 | } 190 | // Key expiry is stored in a hashtable, so we have to pay for the cost of a hashtable entry 191 | // The timestamp itself is stored as an int64, which is a 8 bytes 192 | return o.hashEntry() + 8 193 | } 194 | 195 | func (o overhead) skiplist(size int) uint64 { 196 | return 2*o.arch() + o.hash(size) + 2*o.arch() + 16 197 | } 198 | 199 | func (o overhead) skiplistEntry() uint64 { 200 | return o.hashEntry() + 2*o.arch() + 8 + (o.arch()+8)*o.zsetRandomLevel() 201 | } 202 | 203 | func (o overhead) zsetRandomLevel() uint64 { 204 | level := uint64(1) 205 | rint := rand.Intn(0xFFFF) 206 | max := 0.25 * 0xFFFF 207 | for float64(rint) < max { 208 | level++ 209 | rint = rand.Intn(0xFFFF) 210 | } 211 | if level < 32 { 212 | return level 213 | } 214 | return 32 215 | } 216 | 217 | func (o overhead) hash(size int) uint64 { 218 | return 4 + 7*o.arch() + 4*o.arch() + uint64(float64(o.nextpow(size)*o.arch())*1.5) 219 | } 220 | 221 | func (o overhead) nextpow(size int) uint64 { 222 | power := uint64(1) 223 | for power <= uint64(size) { 224 | power <<= 1 225 | } 226 | return power 227 | } 228 | 229 | func (o overhead) quicklist(size int) uint64 { 230 | return 2*o.arch() + o.arch() + 2*4 + (4*o.arch()+o.arch()+2*4)*uint64(size) 231 | } 232 | 233 | func (o overhead) linkedlist() uint64 { 234 | return o.arch() + 5*o.arch() 235 | } 236 | 237 | func (o overhead) linkedlistEntry() uint64 { 238 | return 3 * o.arch() 239 | } 240 | 241 | func (o overhead) arch() uint64 { 242 | // 64bit only 243 | return 8 244 | } 245 | 246 | func (o overhead) jemalloc(size int) uint64 { 247 | i := sort.Search(len(allocSpec), func(i int) bool { return allocSpec[i] >= uint64(size) }) 248 | if i > len(allocSpec) { 249 | return uint64(size) 250 | } 251 | return allocSpec[i] 252 | } 253 | 254 | var allocSpec = []uint64{ 255 | 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512, 640, 768, 896, 1024, 256 | 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, 20480, 24576, 257 | 28672, 32768, 40960, 49152, 57344, 65536, 81920, 98304, 114688, 131072, 163840, 196608, 229376, 262144, 327680, 258 | 393216, 458752, 524288, 655360, 786432, 917504, 1048576, 1310720, 1572864, 1835008, 2097152, 2621440, 3145728, 259 | 3670016, 4194304, 5242880, 6291456, 7340032, 8388608, 10485760, 12582912, 14680064, 16777216, 20971520, 25165824, 260 | 29360128, 33554432, 41943040, 50331648, 58720256, 67108864, 83886080, 100663296, 117440512, 134217728, 167772160, 261 | 201326592, 234881024, 268435456, 335544320, 402653184, 469762048, 536870912, 671088640, 805306368, 939524096, 262 | 1073741824, 1342177280, 1610612736, 1879048192, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 263 | 5368709120, 6442450944, 7516192768, 8589934592, 10737418240, 12884901888, 15032385536, 17179869184, 21474836480, 264 | 25769803776, 30064771072, 34359738368, 42949672960, 51539607552, 60129542144, 68719476736, 85899345920, 265 | 103079215104, 120259084288, 137438953472, 171798691840, 206158430208, 240518168576, 274877906944, 343597383680, 266 | 412316860416, 481036337152, 549755813888, 687194767360, 824633720832, 962072674304, 1099511627776, 1374389534720, 267 | 1649267441664, 1924145348608, 2199023255552, 2748779069440, 3298534883328, 3848290697216, 4398046511104, 268 | 5497558138880, 6597069766656, 7696581394432, 8796093022208, 10995116277760, 13194139533312, 15393162788864, 269 | 17592186044416, 21990232555520, 26388279066624, 30786325577728, 35184372088832, 43980465111040, 52776558133248, 270 | 61572651155456, 70368744177664, 87960930222080, 105553116266496, 123145302310912, 140737488355328, 175921860444160, 271 | 211106232532992, 246290604621824, 281474976710656, 351843720888320, 422212465065984, 492581209243648, 272 | 562949953421312, 703687441776640, 844424930131968, 985162418487296, 1125899906842624, 1407374883553280, 273 | 1688849860263936, 1970324836974592, 2251799813685248, 2814749767106560, 3377699720527872, 3940649673949184, 274 | 4503599627370496, 5629499534213120, 6755399441055744, 7881299347898368, 9007199254740992, 11258999068426240, 275 | 13510798882111488, 15762598695796736, 18014398509481984, 22517998136852480, 27021597764222976, 31525197391593472, 276 | 36028797018963968, 45035996273704960, 54043195528445952, 63050394783186944, 72057594037927936, 90071992547409920, 277 | 108086391056891904, 126100789566373888, 144115188075855872, 180143985094819840, 216172782113783808, 278 | 252201579132747776, 288230376151711744, 360287970189639680, 432345564227567616, 504403158265495552, 279 | 576460752303423488, 720575940379279360, 864691128455135232, 1008806316530991104, 1152921504606846976, 280 | 1441151880758558720, 1729382256910270464, 2017612633061982208, 2305843009213693952, 2882303761517117440, 281 | 3458764513820540928, 4035225266123964416, 4611686018427387904, 5764607523034234880, 6917529027641081856, 282 | 8070450532247928832, 9223372036854775808, 11529215046068469760, 13835058055282163712, 16140901064495857664, 283 | } 284 | 285 | var _overhead overhead 286 | 287 | func init() { 288 | rand.Seed(time.Now().Unix()) 289 | } 290 | -------------------------------------------------------------------------------- /type.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "strconv" 5 | "sync" 6 | ) 7 | 8 | // Redis value encodings. 9 | const ( 10 | EncodingString = 0 11 | EncodingList = 1 12 | EncodingSet = 2 13 | EncodingSortedSet = 3 14 | EncodingHash = 4 15 | EncodingSortedSet2 = 5 16 | EncodingZipmap = 9 17 | EncodingZiplist = 10 18 | EncodingIntset = 11 19 | EncodingSortedSetZip = 12 20 | EncodingHashZip = 13 21 | EncodingQuicklist = 14 22 | ) 23 | 24 | // Redis types. 25 | const ( 26 | TypeSet = "set" 27 | TypeList = "list" 28 | TypeHash = "hash" 29 | TypeString = "string" 30 | TypeSortedSet = "sortedset" 31 | ) 32 | 33 | // A Filter controls the parser's behaviors. 34 | // 35 | // NOTE: Filter is not safe for concurrent use. 36 | type Filter interface { 37 | Key(key Key) bool 38 | Type(typ Type) bool 39 | Database(db DB) bool 40 | 41 | Set(s *Set) 42 | List(l *List) 43 | Hash(h *Hash) 44 | String(s *String) 45 | SortedSet(s *SortedSet) 46 | } 47 | 48 | // Key represents a redis key. 49 | type Key struct { 50 | Encoding byte 51 | DB int 52 | Expiry int 53 | Idle int 54 | Freq int 55 | Key string 56 | 57 | p *Parser 58 | memory uint64 59 | } 60 | 61 | // Skip sets next item's skipping strategy. 62 | func (k Key) Skip(strategy int) { 63 | k.p.strategy.running = strategy 64 | } 65 | 66 | // Type represents a redis type. 67 | type Type struct { 68 | Encoding byte 69 | 70 | p *Parser 71 | } 72 | 73 | // Skip sets next item's skipping strategy. 74 | func (t Type) Skip(strategy int) { 75 | t.p.strategy.running = strategy 76 | } 77 | 78 | // DB represents a redis database. 79 | type DB struct { 80 | p *Parser 81 | 82 | Num int 83 | } 84 | 85 | // Skip sets next database's skip strategy. 86 | func (db DB) Skip(strategy int) { 87 | db.p.setStrategy(strategy) 88 | } 89 | 90 | // Set represents redis set. 91 | type Set struct { 92 | Key Key 93 | Values map[interface{}]struct{} 94 | memory uint64 95 | } 96 | 97 | // Memory reports memory used by s. 98 | func (s Set) Memory() uint64 { 99 | return s.Key.memory + s.memory 100 | } 101 | 102 | // List represents redis list. 103 | type List struct { 104 | Key Key 105 | Values []string 106 | memory uint64 107 | } 108 | 109 | // Memory reports memory used by l. 110 | func (l List) Memory() uint64 { 111 | return l.Key.memory + l.memory 112 | } 113 | 114 | // Hash represents redis hash. 115 | type Hash struct { 116 | Key Key 117 | Values map[string]string 118 | memory uint64 119 | } 120 | 121 | // Memory reports memory used by l. 122 | func (h Hash) Memory() uint64 { 123 | return h.Key.memory + h.memory 124 | } 125 | 126 | // String represents redis sds. 127 | type String struct { 128 | Key Key 129 | Value string 130 | memory uint64 131 | } 132 | 133 | // Memory reports memory used by s. 134 | func (s String) Memory() uint64 { 135 | return s.Key.memory + s.memory 136 | } 137 | 138 | // SortedSet represents redis sortedset. 139 | type SortedSet struct { 140 | Key Key 141 | Values map[string]float64 142 | memory uint64 143 | } 144 | 145 | // Memory reports memory used by ss. 146 | func (ss SortedSet) Memory() uint64 { 147 | return ss.Key.memory + ss.memory 148 | } 149 | 150 | var ( 151 | redisTypePool = &sync.Pool{ 152 | New: func() interface{} { 153 | return new(redisType) 154 | }, 155 | } 156 | valuePool = &sync.Pool{ 157 | New: func() interface{} { 158 | return new(value) 159 | }, 160 | } 161 | ) 162 | 163 | type redisType struct { 164 | key Key 165 | values []*value 166 | 167 | i interface{} 168 | } 169 | 170 | func (rt *redisType) reset() { 171 | i := rt.i 172 | rt.i = nil 173 | for _, value := range rt.values { 174 | value.reset() 175 | } 176 | redisTypePool.Put(i) 177 | } 178 | 179 | func (rt *redisType) decompress() (err error) { 180 | for _, v := range rt.values { 181 | if v.c { 182 | v.b, err = readLZF(v.b, len(v.b), v.l) 183 | if err != nil { 184 | return err 185 | } 186 | } 187 | } 188 | return nil 189 | } 190 | 191 | func (rt *redisType) set(set *Set) error { 192 | set.memory = 0 193 | set.Key = rt.key 194 | set.Values = make(map[interface{}]struct{}) 195 | switch set.Key.Encoding { 196 | case EncodingSet: 197 | set.memory += _overhead.hash(len(rt.values)) 198 | values := rt.values 199 | for i := 0; i < len(values); i++ { 200 | set.memory += values[i].m + _overhead.hashEntry() + _overhead.root() 201 | if b := values[i].b; b != nil { 202 | set.Values[bytes2string(b)] = struct{}{} 203 | } 204 | } 205 | case EncodingIntset: 206 | set.memory += uint64(rt.values[0].l) 207 | inset, err := rt.values[0].readIntset() 208 | if err != nil { 209 | return err 210 | } 211 | for _, v := range inset { 212 | set.Values[v] = struct{}{} 213 | } 214 | } 215 | return nil 216 | } 217 | 218 | func (rt *redisType) list(list *List) (err error) { 219 | list.memory = 0 220 | list.Key = rt.key 221 | switch list.Key.Encoding { 222 | case EncodingList: 223 | list.Values = make([]string, len(rt.values)) 224 | values := rt.values 225 | list.memory += _overhead.linkedlist() 226 | for i := 0; i < len(values); i++ { 227 | list.memory += values[i].m + _overhead.linkedlistEntry() + _overhead.root() 228 | if b := values[i].b; b != nil { 229 | list.Values[i] = bytes2string(b) 230 | } 231 | } 232 | case EncodingZiplist: 233 | list.memory += uint64(rt.values[0].l) 234 | list.Values, err = rt.values[0].readZiplist() 235 | if err != nil { 236 | return err 237 | } 238 | case EncodingQuicklist: 239 | list.memory += _overhead.quicklist(len(rt.values)) 240 | for _, value := range rt.values { 241 | list.memory += uint64(value.l) 242 | values, err := value.readZiplist() 243 | if err != nil { 244 | return err 245 | } 246 | list.Values = append(list.Values, values...) 247 | } 248 | } 249 | return nil 250 | } 251 | 252 | func (rt *redisType) hash(hash *Hash) error { 253 | hash.memory = 0 254 | hash.Key = rt.key 255 | hash.Values = make(map[string]string) 256 | switch hash.Key.Encoding { 257 | case EncodingHashZip: 258 | hash.memory += uint64(rt.values[0].l) 259 | values, err := rt.values[0].readZiplist() 260 | if err != nil { 261 | return err 262 | } 263 | for i := 0; i < len(values); i += 2 { 264 | hash.Values[values[i]] = values[i+1] 265 | } 266 | case EncodingZipmap: 267 | hash.memory += uint64(rt.values[0].l) 268 | values, err := rt.values[0].readZipmap() 269 | if err != nil { 270 | return err 271 | } 272 | for i := 0; i < len(values); i += 2 { 273 | hash.Values[values[i]] = values[i+1] 274 | } 275 | case EncodingHash: 276 | hash.memory += _overhead.hash(len(rt.values) / 2) 277 | values := rt.values 278 | for i := 0; i < len(values); i += 2 { 279 | hash.memory += values[i].m + values[i+1].m + _overhead.hashEntry() + 2*_overhead.root() 280 | if k, v := values[i].b, values[i+1].b; k != nil && v != nil { 281 | hash.Values[bytes2string(k)] = bytes2string(v) 282 | } 283 | } 284 | } 285 | return nil 286 | } 287 | 288 | func (rt *redisType) string(s *String) *String { 289 | s.memory = 0 290 | s.Key = rt.key 291 | if b := rt.values[0].b; b != nil { 292 | s.Value = bytes2string(b) 293 | } 294 | s.memory = rt.values[0].m 295 | return s 296 | } 297 | 298 | func (rt *redisType) sortedset(ss *SortedSet) error { 299 | ss.memory = 0 300 | ss.Key = rt.key 301 | ss.Values = make(map[string]float64) 302 | switch ss.Key.Encoding { 303 | case EncodingSortedSet, EncodingSortedSet2: 304 | ss.memory += _overhead.skiplist(len(rt.values) / 2) 305 | values := rt.values 306 | for i := 0; i < len(values); i += 2 { 307 | ss.memory += values[i].m + 8 + _overhead.root() + _overhead.skiplistEntry() 308 | if k := values[i].b; k != nil { 309 | ss.Values[bytes2string(k)] = values[i+1].f 310 | } 311 | } 312 | case EncodingSortedSetZip: 313 | ss.memory += uint64(rt.values[0].l) 314 | values, err := rt.values[0].readZiplist() 315 | if err != nil { 316 | return err 317 | } 318 | for i := 0; i < len(values); i += 2 { 319 | f, err := strconv.ParseFloat(values[i+1], 64) 320 | if err != nil { 321 | return err 322 | } 323 | ss.Values[values[i]] = f 324 | } 325 | } 326 | return nil 327 | } 328 | 329 | type value struct { 330 | c bool 331 | l int 332 | m uint64 333 | f float64 334 | b []byte 335 | i interface{} 336 | } 337 | 338 | func newValue(c bool, l int, m uint64, b []byte) *value { 339 | i := valuePool.Get() 340 | v := i.(*value) 341 | v.c = c 342 | v.l = l 343 | v.m = m 344 | v.b = b 345 | v.i = i 346 | return v 347 | } 348 | 349 | func newDoubleValue(f float64) *value { 350 | i := valuePool.Get() 351 | v := i.(*value) 352 | v.m = 8 353 | v.f = f 354 | return v 355 | } 356 | 357 | func (v *value) reset() { 358 | i := v.i 359 | v.l = 0 360 | v.m = 0 361 | v.b = nil 362 | v.c = false 363 | valuePool.Put(i) 364 | } 365 | 366 | func (v *value) readZiplist() ([]string, error) { 367 | if v.b == nil { 368 | return nil, nil 369 | } 370 | 371 | // 372 | 373 | r := &MemReader{b: v.b} 374 | // zlbytes: 4 byte unsigned integer in little endian format 375 | r.Discard(4) 376 | 377 | // zltail: 4 byte unsigned integer in little endian format 378 | r.Discard(4) 379 | 380 | // zllen: 2 byte unsigned integer in little endian format 381 | zllen, err := r.little16() 382 | if err != nil { 383 | return nil, err 384 | } 385 | 386 | values := make([]string, zllen) 387 | for j := 0; j < int(zllen); j++ { 388 | // 389 | b, err := r.ReadByte() 390 | if err != nil { 391 | return nil, err 392 | } 393 | if b == 0xfe { 394 | // ignore length-prev-entry 395 | // if b == 254, next 4 bytes are used to store the length 396 | r.Discard(4) 397 | } 398 | 399 | first, err := r.ReadByte() 400 | if err != nil { 401 | return nil, err 402 | } 403 | switch first >> 6 { 404 | case 0: 405 | // 00: 6 bits string value length 406 | values[j], err = r.readString(int(first) & 0x3f) 407 | if err != nil { 408 | return nil, err 409 | } 410 | case 1: 411 | // 01: 14 bits string value length 412 | second, err := r.ReadByte() 413 | if err != nil { 414 | return nil, err 415 | } 416 | l := int(second) | (int(first)&0x3f)<<8 417 | values[j], err = r.readString(l) 418 | if err != nil { 419 | return nil, err 420 | } 421 | case 2: 422 | // 10: 4 bytes string value length 423 | l, err := r.big32() 424 | if err != nil { 425 | return nil, err 426 | } 427 | values[j], err = r.readString(l) 428 | if err != nil { 429 | return nil, err 430 | } 431 | case 3: 432 | switch (first >> 4) & 0x03 { 433 | case 0: 434 | // 1100: 2 bytes as a 16 bit signed integer 435 | i16, err := r.little16() 436 | if err != nil { 437 | return nil, err 438 | } 439 | values[j] = strconv.Itoa(i16) 440 | case 1: 441 | // 1101: 4 bytes as a 32 bit signed integer 442 | i32, err := r.little32() 443 | if err != nil { 444 | return nil, err 445 | } 446 | values[j] = strconv.Itoa(i32) 447 | case 2: 448 | // 1110: 8 bytes as a 64 bit signed integer 449 | i64, err := r.little64() 450 | if err != nil { 451 | return nil, err 452 | } 453 | values[j] = strconv.Itoa(i64) 454 | } 455 | fallthrough 456 | default: 457 | switch { 458 | case first == 240: 459 | // 11110000: 3 bytes as a 24 bit signed integer 460 | bs, err := r.ReadBytes(3) 461 | if err != nil { 462 | return nil, err 463 | } 464 | i32 := uint32(bs[2])<<24 | uint32(bs[1])<<16 | uint32(bs[0])<<8 465 | values[j] = strconv.Itoa(int(int32(i32) >> 8)) 466 | case first == 254: 467 | // 11111110: 1 bytes as an 8 bit signed integer 468 | b, err = r.ReadByte() 469 | if err != nil { 470 | return nil, err 471 | } 472 | values[j] = strconv.Itoa(int(int8(b))) 473 | case first >= 241 && first <= 253: 474 | // 1111xxxx: 4 bit unsigned integer(0 - 12) 475 | values[j] = strconv.Itoa(int(first) - 241) 476 | } 477 | } 478 | } 479 | // zlend: always 255 480 | 481 | return values, nil 482 | } 483 | 484 | func (v *value) readZipmap() ([]string, error) { 485 | if v.b == nil { 486 | return nil, nil 487 | } 488 | 489 | // "foo""bar""hello""world" 490 | 491 | r := &MemReader{b: v.b} 492 | zmlen, err := r.ReadByte() 493 | if err != nil { 494 | return nil, err 495 | } 496 | 497 | var str string 498 | var values []string 499 | if zmlen <= 254 { 500 | values = make([]string, zmlen*2) 501 | } else { 502 | values = make([]string, 512*2) 503 | } 504 | for { 505 | str, err = readZipmapEntry(r, false) 506 | if err != nil { 507 | return nil, err 508 | } 509 | values = append(values, str) 510 | 511 | str, err = readZipmapEntry(r, true) 512 | if err != nil { 513 | return nil, err 514 | } 515 | values = append(values, str) 516 | 517 | if r.i < len(r.b) && r.b[r.i] == 255 { 518 | // zmend: always 255 519 | return values, nil 520 | } 521 | } 522 | } 523 | 524 | func (v *value) readIntset() ([]int, error) { 525 | if v.b == nil { 526 | return nil, nil 527 | } 528 | 529 | // 530 | 531 | r := &MemReader{b: v.b} 532 | // encoding: 32 bit unsigned integer 533 | encoding, err := r.little32() 534 | if err != nil { 535 | return nil, err 536 | } 537 | 538 | // length-of-contents: 32 bit unsigned integer 539 | lengthOfContents, err := r.little32() 540 | if err != nil { 541 | return nil, err 542 | } 543 | 544 | values := make([]int, lengthOfContents) 545 | for j := 0; j < int(lengthOfContents); j++ { 546 | switch encoding { 547 | case 2: 548 | values[j], err = r.little16() 549 | case 4: 550 | values[j], err = r.little32() 551 | case 8: 552 | values[j], err = r.little64() 553 | } 554 | if err != nil { 555 | return nil, err 556 | } 557 | } 558 | return values, nil 559 | } 560 | -------------------------------------------------------------------------------- /rdb.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | stderr "errors" 7 | "fmt" 8 | "io" 9 | "log" 10 | "math" 11 | "regexp" 12 | "runtime" 13 | "strconv" 14 | "sync" 15 | 16 | "github.com/pkg/errors" 17 | ) 18 | 19 | const ( 20 | tokenIdle = 0xF8 // number of seconds since the object stored at the specified key is idle 21 | tokenFreq = 0xF9 // logarithmic access frequency counter of the object stored at the specified key 22 | tokenAUX = 0xFA // information about the RDB generated 23 | tokenResize = 0xFB // hint about the size of the keys in the currently selected database 24 | tokenExpMSec = 0xFC // expiry time in ms 25 | tokenExpSec = 0xFD // expiry time in seconds 26 | tokenDB = 0xFE // database selector 27 | tokenEOF = 0xFF // end of RDB file 28 | ) 29 | 30 | // Parse strategies 31 | const ( 32 | SkipMeta = 1 << iota 33 | SkipExpiry 34 | SkipIdle 35 | SkipFreq 36 | SkipValue 37 | SkipAll 38 | ) 39 | 40 | // Parse errors 41 | var ( 42 | ErrInvalidRDB = stderr.New("Invalid RDB file") 43 | ErrUnsupportedRDB = stderr.New("Unsupported RDB version") 44 | ErrInvalidZipmapEntry = stderr.New("Invalid zipmap entry") 45 | ErrInvalidLengthEncoding = stderr.New("Invalid length encoding") 46 | ErrInvalidCompressedData = stderr.New("Invalid compressed data") 47 | ) 48 | 49 | // ParseOption configures the behaviors when parsing a rdb file. 50 | type ParseOption func(*Parser) 51 | 52 | // WithFilter returns a ParseOption which sets a parse filter. 53 | func WithFilter(filter Filter) ParseOption { 54 | return func(p *Parser) { 55 | p.filter = filter 56 | } 57 | } 58 | 59 | // WithStrategy returns a ParseOption which sets default parse strategy. 60 | func WithStrategy(strategy int) ParseOption { 61 | return func(p *Parser) { 62 | p.setStrategy(strategy) 63 | } 64 | } 65 | 66 | const ( 67 | filterBufferSize = 512 68 | ) 69 | 70 | // EnableSync returns a ParseOption which disable async filtering. 71 | func EnableSync() ParseOption { 72 | return func(p *Parser) { 73 | p.sync = make(chan *redisType, filterBufferSize) 74 | } 75 | } 76 | 77 | // state represents parser's current state. 78 | type state struct { 79 | skip bool // skipping current key or value? 80 | compressed bool // current key or value is compressed? 81 | memory uint64 // current key or value memory usage 82 | } 83 | 84 | // state represents parser's skipping strategy. 85 | type strategy struct { 86 | running int 87 | global int 88 | } 89 | 90 | // Parser represents a Redis RDB parser. 91 | type Parser struct { 92 | sync.WaitGroup 93 | Reader 94 | 95 | version string 96 | filter Filter 97 | state state 98 | strategy strategy 99 | 100 | sync chan *redisType 101 | async chan *redisType 102 | 103 | err chan error 104 | sizeint uint64 105 | } 106 | 107 | // Parse parses a Redis RDB file. 108 | func Parse(r Reader, opts ...ParseOption) error { 109 | p := new(Parser) 110 | p.Reader = r 111 | p.err = make(chan error, 1) 112 | p.sizeint = 8 113 | 114 | // "REDIS" string 115 | magic, err := p.readString(5) 116 | if err != nil { 117 | return err 118 | } 119 | if magic != "REDIS" { 120 | return errors.WithStack(ErrInvalidRDB) 121 | } 122 | 123 | version, err := p.readString(4) 124 | if err != nil { 125 | return err 126 | } 127 | v, err := strconv.Atoi(version) 128 | if err != nil { 129 | return err 130 | } 131 | if v < 1 || v > 9 { 132 | return errors.WithStack(ErrUnsupportedRDB) 133 | } 134 | p.version = version 135 | 136 | for _, opt := range opts { 137 | opt(p) 138 | } 139 | 140 | if p.filter != nil { 141 | ch := p.sync 142 | workers := 1 143 | if p.sync == nil { 144 | p.async = make(chan *redisType, filterBufferSize) 145 | ch = p.async 146 | workers = runtime.NumCPU() 147 | } 148 | if workers == 1 { 149 | // at least on worker 150 | workers++ 151 | } 152 | 153 | for i := 1; i < workers; i++ { 154 | p.Add(1) 155 | go p.filterWorker(ch) 156 | } 157 | } 158 | return p.Parse() 159 | } 160 | 161 | func (p *Parser) filterWorker(ch <-chan *redisType) { 162 | var ( 163 | set = new(Set) 164 | list = new(List) 165 | hash = new(Hash) 166 | sds = new(String) 167 | sortedset = new(SortedSet) 168 | ) 169 | 170 | defer p.Done() 171 | 172 | for { 173 | rt, ok := <-ch 174 | if !ok { 175 | return 176 | } 177 | if err := rt.decompress(); err != nil { 178 | p.close(err) 179 | return 180 | } 181 | switch Encoding2Type(rt.key.Encoding) { 182 | case TypeSet: 183 | if err := rt.set(set); err != nil { 184 | p.close(err) 185 | return 186 | } 187 | p.filter.Set(set) 188 | case TypeList: 189 | if err := rt.list(list); err != nil { 190 | p.close(err) 191 | return 192 | } 193 | p.filter.List(list) 194 | case TypeHash: 195 | if err := rt.hash(hash); err != nil { 196 | p.close(err) 197 | return 198 | } 199 | p.filter.Hash(hash) 200 | case TypeString: 201 | p.filter.String(rt.string(sds)) 202 | case TypeSortedSet: 203 | if err := rt.sortedset(sortedset); err != nil { 204 | p.close(err) 205 | return 206 | } 207 | p.filter.SortedSet(sortedset) 208 | } 209 | 210 | rt.reset() 211 | } 212 | } 213 | 214 | func (p *Parser) close(err error) { 215 | select { 216 | case p.err <- err: 217 | default: 218 | } 219 | } 220 | 221 | func (p *Parser) skipStage(strategies ...int) bool { 222 | p.state.skip = false 223 | for _, strategy := range strategies { 224 | if p.strategy.running&strategy != 0 { 225 | p.state.skip = true 226 | return true 227 | } 228 | } 229 | return false 230 | } 231 | 232 | func (p *Parser) getMemory() uint64 { 233 | m := p.state.memory 234 | p.state.memory = 0 235 | return m 236 | } 237 | 238 | func (p *Parser) readString(n int) (string, error) { 239 | b, err := p.ReadBytes(n) 240 | if err != nil { 241 | return "", err 242 | } 243 | return bytes2string(b), nil 244 | } 245 | 246 | func (p *Parser) readRawString(memory bool) (string, error) { 247 | b, length, err := p.readRawBytes(memory) 248 | if err != nil { 249 | return "", err 250 | } 251 | if b == nil { 252 | return "", nil 253 | } 254 | if p.state.compressed { 255 | p.state.compressed = false 256 | b, err = readLZF(b, len(b), length) 257 | if err != nil { 258 | return "", err 259 | } 260 | } 261 | return bytes2string(b), nil 262 | } 263 | 264 | func (p *Parser) readValue(memory bool) (*value, error) { 265 | b, length, err := p.readRawBytes(memory) 266 | if err != nil { 267 | return nil, err 268 | } 269 | c := p.state.compressed 270 | p.state.compressed = false 271 | return newValue(c, length, p.getMemory(), b), nil 272 | } 273 | 274 | func (p *Parser) readDoubleValue(encoding byte) (*value, error) { 275 | f, err := p.readDouble(encoding) 276 | if err != nil { 277 | return nil, err 278 | } 279 | return newDoubleValue(f), nil 280 | } 281 | 282 | func (p *Parser) readDouble(encoding byte) (float64, error) { 283 | if encoding == EncodingSortedSet2 { 284 | if p.state.skip { 285 | p.Discard(8) 286 | return 0, nil 287 | } 288 | bs, err := p.ReadBytes(8) 289 | if err != nil { 290 | return 0, err 291 | } 292 | var f64 float64 293 | err = binary.Read(bytes.NewReader(bs), binary.LittleEndian, &f64) 294 | return f64, err 295 | } 296 | 297 | b, err := p.ReadByte() 298 | if err != nil { 299 | return 0, err 300 | } 301 | switch b { 302 | case 253: 303 | return math.NaN(), nil 304 | case 254: 305 | return math.Inf(0), nil 306 | case 255: 307 | return math.Inf(-1), nil 308 | default: 309 | if p.state.skip { 310 | p.Discard(int(b)) 311 | return 0, nil 312 | } 313 | str, err := p.readString(int(b)) 314 | if err != nil { 315 | return 0, err 316 | } 317 | var f float64 318 | fmt.Sscanf(str, "%lg", &f) 319 | return f, nil 320 | } 321 | } 322 | 323 | var ( 324 | isInt = regexp.MustCompile("^(?:[-+]?(?:0|[1-9][0-9]*))$") 325 | ) 326 | 327 | // readRawBytes reads a redis string from input. 328 | // It returns the string as bytes format along with how much space it consumed. 329 | // If memory is true, it calculates memory used in redis instance by this string. 330 | func (p *Parser) readRawBytes(memory bool) ([]byte, int, error) { 331 | length, encoded, err := p.readLength(true) 332 | if err != nil { 333 | return nil, 0, err 334 | } 335 | 336 | if !encoded { 337 | bs, err := p.ReadBytes(length) 338 | if err != nil { 339 | return nil, 0, err 340 | } 341 | if memory { 342 | if length <= 32 && isInt.Match(bs) { 343 | p.state.memory = p.sizeint 344 | } else { 345 | p.state.memory = _overhead.alloc(length) 346 | } 347 | } 348 | if p.state.skip { 349 | return nil, length, nil 350 | } 351 | return bs, length, nil 352 | } 353 | 354 | switch length { 355 | case 3: 356 | return p.readCompressed(memory) 357 | case 2: 358 | // 1110: an 32 bit integer 359 | i32, err := p.little32() 360 | if err != nil { 361 | return nil, 0, err 362 | } 363 | var buf = make([]byte, 0, 11) 364 | bs, err := strconv.AppendInt(buf[:], int64(i32), 10), nil 365 | if err != nil { 366 | return nil, 0, err 367 | } 368 | p.state.memory += 8 369 | if p.state.skip { 370 | return nil, len(bs), nil 371 | } 372 | return bs, len(bs), nil 373 | case 1: 374 | // 1101: an 16 bit integer 375 | i16, err := p.little16() 376 | if err != nil { 377 | return nil, 0, err 378 | } 379 | var buf = make([]byte, 0, 6) 380 | bs, err := strconv.AppendInt(buf[:], int64(i16), 10), nil 381 | if err != nil { 382 | return nil, 0, err 383 | } 384 | if i16 < 0 || i16 >= 10000 { 385 | p.state.memory += 8 386 | } 387 | if p.state.skip { 388 | return nil, len(bs), nil 389 | } 390 | return bs, len(bs), nil 391 | case 0: 392 | // 1100: an 8 bit integer 393 | b, err := p.ReadByte() 394 | if err != nil { 395 | return nil, 0, err 396 | } 397 | var buf = make([]byte, 0, 4) 398 | bs, err := strconv.AppendInt(buf[:], int64(int8(b)), 10), nil 399 | if err != nil { 400 | return nil, 0, err 401 | } 402 | if int8(b) < 0 { 403 | p.state.memory += 8 404 | } 405 | if p.state.skip { 406 | return nil, len(bs), nil 407 | } 408 | return bs, len(bs), nil 409 | } 410 | return nil, 0, errors.WithStack(ErrInvalidLengthEncoding) 411 | } 412 | 413 | func (p *Parser) readCompressed(memory bool) ([]byte, int, error) { 414 | // 415 | clen, _, err := p.readLength(false) 416 | if err != nil { 417 | return nil, 0, err 418 | } 419 | ulen, _, err := p.readLength(false) 420 | if err != nil { 421 | return nil, 0, err 422 | } 423 | 424 | if memory { 425 | p.state.memory += _overhead.alloc(ulen) 426 | } 427 | if p.state.skip { 428 | p.Discard(clen) 429 | return nil, ulen, nil 430 | } 431 | 432 | buf, err := p.ReadBytes(clen) 433 | if err != nil { 434 | return nil, 0, err 435 | } 436 | p.state.compressed = true 437 | return buf, ulen, nil 438 | } 439 | 440 | func (p *Parser) readLength(withEncoding bool) (int, bool, error) { 441 | first, err := p.ReadByte() 442 | if err != nil { 443 | return 0, false, err 444 | } 445 | switch first { 446 | case 0x80: 447 | i32, err := p.big32() 448 | if err != nil { 449 | return 0, false, err 450 | } 451 | return i32, false, nil 452 | case 0x81: 453 | i64, err := p.big64() 454 | if err != nil { 455 | return 0, false, err 456 | } 457 | return i64, false, nil 458 | default: 459 | switch first >> 6 { 460 | case 0: 461 | // 00: 6 bits 462 | return int(first & 0x3f), false, nil 463 | case 1: 464 | // 01: 14 bits 465 | next, err := p.ReadByte() 466 | if err != nil { 467 | return 0, false, err 468 | } 469 | return int(next) | int(first&0x3f)<<8, false, nil 470 | case 3: 471 | // 11: encoded in a special format 472 | if !withEncoding { 473 | return 0, false, errors.WithStack(ErrInvalidLengthEncoding) 474 | } 475 | return int(first & 0x3f), true, nil 476 | } 477 | } 478 | return 0, false, errors.WithStack(ErrInvalidLengthEncoding) 479 | } 480 | 481 | // Parse parses a Redis RDB file. 482 | func (p *Parser) Parse() error { 483 | var ( 484 | exp = -1 485 | idle = -1 486 | freq = -1 487 | currentDB = DB{p: p} 488 | currentKey = Key{p: p} 489 | currentType = Type{p: p} 490 | defaultStrategy = p.strategy.global 491 | ) 492 | 493 | if c, ok := p.Reader.(io.Closer); ok { 494 | defer c.Close() 495 | } 496 | 497 | defer func() { 498 | if p.sync != nil { 499 | close(p.sync) 500 | } 501 | if p.async != nil { 502 | close(p.async) 503 | } 504 | p.Wait() 505 | }() 506 | 507 | for { 508 | select { 509 | case err := <-p.err: 510 | return err 511 | default: 512 | } 513 | 514 | b, err := p.ReadByte() 515 | if err != nil { 516 | return err 517 | } 518 | 519 | switch b { 520 | case tokenDB: 521 | // restore default strategy 522 | p.setStrategy(defaultStrategy) 523 | num, _, err := p.readLength(false) 524 | if err != nil { 525 | return err 526 | } 527 | currentKey.DB = num 528 | currentDB.Num = num 529 | if p.database(currentDB) { 530 | return nil 531 | } 532 | 533 | case tokenAUX: 534 | p.skipStage(SkipMeta, SkipAll) 535 | key, err := p.readRawString(false) 536 | if err != nil { 537 | return err 538 | } 539 | value, err := p.readRawString(false) 540 | if err != nil { 541 | return err 542 | } 543 | if !p.skipStage(SkipMeta, SkipAll) { 544 | fmt.Printf("AUX: %s: %s\n", key, value) 545 | } 546 | 547 | case tokenIdle: 548 | i, _, err := p.readLength(false) 549 | if err != nil { 550 | return err 551 | } 552 | if !p.skipStage(SkipIdle, SkipAll) { 553 | idle = i 554 | } 555 | 556 | case tokenFreq: 557 | f, _, err := p.readLength(false) 558 | if err != nil { 559 | return err 560 | } 561 | if !p.skipStage(SkipFreq, SkipAll) { 562 | freq = f 563 | } 564 | 565 | case tokenResize: 566 | dbSize, _, err := p.readLength(false) 567 | if err != nil { 568 | return err 569 | } 570 | expiresSize, _, err := p.readLength(false) 571 | if err != nil { 572 | return err 573 | } 574 | if !p.skipStage(SkipMeta, SkipAll) { 575 | fmt.Printf("db_size: %d, expires_size: %d\n", dbSize, expiresSize) 576 | } 577 | 578 | case tokenExpMSec: 579 | if p.skipStage(SkipExpiry, SkipAll) { 580 | p.Discard(8) 581 | break 582 | } 583 | exp, err = p.little64() 584 | if err != nil { 585 | return err 586 | } 587 | 588 | case tokenExpSec: 589 | if p.skipStage(SkipExpiry, SkipAll) { 590 | p.Discard(4) 591 | break 592 | } 593 | exp, err = p.little32() 594 | if err != nil { 595 | return err 596 | } 597 | 598 | case tokenEOF: 599 | return nil 600 | 601 | default: 602 | p.skipStage(SkipAll) 603 | currentType.Encoding = b 604 | if p.typ(currentType) { 605 | return nil 606 | } 607 | 608 | p.skipStage(SkipAll) 609 | key, err := p.readRawString(true) 610 | if err != nil { 611 | return err 612 | } 613 | currentKey.Key = key 614 | currentKey.Expiry = exp 615 | currentKey.Idle = idle 616 | currentKey.Freq = freq 617 | currentKey.Encoding = b 618 | currentKey.memory = p.getMemory() + _overhead.top(exp) 619 | if p.key(currentKey) { 620 | return nil 621 | } 622 | exp = -1 623 | idle = -1 624 | freq = -1 625 | 626 | p.skipStage(SkipValue, SkipAll) 627 | switch b { 628 | case EncodingString: 629 | value, err := p.readValue(true) 630 | if err != nil { 631 | return err 632 | } 633 | p.filterRedisType(currentKey, value) 634 | 635 | case EncodingSet, EncodingList: 636 | size, _, err := p.readLength(false) 637 | if err != nil { 638 | return err 639 | } 640 | 641 | if b == EncodingList { 642 | p.sizeint = 4 643 | } 644 | values := make([]*value, size) 645 | for i := 0; i < size; i++ { 646 | values[i], err = p.readValue(true) 647 | if err != nil { 648 | return err 649 | } 650 | } 651 | p.filterRedisType(currentKey, values...) 652 | 653 | case EncodingHash: 654 | size, _, err := p.readLength(false) 655 | if err != nil { 656 | return err 657 | } 658 | 659 | values := make([]*value, size*2) 660 | for i := 0; i < size*2; i++ { 661 | values[i], err = p.readValue(true) 662 | if err != nil { 663 | return err 664 | } 665 | } 666 | p.filterRedisType(currentKey, values...) 667 | 668 | case EncodingSortedSet, EncodingSortedSet2: 669 | size, _, err := p.readLength(false) 670 | if err != nil { 671 | return err 672 | } 673 | values := make([]*value, size*2) 674 | for i := 0; i < size; i++ { 675 | values[2*i], err = p.readValue(true) 676 | if err != nil { 677 | return err 678 | } 679 | values[2*i+1], err = p.readDoubleValue(b) 680 | if err != nil { 681 | return err 682 | } 683 | } 684 | p.filterRedisType(currentKey, values...) 685 | 686 | case EncodingZipmap, EncodingZiplist, EncodingHashZip, 687 | EncodingSortedSetZip, EncodingIntset: 688 | value, err := p.readValue(false) 689 | if err != nil { 690 | return err 691 | } 692 | p.filterRedisType(currentKey, value) 693 | 694 | case EncodingQuicklist: 695 | // quicklist ziplist size 696 | size, _, err := p.readLength(false) 697 | if err != nil { 698 | return err 699 | } 700 | values := make([]*value, size) 701 | for i := 0; i < size; i++ { 702 | values[i], err = p.readValue(false) 703 | if err != nil { 704 | return err 705 | } 706 | } 707 | p.filterRedisType(currentKey, values...) 708 | 709 | default: 710 | log.Printf("unsupported encoding: %d, %x\n", b, b) 711 | return nil 712 | } 713 | } 714 | // restore default state 715 | p.clearstate() 716 | } 717 | } 718 | 719 | func (p *Parser) clearstate() { 720 | p.state.memory = 0 721 | p.state.skip = false 722 | p.state.compressed = false 723 | p.strategy.running = p.strategy.global 724 | p.sizeint = 8 725 | } 726 | 727 | func (p *Parser) setStrategy(strategy int) { 728 | p.strategy.global = strategy 729 | p.strategy.running = strategy 730 | } 731 | 732 | // filter callback helpers 733 | 734 | func (p *Parser) key(key Key) bool { 735 | if p.filter != nil && !p.state.skip { 736 | return p.filter.Key(key) 737 | } 738 | return false 739 | } 740 | 741 | func (p *Parser) typ(typ Type) bool { 742 | if p.filter != nil && !p.state.skip { 743 | return p.filter.Type(typ) 744 | } 745 | return false 746 | } 747 | 748 | func (p *Parser) database(db DB) bool { 749 | if p.filter != nil && !p.state.skip { 750 | return p.filter.Database(db) 751 | } 752 | return false 753 | } 754 | 755 | func (p *Parser) filterRedisType(key Key, values ...*value) { 756 | p.skipStage(SkipAll) 757 | if p.filter == nil || p.state.skip { 758 | return 759 | } 760 | i := redisTypePool.Get() 761 | rt := i.(*redisType) 762 | rt.key = key 763 | rt.values = values 764 | rt.i = i 765 | 766 | select { 767 | case p.async <- rt: 768 | case p.sync <- rt: 769 | case err := <-p.err: 770 | p.close(err) 771 | } 772 | } 773 | -------------------------------------------------------------------------------- /testdata/.20kbytes: -------------------------------------------------------------------------------- 1 | TO29G8HV1EAC44Z6NZBLD06R6P6Q4271M6AOS702PSZ2NKWF6G011IXPU4CX0UCJ1V9HUDZVSXCX2GM1N9HT8FR4W5XUPWQDO0Q5Q6PI9H1J42RT5V0X8CYK80B9YYSIQBYWA2878W8QVCIM7U2YVEFXBKOQIKA1AEUX86JDRG6AWWA7GJFNWUD6T4T5TRICEG2S9LSBUCLCAEQ7JFET8DUFN1RN1PY7NJDM2YN5JWNZHHD9IZ1HN7NUMTBQZ4GU5MPOXY84BMDRH27MLNCTT8H817WUUCRMYGJHMRDVQQ3NDPSQOWPSS9JJYOFCZY0ZA6R7K9FRF7SAX2FOAKCK1Z3RR0QN8FT508WAWYAYOOPYC9S7CK9JSR6QF046495KTO0EINUDNWJKS0GWJEG62VRWS52IWPVSUL76CKG9HPYK2UZZBSDJIJB0QH0HDGJVYWOXM87LH8FK9BD1G4YW32PT5QPIZDGBC11A8BHIUX855X29B0E2D8AZ4EIGWENSCCAS3PE2VD59Z4OGULDIMVK33HI3ZU2BOSLGJQHSLCWQUKVK0QKNV0MZCU792T5MECW8JVME719MU56GFF4C51KVR0UWEIS887VBFC5MFXPZATDORK52PJSN1I1SYH48S65YWD10S5IDXD29MCTBDKKD3ZRGOG1RLXZ7AE7U3UKR3E1B57APIC2UGFNY8WI5K757QS0F529JMCW3M97YX9WPQ1BS3BCO1GOL9T4OGA6B132JZWI0G137V08S134E94LOFX9WCW0BXJ3OX4KLTAVLIBG8X3IBJK3AONMB7SR8QST6RNQF068QKGLGIWD70I1JSNA6JS50ESMGE4NUXY0783VY5LZZPTUG6B8GTQF5I2OCJSVS7VOSH0XB6L8JRP434HBAXZPBB52I95CTLGFBIIPNERNQQS4VWRXNK4KF0MEG8VBE9XFMYL4TQL3WMZDD88WZUK3D0YRM13Y83D72ER01IGE05MSYVY3W8ZWOCH046O8TER8EQ79UUO32BXVKSASMMSGUFVLTV6EIWVVJ8H9K37Z7BLEGA5FZ3BID0NC18J86XNGOCWALKM9PUZF57HGPG2OCVYPVWSO2904CL1JJOVE5DN3WQBS47QI7YWPKP9FKPA8CJHH6T5LC93OYE02OO3SP30ASXKOOI1MYNWQQRTKLFA6IRGEZZOH8IWNXC5FIXCOYJ2THG43OLZEWV7MES4JEH30BPRF6KG0SM4B1KXTTOQIK9HA3DXTTFLAGB9IJFFD2AAXO802SB68DVWI7OLMUWWUQCJZKIOJBXRYSS2NZXNEHM9SI4UVNT49CFHAAVXPAK6GYXTNOYR6156CM3CA5CLO6M7GNTK1AXCS1A6ODR75O9JO8BN76KGCTHR4IIGM7GTO0V3304G4HMEGQHOS1KXODD32F2X5RDXZZSDNKKC3JB2DVERRR837PL5GXS48YCNMU8VZMRL9NXAXTA3JP7OINI02UXLMK0MJUTG1HTRU2VMUBSWHP3V4HXZT2BZG2WNMCZI54ZS2ZOKBLLU7Z8HUUQ6VBK70QMVJA49XA59T55TVVBFGJ7X3CTHF39Y95E7TEODDLK7CJ6XJHTESI7AUA19KT9AFOMDNE0IVQVZID3EYA4QOEDNJ5Q1OFOANHC1XP1MHLFZQ3NL9LOQ1MQIO5OWXTZZ6WK1GBD6O4TY35HWN0FMS28Z4ZPAYC4RXOGCWDIIL7PSOESVKG53HGG1298MB195A0UZE36ID49KVOGEH3YS6VWOFTZEE6AFULWLRZK0KKI0PLQI3UV1DXUMHRD5982ZUJAQU2DGBQIZPFQJ53R0H3KECVYKEYAK81VND80Z1F6SN3BTYXFB0XK696QQH5ELQ6CHKWZFBZYL0HFA4TUHJ1FPQBLGVGD5CE5S0U030ET2HLC50LKQ87CCCRZAFLDHWB6SNUTP19LH4NY85HA4I4LK8GJNZ5Y89YHP9G4KTJ4OEZ8QY0FAAN5DBSMV6YDHTQ60OV26RUYOIQAT899IYTAI4RCV8DPG2WUM9FE78YRIIPEUA84OND8U33ZN7SRADEWN313OZWDM7LSKYSUIHG1DXKLRMZZCP869ZXMEA2P3F8J30UJCTAZLK5D3AZLGOWCFUL4ZC3YV9WE3FFER8POIVKURXV3TYF2PGUTF7Y977DM3D3DNBMZTPJW3Z0NKBSJAX7MM92D3T4Q73ZYJJCZNQZ3P4Y7IIWLTEI6W8KGY1Q53BL8W127ZBPGZGYYWC0ZI5PBGVXCQSWXLNOY0GSRBVK4NAZD12IHL6AKMFDESLE50G5EJT6L9RKTZOH87J0RD283CUTCE6A9231UVWDFQQLBQ5PL0R7DO4P0D9LLB1D5Q56AOQ747M9TWK7IZ6VM273HW4OJF2FLME8EQ8LD211MR4S1AF3F1XIZ2A9R43WNB88H0FTGL7UV18EGJ9TG3P56JA6M9G3724SGJIZTJYYSOH18GE7SKK30M6JH3TPH5Q62YTKQ5HEOPTRLMELN302DP3OPTBB8ETBI9CFZ2IHH6TD4Q7HZSCDBH1KLF9X1B3PRI9WM27G2AOSP7TSTF0JC0GSPPF98PH8WSNAB17DXZPHK4RAYC4XOEN1MJ9XVN8FZKIXNF8NW8IVVXUKJ6GS5NQLX5BQ2DLUW4T99XZ5H9VQ9D5JHU766RMYM0LG9OA3FK4CUGGX7JJ0OE9BYBDCPHNMB7494K2HIXKZMBHV6KW6KESHF9QQWDAM6I716OMTNA42HTH8LJW3DPCLCVWU25GHP6MHBYY3W0LFT7YJWGP0VTKOC6BEF2CA4IYSRLQVO7EXWJULD829TA6J9VFJWL5D271HTFYIN7YP8XX33VQ187UIV9KFVWO5YR4DVU7NYRT9RM39MRYK36IBWT45YY7F0U2V4SQBJTZLK3RV35EAFC3KFBY8LZYVUN8UR2D4U044XOIYGEMN3Z3VR7413TJEZO12IQ53E5MCMHBHNHZF8MNMELEUD6BKEONC3J15FVYBFY9F1HRR4QXORSZUYM02SLYDG9VSF6YIX98LBFIHYBONUFY8U5X2H28LT26IA8R845T4K3BMF95AR8LS8C9I88C5MXCQY71UQEM6UP9VC6RZRKT08OFL99I5MM51TZWQC7X5NQ6P1EJR7KTR197TMET2NQAAZIL0SJLOKMVE53WE01PRQFNSIR4HP9S2E14GRQE50HIODG1B8BG9BG4ZK8RUC390H0YZVH736WHE2O4819TRHO198LHBXCNP71HJSYRQJHZO0X7QP4070FU1ZXN5CLYKMFC37MM5G95WRC8Q63I2MEOC7PL6V7L3FQMCYXG52840ABODC188W0HTVZYVVXY6PX5PN029CY3ZWRQF7VDZUZ2UAQOI3ATJGRYNGTSLJGNUDM8GKXR9IG6C36LV1OPW2AJINUKJG7HNK7MRIEMNWF16GP3KIG72R5826SFFGQNFU7OQ9H2OIF8DPU93AOA7KQB01D5VETLO42P5XJT1O60PN83OWXMEZG5876U1C8RK6203JS2Y9CZGSWBSB1D543SOYXKYCOBQU04GBQQ1WZZVLNRWWMX9NIATY4L2OV99UFPMLKS1DY4EZT3RVZQ1QYKT4A7N4X0713PQ6YQO9N8AE5YLAOOT7IUR3VDOE0HCCAFG0VZ42HP6OK1R8MCRO37AS2QESD1CIZPNCAPMPUVCEZK1197TQ4A4EV9LGRJAGAGD7VKI7WXO0FS8504UQ2D3LINE2RO72O1FMQ8LMLU150OTUUXAA6PIBSOMZR1S3YLCRWVQ36SBTI9T08LZ6U9OI32VFJ19H3Y0PQJ3488ELUXVJFCDQ8D4WYBZW8MW9IFP270TEVU34TXI98UL9V67EK24FCG18SHMD075ABXMFI1PVGHQVJXSNGSJJSGY5YLBDYURG2FKV9BWX77M300IKSZTMA7A9SGQHL6ZTTZVBMOH0Y3T1PXUWK4IRJCD8BJ5WNU3P4394QSSM31JPA9WONSN1AUMNCFVKGPE32UJGNPSSLECZZYU4X1RKUH68PJD85IAM8AD2NEGQU71B5A1IETA89D6R2MLZFER6GLGR1OUCUZXRKDUXSB5WIUZSJW08LJNGWI0BCJEYCCXL8DAF2QQEWL0M1TAFDM2NTB3FXITJYBRTK9ZQXE7SB5CN5EK2U523M1NLWXF15J283T34WK2V7A9Y2FIGPGNLCVP6MZ5VD2ID91MNEEEJHGZHQFE9OVM48IDQGDBT1LAP2ZI61T7PMFA9QLLV14OUJ2I500RR59O1AIBTCGJM91XVE1RS6NZ7ZXT1OUHEGXWIXV8QC5XW51ZZKXN8M0UQ38NV5QQRA5HGM0W28GN971PMKQIXII69YYHY6E02PVBO2GFUIEAQEUUV9BQXCOY01OG9XME9PDPQU86CJY3E38OG3B149ZGESAYXASGZF69X0E7MR24OB97HQ31INT39TS5PR3ACQS7ZA3K3N8PZARGQYSACFGMN6MZKLVWV6K21ARY8OT5C8Y8FHRTGC3U1HJVRFB37HPUC2488E7HICYLZVYZWH3D7P1LTTMD1GDD1OL3IWL21PQ2TH5YTXN32N57564XQV13KRSWOUMPEY9A877RPIH14CY4KT4U3WRBG9LJ389Y00LSEOVUF3VCONXT1R0X4Y6TP64LX2VEWHIHWPPHUC04PY1VGV1ID5SYZPQ3XP6XLUP9U9MXBFBPGNZV6X43G6FAN92ZVP0NS81LO1ENJ2PGWKAJZY904WV0F6GPP0HU0YDHFEMESVCNJWYCF2REA9KK3Q95ZU34GUFBFFI2YBVPU1VQ2Q8YQFHQWQKCJSR2UACOQ3CMM3YBFX1YTBEJNDHGYV8XO3M8HEJQWI5UT5CYO7PFVUMQQ4H1P9ITTHERX7NRD4Y4A23OQACYAVVKVR5EUS3PSGBX370ATD2NLG8V99S5TYXXUBJXMC6QF80BAHV2Z2VRM2XQOTEZFTXXN9Q1ZHGKXK7X3OYT36DDH08C7AYFWTF0XQ0NVZUB7A09U64MQJ6MDLLZW0GTN8ROVNSTIE44UQYDSF5Z5WTCY3AC3DGV5RLC3HBCDHOIXMWIFLWOUGUGOC7YH87OUKLVFLABZH2HLLECDZIGN6CA9CWJ81JBIX6W6MWXEEUX4ROM0830B4QTOI9LD6S31QD8PP75VTMKTCA71XSXSK0K8W1VEWOP265N2MQFRMNCFI3HUSOH8Z0GEQ2UHDVPA7L2WIOVTDYD3QWNWI2GJFFKB9WT81IBO99VLNV01AL40AEEQDKORX77XPCCSOXQ6GK8DHFOC19OPO0G1SV9L3A8IMDWZX68SG55P3PUVZIGSWZULY62X1NVZCKGFA56BBFDE04U26SW4D2P8ZE9O0PNQBP5PQEI90VFUZJC2WAW8DX3GE61V2Z862D8SN99PZBV838GJF3C3VCVWTYVDK0Q5XTRUDNCHP006VZSH2AN8G4GBFT4R58UK4DAAIGTU7DZMVQO1VI27EU16BOW4QXDZNTAL93B6OY810LUOSYOCOIQFO6YW8J11G4OJAKHAGHY68O9MTKCI7242SJSWK7WWI82OWC1BNL537M3M364TR9QE53IKG64PV4VOYWJTQ360CAGAAN2316Q1L8EIUIUYQEE82TY4GI2K1ULBE6YSUGCYG67NWBTNXSH933YWY1GLLR2UCGF4M26SAP8NOD1X3THI9T3QIJCRA2OHMSTL9KBAU9G4397T0CE29W976METU1LGIUXF3AQWQIW05NJSD2HAY2RBEY96M1QL6SOOK5MPUE743YGED4G367ZV3DV6H7SAUSORS5N3UCND7HUNHFYBN5UIE7W99MB72OVXUXHSNEM7T3P4NA49ZMSJM1NP9UW57VSX9DS1DH2YK4HZ41G6OAM306R4MH3F2FCW02MP9IM51QA8FIBYW64V41QTPNNICGZA0NZGJ93ORXKC1VK392JUY2K9V7WPRJH0RMD03NZ84KMGEF9GJK0685XDEBGMC0YAKXUW4CSJTJG9XLQ0YUCZTPJ4EH49B5T5DNXZXBMYC5E5DC9SGJOP0SM9HASZMYMH8J8J0CL5I7HKIAWC1UZZA0H59VKN0PCWOWE9JDOW1MR1LK6F57TJ4M0MFJKFRZJE9W8RRXADHCFSKHDACXLJCSKHQUH2X9KZEPIFWNK699IUMUS18BLNWUPMJB3231V5I4JVSF5WYOSHDFPSVRDEEI9DTIIKXXEM7F0XHR6P8S7635AQT5B04QQEYSJYX6FVT63MKXAOPIXWPQ9Q2L6IY7JNBFKMIGA6QTBNVX2038P7FW6I57GEQP27P115CZ9NYFQH3RGE4RV6GB27PNAO63Q5FKYYSW8V9IIXLNDWAI8I3OIVKT6FBNYDQ70URYP1OJUZPG0HS07YKCOLOSYHU5C8F83YV2AGGNUZW0FAIXZ65W96V74EINI9M5F3GVR594US0WDCZUWH3OLQQWVF8ISV33KQU0JEW2RWCPYZKAOZYVNDN2A27EU8AME4PKZ4KF4ANOW82KX7H1JL038C2R1NUP80UHN9ECK8W8BEGCRVHH223W7GN1XD1EICRXHS7A9JXFUE3D6LWHIJ0ZDYLMIIBH4BFXO5RCDJE1R2E9V4JPYBC3QM53G8ULUR31WHZUUQAB862H4WUNIIH2II8IH7E8QULGGXK7P31L5CLL3K8GM88ELJ2KBQXZ8BVWVXVL08A1GRHTWXFBBVGCIBXK9Y4NLKCC2CCGLIQCKAZB1AYJGBXDH8A49XG5QMJVCKOUCKG5QDH84YMCICNXROIHK8Y9LZ4YEKQZLVW1O1WR2XDM8T04QNVP24IRAGME9743GY5WMDXHT66X3GL9BJTDIM7428B4IJ93TUZB2BSQB1SYXD7IZEWHG1U5L966KEEPY08QK4UONKT3ZUPCULWV9EY0EAMI6SJINWPUJ4DKF9U4HVEIU1UI924RDPYWKIML5CAA6FFZD4SHB1SXA9BQPVBN214NLDLRZ2A5GB5TTRNDSX917XGV5KOEQBSCESD7D0SFI22N3Z082PVQT73KLXL2IFGA6HI8PM08CEJCGIGYYSP8NM57MPT76RBQKGXHG2U4I2UR10MGP4K8E4C6IQAL1B5RWK45Q5NCMZY3HZ35YG0TCAB8WQCHK9WVTZBVMG7794FIEVYO0U1B359A89I33LD0LINGDKPNT6I4ZUKKF57WPEI13AW1MN0OLYM4UELKFUI3TWPH9I7J7E6SXO8ZLQP6DZN6M6FOQD99U4N8ILZA0Y9SZMMZOVA6PT5FV9RGK09IZ4XOICH1VQUU6N7XEHK4YKMXRX4YOFR5S5SY4PRAXME1VN07WQVAI3Z7XU0QD52J94EJ81GQEWX00UP3O95HUID0NZV1WAQNXR6TMDG04MU9Z93F0PI1XYNU6QIDN0PA6E5N9SCTPBN6EVMIP8HE1QH3B8CL8NF1TURI7DTGBX2MJMY7VZUXI62Z7SPXB43AQE7F22JAS06YF1QTA9G9Z8IXWVF3G87P847BLLTZ02ERH2XRP750OA1AD55ERSKQLJENHISJMCK2IDUEA79LAYNJICALTDYSSW4AKBX2OV1ZGU3EU4WAIJZZDOXHQAG1PTP7FJKZN1O84EAQVIKKF5H2108VAT1K7MB7F3TZLVSUHFHELQJ6VZUS015FJ8P9RFBNBQ15GPC64HAIBFXDHWFZ98XWDTTY0U6WBKZ4F8894DAFOI0I1VZQL8BQ749CWLLK89CBGIIPPXNMJTGX8IF2XK7D1NNSWIJYLEXBB3QK29IPZX9XPBJ6ZAUYS12RWV7EK856GG3DLO59B4I11T5XOXTRMFPXLF6E5RJIS1L28PAQ7VMAL66M14W3BLCA5Q8NB0W8A2T9UVJRVH35GVJ91TC717NVDFY1TAISWJQTESGUO2TKNZ3P3W1X8CD9CY3D8DV9DIKD1HLK0M7P46IZ95ULPAFALTUSGXVXDUZCQKWQ6AZ96D2HK4YQWX25V0HL20BQ5YHYHT04COICIHHFPJ6UTCK1NPR7VISTUMU3N5MNU2TE685PXSWHRC83YQ8ENN1RICJH2556GXK326OYZ7507Z0UJGOYKI4OE48O2SHXX3CWNLW62YZ33Y2Z4QB3RMXBX5RRN6LQFRUY7ZVIP4H8YGYEMAOB9JYVR3Q0220MLPNNTYSH2A3JLKJCFHIWFXR96R24VQY5JJRE8GOWMHAYHF7VOQU6FIWS2O3X28D29X1M0I1QJMUOXA8RBB5N3LFYMZ799RQX1EW93TPWWSZ7M0F4EOKJFIHQVSRR41H1CZ84VCZK14BAG8WZKNHBEHM1FNLFQ2VUD1NTP02R2HV2QJ4N9UT6AIAH11T5CH5PR1GUPR1C0UIMI9S2EZU7UMA1FVIG28FOG4PSPRHN9DDC803URCWCDLHNL30TAHS2IETNGBL9B2LM4EDQSR1E88VHD0L82WDFXC4GKZ32SPQVZ1XMPG08VQHZZWSQAQIW90ZAZESNKI67I6E1XFFMHIDR47KXUINHF2AOFWUVL10NTDG0PKWA5SAU5QE260RZ3OZSIXYNZYCX4FYIYUXVIHZAHGAM2QQB1B42ZLYT6MOHZVQD7T95MBR62RYIC28WNDPPP68RYQXGWHJCB3JTQ40B9ML1CLMM05F22LHBNL3Y0XFHKBC8QY72ZLNF54N4QDS0TSR770UGF3KA05NOWMWMADC6UXF0GPB6BJTGA0GZE9OEI1HJQ6L9HTW8C5WW900PDIC1MUJWPWSY4E41TRGGX8FSI7T2C9GWV29MDC2QSK99JE7U2HXFATQI7UT9CMFJSYW0E009N1IU700I3NHKJAZWDDL5T7MCLC9ZM6YB6QWZSGEGDC5SW8CQZVM6GWVJFPZX94GGY4SBIX6CLSKZKJTFOPFN9QPDV816W1L1HJQV280ER53EDEN46XEXF85UOB1HT6TLY3SKTYDXW8ACMDJTUR5HWU2I2W8KDW8G4KGUJWY7JO9NQMZPUS47MUZAEGE97Q9RD982GGJDPGBJXXG6UCMO3NNJRVN45L286F02RVI5BUX9L7GESVHB4HA71R6XN22PG2J4EIGJ476T5KX776J7UQ0LDN6ILSWXRYQ25VCHAWE50U2R0L6U8E3C8ULDNXFU747PKHFT21ND2277JMX3KIL578A7AKQX25JUA2Q9KBLI30XM41PCJXQ9QGA82C0XLN9EVRAMAQAJF8N6WJP1YATKRQA7KSVWAH9U0O0PCX58IQTHKQITWOJSM3LR4BI8E81MP8PMJBX521OJYCASPLBVNFLN9WSHLAMEE3GIO72A54EWPRDN6Y3FTSFKC6ILBSCZDNEZVK9VP8VSO4YMH6NR6Q6AX3G7TCGD50UCRY30WTU7PFQIYNZRFEVK7ABS85P6D5GW15NOP0R9ELG8COMPBGGYD4KZV601IV7FXTP8Q29OO4JPZX23NH43D0836DEOX8IU0GQUMEYSFDMIOAK1TZ3TZE9ZOOECZ00GNID3ZK74X9AI75W8WDVYS931CGXSW0JYPJUX528OXH8W0I3LGCK68B0LEA05QX95TWBVXY68S26I5QNTERZ83XUZ1Y9N2KZIJ1LV88Y01X64R803ZFKGGMABVA07CN1ZE1LAXFEPNRDYS8XU98TENE5MBQZMQ826G79UELNRXA77GYPG7NZA17H5BIKW6UX4OZ394WTJD85OGFK9IEBI2T71RTGE6GZD2FUXX0J8P1C8JX66MS3Z1MKBOY92OOZM815UOTTPYE96SLZUQE15RG1PMPY7NJDZU5HH0Y13PNY6BF9ND2SR6LF8BC8IG8GRM0AS5HFSU5XUVD73PAHMABXEQ9YPV1GE8BG1S5BB8YMAUJXV1DTKJBTNIZ3QJ9NPF889FLDQ1COEFNGWSKF3CV1OLV1QUMSEGQR7HBDNTBG1OGZU06AVM0YPFM4W0LM00VMRY469S99Y1NLX5R5UXNWZEV264TWDHC2POKXUGX9MCV6ROCCT325SZ0318WAKOR30QWE347PSWUN4MLV3WSPRLXVHE67IQT2F2BC5I009I4F2EXA9U9LD85AY8D5XVWNFG1JUOSOBB5K7XLEH0234RELJDQA1EHOPZQ5DBDXQGQ9OCJX1HH2DZS1HQGNMKKX1JNT522SA48O1TYSOCOBPTDMGCI10WUYQY1Q3OP2BYDOPILQLPQE08K5E3AUMYID8NOH0IFOQMNO3HE93PB69P3LER8CIGCERV8SC161HK0WGSL1KQJ5EC6F9UCLZQQLAXJQO4M1VRRL5EN7WEG6NKM9937Z84RIET0DR6D97E8FC1OJ3C5BTT18IDGK7OQFANFWT8W3H86P8CMHFAWY1MMG6XGG34KMLGJPF9WXSWLW6OS4321B94U792YZA6FEJXKWHQK2ZW6MACJ09F3A50EAAXPQ2E5F05QEZNBIOBXT0L69511UFY3A5L34TVM2IUWYWERU8WSZ36YQ2O720NCAKT6B5FR1VB1ARCBBKTS5C2Q352JRHF7S1A74SF6LE56USIB3I1UCVQ13OYC0OJ622KNPKUI78R8URG4HSU1MQ0OT3K5BX6A9RILPGZH41M5PIHJZJY59TKUAIYJJLLZR1Z2EKO92XH7TL7RPYL3HP05T8XEFE5C4Q8TWZU94PVAOZQCBMQRXJZT6W5SBSS1PALJMF1MLPPHJKC93X8L16X8WGYG3OXEFHXXK79ME3OVA735YAP5POMKUGZOYT5BV2ER9V8B1L64L61VMQ43Y1QEX1EPIS7W8MW5Q80YI917DXUV6OP5LND9KQ7PD669DO9UZZTH91SQ9MG9E43J5FC1ANJYYQDWE3A76WVJ7LZ3EA1G9I4L4MU5XMC2RCUYE5KHXJ4PZ0SHMDS0UFBJ481QKTJ7P48WWUOP691RYXU1M3ESZVTDX4PD6IB11Y8NEHIJF5JDKSJXETD09B3HNMA7AUGH3YXK5YQFDHRRAI52L4U877FAKDA10J0FL7QAZ6VO3OLB7CNFHXXOVQ12T6XXKRW8S7JRBQXD6DNV1XNM3G4D93DGOGBIP8UNWBBIITLKAOFZJGF1G1K0W6IH796B22HGZ2QDA14UZG7XCV9ZTMUDBJKB4HVS5ACFY0XIPN2BZNYS7SG1WAS2113AMUWPTPY4SK5GGK0WZKB2RT1TOUDQSPHRFOQWFEXM30PL1WPGDQGC638CDDGON5SOZU3NG2V2MI58WEUMCPNXIUVMYX6IIP28JFP55K9FW40E0XLU9SZL6BABN18S0HOXKY8SZ2CRI5K84TFYEKZU9V0G8OTXHFJ91GS4C20B988799SLTFYRKGDNVA86P0N6KTDOL30W9MO38RX819QHSUKTBT395VXTD5VTUGEDCEJBGKEMWS53QZJBIKS9VFY4KOCHO666NATRV3LK3N5JV2S7SD1F7W51EEPU4XWV8YHZ3GQ8T8EOML065YQYD737VSKS9ITS8L27XMS60AWFQC2XCGTDHFWEPLJHO0JPOEV6MWVTE9BKK9EXZS26AHC3EDAVK56NAW0S7991ODAKF56IOGONHAIWQQ492LZCU2AWLKO22G1GLUP8EMR1O42WRMHTAA1IFX9T6FWPKZYZF7YWNW8VAF4ZES4DRQH3L41V3W3LOHH9QFXLNJ7YP4BDO34Z1IP82UDBQ34GTB0EEHH0HBDGL9HH7ZQYW27NL052U19E9QIYTJ2RDQIPDAFXCJD7ICRQMJDFWFEV80Y0ZUIBJ241BY5NPQ0RR6P8FPHM4VLU4KRR682ALONKCFPZ3CAP6GZFPAOM0CXBAEJHHK3OH8IXOLIOFA5T4NIGG2ESUT65FKELTSXCWCHQX5DDRLJVBAWP1QITRI81HINAA6C9QTAJ7OB8E4L6BDZUB9RRB9GC4F7NQW54DBA1BY1S8Y74TRHCXKSEN9P67LJNR6GA4YKK3V675CFK2OIPAD1FOAYIUBXN0AUM8XLRR9DKOF62VQ2R9OX2BUK1C3VDVL4T5B7WU4V4Q30952E5QDYV36C1J375FS5PR9S7FVYYP7BTSS3NSWHG1IL0LOPYXL8DK5T85005AJKLV7LZQX4J1P6KMYCR5VHZRKZBNL0BEWPVR3KKKXUUHP2D798YQS803Y74N40PP2O3A2F2Z5YNE7AHVRB0BQHKB92QE27PKIRJESGRZACGUHKDVKPD9V67GRTAD3ZEREMLL2J9GQXQIFG5UXZQK3N0VCWO967PGSR3S6EJI23ZYUDVEOOLM80UUJLDDXR1FROZKH4Y7GMFF6FU1HE0JXXWMRR7LC9T17GFXEPFJSEF37C1EJYLE11WZCY7GC9NIDYDQVYQQYE9WUK7ZQ26TY91S6W10A67MYGCOFWCFNA2XQ53N2U5C4T6INLQDKHCA5E34WTF467Z61LFE22RGZM07KDUNVQVZ5ICMVGP8HOTZ34K1TW1O05DJNYQPHU0A2E5K5M2XXAHMZREOUMHCOK10U7WK8O91JXVOFAKAG4Z1J7TEN42XZJDG6L8MKB7QOXVZPNX4P9GTTGXZ7MRBHZ63PCKXXLBCHZ7ETINC5AX8CRZ701YKRH2NVN6FWPI5AF9328735I69POY5IBFVGUV9DTV98CNDN617IWE4PE7LML4ERVFUKEJRGHSL6WHH8YMWVP8G2NWQ6AHT5F8XA0BRT42IKRDUXDSC0ZJHHMQERDWR5OBLXITBKG4OD40GD7GI428SHM06JRLAHWBVGOFM133T8K3YE9ORQ7FTG9Z1UYPG16SVS3545BI85P027FW8UKE29H6VS29ZHB6SP30OXQDGT2UH2LXU3IBJQUOTL1DK8ZNMUFXVN0QDD17W8Q4F7HQKNVVBDKAILOYDXVMPY9CVWA3JDNSLI3H4IX939W2CW66RIGFH78O1EAQZUV4GHKQIR06G5OB3Y3KIACEQIFQHK6QA7N2UWY8MXGWUVXQREU9YS1SJ9GI6MKK0CXTJTFVEDG198MYJRDZ0ATIQMF8TMYVXZWS36R2GW5AQ09AYRPGTDDXZ016V2N82V43QTA6FMC1O5RMFK1X3IHFLKK1VH03XYI7SGGI29XVPDDAE645ZMU3MJF2600Z1BJV599P3EAK3T6ZYBWF3XTB4SZ9K0SNL5FVDQ0UOI5ODCMPSB69HTCF9OQ5FR99UV2ZGPRCSCK1JTU98GCEWSIVBQ7WIK0VDGRHT1BSRJKZWESK82DE4OB4E2FVAQ744XPWFWAKND6ZWPB2XSGGYFOV9VCD0BRHCLSRBNBTZFPQTHCJHMZH1JMXTRFZIOJKO0B86IQHP0ZE0UH7MPHPN4X5ZJEZDCSN8XU8RTJV7DX05WKHKW3NKCPY5MP7XHQOV1Z7HN5NIDS0C3CIBE0KEHX9LLF04CWONIX150M0KTQVA8TKRYB9UHX8CDRF6J7X6U44AYOMQIT4SU0F1YF1HS9UR3BIDY42324LST8PDC3GXPAI3OPD8OGO75OUL72KP5ZP4DKRQG6WGOVH4T77FSXHSN89M44YKL12603U9RXLJA0N98HNSWXF638U3TANA6TB14YHAT5WDDHK1KM2CTCSZQTQYUE6TC78QXSAEDD3V0JK07521O0UVBB0V6D22II1506X1F0DOIJASN88V6EKHLS6KB4WU53XGMWWR2CDDOCBJSJCMA4TAIRG7SY3LD2QKJBDQ6N4ERS2ZEN1I2SHO6HMLCYILG7AEY9TPRAKBADFYB7UKWL2E2CZVJ71TVAMWE3ADK0HNLMV5V6EQND1IM1H8S4M6LFOPFHS9DJIFC28RON9F9YZ9VVDIZ82OC1TTI91GGO0UDVEQR499G3EY4D0ZF8R40XOEL4RRAK0XSWNADLCWO0HV6X7W53NHIVKMJQF229SBRQYVIS5R2QYXPBR6LD1HIC45QVCNXJR7EF5LKF32UMSY0W3XMX374LWBFBGQEK31XDC7V0VT4ITKRIF4JRQUGTJTG358HNYXUBYDEXWBOV7ORSWDC316QNM95V0BAUGYF9RPJXZT90TEUK0OD1OOLY7M3ZPVC8N2Z25GSZHAKMLLASBS167K27XY31NHGC7RTBGIQVO4ZJB8WOWKHSWVSL9503D61S08GVB8WGIGQCO5OWIJV5TAUANVOGYTQ0J6F010VZA1T49C0PZ36UO7EK1Q4E8D31R1O3A644TTQ64JUWUB43DN1SI1KO569SE13GUQSTYI7M92E72O9B2JW7NSYF8Q99IIR9RI4CCI8XLK57D3FXKZS95EHQ813XAAEUU51XM0BWCQGGKYC15GIR51RUS6N1O1KR8NVUVPLHY667XFT6S9Z1RUUGF6KMTBS2750HQ822SI79R1GTLR3TN7P1X6KKA7OF0FFT8OQ8M98RCNIJDRMTPZM3GZURYRJ2HRQFEGX14RYWR3PONHEQWTAHW8Q4X0MYFG5G35QYCNHBRKGPR8QUH8QUZOJQOLDJ8AP8G4ARDXL6XCYK6OGRUHH2A3POBKH2W9AJ04TLZ7FNOSL0767AF5W76UYYZYHCQPUFKNRU4I4NO0BPB65H9BGUUB5IK1MJYCUEITSRRXOHAU3HU2X3ZQ3DIXCEABGC5YIKN031AO5OFWCP3274C5NC48G1Q3NC1VPGF182RQ5ZH7RZX4L9GEHUJ9NBXKWRYEF05HJ4GB387XR7HUJGYQT6GD6RM6SRPAJNI5VEU1QD3JGMWY1I0ZVG97WP01F0HPV1DO8RK4MWGDFZ2FK5O9FMJGZXJM1AESB6TUGJO704Z1M0PAOWN27OZ0R8N74S7LGVMFBUMEXQRQHQ1Q4L4PKYVVPF9B46A94HG9O5F7VT18PP6WFYEXAJS5AIBUQVPQJS2R5UM7VDJAH559QZA4R7C5IX1SKUDABA1XFUZGNR34HKQLH64PKWZPJJLT60T9B490II42WZEYGAXBFKZCCRRR5WO5A9G82BO8WDQFO70YO2G739I7RFCJ73Y5TY1FU1F65NHDD2FOFO1BUM9LXAUAZWFVIXOGGHPGVYB5LDCXBDWFXTMBPG2DHZM4GHRKAZDGTCYS9CJMRXL6GLPI6PGLF0GFMX7KRZ3ZGIBPY3HDX1EXAOYT0G61MKXM02H4Q0TICA9GZOQBIOIH2OUXDWBK22CX0GW5KHVDRNLMH9UN6ZSGDBFVABVQYCM09SIJV5IY6S37BXWDJETJTENFDKYK8KIUMBZTXJ6C7EX4KKA6VQTLMQ3ZU74K5OJ3YWT0MTYRZPFH8NY07FOFVXEYOK34LTLLD18ECXKAFF97ISXE4T6UM131EW8X5TMUNIKMPW8LOHSFSCS8DXQAE64BWCKRW62PAZ43FHV9BQSNHLJ6TN7C3GW5VOPP9SU805F2M56MQGZ67AGPT770V7PKSUK7JL4P3UWVJZ843T86DLDFXS32NFB6M59598DCUHXNMXKPRRI2UPGPJNV8G3FFD7L6ZTECYX791GX7IYABA7FHQIIDTOR7TDRSKFCJZ5HE870DCTOMD4FFCHRIDEMY81RME3NKMADE8TCDNKEES8WDLLYNDYDLTU7W31WS3KPU4BB7QBA2EO67LILEUJHR9EUOEUVKZKSI0GJAU855XF8BA8IK95DJ1WR5AVZOEQT7LGPBTQL28ZJI0CTCTQJFGRZ1DJQHEDW8JP4U5916RAHY5OHHI6RF4D5M6BB1BWHHFCNJM9G8XRJN8UVB3BXPOHHMDCBYJP58Q2WVHJRN4JC9FM1P10WZQSY53BLBOLL8KW6YSZKL60ZVN38GP6OYNCKC2I2EPK7TWKDVCHNFY279JOJOOCAV9HQAYVZAFC268A0B6JX44SI5SCUZAYY71NDW1H2G3IM64W43KIH284U2M9UAO68XB6AQKY8544X3Q0B6LHPPNSHM13F09QF2C8PVXN90M5LUA71CVV0L5FYOQP7AB2X0PFBJXY9O2NKBVGAYNDJ36GLLJWQUA4ZI77O3W9EMO2N3FVJS00OX4U04126QPQL06DKAKCXWLORJJP5R182OC2A9OEJ0OZXTWRKHLBDK5TCA9QPFFC1LORHREIFSQQ5UEPDV2HOLTJFPWMLJWQ8L87V70SHM43B0J3TJOXB96TWV8P797JU51E6J1USPKSPL39XWJ0439NBT7PCZGBEKNON3X9YJGIDY3PN8M2UY21BLUUKQUL7Q19AEMPSJ3LER7O9F4B26YTMJQO6X3HKL6KPD8PAT5OTS24T0JLYF45V8N52J2FLPUG3X2RZ31TCAD87BZXROGS0CGX3QNUKOYGV20FJS093CQKNJABV7QCEGRUCHKSJNZKBGOYXOCHR8H3F9R3T20GNXPGM9Z25FQ3X9PWW4MKXTMIJE943GHMGGUYCI10N52PS5HC9BY2AOUWLKAT46U1AOJE25YMMR1F962NYHAFLLFUPEPDL9V0X5FKHWTSQKPCYLZ1WF4V7FCNRKYNFS6SMD54DKBNOEV8LVPI2YDYN533V2PO7ORN028GLIWH55M7Z5T41ACPW6UHNQLPWTQO6O9AOYIOCSLEMNTMY0IJI9E0UFIOGOWQLH0H4KEUWFJ3DIP6UJBV47U71HY3PW0P8C9PRKPWOCC79X60GSOINZ3U6RVIRT5X3J2FW5YJG2W1Z6OQOXWJ3QSDTE3VZOS57RL1IO39XQM88XRSKZ6Z3U6A6B8ZWD32QLTSS9R82QYKJMG5LL95VTPW2OM5SQT4GZZWTH3CY53PC0PWKPQKHZ6SYWZ3KMY2L7WQVK4JVA94DFJWLTUU0TB28QKSPZTRGDQ5G9BFDLCOESM7E6M4Z7ITKAYX2JXWANQU5Q3T4O4ONFP7SS5HUIY7HD41D2B07BXYDH43BRGMRYWSTIP853EMYMDC3E90TCIEFHRGIO55UR8R1D6T6RSZ4626WGLAXLFNF1SR7P9BJW5MJRW65SZSP8Z3JRK6KVSW4TF29U8GATCYRV4KUU8QG9DHY1XD9NYE0PF0PLCAYWB5IKYGYMD3KWS8BT3ITA8MDE8HQMD8ER5MN9NM6G1SUYT8MVWE2OKQK4HYOP79T4MXR3H4FW4D8D8BC3NSX58HUXN99XU20YOFF1EDPJCE5KVAKZVPFA046KXP9YM91ZKLK2F04VUC2TRARP2M0TD6NTDOT24PCS4RFQHA64X2TZSXOBZ3PSEG9PAQ5F08HFV7NJXYM1VHTD6HONDBIGZR5XDNX3TTIBVTKIP9FC9VYPXOOIKPJWHFCJYUTDBWHR4YQYT4KLH59DW0ED0RNBCNTL6L1ZK0NPPN8VNNIJ34SLH248JY6BWX2PTMAUU4V9V436RVUT868660VAGEMC5GC403R0E9PYVG9VHTGTMAO1FJQTSNZP70YXNXQ6DJJG10MWVVMABBDISDYH6W61D3RNJWIBV7GH34894FBFRX6MU0Y4TB080AOJFEXM2KITB060K00NOTE80HZZ4B5PO1HFGD5XYDV07V0UCZ1HE448R6NFAILF5NBIEICJHJDOYTPCQSXV304K3MYXIRNY1C7LHB9YQ4RSM55EQUS4E2QLYAR756CN4M3GZAXOFH30T1NEINLG8BSEWYAF6UPK6404L3DX80CHRKH42M5R5YMAFF0WUDRJRTXSIDLFKPHYYTH0X6AUY4BXEHZRBUBXRCZJ92C7FISSOFTTLLXIGRFOZI7NSG1E2I2AQYBHKPWU5E2CVV2LYH7Y4XHZQ7C4P31KD5IRHZH0G5O16QAAYCM4S6DAYZOSSV5ORXS6HOPMCS4HM9AI9BE74ICGA3ON1Q1D2MID6XUZFNR09M3E84XZOJ60ZAKQXQ6OJRMZZ2ASA0HSF41XZPI70BYO89U3W0RXJHMCX72LNGKRPGRLQM688PUQHK4UFY1Q34GJ943C5RGFNPBN84SXUT4DJJ89V9IOC2GDLZTYW72USJWBUDLRBD7ZA1YECQTNF36RM4547VXW5PYSSR0BQP80525Y4I6HWL8UF7P44SS1TDA1SAIF73OBA7CGJ93QEPSKHFOJRONHFCZ1FZLNJHK8FYMRK4I1TU9RG4YH55AOWXC12WB82BRLJ5FVKCU4YWUBHK9ZLJHEI54LGSY8FR9A5AMA73UVZCO4KBDSYXIBI7L4SJ88S8D20VLYTF3UDKY04DNTL0P1Z67MSX803IQW79NL6Q1ZOD0CFHNHNS7V9BHFTD1P9Z7053OKHTZEYLS2HOD943CPJEIQP1ALE20UET65LIHDRV5E7WLFDHOQPED5BRN27NBFVX77Q2J7TVSBIP4WKHS9ODM9O6V7GKXDD9VU7KEVJZZ32GQQFH69B805JTIEG47QL8L3JU843KPVNC43VVG1GJMDJKTYGOTA9AXL510PVS9YC2DSBF413TKUBD9CGTL7KF6S3ZI84FYTRS1K1894PUVOGAIP4FKWPXL1GYTUSBQJZJXVDAW6G828FM7EYPP39MG4XD7N71P9RFYJKKMC0R596JL87Z3XV7SD38S1XBT5OHG0QLM6ZF4ZOX0H6XIFUFWSN70SJA2X0XTLLUPZRUNCBFXKT1IL68X851WA40AZDZLU9M1SJUS654ZQDOXZQ9LEJHX91IQKOOWMOU8L3MACX3QSA5W3OPOLCQ3TOAQ4OLIG9FDMR8KGXUUCO7TPPG8N48ZYU1XPIQTYKK1MTQDSC7IAS1UFX2AJ8VPM57DUS1U76X3H1WHH86R2WV08E8EEZRDRAVM4WIFB5GK3H670993WISY26CQIV6WCOJUOA2V6SFZ1HP33I6U5C6IIYLE2TG9WSIZ9NHHHIACLTQA3S2NSMENTVV888LHHQ9CCV8KKVI6WT714G1TZD38TRQFLJ4QQ0KGEDPRNG7O7KRKJJQ785LJEB1DET6EP2TSLEZ51VDOXYFYXXHQKDGMI5XTR0PL0D0WPHDOOIA9A7GO8HR4LLZ5K7SK9XL2VVODY6S7VMKAUFK9UJUIPBX5287OQI4YGF5XXWXZRN1EN40NHNH36E0S1KKG97ATG7PRUJMYTCFFYZ6XYB3I0D71CBJR4TEK9E8EF3LS7SJJRL1I6CYD47NMDIBAGS8EEB3TCDMII4STVKMLTH9ZYQHFZ0SV6DN7XJOZI1JQSIOZJD0O8F0QVNLZS22LG7YED2VYFX7R7XOPRSYDWAVP05ND3XK20JUMNVX01YTK1GVO42JMO0QIDLU6TEUS77OEEGZLCL4PC5ODRO6MTFQ8MGK3W4ZH9IW1F3IKXUE4OMSONWEOFRB61TDH2A30WLBL7YFG5VYC1V73IZTOY7P0N91MCZKGR4O6GEDTHE1GAP1ITTDYR5TKCT9JNGMS7I6E8O5VZ1PZ1GGR4MDYUM55F6CMDNKWLLTWOF3T1LM06I8EHS4LZ9IAWSTVA7GR2MK7C5WHJH3YPV35S05O33RLMMLUKW3NOAHSRLSOPTNMWQRKVF18RU1AQOBRK0X2IXWN2BILCN9678O566RI5BTBEG7M781CQA61DWD0XQ780W9UB8IKFKEGFAN2PLRQ05RG5RCFSH7MPEXN0X27OS0D124UQ5FN5SMKGPLSNFJYX1OAI63YSACPMKS7L5I4WSOMTF8ILTE7CW81E7OPLU7BJ2YG36RG9QLZ8REWL3K6VR0REO365HFEGG1VFJG5TY7625FYNEUC68MVSOPL2HO7XZS1GOTZVBXB7I1HLF0BGXK3NW3JSEVLYDP20AWXZTKI4C1JLX6WJM2X5YSKTNVANWOC7W85RUXY6IIAD5ZAMD3JWF60B4L49IFTLC95D91O4YH12174I6VGKJ94DNGBH83C9YMYZTSBKBL5JLUR7HF61Y7YTIQSQV2VBR7J5CXUTTJFG0L29LK4JLQ9V7W8IJ60X23INRFIP78V0MW5474X2HGDMV3I4NLZDIIS5SH3MJASPTHZ9JWXYDCFIZPSNBYI7LT34DS9WVEACY9A26OMBM4OU77YMPAT0GO676EW8655NRKFIETIACNY885ENB5HGA5ISA3Y8OW 2 | -------------------------------------------------------------------------------- /rdb_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "testing" 11 | 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | const ( 16 | debug = false 17 | ) 18 | 19 | func TestMain(m *testing.M) { 20 | data, err := ioutil.ReadFile("testdata/.20kbytes") 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | _20kbytes = strings.TrimSpace(string(data)) 25 | initTestCases() 26 | os.Exit(m.Run()) 27 | } 28 | 29 | func TestParse(t *testing.T) { 30 | for i, test := range testParseCases { 31 | test.reset() 32 | mem, err := NewMemReader(test.file) 33 | if err != nil { 34 | t.Fatal(err, i) 35 | } 36 | if got := Parse(mem, test.options...); errors.Cause(got) != test.want { 37 | t.Fatalf("index: %v, got: %+v, want: %v", i, got, test.want) 38 | } 39 | test.validate(t) 40 | 41 | test.reset() 42 | buffer, err := NewBufferReader(test.file, 0) 43 | if err != nil { 44 | t.Fatal(err, i) 45 | } 46 | if got := Parse(buffer, test.options...); errors.Cause(got) != test.want { 47 | t.Fatalf("index: %v, got: %+v, want: %v", i, got, test.want) 48 | } 49 | test.validate(t) 50 | } 51 | } 52 | 53 | var testParseCases []testParseCase 54 | 55 | func add(tests ...testParseCase) { 56 | testParseCases = append(testParseCases, tests...) 57 | } 58 | 59 | func initTestCases() { 60 | bigString := &stringMapFilter{ 61 | want: map[string]string{ 62 | "20kbytes": _20kbytes, 63 | "40kbytes": _20kbytes + _20kbytes, 64 | "80kbytes": _20kbytes + _20kbytes + _20kbytes + _20kbytes, 65 | }, 66 | wantEncoding: "string", 67 | } 68 | add(testParseCase{ 69 | want: nil, 70 | file: "testdata/dumps/big_string.rdb", 71 | options: []ParseOption{WithFilter(bigString)}, 72 | validators: []validator{bigString}, 73 | }) 74 | 75 | quicklist := &listFilter{ 76 | total: 1806, 77 | wantEncoding: "quicklist", 78 | in: []string{"1470323953955869026", "1470323959757084081"}, 79 | } 80 | add(testParseCase{ 81 | want: nil, 82 | file: "testdata/dumps/quicklist.rdb", 83 | options: []ParseOption{WithFilter(quicklist)}, 84 | validators: []validator{quicklist}, 85 | }) 86 | 87 | version9Idle := &keysWithIdleFilter{ 88 | want: map[string]int{ 89 | "foo": 9, 90 | }, 91 | } 92 | add(testParseCase{ 93 | want: nil, 94 | file: "testdata/dumps/rdb_version_9_with_idle.rdb", 95 | options: []ParseOption{WithFilter(version9Idle)}, 96 | validators: []validator{version9Idle}, 97 | }) 98 | 99 | version9Freq := &keysWithFreqFilter{ 100 | want: map[string]int{ 101 | "foo": 5, 102 | }, 103 | } 104 | add(testParseCase{ 105 | want: nil, 106 | file: "testdata/dumps/rdb_version_9_with_freq.rdb", 107 | options: []ParseOption{WithFilter(version9Freq)}, 108 | validators: []validator{version9Freq}, 109 | }) 110 | 111 | version8 := &sortedsetFilter{ 112 | want: map[string]float64{ 113 | "finalfield": 2.718, 114 | }, 115 | wantEncoding: "skiplist", 116 | } 117 | add(testParseCase{ 118 | want: nil, 119 | file: "testdata/dumps/rdb_version_8_with_64b_length_and_scores.rdb", 120 | options: []ParseOption{WithFilter(version8)}, 121 | validators: []validator{version8}, 122 | }) 123 | 124 | version5 := &stringMapFilter{ 125 | want: map[string]string{ 126 | "abcd": "efgh", 127 | "abcdef": "abcdef", 128 | "foo": "bar", 129 | "bar": "baz", 130 | "longerstring": "thisisalongerstring.idontknowwhatitmeans", 131 | }, 132 | wantEncoding: "string", 133 | } 134 | add(testParseCase{ 135 | want: nil, 136 | file: "testdata/dumps/rdb_version_5_with_checksum.rdb", 137 | options: []ParseOption{WithFilter(version5)}, 138 | validators: []validator{version5}, 139 | }) 140 | 141 | sortedsetAsZiplist := &sortedsetFilter{ 142 | want: map[string]float64{ 143 | "8b6ba6718a786daefa69438148361901": 1, 144 | "cb7a24bb7528f934b841b34c3a73e0c7": 2.37, 145 | "523af537946b79c4f8369ed39ba78605": 3.423, 146 | }, 147 | wantEncoding: "ziplist", 148 | } 149 | add(testParseCase{ 150 | want: nil, 151 | file: "testdata/dumps/sorted_set_as_ziplist.rdb", 152 | options: []ParseOption{WithFilter(sortedsetAsZiplist)}, 153 | validators: []validator{sortedsetAsZiplist}, 154 | }) 155 | 156 | set := &setFilter{ 157 | want: map[interface{}]struct{}{ 158 | "alpha": {}, 159 | "beta": {}, 160 | "gamma": {}, 161 | "delta": {}, 162 | "phi": {}, 163 | "kappa": {}, 164 | }, 165 | wantEncoding: "hashtable", 166 | } 167 | add(testParseCase{ 168 | want: nil, 169 | file: "testdata/dumps/regular_set.rdb", 170 | options: []ParseOption{WithFilter(set)}, 171 | validators: []validator{set}, 172 | }) 173 | 174 | intset64 := &setFilter{ 175 | want: map[interface{}]struct{}{ 176 | 0x7ffefffefffefffe: {}, 177 | 0x7ffefffefffefffd: {}, 178 | 0x7ffefffefffefffc: {}, 179 | }, 180 | wantEncoding: "intset", 181 | } 182 | add(testParseCase{ 183 | want: nil, 184 | file: "testdata/dumps/intset_64.rdb", 185 | options: []ParseOption{WithFilter(intset64)}, 186 | validators: []validator{intset64}, 187 | }) 188 | 189 | intset32 := &setFilter{ 190 | want: map[interface{}]struct{}{ 191 | 0x7ffefffe: {}, 192 | 0x7ffefffd: {}, 193 | 0x7ffefffc: {}, 194 | }, 195 | wantEncoding: "intset", 196 | } 197 | add(testParseCase{ 198 | want: nil, 199 | file: "testdata/dumps/intset_32.rdb", 200 | options: []ParseOption{WithFilter(intset32)}, 201 | validators: []validator{intset32}, 202 | }) 203 | 204 | intset16 := &setFilter{ 205 | want: map[interface{}]struct{}{ 206 | 0x7ffe: {}, 207 | 0x7ffd: {}, 208 | 0x7ffc: {}, 209 | }, 210 | wantEncoding: "intset", 211 | } 212 | add(testParseCase{ 213 | want: nil, 214 | file: "testdata/dumps/intset_16.rdb", 215 | options: []ParseOption{WithFilter(intset16)}, 216 | validators: []validator{intset16}, 217 | }) 218 | 219 | linkedlist := &listFilter{ 220 | total: 1000, 221 | wantEncoding: "linkedlist", 222 | in: []string{"JYY4GIFI0ETHKP4VAJF5333082J4R1UPNPLE329YT0EYPGHSJQ", "TKBXHJOX9Q99ICF4V78XTCA2Y1UYW6ERL35JCIL1O0KSGXS58S"}, 223 | } 224 | add(testParseCase{ 225 | want: nil, 226 | file: "testdata/dumps/linkedlist.rdb", 227 | options: []ParseOption{WithFilter(linkedlist)}, 228 | validators: []validator{linkedlist}, 229 | }) 230 | 231 | expectedNums := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -2, 13, 25, -61, 63, 16380, -16000, 65535, -65523, 4194304, 0x7fffffffffffffff} 232 | ziplistWithInteger := &listFilter{ 233 | wantEncoding: "ziplist", 234 | } 235 | for _, v := range expectedNums { 236 | ziplistWithInteger.want = append(ziplistWithInteger.want, strconv.Itoa(v)) 237 | } 238 | add(testParseCase{ 239 | want: nil, 240 | file: "testdata/dumps/ziplist_with_integers.rdb", 241 | options: []ParseOption{WithFilter(ziplistWithInteger)}, 242 | validators: []validator{ziplistWithInteger}, 243 | }) 244 | 245 | ziplistSimple := &listFilter{ 246 | wantEncoding: "ziplist", 247 | want: []string{"aj2410", "cc953a17a8e096e76a44169ad3f9ac87c5f8248a403274416179aa9fbd852344"}, 248 | } 249 | add(testParseCase{ 250 | want: nil, 251 | file: "testdata/dumps/ziplist_that_doesnt_compress.rdb", 252 | options: []ParseOption{WithFilter(ziplistSimple)}, 253 | validators: []validator{ziplistSimple}, 254 | }) 255 | 256 | ziplistCompression := &listFilter{ 257 | wantEncoding: "ziplist", 258 | want: []string{"aaaaaa", "aaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, 259 | } 260 | add(testParseCase{ 261 | want: nil, 262 | file: "testdata/dumps/ziplist_that_compresses_easily.rdb", 263 | options: []ParseOption{WithFilter(ziplistCompression)}, 264 | validators: []validator{ziplistCompression}, 265 | }) 266 | 267 | dictionary := &stringMapFilter{ 268 | want: map[string]string{ 269 | "ZMU5WEJDG7KU89AOG5LJT6K7HMNB3DEI43M6EYTJ83VRJ6XNXQ": "T63SOS8DQJF0Q0VJEZ0D1IQFCYTIPSBOUIAI9SB0OV57MQR1FI", 270 | "UHS5ESW4HLK8XOGTM39IK1SJEUGVV9WOPK6JYA5QBZSJU84491": "6VULTCV52FXJ8MGVSFTZVAGK2JXZMGQ5F8OVJI0X6GEDDR27RZ", 271 | }, 272 | wantEncoding: "hashtable", 273 | } 274 | add(testParseCase{ 275 | want: nil, 276 | file: "testdata/dumps/dictionary.rdb", 277 | options: []ParseOption{WithFilter(dictionary)}, 278 | validators: []validator{dictionary}, 279 | }) 280 | 281 | hashAsZiplist := &stringMapFilter{ 282 | want: map[string]string{ 283 | "a": "aa", 284 | "aa": "aaaa", 285 | "aaaaa": "aaaaaaaaaaaaaa", 286 | }, 287 | wantEncoding: "ziplist", 288 | } 289 | add(testParseCase{ 290 | want: nil, 291 | file: "testdata/dumps/hash_as_ziplist.rdb", 292 | options: []ParseOption{WithFilter(hashAsZiplist)}, 293 | validators: []validator{hashAsZiplist}, 294 | }) 295 | 296 | zipmapBigValue := &stringMapFilter{ 297 | want: map[string]string{ 298 | "253bytes": _253bytes, 299 | "254bytes": _254bytes, 300 | "255bytes": _255bytes, 301 | "300bytes": _300bytes, 302 | "20kbytes": _20kbytes, 303 | }, 304 | wantEncoding: "zipmap", 305 | } 306 | add(testParseCase{ 307 | want: nil, 308 | file: "testdata/dumps/zipmap_with_big_values.rdb", 309 | options: []ParseOption{WithFilter(zipmapBigValue)}, 310 | validators: []validator{zipmapBigValue}, 311 | }) 312 | 313 | zipmapSimple := &stringMapFilter{ 314 | want: map[string]string{ 315 | "MKD1G6": "2", 316 | "YNNXK": "F7TI", 317 | }, 318 | wantEncoding: "zipmap", 319 | } 320 | add(testParseCase{ 321 | want: nil, 322 | file: "testdata/dumps/zipmap_that_doesnt_compress.rdb", 323 | options: []ParseOption{WithFilter(zipmapSimple)}, 324 | validators: []validator{zipmapSimple}, 325 | }) 326 | 327 | zipmapCompression := &stringMapFilter{ 328 | want: map[string]string{ 329 | "a": "aa", 330 | "aa": "aaaa", 331 | "aaaaa": "aaaaaaaaaaaaaa", 332 | }, 333 | wantEncoding: "zipmap", 334 | } 335 | add(testParseCase{ 336 | want: nil, 337 | file: "testdata/dumps/zipmap_that_compresses_easily.rdb", 338 | options: []ParseOption{WithFilter(zipmapCompression)}, 339 | validators: []validator{zipmapCompression}, 340 | }) 341 | 342 | stringKeyWithCompression := &stringMapFilter{ 343 | want: map[string]string{ 344 | strings.Repeat("a", 200): "Key that redis should compress easily", 345 | }, 346 | wantEncoding: "string", 347 | } 348 | add(testParseCase{ 349 | want: nil, 350 | file: "testdata/dumps/easily_compressible_string_key.rdb", 351 | options: []ParseOption{WithFilter(stringKeyWithCompression)}, 352 | validators: []validator{stringKeyWithCompression}, 353 | }) 354 | 355 | integerKeys := &stringMapFilter{ 356 | want: map[string]string{ 357 | strconv.Itoa(125): "Positive 8 bit integer", 358 | strconv.Itoa(0xABAB): "Positive 16 bit integer", 359 | strconv.Itoa(0x0AEDD325): "Positive 32 bit integer", 360 | strconv.Itoa(-123): "Negative 8 bit integer", 361 | strconv.Itoa(-0x7325): "Negative 16 bit integer", 362 | strconv.Itoa(-0x0AEDD325): "Negative 32 bit integer", 363 | }, 364 | wantEncoding: "string", 365 | } 366 | add(testParseCase{ 367 | want: nil, 368 | file: "testdata/dumps/integer_keys.rdb", 369 | options: []ParseOption{WithFilter(integerKeys)}, 370 | validators: []validator{integerKeys}, 371 | }) 372 | 373 | keysWithExpiry := &keysWithExpiryFilter{ 374 | want: map[string]int{ 375 | "expires_ms_precision": 1671963072573, 376 | }, 377 | } 378 | add(testParseCase{ 379 | want: nil, 380 | file: "testdata/dumps/keys_with_expiry.rdb", 381 | options: []ParseOption{WithFilter(keysWithExpiry)}, 382 | validators: []validator{keysWithExpiry}, 383 | }) 384 | 385 | multipleDatabase := &multipleDatabaseFilter{ 386 | want: []int{0, 2}, 387 | } 388 | add(testParseCase{ 389 | want: nil, 390 | file: "testdata/dumps/multiple_databases.rdb", 391 | options: []ParseOption{ 392 | EnableSync(), 393 | WithStrategy(0), 394 | WithFilter(multipleDatabase), 395 | }, 396 | validators: []validator{multipleDatabase}, 397 | }) 398 | 399 | add(testParseCase{ 400 | want: nil, 401 | file: "testdata/dumps/empty_database.rdb", 402 | }) 403 | } 404 | 405 | // multiple databases 406 | 407 | type multipleDatabaseFilter struct { 408 | testEmptyFilter 409 | 410 | got []int 411 | want []int 412 | wantOnly map[int]struct{} 413 | } 414 | 415 | func (f *multipleDatabaseFilter) Database(db DB) bool { 416 | if _, ok := f.wantOnly[db.Num]; !ok && len(f.wantOnly) > 0 { 417 | return false 418 | } 419 | f.got = append(f.got, db.Num) 420 | return false 421 | } 422 | 423 | func (f *multipleDatabaseFilter) reset() { 424 | if f.got != nil { 425 | f.got = f.got[:0] 426 | } 427 | } 428 | 429 | func (f *multipleDatabaseFilter) validate(t *testing.T) { 430 | if len(f.got) != len(f.want) { 431 | t.Fatalf("want: %v, got: %v", f.want, f.got) 432 | } 433 | for i, w := range f.want { 434 | if f.got[i] != w { 435 | t.Fatalf("want: %v, got: %v", f.want, f.got) 436 | } 437 | } 438 | } 439 | 440 | // keys with expiry 441 | 442 | type keysWithExpiryFilter struct { 443 | testEmptyFilter 444 | 445 | got map[string]int 446 | want map[string]int 447 | } 448 | 449 | func (f *keysWithExpiryFilter) String(s *String) { 450 | f.Lock() 451 | defer f.Unlock() 452 | if f.got == nil { 453 | f.got = make(map[string]int) 454 | } 455 | f.got[s.Key.Key] = s.Key.Expiry 456 | } 457 | 458 | func (f *keysWithExpiryFilter) reset() { 459 | for k := range f.got { 460 | delete(f.got, k) 461 | } 462 | } 463 | 464 | func (f *keysWithExpiryFilter) validate(t *testing.T) { 465 | for k, v := range f.want { 466 | if g, ok := f.got[k]; !ok || g != v { 467 | t.Fatalf("key: %v, want: %v, got: %v", k, v, g) 468 | } 469 | } 470 | } 471 | 472 | // keys with idle 473 | 474 | type keysWithIdleFilter struct { 475 | testEmptyFilter 476 | 477 | got map[string]int 478 | want map[string]int 479 | } 480 | 481 | func (f *keysWithIdleFilter) String(s *String) { 482 | f.Lock() 483 | defer f.Unlock() 484 | if f.got == nil { 485 | f.got = make(map[string]int) 486 | } 487 | f.got[s.Key.Key] = s.Key.Idle 488 | } 489 | 490 | func (f *keysWithIdleFilter) reset() { 491 | for k := range f.got { 492 | delete(f.got, k) 493 | } 494 | } 495 | 496 | func (f *keysWithIdleFilter) validate(t *testing.T) { 497 | for k, v := range f.want { 498 | if g, ok := f.got[k]; !ok || g != v { 499 | t.Fatalf("key: %v, want: %v, got: %v", k, v, g) 500 | } 501 | } 502 | } 503 | 504 | // keys with freq 505 | 506 | type keysWithFreqFilter struct { 507 | testEmptyFilter 508 | 509 | got map[string]int 510 | want map[string]int 511 | } 512 | 513 | func (f *keysWithFreqFilter) String(s *String) { 514 | f.Lock() 515 | defer f.Unlock() 516 | if f.got == nil { 517 | f.got = make(map[string]int) 518 | } 519 | f.got[s.Key.Key] = s.Key.Freq 520 | } 521 | 522 | func (f *keysWithFreqFilter) reset() { 523 | for k := range f.got { 524 | delete(f.got, k) 525 | } 526 | } 527 | 528 | func (f *keysWithFreqFilter) validate(t *testing.T) { 529 | for k, v := range f.want { 530 | if g, ok := f.got[k]; !ok || g != v { 531 | t.Fatalf("key: %v, want: %v, got: %v", k, v, g) 532 | } 533 | } 534 | } 535 | 536 | // string map 537 | 538 | type stringMapFilter struct { 539 | testEmptyFilter 540 | 541 | got map[string]string 542 | want map[string]string 543 | 544 | gotEncoding string 545 | wantEncoding string 546 | } 547 | 548 | func (f *stringMapFilter) String(s *String) { 549 | f.Lock() 550 | defer f.Unlock() 551 | if f.got == nil { 552 | f.got = make(map[string]string) 553 | } 554 | f.gotEncoding = Encoding2String(s.Key.Encoding) 555 | f.got[s.Key.Key] = s.Value 556 | } 557 | 558 | func (f *stringMapFilter) Hash(h *Hash) { 559 | f.Lock() 560 | defer f.Unlock() 561 | if f.got == nil { 562 | f.got = make(map[string]string) 563 | } 564 | f.gotEncoding = Encoding2String(h.Key.Encoding) 565 | for k, v := range h.Values { 566 | f.got[k] = v 567 | } 568 | } 569 | 570 | func (f *stringMapFilter) reset() { 571 | for k := range f.got { 572 | delete(f.got, k) 573 | } 574 | } 575 | 576 | func (f *stringMapFilter) validate(t *testing.T) { 577 | for k, v := range f.want { 578 | if g, ok := f.got[k]; !ok || g != v { 579 | t.Fatalf("key: %v, want: %v, got: %v", k, v, g) 580 | } 581 | } 582 | if f.wantEncoding != "" && f.wantEncoding != f.gotEncoding { 583 | t.Fatalf("want: %v, got: %v", f.wantEncoding, f.gotEncoding) 584 | } 585 | } 586 | 587 | // list 588 | 589 | type listFilter struct { 590 | testEmptyFilter 591 | 592 | got []string 593 | want []string 594 | 595 | in []string 596 | total int 597 | 598 | gotEncoding string 599 | wantEncoding string 600 | } 601 | 602 | func (f *listFilter) List(l *List) { 603 | f.Lock() 604 | defer f.Unlock() 605 | f.gotEncoding = Encoding2String(l.Key.Encoding) 606 | for _, v := range l.Values { 607 | f.got = append(f.got, v) 608 | } 609 | } 610 | 611 | func (f *listFilter) reset() { 612 | if f.got != nil { 613 | f.got = f.got[:0] 614 | } 615 | } 616 | 617 | func (f *listFilter) validate(t *testing.T) { 618 | for i, v := range f.want { 619 | if f.got[i] != v { 620 | t.Fatalf("want: %v, got: %v", f.want, f.got) 621 | } 622 | } 623 | if f.wantEncoding != "" && f.wantEncoding != f.gotEncoding { 624 | t.Fatalf("want: %v, got: %v", f.wantEncoding, f.gotEncoding) 625 | } 626 | if len(f.in) == 0 { 627 | return 628 | } 629 | if f.total != len(f.got) { 630 | t.Fatalf("want: %v, got: %v", f.total, len(f.got)) 631 | } 632 | for _, w := range f.in { 633 | found := false 634 | for _, g := range f.got { 635 | if g == w { 636 | found = true 637 | break 638 | } 639 | } 640 | if !found { 641 | t.Fatalf("want: %v, got: %v", f.want, f.got) 642 | } 643 | } 644 | } 645 | 646 | // set 647 | 648 | type setFilter struct { 649 | testEmptyFilter 650 | 651 | got map[interface{}]struct{} 652 | want map[interface{}]struct{} 653 | 654 | gotEncoding string 655 | wantEncoding string 656 | } 657 | 658 | func (f *setFilter) Set(s *Set) { 659 | f.Lock() 660 | defer f.Unlock() 661 | if f.got == nil { 662 | f.got = make(map[interface{}]struct{}) 663 | } 664 | f.gotEncoding = Encoding2String(s.Key.Encoding) 665 | for k, v := range s.Values { 666 | f.got[k] = v 667 | } 668 | } 669 | 670 | func (f *setFilter) reset() { 671 | for k := range f.got { 672 | delete(f.got, k) 673 | } 674 | } 675 | 676 | func (f *setFilter) validate(t *testing.T) { 677 | for k, v := range f.want { 678 | if g, ok := f.got[k]; !ok || g != v { 679 | t.Fatalf("key: %v, want: %v, got: %v", k, v, g) 680 | } 681 | } 682 | if f.wantEncoding != "" && f.wantEncoding != f.gotEncoding { 683 | t.Fatalf("want: %v, got: %v", f.wantEncoding, f.gotEncoding) 684 | } 685 | } 686 | 687 | // sortedset 688 | 689 | type sortedsetFilter struct { 690 | testEmptyFilter 691 | 692 | got map[string]float64 693 | want map[string]float64 694 | 695 | gotEncoding string 696 | wantEncoding string 697 | } 698 | 699 | func (f *sortedsetFilter) SortedSet(ss *SortedSet) { 700 | f.Lock() 701 | defer f.Unlock() 702 | if f.got == nil { 703 | f.got = make(map[string]float64) 704 | } 705 | f.gotEncoding = Encoding2String(ss.Key.Encoding) 706 | for k, v := range ss.Values { 707 | f.got[k] = v 708 | } 709 | } 710 | 711 | func (f *sortedsetFilter) reset() { 712 | for k := range f.got { 713 | delete(f.got, k) 714 | } 715 | } 716 | 717 | func (f *sortedsetFilter) validate(t *testing.T) { 718 | for k, v := range f.want { 719 | if g, ok := f.got[k]; !ok || g != v { 720 | t.Fatalf("key: %v, want: %v, got: %v", k, v, g) 721 | } 722 | } 723 | if f.wantEncoding != "" && f.wantEncoding != f.gotEncoding { 724 | t.Fatalf("want: %v, got: %v", f.wantEncoding, f.gotEncoding) 725 | } 726 | } 727 | 728 | // test helpers 729 | 730 | type testEmptyFilter struct { 731 | sync.Mutex 732 | t *testing.T 733 | } 734 | 735 | func (f *testEmptyFilter) Key(k Key) bool { return false } 736 | func (f *testEmptyFilter) Type(_ Type) bool { return false } 737 | func (f *testEmptyFilter) Database(_ DB) bool { return false } 738 | func (f *testEmptyFilter) Set(v *Set) { 739 | if debug { 740 | log.Println("set:", v.Key.Key, v.Values) 741 | } 742 | } 743 | func (f *testEmptyFilter) List(v *List) { 744 | if debug { 745 | log.Println("list:", v.Key.Key, v.Values) 746 | } 747 | } 748 | func (f *testEmptyFilter) Hash(v *Hash) { 749 | if debug { 750 | log.Println("hash:", v.Key.Key, v.Values) 751 | } 752 | } 753 | func (f *testEmptyFilter) String(v *String) { 754 | if debug { 755 | log.Println("string:", v.Key.Key, v.Value) 756 | } 757 | } 758 | func (f *testEmptyFilter) SortedSet(v *SortedSet) { 759 | if debug { 760 | log.Println("sortedset:", v.Key.Key, v.Values) 761 | } 762 | } 763 | 764 | type validator interface { 765 | reset() 766 | validate(t *testing.T) 767 | } 768 | 769 | type testParseCase struct { 770 | want error 771 | file string 772 | options []ParseOption 773 | validators []validator 774 | } 775 | 776 | func (tc testParseCase) reset() { 777 | for _, v := range tc.validators { 778 | v.reset() 779 | } 780 | } 781 | 782 | func (tc testParseCase) validate(t *testing.T) { 783 | for _, v := range tc.validators { 784 | v.validate(t) 785 | } 786 | } 787 | 788 | // test data 789 | 790 | var ( 791 | _253bytes = `NYKK5QA4TDYJFZH0FCVT39DWI89IH7HV9HV162MULYY9S6H67MGS6YZJ54Q2NISW9U69VC6ZK3OJV6J095P0P5YNSEHGCBJGYNZ8BPK3GEFBB8ZMGPT2Y33WNSETHINMSZ4VKWUE8CXE0Y9FO7L5ZZ02EO26TLXF5NUQ0KMA98973QY62ZO1M1WDDZNS25F37KGBQ8W4R5V1YJRR2XNSQKZ4VY7GW6X038UYQG30ZM0JY1NNMJ12BKQPF2IDQ` 792 | _254bytes = `IZ3PNCQQV5RG4XOAXDN7IPWJKEK0LWRARBE3393UYD89PSQFC40AG4RCNW2M4YAVJR0WD8AVO2F8KFDGUV0TGU8GF8M2HZLZ9RDX6V0XKIOXJJ3EMWQGFEY7E56RAOPTA60G6SQRZ59ZBUKA6OMEW3K0LH464C7XKAX3K8AXDUX63VGX99JDCW1W2KTXPQRN1R1PY5LXNXPW7AAIYUM2PUKN2YN2MXWS5HR8TPMKYJIFTLK2DNQNGTVAWMULON` 793 | _255bytes = `6EUW8XSNBHMEPY991GZVZH4ITUQVKXQYL7UBYS614RDQSE7BDRUW00M6Y4W6WUQBDFVHH6V2EIAEQGLV72K4UY7XXKL6K6XH6IN4QVS15GU1AAH9UI40UXEA8IZ5CZRRK6SAV3R3X283O2OO9KG4K0DG0HZX1MLFDQHXGCC96M9YUVKXOEC5X35Q4EKET0SDFDSBF1QKGAVS9202EL7MP2KPOYAUKU1SZJW5OP30WAPSM9OG97EBHW2XOWGICZG` 794 | _300bytes = `IJXP54329MQ96A2M28QF6SFX3XGNWGAII3M32MSIMR0O478AMZKNXDUYD5JGMHJRB9A85RZ3DC3AIS62YSDW2BDJ97IBSH7FKOVFWKJYS7XBMIBX0Z1WNLQRY7D27PFPBBGBDFDCKL0FIOBYEADX6G5UK3B0XYMGS0379GRY6F0FY5Q9JUCJLGOGDNNP8XW3SJX2L872UJZZL8G871G9THKYQ2WKPFEBIHOOTIGDNWC15NL5324W8FYDP97JHKCSMLWXNMSTYIUE7F22ZGR4NZK3T0UTBZ2AFRCT5LMT3P6B` 795 | _20kbytes string 796 | ) 797 | --------------------------------------------------------------------------------