├── .github └── workflows │ └── main.yml ├── .gitignore ├── .travis.yml ├── .vscode └── launch.json ├── LICENSE ├── README.md ├── README_CN.md ├── build.sh ├── bytefmt ├── bytefmt.go └── bytefmt_test.go ├── cases ├── easily_compressible_string_key.json ├── easily_compressible_string_key.rdb ├── empty_database.json ├── empty_database.rdb ├── hash.json ├── hash.rdb ├── hash_as_ziplist.json ├── hash_as_ziplist.rdb ├── integer_keys.json ├── integer_keys.rdb ├── intset_16.json ├── intset_16.rdb ├── intset_32.json ├── intset_32.rdb ├── intset_64.json ├── intset_64.rdb ├── issue27.json ├── issue27.rdb ├── keys_with_expiry.json ├── keys_with_expiry.rdb ├── largest.csv ├── linkedlist.json ├── linkedlist.rdb ├── listpack.csv ├── listpack.json ├── listpack.rdb ├── memory.aof ├── memory.csv ├── memory.json ├── memory.rdb ├── memory_expired.csv ├── memory_regex.aof ├── memory_regex.biggest.csv ├── memory_regex.csv ├── memory_regex.json ├── multiple_databases.json ├── multiple_databases.rdb ├── non_ascii_values.json ├── non_ascii_values.rdb ├── parser_filters.json ├── parser_filters.rdb ├── quicklist.json ├── quicklist.rdb ├── rdb_version_5_with_checksum.json ├── rdb_version_5_with_checksum.rdb ├── rdb_version_8_with_64b_length_and_scores.json ├── rdb_version_8_with_64b_length_and_scores.rdb ├── regular_set.json ├── regular_set.rdb ├── regular_sorted_set.json ├── regular_sorted_set.rdb ├── set_listpack.csv ├── set_listpack.json ├── set_listpack.rdb ├── sorted_set_as_ziplist.json ├── sorted_set_as_ziplist.rdb ├── stream_listoacks_3.json ├── stream_listoacks_3.rdb ├── stream_listpacks_1.csv ├── stream_listpacks_1.json ├── stream_listpacks_1.rdb ├── stream_listpacks_2.csv ├── stream_listpacks_2.json ├── stream_listpacks_2.rdb ├── tree.csv ├── tree.rdb ├── tree2.csv ├── uncompressible_string_keys.json ├── uncompressible_string_keys.rdb ├── ziplist_that_compresses_easily.json ├── ziplist_that_compresses_easily.rdb ├── ziplist_that_doesnt_compress.json ├── ziplist_that_doesnt_compress.rdb ├── ziplist_with_integers.json ├── ziplist_with_integers.rdb ├── zipmap_big_len.json ├── zipmap_big_len.rdb ├── zipmap_that_compresses_easily.json ├── zipmap_that_compresses_easily.rdb ├── zipmap_that_doesnt_compress.json ├── zipmap_that_doesnt_compress.rdb ├── zipmap_with_big_values.json └── zipmap_with_big_values.rdb ├── cmd.go ├── cmd_test.go ├── core ├── decoder.go ├── encoder.go ├── encoder_test.go ├── hash.go ├── hash_test.go ├── list.go ├── list_test.go ├── listpack.go ├── module.go ├── module_test.go ├── set.go ├── set_test.go ├── specialobj_test.go ├── stream.go ├── string.go ├── string_test.go ├── utils.go ├── ziplist.go ├── zset.go └── zset_test.go ├── crc64jones ├── crc64.go └── crc64_test.go ├── d3flame ├── bootstrap.min.css ├── d3-flamegraph.css ├── d3-flamegraph.min.js ├── d3-tip.min.js ├── d3.v4.min.js ├── template.go └── web.go ├── encoder └── portal.go ├── examples ├── decode │ ├── example.rdb │ └── main.go └── encode │ └── main.go ├── go.mod ├── go.sum ├── helper ├── bigkey.go ├── bigkey_test.go ├── converter.go ├── converter_test.go ├── flamegraph.go ├── flamegraph_test.go ├── json.go ├── memory.go ├── memory_test.go ├── prefix.go ├── prefix_test.go ├── radix.go ├── radix_test.go ├── regex.go ├── resp.go └── toplist.go ├── lzf ├── lzf.go └── lzf_test.go ├── memprofiler ├── common.go ├── hash.go ├── list.go ├── memprofiler.go ├── stream.go ├── zset.go └── zset_test.go ├── model ├── detail.go ├── model.go └── stream.go ├── parser └── portal.go └── tree.csv /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "master", "github-actions" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | workflow_dispatch: 12 | 13 | 14 | jobs: 15 | 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | 21 | - uses: actions/setup-go@v3 22 | with: 23 | go-version: '1.18' 24 | - run: go test -v -coverpkg=./core,./ -coverprofile=profile.cov ./... 25 | 26 | - uses: shogo82148/actions-goveralls@v1 27 | with: 28 | path-to-profile: profile.cov 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .idea 17 | target 18 | large.rdb 19 | /large.aof 20 | /large.json 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.16.x 5 | before_install: 6 | - go install github.com/mattn/goveralls@latest 7 | script: 8 | - $GOPATH/bin/goveralls -service=travis-ci 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // 使用 IntelliSense 了解相关属性。 3 | // 悬停以查看现有属性的描述。 4 | // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Package", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceFolder}", 13 | "args": ["-c", "prefix", "-n", "20", "-o", "tree.csv", "large.rdb"] 14 | } 15 | 16 | ] 17 | } -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o target/rdb-linux-amd64 ./ 4 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o target/rdb-darwin-amd64 ./ 5 | CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o target/rdb-darwin-arm64 ./ 6 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o target/rdb-windows-amd64.exe ./ -------------------------------------------------------------------------------- /bytefmt/bytefmt.go: -------------------------------------------------------------------------------- 1 | package bytefmt 2 | 3 | // from: github.com/cloudfoundry/bytefmt by Apache License 4 | 5 | import ( 6 | "errors" 7 | "strconv" 8 | "strings" 9 | "unicode" 10 | ) 11 | 12 | const ( 13 | sizeByte = 1 << (10 * iota) 14 | sizeKilo 15 | sizeMega 16 | sizeGiga 17 | sizeTera 18 | sizePeta 19 | sizeExa 20 | ) 21 | 22 | var errInvalidByteQuantity = errors.New("byte quantity must be a positive integer with a unit of measurement like M, MB, MiB, G, GiB, or GB") 23 | 24 | // FormatSize returns a human-readable byte string of the form 10M, 12.5K, and so forth. The following units are available: 25 | // E: Exabyte 26 | // P: Petabyte 27 | // T: Terabyte 28 | // G: Gigabyte 29 | // M: Megabyte 30 | // K: Kilobyte 31 | // B: Byte 32 | // The unit that results in the smallest number greater than or equal to 1 is always chosen. 33 | func FormatSize(bytes uint64) string { 34 | unit := "" 35 | value := float64(bytes) 36 | 37 | switch { 38 | case bytes >= sizeExa: 39 | unit = "E" 40 | value = value / sizeExa 41 | case bytes >= sizePeta: 42 | unit = "P" 43 | value = value / sizePeta 44 | case bytes >= sizeTera: 45 | unit = "T" 46 | value = value / sizeTera 47 | case bytes >= sizeGiga: 48 | unit = "G" 49 | value = value / sizeGiga 50 | case bytes >= sizeMega: 51 | unit = "M" 52 | value = value / sizeMega 53 | case bytes >= sizeKilo: 54 | unit = "K" 55 | value = value / sizeKilo 56 | case bytes >= sizeByte: 57 | unit = "B" 58 | case bytes == 0: 59 | return "0" 60 | } 61 | 62 | result := strconv.FormatFloat(value, 'f', 1, 64) 63 | result = strings.TrimSuffix(result, ".0") 64 | return result + unit 65 | } 66 | 67 | // ParseSize parses a string formatted by FormatSize as bytes. Note binary-prefixed and SI prefixed units both mean a base-2 units 68 | // KB = K = KiB = 1024 69 | // MB = M = MiB = 1024 * K 70 | // GB = G = GiB = 1024 * M 71 | // TB = T = TiB = 1024 * G 72 | // PB = P = PiB = 1024 * T 73 | // EB = E = EiB = 1024 * P 74 | func ParseSize(s string) (uint64, error) { 75 | s = strings.TrimSpace(s) 76 | s = strings.ToUpper(s) 77 | 78 | i := strings.IndexFunc(s, unicode.IsLetter) 79 | 80 | if i == -1 { 81 | return 0, errInvalidByteQuantity 82 | } 83 | 84 | bytesString, multiple := s[:i], s[i:] 85 | bytes, err := strconv.ParseFloat(bytesString, 64) 86 | if err != nil || bytes <= 0 { 87 | return 0, errInvalidByteQuantity 88 | } 89 | 90 | switch multiple { 91 | case "E", "EB", "EIB": 92 | return uint64(bytes * sizeExa), nil 93 | case "P", "PB", "PIB": 94 | return uint64(bytes * sizePeta), nil 95 | case "T", "TB", "TIB": 96 | return uint64(bytes * sizeTera), nil 97 | case "G", "GB", "GIB": 98 | return uint64(bytes * sizeGiga), nil 99 | case "M", "MB", "MIB": 100 | return uint64(bytes * sizeMega), nil 101 | case "K", "KB", "KIB": 102 | return uint64(bytes * sizeKilo), nil 103 | case "B": 104 | return uint64(bytes), nil 105 | default: 106 | return 0, errInvalidByteQuantity 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /bytefmt/bytefmt_test.go: -------------------------------------------------------------------------------- 1 | package bytefmt 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | ) 7 | 8 | func TestFormatSize(t *testing.T) { 9 | if FormatSize(0) != "0" { 10 | t.Error("format error") 11 | } 12 | if FormatSize(123) != "123B" { 13 | t.Error("format error") 14 | } 15 | if FormatSize(123*(1<<10)) != "123K" { 16 | t.Error("format error") 17 | } 18 | if FormatSize(123*(1<<20)) != "123M" { 19 | t.Error("format error") 20 | } 21 | if FormatSize(123*(1<<30)) != "123G" { 22 | t.Error("format error") 23 | } 24 | if FormatSize(123*(1<<40)) != "123T" { 25 | t.Error("format error") 26 | } 27 | if FormatSize(123*(1<<50)) != "123P" { 28 | t.Error("format error") 29 | } 30 | if FormatSize(math.MaxUint64) != "16E" { 31 | t.Error("format error") 32 | } 33 | } 34 | 35 | func TestParseSize(t *testing.T) { 36 | if _, err := ParseSize("0"); err != errInvalidByteQuantity { 37 | t.Error("parse error") 38 | } 39 | if _, err := ParseSize("0B"); err != errInvalidByteQuantity { 40 | t.Error("parse error") 41 | } 42 | if _, err := ParseSize("1A"); err != errInvalidByteQuantity { 43 | t.Error("parse error") 44 | } 45 | if b, err := ParseSize("123B"); err != nil || b != 123 { 46 | t.Error("parse error") 47 | } 48 | if b, err := ParseSize("123K"); err != nil || b != 123*(1<<10) { 49 | t.Error("format error") 50 | } 51 | if b, err := ParseSize("123M"); err != nil || b != 123*(1<<20) { 52 | t.Error("format error") 53 | } 54 | if b, err := ParseSize("123G"); err != nil || b != 123*(1<<30) { 55 | t.Error("format error") 56 | } 57 | if b, err := ParseSize("123T"); err != nil || b != 123*(1<<40) { 58 | t.Error("format error") 59 | } 60 | if b, err := ParseSize("123P"); err != nil || b != 123*(1<<50) { 61 | t.Error("format error") 62 | } 63 | if b, err := ParseSize("1E"); err != nil || b != 1<<60 { 64 | t.Error("format error") 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /cases/easily_compressible_string_key.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":304,"type":"string","encoding":"string","value":"Key that redis should compress easily"} 3 | ] -------------------------------------------------------------------------------- /cases/easily_compressible_string_key.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/easily_compressible_string_key.rdb -------------------------------------------------------------------------------- /cases/empty_database.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | ] -------------------------------------------------------------------------------- /cases/empty_database.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/empty_database.rdb -------------------------------------------------------------------------------- /cases/hash.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/hash.rdb -------------------------------------------------------------------------------- /cases/hash_as_ziplist.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"zipmap_compresses_easily","size":123,"type":"hash","encoding":"ziplist","hash":{"a":"aa","aa":"aaaa","aaaaa":"aaaaaaaaaaaaaa"}} 3 | ] -------------------------------------------------------------------------------- /cases/hash_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/hash_as_ziplist.rdb -------------------------------------------------------------------------------- /cases/integer_keys.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"183358245","size":72,"type":"string","encoding":"string","value":"Positive 32 bit integer"}, 3 | {"db":0,"key":"125","size":64,"type":"string","encoding":"string","value":"Positive 8 bit integer"}, 4 | {"db":0,"key":"-29477","size":72,"type":"string","encoding":"string","value":"Negative 16 bit integer"}, 5 | {"db":0,"key":"-123","size":64,"type":"string","encoding":"string","value":"Negative 8 bit integer"}, 6 | {"db":0,"key":"43947","size":72,"type":"string","encoding":"string","value":"Positive 16 bit integer"}, 7 | {"db":0,"key":"-183358245","size":72,"type":"string","encoding":"string","value":"Negative 32 bit integer"} 8 | ] -------------------------------------------------------------------------------- /cases/integer_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/integer_keys.rdb -------------------------------------------------------------------------------- /cases/intset_16.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"intset_16","size":70,"type":"set","encoding":"intset","members":["32764","32765","32766"]} 3 | ] -------------------------------------------------------------------------------- /cases/intset_16.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/intset_16.rdb -------------------------------------------------------------------------------- /cases/intset_32.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"intset_32","size":76,"type":"set","encoding":"intset","members":["2147418108","2147418109","2147418110"]} 3 | ] -------------------------------------------------------------------------------- /cases/intset_32.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/intset_32.rdb -------------------------------------------------------------------------------- /cases/intset_64.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"intset_64","size":88,"type":"set","encoding":"intset","members":["9223090557583032316","9223090557583032317","9223090557583032318"]} 3 | ] -------------------------------------------------------------------------------- /cases/intset_64.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/intset_64.rdb -------------------------------------------------------------------------------- /cases/issue27.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/issue27.rdb -------------------------------------------------------------------------------- /cases/keys_with_expiry.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"expires_ms_precision","expiration":"2022-12-25T18:11:12.573+08:00","size":128,"type":"string","encoding":"string","value":"2022-12-25 10:11:12.573 UTC"} 3 | ] -------------------------------------------------------------------------------- /cases/keys_with_expiry.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/keys_with_expiry.rdb -------------------------------------------------------------------------------- /cases/largest.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count 2 | 0,large,string,2608,2.5K,0 3 | 0,set,set,284,284B,2 4 | 0,list,list,203,203B,4 5 | 0,hash,hash,131,131B,2 6 | 0,zset,zset,99,99B,2 7 | -------------------------------------------------------------------------------- /cases/linkedlist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/linkedlist.rdb -------------------------------------------------------------------------------- /cases/listpack.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,l,list,124,124B,9,quicklist2, 3 | 0,z,zset,139,139B,12,listpack, 4 | 0,h,hash,150,150B,11,listpack, 5 | -------------------------------------------------------------------------------- /cases/listpack.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"l","size":124,"type":"list","encoding":"quicklist2","values":["1","20000","aaaa","4","16380","-16380","1048576","268435456","8589934592"]}, 3 | {"db":0,"key":"z","size":139,"type":"zset","encoding":"listpack","entries":[{"member":"11","score":-8589934592},{"member":"9","score":-268435456},{"member":"7","score":-1048576},{"member":"5","score":-16380},{"member":"12","score":-2000},{"member":"3","score":0},{"member":"1","score":1},{"member":"2","score":2000},{"member":"4","score":16380},{"member":"6","score":1048576},{"member":"8","score":268435456},{"member":"10","score":8589934592}]}, 4 | {"db":0,"key":"h","size":150,"type":"hash","encoding":"listpack","hash":{"1":"1","10":"8589934592","11":"8589934592","2":"2000","3":"aaaaaaaaaaaaaaaa","4":"16380","5":"-16380","6":"1048576","7":"-1048576","8":"268435456","9":"-268435456"}} 5 | ] -------------------------------------------------------------------------------- /cases/listpack.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/listpack.rdb -------------------------------------------------------------------------------- /cases/memory.aof: -------------------------------------------------------------------------------- 1 | *6 2 | $5 3 | HMSET 4 | $4 5 | hash 6 | $16 7 | ca32mbn2k3tp41iu 8 | $16 9 | ca32mbn2k3tp41iu 10 | $16 11 | mddbhxnzsbklyp8c 12 | $16 13 | mddbhxnzsbklyp8c 14 | *3 15 | $3 16 | SET 17 | $1 18 | s 19 | $7 20 | aaaaaaa 21 | *3 22 | $3 23 | SET 24 | $1 25 | e 26 | $5 27 | zxcvb 28 | *3 29 | $9 30 | PEXPIREAT 31 | $1 32 | e 33 | $13 34 | 1645136129180 35 | *6 36 | $5 37 | RPUSH 38 | $4 39 | list 40 | $10 41 | 7fbn7xhcnu 42 | $10 43 | lmproj6c2e 44 | $10 45 | e5lom29act 46 | $10 47 | yy3ux925do 48 | *6 49 | $4 50 | ZADD 51 | $4 52 | zset 53 | $1 54 | 1 55 | $16 56 | zn4ejjo4ths63irg 57 | $1 58 | 2 59 | $16 60 | 1ik4jifkg6olxf5n 61 | *3 62 | $3 63 | SET 64 | $5 65 | large 66 | $2048 67 | 7sqlkn50jsn9zh2hrp3kj9tvumyoj7cdzolisj6y59ev3ymdy8ffne1nxzzbb4bg0pnvuk1gikwj68ig0wl2s5az25ffldquavkuh5k4tcsrcmph6ubcjb5lk1i2rq4qs41p7j9tj34ek3dj9fu8zw72qfdkr7clk9y0le6rj58krfx0to33wr4fn0t2sq82hrdrdetr60l6bbttsxi4b8z4hs7xd0fu63i2xa511odmmjj1mcpz2bcqohdjx1jcwntu0kttwq0ov3jh9252yqe3z8cz8dml7mrd21brndspix586jk9rd9f872177hvfzm08ai4uosqhdkjrecgududl3yry0rha8gyhheb5c8x3rjjnne4737u1pnwfhg0tdrg3mg8ar4ktcqsifr5ooed40jrrncnr6b5q34vnkrdck8t079nbq69183lh3c1z6xylxc9anxxbu6l9bcpwgltsxi3ovr4dj2l5tkj4mdbymtvfdufc9zh23l8q5kjhdys8g1d2hitk8u39q0jgaka0w9wx5xucdlqc5dwi5mxxviaob3061dcutmfmow0vc10drmp7qq9c9gtb77fnwv6tl9jpkw7duwibo4lmk8hjhboup8mhctinkw3zzy1m84apzyl453ldcako2vok0enohxwwsc2fszxaqnayoyda1y2tqa6wf60d8y8pbi2m4csffo2l1crv8cpoo5gwt6amkcj8esa8h2vewmzago74bnbcng3jbgmrmvhtd3xikpu3q8xw3ri7t2eh2kof28y221247z94uppka0e97dp0bs8by5512xbwuqt5r3s3yb5zk4ytz9c1iadsv8b717enhfkeaimptw8rzvwkd5kx6q8gymd893umlfvmpnho3tcx7wslukp4nuclhonod9k2lojya8h4nswxlegewgj9pswpnhbd6itty5xm4q5w0n1omwdtb5ccnxp9hwf3yme64anp8xk7q81bmt6gmv0zoreyjwjcjrlebrgpv9etsie3eyffrb8fzgtnqa086j0yhyz9emcjaexsvrspiupmilu1v8kc7udh1xnte0flzolol7xyvr56u1otsp1lujhzm0pq4oxnkaw930l5g2s8iz3zmfmuhzzwtrli3mnmjhj5dajbk3xz9yjxttwredz00f1r8gyme5x0r52xmeklq24huoyuon4x1w1tb5psq73nn9444dzlx2guahyvu6isb4di8dg0c7yphzah1co8y76qb0098atf0pxfbr37ff2hlvqfqun48yh8qw263p0rxp57antnbkyzu1b6rmh344893oca9dp8ce5wcsterbyjnpgpaf9e4lx5a9tkz3eh3gwqssu9pn3hnb8wd6kaxr2w6bak1r8n45lsxq3guigerlfcgpg0bozyvfq7xg89t7credt8qs3ic6c3u918o8rr1zcewhongee8b8g0ae0wme8tikzovxi2n5hhzffmdi2blfn1ko7g7gy1l406oac4nsh1ri66pfv13mox915lywmv9cis2zfpmj1an4zz3xbvchivzgl8v71c4mt8n6j9j5yqs1cuw93kgzr1sm44cl885jj96d6k7olxodkwpkl7gkgibxwwkwoy1n47iput8kyee9slpneuqac0yccrg09tebu9qqoczh9i6obsngvmg8yjsee2usp450n736i3i2wcznhyyj72cdzkik4t9sdpg08k0tu5y6xmta77mchylh3vf9y9hqsxdul84kdzg663dtxoms766evqe1mpcy3pnhr9bmhpg70kp0tdvem31n3dzw3e4dqxpwkpm6fy5sjw1gtw4nlcn6dnqrcplynksoxeut4o228uaf6341cwi4oakavnot5sk03o77b7gnnz60arimo52wfjzg8us2j4pqpvysdgiuv76fn404gohyepyz0r0vqbf63ir51sdsv0veywyc2ikmmtifankyzi530juj437pzmenbv7nd3ir21mf3m90tav8dwy6zb0c4lbexsqwzmrzq 68 | *4 69 | $4 70 | SADD 71 | $3 72 | set 73 | $16 74 | 2hzm5rnmkmwb3zqd 75 | $16 76 | tdje6bk22c6ddlrw 77 | -------------------------------------------------------------------------------- /cases/memory.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,hash,hash,131,131B,2,ziplist, 3 | 0,s,string,64,64B,0,string, 4 | 0,e,string,88,88B,0,string,2022-02-18T06:15:29+08:00 5 | 0,list,list,203,203B,4,quicklist, 6 | 0,zset,zset,99,99B,2,ziplist, 7 | 0,large,string,2608,2.5K,0,string, 8 | 0,set,set,284,284B,2,set, 9 | -------------------------------------------------------------------------------- /cases/memory.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"hash","size":131,"type":"hash","encoding":"ziplist","hash":{"ca32mbn2k3tp41iu":"ca32mbn2k3tp41iu","mddbhxnzsbklyp8c":"mddbhxnzsbklyp8c"}}, 3 | {"db":0,"key":"s","size":64,"type":"string","encoding":"string","value":"aaaaaaa"}, 4 | {"db":0,"key":"e","expiration":"2022-02-18T06:15:29.18+08:00","size":88,"type":"string","encoding":"string","value":"zxcvb"}, 5 | {"db":0,"key":"list","size":203,"type":"list","encoding":"quicklist","values":["7fbn7xhcnu","lmproj6c2e","e5lom29act","yy3ux925do"]}, 6 | {"db":0,"key":"zset","size":99,"type":"zset","encoding":"ziplist","entries":[{"member":"zn4ejjo4ths63irg","score":1},{"member":"1ik4jifkg6olxf5n","score":2}]}, 7 | {"db":0,"key":"large","size":2608,"type":"string","encoding":"string","value":"7sqlkn50jsn9zh2hrp3kj9tvumyoj7cdzolisj6y59ev3ymdy8ffne1nxzzbb4bg0pnvuk1gikwj68ig0wl2s5az25ffldquavkuh5k4tcsrcmph6ubcjb5lk1i2rq4qs41p7j9tj34ek3dj9fu8zw72qfdkr7clk9y0le6rj58krfx0to33wr4fn0t2sq82hrdrdetr60l6bbttsxi4b8z4hs7xd0fu63i2xa511odmmjj1mcpz2bcqohdjx1jcwntu0kttwq0ov3jh9252yqe3z8cz8dml7mrd21brndspix586jk9rd9f872177hvfzm08ai4uosqhdkjrecgududl3yry0rha8gyhheb5c8x3rjjnne4737u1pnwfhg0tdrg3mg8ar4ktcqsifr5ooed40jrrncnr6b5q34vnkrdck8t079nbq69183lh3c1z6xylxc9anxxbu6l9bcpwgltsxi3ovr4dj2l5tkj4mdbymtvfdufc9zh23l8q5kjhdys8g1d2hitk8u39q0jgaka0w9wx5xucdlqc5dwi5mxxviaob3061dcutmfmow0vc10drmp7qq9c9gtb77fnwv6tl9jpkw7duwibo4lmk8hjhboup8mhctinkw3zzy1m84apzyl453ldcako2vok0enohxwwsc2fszxaqnayoyda1y2tqa6wf60d8y8pbi2m4csffo2l1crv8cpoo5gwt6amkcj8esa8h2vewmzago74bnbcng3jbgmrmvhtd3xikpu3q8xw3ri7t2eh2kof28y221247z94uppka0e97dp0bs8by5512xbwuqt5r3s3yb5zk4ytz9c1iadsv8b717enhfkeaimptw8rzvwkd5kx6q8gymd893umlfvmpnho3tcx7wslukp4nuclhonod9k2lojya8h4nswxlegewgj9pswpnhbd6itty5xm4q5w0n1omwdtb5ccnxp9hwf3yme64anp8xk7q81bmt6gmv0zoreyjwjcjrlebrgpv9etsie3eyffrb8fzgtnqa086j0yhyz9emcjaexsvrspiupmilu1v8kc7udh1xnte0flzolol7xyvr56u1otsp1lujhzm0pq4oxnkaw930l5g2s8iz3zmfmuhzzwtrli3mnmjhj5dajbk3xz9yjxttwredz00f1r8gyme5x0r52xmeklq24huoyuon4x1w1tb5psq73nn9444dzlx2guahyvu6isb4di8dg0c7yphzah1co8y76qb0098atf0pxfbr37ff2hlvqfqun48yh8qw263p0rxp57antnbkyzu1b6rmh344893oca9dp8ce5wcsterbyjnpgpaf9e4lx5a9tkz3eh3gwqssu9pn3hnb8wd6kaxr2w6bak1r8n45lsxq3guigerlfcgpg0bozyvfq7xg89t7credt8qs3ic6c3u918o8rr1zcewhongee8b8g0ae0wme8tikzovxi2n5hhzffmdi2blfn1ko7g7gy1l406oac4nsh1ri66pfv13mox915lywmv9cis2zfpmj1an4zz3xbvchivzgl8v71c4mt8n6j9j5yqs1cuw93kgzr1sm44cl885jj96d6k7olxodkwpkl7gkgibxwwkwoy1n47iput8kyee9slpneuqac0yccrg09tebu9qqoczh9i6obsngvmg8yjsee2usp450n736i3i2wcznhyyj72cdzkik4t9sdpg08k0tu5y6xmta77mchylh3vf9y9hqsxdul84kdzg663dtxoms766evqe1mpcy3pnhr9bmhpg70kp0tdvem31n3dzw3e4dqxpwkpm6fy5sjw1gtw4nlcn6dnqrcplynksoxeut4o228uaf6341cwi4oakavnot5sk03o77b7gnnz60arimo52wfjzg8us2j4pqpvysdgiuv76fn404gohyepyz0r0vqbf63ir51sdsv0veywyc2ikmmtifankyzi530juj437pzmenbv7nd3ir21mf3m90tav8dwy6zb0c4lbexsqwzmrzq"}, 8 | {"db":0,"key":"set","size":284,"type":"set","encoding":"set","members":["2hzm5rnmkmwb3zqd","tdje6bk22c6ddlrw"]} 9 | ] -------------------------------------------------------------------------------- /cases/memory.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/memory.rdb -------------------------------------------------------------------------------- /cases/memory_expired.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,hash,hash,131,131B,2,ziplist, 3 | 0,s,string,64,64B,0,string, 4 | 0,list,list,203,203B,4,quicklist, 5 | 0,zset,zset,99,99B,2,ziplist, 6 | 0,large,string,2608,2.5K,0,string, 7 | 0,set,set,284,284B,2,set, 8 | -------------------------------------------------------------------------------- /cases/memory_regex.aof: -------------------------------------------------------------------------------- 1 | *6 2 | $5 3 | RPUSH 4 | $4 5 | list 6 | $10 7 | 7fbn7xhcnu 8 | $10 9 | lmproj6c2e 10 | $10 11 | e5lom29act 12 | $10 13 | yy3ux925do 14 | *3 15 | $3 16 | SET 17 | $5 18 | large 19 | $2048 20 | 7sqlkn50jsn9zh2hrp3kj9tvumyoj7cdzolisj6y59ev3ymdy8ffne1nxzzbb4bg0pnvuk1gikwj68ig0wl2s5az25ffldquavkuh5k4tcsrcmph6ubcjb5lk1i2rq4qs41p7j9tj34ek3dj9fu8zw72qfdkr7clk9y0le6rj58krfx0to33wr4fn0t2sq82hrdrdetr60l6bbttsxi4b8z4hs7xd0fu63i2xa511odmmjj1mcpz2bcqohdjx1jcwntu0kttwq0ov3jh9252yqe3z8cz8dml7mrd21brndspix586jk9rd9f872177hvfzm08ai4uosqhdkjrecgududl3yry0rha8gyhheb5c8x3rjjnne4737u1pnwfhg0tdrg3mg8ar4ktcqsifr5ooed40jrrncnr6b5q34vnkrdck8t079nbq69183lh3c1z6xylxc9anxxbu6l9bcpwgltsxi3ovr4dj2l5tkj4mdbymtvfdufc9zh23l8q5kjhdys8g1d2hitk8u39q0jgaka0w9wx5xucdlqc5dwi5mxxviaob3061dcutmfmow0vc10drmp7qq9c9gtb77fnwv6tl9jpkw7duwibo4lmk8hjhboup8mhctinkw3zzy1m84apzyl453ldcako2vok0enohxwwsc2fszxaqnayoyda1y2tqa6wf60d8y8pbi2m4csffo2l1crv8cpoo5gwt6amkcj8esa8h2vewmzago74bnbcng3jbgmrmvhtd3xikpu3q8xw3ri7t2eh2kof28y221247z94uppka0e97dp0bs8by5512xbwuqt5r3s3yb5zk4ytz9c1iadsv8b717enhfkeaimptw8rzvwkd5kx6q8gymd893umlfvmpnho3tcx7wslukp4nuclhonod9k2lojya8h4nswxlegewgj9pswpnhbd6itty5xm4q5w0n1omwdtb5ccnxp9hwf3yme64anp8xk7q81bmt6gmv0zoreyjwjcjrlebrgpv9etsie3eyffrb8fzgtnqa086j0yhyz9emcjaexsvrspiupmilu1v8kc7udh1xnte0flzolol7xyvr56u1otsp1lujhzm0pq4oxnkaw930l5g2s8iz3zmfmuhzzwtrli3mnmjhj5dajbk3xz9yjxttwredz00f1r8gyme5x0r52xmeklq24huoyuon4x1w1tb5psq73nn9444dzlx2guahyvu6isb4di8dg0c7yphzah1co8y76qb0098atf0pxfbr37ff2hlvqfqun48yh8qw263p0rxp57antnbkyzu1b6rmh344893oca9dp8ce5wcsterbyjnpgpaf9e4lx5a9tkz3eh3gwqssu9pn3hnb8wd6kaxr2w6bak1r8n45lsxq3guigerlfcgpg0bozyvfq7xg89t7credt8qs3ic6c3u918o8rr1zcewhongee8b8g0ae0wme8tikzovxi2n5hhzffmdi2blfn1ko7g7gy1l406oac4nsh1ri66pfv13mox915lywmv9cis2zfpmj1an4zz3xbvchivzgl8v71c4mt8n6j9j5yqs1cuw93kgzr1sm44cl885jj96d6k7olxodkwpkl7gkgibxwwkwoy1n47iput8kyee9slpneuqac0yccrg09tebu9qqoczh9i6obsngvmg8yjsee2usp450n736i3i2wcznhyyj72cdzkik4t9sdpg08k0tu5y6xmta77mchylh3vf9y9hqsxdul84kdzg663dtxoms766evqe1mpcy3pnhr9bmhpg70kp0tdvem31n3dzw3e4dqxpwkpm6fy5sjw1gtw4nlcn6dnqrcplynksoxeut4o228uaf6341cwi4oakavnot5sk03o77b7gnnz60arimo52wfjzg8us2j4pqpvysdgiuv76fn404gohyepyz0r0vqbf63ir51sdsv0veywyc2ikmmtifankyzi530juj437pzmenbv7nd3ir21mf3m90tav8dwy6zb0c4lbexsqwzmrzq 21 | -------------------------------------------------------------------------------- /cases/memory_regex.biggest.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count 2 | 0,large,string,2608,2.5K,0 3 | 0,list,list,203,203B,4 4 | -------------------------------------------------------------------------------- /cases/memory_regex.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,list,list,203,203B,4,quicklist, 3 | 0,large,string,2608,2.5K,0,string, 4 | -------------------------------------------------------------------------------- /cases/memory_regex.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"list","size":203,"type":"list","encoding":"quicklist","values":["7fbn7xhcnu","lmproj6c2e","e5lom29act","yy3ux925do"]}, 3 | {"db":0,"key":"large","size":2608,"type":"string","encoding":"string","value":"7sqlkn50jsn9zh2hrp3kj9tvumyoj7cdzolisj6y59ev3ymdy8ffne1nxzzbb4bg0pnvuk1gikwj68ig0wl2s5az25ffldquavkuh5k4tcsrcmph6ubcjb5lk1i2rq4qs41p7j9tj34ek3dj9fu8zw72qfdkr7clk9y0le6rj58krfx0to33wr4fn0t2sq82hrdrdetr60l6bbttsxi4b8z4hs7xd0fu63i2xa511odmmjj1mcpz2bcqohdjx1jcwntu0kttwq0ov3jh9252yqe3z8cz8dml7mrd21brndspix586jk9rd9f872177hvfzm08ai4uosqhdkjrecgududl3yry0rha8gyhheb5c8x3rjjnne4737u1pnwfhg0tdrg3mg8ar4ktcqsifr5ooed40jrrncnr6b5q34vnkrdck8t079nbq69183lh3c1z6xylxc9anxxbu6l9bcpwgltsxi3ovr4dj2l5tkj4mdbymtvfdufc9zh23l8q5kjhdys8g1d2hitk8u39q0jgaka0w9wx5xucdlqc5dwi5mxxviaob3061dcutmfmow0vc10drmp7qq9c9gtb77fnwv6tl9jpkw7duwibo4lmk8hjhboup8mhctinkw3zzy1m84apzyl453ldcako2vok0enohxwwsc2fszxaqnayoyda1y2tqa6wf60d8y8pbi2m4csffo2l1crv8cpoo5gwt6amkcj8esa8h2vewmzago74bnbcng3jbgmrmvhtd3xikpu3q8xw3ri7t2eh2kof28y221247z94uppka0e97dp0bs8by5512xbwuqt5r3s3yb5zk4ytz9c1iadsv8b717enhfkeaimptw8rzvwkd5kx6q8gymd893umlfvmpnho3tcx7wslukp4nuclhonod9k2lojya8h4nswxlegewgj9pswpnhbd6itty5xm4q5w0n1omwdtb5ccnxp9hwf3yme64anp8xk7q81bmt6gmv0zoreyjwjcjrlebrgpv9etsie3eyffrb8fzgtnqa086j0yhyz9emcjaexsvrspiupmilu1v8kc7udh1xnte0flzolol7xyvr56u1otsp1lujhzm0pq4oxnkaw930l5g2s8iz3zmfmuhzzwtrli3mnmjhj5dajbk3xz9yjxttwredz00f1r8gyme5x0r52xmeklq24huoyuon4x1w1tb5psq73nn9444dzlx2guahyvu6isb4di8dg0c7yphzah1co8y76qb0098atf0pxfbr37ff2hlvqfqun48yh8qw263p0rxp57antnbkyzu1b6rmh344893oca9dp8ce5wcsterbyjnpgpaf9e4lx5a9tkz3eh3gwqssu9pn3hnb8wd6kaxr2w6bak1r8n45lsxq3guigerlfcgpg0bozyvfq7xg89t7credt8qs3ic6c3u918o8rr1zcewhongee8b8g0ae0wme8tikzovxi2n5hhzffmdi2blfn1ko7g7gy1l406oac4nsh1ri66pfv13mox915lywmv9cis2zfpmj1an4zz3xbvchivzgl8v71c4mt8n6j9j5yqs1cuw93kgzr1sm44cl885jj96d6k7olxodkwpkl7gkgibxwwkwoy1n47iput8kyee9slpneuqac0yccrg09tebu9qqoczh9i6obsngvmg8yjsee2usp450n736i3i2wcznhyyj72cdzkik4t9sdpg08k0tu5y6xmta77mchylh3vf9y9hqsxdul84kdzg663dtxoms766evqe1mpcy3pnhr9bmhpg70kp0tdvem31n3dzw3e4dqxpwkpm6fy5sjw1gtw4nlcn6dnqrcplynksoxeut4o228uaf6341cwi4oakavnot5sk03o77b7gnnz60arimo52wfjzg8us2j4pqpvysdgiuv76fn404gohyepyz0r0vqbf63ir51sdsv0veywyc2ikmmtifankyzi530juj437pzmenbv7nd3ir21mf3m90tav8dwy6zb0c4lbexsqwzmrzq"} 4 | ] -------------------------------------------------------------------------------- /cases/multiple_databases.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"key_in_zeroth_database","size":72,"type":"string","encoding":"string","value":"zero"}, 3 | {"db":2,"key":"key_in_second_database","size":72,"type":"string","encoding":"string","value":"second"} 4 | ] -------------------------------------------------------------------------------- /cases/multiple_databases.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/multiple_databases.rdb -------------------------------------------------------------------------------- /cases/non_ascii_values.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"int_value","size":56,"type":"string","encoding":"string","value":"123"}, 3 | {"db":0,"key":"ascii","size":64,"type":"string","encoding":"string","value":"\u0000! ~0\n\t\rAb"}, 4 | {"db":0,"key":"bin","size":64,"type":"string","encoding":"string","value":"\u0000$ ~0\ufffd\n\ufffd\t\ufffd\rAb"}, 5 | {"db":0,"key":"printable","size":72,"type":"string","encoding":"string","value":"!+ Ab^~"}, 6 | {"db":0,"key":"378","size":56,"type":"string","encoding":"string","value":"int_key_name"}, 7 | {"db":0,"key":"utf8","size":80,"type":"string","encoding":"string","value":"בדיקה𐀏123עברית"} 8 | ] -------------------------------------------------------------------------------- /cases/non_ascii_values.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/non_ascii_values.rdb -------------------------------------------------------------------------------- /cases/parser_filters.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"k1","size":64,"type":"string","encoding":"string","value":"ssssssss"}, 3 | {"db":0,"key":"k3","size":64,"type":"string","encoding":"string","value":"wwwwwwww"}, 4 | {"db":0,"key":"s1","size":688,"type":"string","encoding":"string","value":".ahaa bit longer and with spaceslonger than 256 characters and trivially compressible --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"}, 5 | {"db":0,"key":"s2","size":64,"type":"string","encoding":"string","value":"now_exists"}, 6 | {"db":0,"key":"n5b","size":48,"type":"string","encoding":"string","value":"1000"}, 7 | {"db":0,"key":"l10","size":95,"type":"list","encoding":"ziplist","values":["100001","100002","100003","100004"]}, 8 | {"db":0,"key":"l11","size":101,"type":"list","encoding":"ziplist","values":["9999999999","9999999998","9999999997"]}, 9 | {"db":0,"key":"l12","size":101,"type":"list","encoding":"ziplist","values":["9999999997","9999999998","9999999999"]}, 10 | {"db":0,"key":"b1","size":56,"type":"string","encoding":"string","value":"\ufffd"}, 11 | {"db":0,"key":"b2","size":56,"type":"string","encoding":"string","value":"\u0000\ufffd"}, 12 | {"db":0,"key":"b3","size":56,"type":"string","encoding":"string","value":"\u0000\u0000\ufffd"}, 13 | {"db":0,"key":"b4","size":56,"type":"string","encoding":"string","value":"\u0000\u0000\u0000\ufffd"}, 14 | {"db":0,"key":"b5","size":56,"type":"string","encoding":"string","value":"\u0000\u0000\u0000\u0000\ufffd"}, 15 | {"db":0,"key":"h1","size":780,"type":"hash","encoding":"hash","hash":{"a":"aha","b":"a bit longer, but not very much","c":"now this is quite a bit longer, but sort of boring...................................................................................................................................................................................................................................................................................................................................................................."}}, 16 | {"db":0,"key":"h2","size":196,"type":"hash","encoding":"zipmap","hash":{"a":"101010"}}, 17 | {"db":0,"key":"h3","size":308,"type":"hash","encoding":"zipmap","hash":{"b":"b2","c":"c2","d":"d"}}, 18 | {"db":0,"key":"l1","size":77,"type":"list","encoding":"ziplist","values":["yup","aha"]}, 19 | {"db":0,"key":"set1","size":364,"type":"set","encoding":"set","members":["c","d","a","b"]}, 20 | {"db":0,"key":"l2","size":125,"type":"list","encoding":"ziplist","values":["something","now a bit longer and perhaps more interesting"]}, 21 | {"db":0,"key":"set2","size":252,"type":"set","encoding":"set","members":["d","a"]}, 22 | {"db":0,"key":"n1","size":48,"type":"string","encoding":"string","value":"-6"}, 23 | {"db":0,"key":"l3","size":800,"type":"list","encoding":"list","values":["this one is going to be longer -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------","a bit more"]}, 24 | {"db":0,"key":"set3","size":196,"type":"set","encoding":"set","members":["b"]}, 25 | {"db":0,"key":"set4","size":76,"type":"set","encoding":"intset","members":["1","2","3","4","5","6","7","8","9","10"]}, 26 | {"db":0,"key":"n2","size":48,"type":"string","encoding":"string","value":"501"}, 27 | {"db":0,"key":"l4","size":80,"type":"list","encoding":"ziplist","values":["b","c","d"]}, 28 | {"db":0,"key":"set5","size":72,"type":"set","encoding":"intset","members":["100000","100001","100002","100003"]}, 29 | {"db":0,"key":"n3","size":48,"type":"string","encoding":"string","value":"500001"}, 30 | {"db":0,"key":"l5","size":73,"type":"list","encoding":"ziplist","values":["c","a"]}, 31 | {"db":0,"key":"set6","size":80,"type":"set","encoding":"intset","members":["9999999997","9999999998","9999999999"]}, 32 | {"db":0,"key":"n4","size":48,"type":"string","encoding":"string","value":"1"}, 33 | {"db":0,"key":"l6","size":66,"type":"list","encoding":"ziplist","values":["b"]}, 34 | {"db":0,"key":"n5","size":48,"type":"string","encoding":"string","value":"1000"}, 35 | {"db":0,"key":"l7","size":73,"type":"list","encoding":"ziplist","values":["a","b"]}, 36 | {"db":0,"key":"n6","size":48,"type":"string","encoding":"string","value":"1000000"}, 37 | {"db":0,"key":"n4b","size":48,"type":"string","encoding":"string","value":"1"}, 38 | {"db":0,"key":"l8","size":90,"type":"list","encoding":"ziplist","values":["c","1","2","3","4"]}, 39 | {"db":0,"key":"l9","size":91,"type":"list","encoding":"ziplist","values":["10001","10002","10003","10004"]}, 40 | {"db":0,"key":"n6b","size":48,"type":"string","encoding":"string","value":"1000000"}, 41 | {"db":0,"key":"z1","size":73,"type":"zset","encoding":"ziplist","entries":[{"member":"a","score":1},{"member":"c","score":13}]}, 42 | {"db":0,"key":"z2","size":83,"type":"zset","encoding":"ziplist","entries":[{"member":"1","score":1},{"member":"2","score":2},{"member":"3","score":3}]}, 43 | {"db":0,"key":"z3","size":75,"type":"zset","encoding":"ziplist","entries":[{"member":"10002","score":10001},{"member":"10003","score":10003}]}, 44 | {"db":0,"key":"z4","size":119,"type":"zset","encoding":"ziplist","entries":[{"member":"10000000001","score":10000000001},{"member":"10000000002","score":10000000002},{"member":"10000000003","score":10000000003}]} 45 | ] -------------------------------------------------------------------------------- /cases/parser_filters.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/parser_filters.rdb -------------------------------------------------------------------------------- /cases/quicklist.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"list","size":267,"type":"list","encoding":"quicklist","values":["eb5foapxep8846is","ns8ra7iy34tpvt","2dmoobfe4vlmok1f","bmnctno6rrxjs5yl","sq1c36x0ixv50jqm","jfds2extynrj6l"]} 3 | ] -------------------------------------------------------------------------------- /cases/quicklist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/quicklist.rdb -------------------------------------------------------------------------------- /cases/rdb_version_5_with_checksum.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"abcd","size":56,"type":"string","encoding":"string","value":"efgh"}, 3 | {"db":0,"key":"foo","size":56,"type":"string","encoding":"string","value":"bar"}, 4 | {"db":0,"key":"bar","size":56,"type":"string","encoding":"string","value":"baz"}, 5 | {"db":0,"key":"abcdef","size":56,"type":"string","encoding":"string","value":"abcdef"}, 6 | {"db":0,"key":"longerstring","size":104,"type":"string","encoding":"string","value":"thisisalongerstring.idontknowwhatitmeans"}, 7 | {"db":0,"key":"abc","size":56,"type":"string","encoding":"string","value":"def"} 8 | ] -------------------------------------------------------------------------------- /cases/rdb_version_5_with_checksum.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/rdb_version_5_with_checksum.rdb -------------------------------------------------------------------------------- /cases/rdb_version_8_with_64b_length_and_scores.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/rdb_version_8_with_64b_length_and_scores.rdb -------------------------------------------------------------------------------- /cases/regular_set.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"regular_set","size":436,"type":"set","encoding":"set","members":["beta","delta","alpha","phi","gamma","kappa"]} 3 | ] -------------------------------------------------------------------------------- /cases/regular_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/regular_set.rdb -------------------------------------------------------------------------------- /cases/regular_sorted_set.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/regular_sorted_set.rdb -------------------------------------------------------------------------------- /cases/set_listpack.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,s,set,67,67B,4,listpack, 3 | -------------------------------------------------------------------------------- /cases/set_listpack.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"s","size":67,"type":"set","encoding":"listpack","members":["a","b","c","d"]} 3 | ] -------------------------------------------------------------------------------- /cases/set_listpack.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/set_listpack.rdb -------------------------------------------------------------------------------- /cases/sorted_set_as_ziplist.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"sorted_set_as_ziplist","size":208,"type":"zset","encoding":"ziplist","entries":[{"member":"8b6ba6718a786daefa69438148361901","score":1},{"member":"cb7a24bb7528f934b841b34c3a73e0c7","score":2.37},{"member":"523af537946b79c4f8369ed39ba78605","score":3.423}]} 3 | ] -------------------------------------------------------------------------------- /cases/sorted_set_as_ziplist.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/sorted_set_as_ziplist.rdb -------------------------------------------------------------------------------- /cases/stream_listoacks_3.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"mystream","size":1776,"type":"stream","encoding":"","version":3,"entries":[{"firstMsgId":"1704557973866-0","fields":["name","surname"],"msgs":[{"id":"1704557973866-0","fields":{"name":"Sara","surname":"OConnor"},"deleted":false}]}],"groups":[{"name":"consumer-group-name","lastId":"1704557973866-0","pending":[{"id":"1704557973866-0","deliveryTime":1704557998397,"deliveryCount":1}],"consumers":[{"name":"consumer-name","seenTime":1704557998397,"pending":["1704557973866-0"],"activeTime":1704557998397}],"entriesRead":1}],"len":1,"lastId":"1704557973866-0","firstId":"1704557973866-0","maxDeletedId":"0-0","addedEntriesCount":1} 3 | ] -------------------------------------------------------------------------------- /cases/stream_listoacks_3.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/stream_listoacks_3.rdb -------------------------------------------------------------------------------- /cases/stream_listpacks_1.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,test,stream,616,616B,0,listpack, 3 | 0,my,stream,616,616B,0,listpack, 4 | 0,trim,stream,1868,1.8K,0,listpack, 5 | 0,listpack,stream,10852,10.6K,0,listpack, 6 | 0,nums,stream,616,616B,0,listpack, 7 | -------------------------------------------------------------------------------- /cases/stream_listpacks_1.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/stream_listpacks_1.rdb -------------------------------------------------------------------------------- /cases/stream_listpacks_2.csv: -------------------------------------------------------------------------------- 1 | database,key,type,size,size_readable,element_count,encoding,expiration 2 | 0,astream,stream,664,664B,0,listpack, 3 | -------------------------------------------------------------------------------- /cases/stream_listpacks_2.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"astream","size":664,"type":"stream","encoding":"listpack","version":2,"entries":[{"firstMsgId":"1681085300799-0","fields":["a","b","c"],"msgs":[{"id":"1681085300799-0","fields":{"a":"1","b":"2","c":"3"},"deleted":false},{"id":"1681085312465-0","fields":{"a":"2","b":"3","c":"4"},"deleted":false}]}],"len":2,"lastId":"1681085312465-0","firstId":"1681085300799-0","maxDeletedId":"0-0","addedEntriesCount":2} 3 | ] -------------------------------------------------------------------------------- /cases/stream_listpacks_2.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/stream_listpacks_2.rdb -------------------------------------------------------------------------------- /cases/tree.csv: -------------------------------------------------------------------------------- 1 | database,prefix,size,size_readable,key_count 2 | 0,a,424,424B,6 3 | 0,ab,368,368B,5 4 | 0,abb,232,232B,3 5 | -------------------------------------------------------------------------------- /cases/tree.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/tree.rdb -------------------------------------------------------------------------------- /cases/tree2.csv: -------------------------------------------------------------------------------- 1 | database,prefix,size,size_readable,key_count 2 | 0,a,424,424B,6 3 | 0,ab,368,368B,5 4 | 0,b,64,64B,1 5 | -------------------------------------------------------------------------------- /cases/uncompressible_string_keys.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/uncompressible_string_keys.rdb -------------------------------------------------------------------------------- /cases/ziplist_that_compresses_easily.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"ziplist_compresses_easily","size":245,"type":"list","encoding":"ziplist","values":["aaaaaa","aaaaaaaaaaaa","aaaaaaaaaaaaaaaaaa","aaaaaaaaaaaaaaaaaaaaaaaa","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]} 3 | ] -------------------------------------------------------------------------------- /cases/ziplist_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/ziplist_that_compresses_easily.rdb -------------------------------------------------------------------------------- /cases/ziplist_that_doesnt_compress.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"ziplist_doesnt_compress","size":169,"type":"list","encoding":"ziplist","values":["aj2410","cc953a17a8e096e76a44169ad3f9ac87c5f8248a403274416179aa9fbd852344"]} 3 | ] -------------------------------------------------------------------------------- /cases/ziplist_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/ziplist_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /cases/ziplist_with_integers.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"ziplist_with_integers","size":238,"type":"list","encoding":"ziplist","values":["0","1","2","3","4","5","6","7","8","9","10","11","12","-2","13","25","-61","63","16380","-16000","65535","-65523","4194304","9223372036854775807"]} 3 | ] -------------------------------------------------------------------------------- /cases/ziplist_with_integers.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/ziplist_with_integers.rdb -------------------------------------------------------------------------------- /cases/zipmap_big_len.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"zimap_doesnt_compress","size":276,"type":"hash","encoding":"zipmap","hash":{"MKD1G6":"2","YNNXK":"F7TI"}} 3 | ] -------------------------------------------------------------------------------- /cases/zipmap_big_len.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/zipmap_big_len.rdb -------------------------------------------------------------------------------- /cases/zipmap_that_compresses_easily.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"zipmap_compresses_easily","size":340,"type":"hash","encoding":"zipmap","hash":{"a":"aa","aa":"aaaa","aaaaa":"aaaaaaaaaaaaaa"}} 3 | ] -------------------------------------------------------------------------------- /cases/zipmap_that_compresses_easily.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/zipmap_that_compresses_easily.rdb -------------------------------------------------------------------------------- /cases/zipmap_that_doesnt_compress.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"db":0,"key":"zimap_doesnt_compress","size":276,"type":"hash","encoding":"zipmap","hash":{"MKD1G6":"2","YNNXK":"F7TI"}} 3 | ] -------------------------------------------------------------------------------- /cases/zipmap_that_doesnt_compress.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/zipmap_that_doesnt_compress.rdb -------------------------------------------------------------------------------- /cases/zipmap_with_big_values.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/cases/zipmap_with_big_values.rdb -------------------------------------------------------------------------------- /cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/hdt3213/rdb/helper" 10 | ) 11 | 12 | const help = ` 13 | This is a tool to parse Redis' RDB files 14 | Options: 15 | -c command, including: json/memory/aof/bigkey/prefix/flamegraph 16 | -o output file path 17 | -n number of result, using in command: bigkey/prefix 18 | -port listen port for flame graph web service 19 | -sep separator for flamegraph, rdb will separate key by it, default value is ":". 20 | supporting multi separators: -sep sep1 -sep sep2 21 | -regex using regex expression filter keys 22 | -no-expired filter expired keys 23 | -concurrent The number of concurrent json converters. 4 by default. 24 | 25 | Examples: 26 | parameters between '[' and ']' is optional 27 | 1. convert rdb to json 28 | rdb -c json -o dump.json dump.rdb 29 | 2. generate memory report 30 | rdb -c memory -o memory.csv dump.rdb 31 | 3. convert to aof file 32 | rdb -c aof -o dump.aof dump.rdb 33 | 4. get largest keys 34 | rdb -c bigkey [-o dump.aof] [-n 10] dump.rdb 35 | 5. get number and memory size by prefix 36 | rdb -c prefix [-n 10] [-max-depth 3] [-o prefix-report.csv] dump.rdb 37 | 6. draw flamegraph 38 | rdb -c flamegraph [-port 16379] [-sep :] dump.rdb 39 | ` 40 | 41 | type separators []string 42 | 43 | func (s *separators) String() string { 44 | return strings.Join(*s, " ") 45 | } 46 | 47 | func (s *separators) Set(value string) error { 48 | *s = append(*s, value) 49 | return nil 50 | } 51 | 52 | func main() { 53 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 54 | var cmd string 55 | var output string 56 | var n int 57 | var port int 58 | var seps separators 59 | var regexExpr string 60 | var noExpired bool 61 | var maxDepth int 62 | var concurrent int 63 | var err error 64 | flagSet.StringVar(&cmd, "c", "", "command for rdb: json") 65 | flagSet.StringVar(&output, "o", "", "output file path") 66 | flagSet.IntVar(&n, "n", 0, "") 67 | flagSet.IntVar(&maxDepth, "max-depth", 0, "max depth of prefix tree") 68 | flagSet.IntVar(&port, "port", 0, "listen port for web") 69 | flagSet.IntVar(&concurrent, "concurrent", 0, "concurrent number for json converter") 70 | flagSet.Var(&seps, "sep", "separator for flame graph") 71 | flagSet.StringVar(®exExpr, "regex", "", "regex expression") 72 | flagSet.BoolVar(&noExpired, "no-expired", false, "filter expired keys") 73 | _ = flagSet.Parse(os.Args[1:]) // ExitOnError 74 | src := flagSet.Arg(0) 75 | 76 | if cmd == "" { 77 | println(help) 78 | return 79 | } 80 | if src == "" { 81 | println("src file is required") 82 | return 83 | } 84 | 85 | var options []interface{} 86 | if regexExpr != "" { 87 | options = append(options, helper.WithRegexOption(regexExpr)) 88 | } 89 | if noExpired { 90 | options = append(options, helper.WithNoExpiredOption()) 91 | } 92 | if concurrent != 0 { 93 | options = append(options, helper.WithConcurrent(concurrent)) 94 | } 95 | 96 | var outputFile *os.File 97 | if output == "" { 98 | outputFile = os.Stdout 99 | } else { 100 | outputFile, err = os.Create(output) 101 | if err != nil { 102 | fmt.Printf("open output faild: %v", err) 103 | } 104 | defer func() { 105 | _ = outputFile.Close() 106 | }() 107 | } 108 | 109 | switch cmd { 110 | case "json": 111 | err = helper.ToJsons(src, output, options...) 112 | case "memory": 113 | err = helper.MemoryProfile(src, output, options...) 114 | case "aof": 115 | err = helper.ToAOF(src, output, options) 116 | case "bigkey": 117 | err = helper.FindBiggestKeys(src, n, outputFile, options...) 118 | case "prefix": 119 | err = helper.PrefixAnalyse(src, n, maxDepth, outputFile, options...) 120 | case "flamegraph": 121 | _, err = helper.FlameGraph(src, port, seps, options...) 122 | if err != nil { 123 | fmt.Printf("error: %v\n", err) 124 | return 125 | } 126 | <-make(chan struct{}) 127 | default: 128 | println("unknown command") 129 | return 130 | } 131 | if err != nil { 132 | fmt.Printf("error: %v\n", err) 133 | return 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | // just make sure it can parse command line args correctly 9 | func TestCmd(t *testing.T) { 10 | err := os.MkdirAll("tmp", os.ModePerm) 11 | if err != nil { 12 | return 13 | } 14 | defer func() { 15 | err := os.RemoveAll("tmp") 16 | if err != nil { 17 | t.Logf("remove tmp directory failed: %v", err) 18 | } 19 | }() 20 | // test command line parser only 21 | os.Args = []string{"", "-c", "json", "-o", "tmp/cmd.json", "cases/memory.rdb"} 22 | main() 23 | if f, _ := os.Stat("tmp/cmd.json"); f == nil { 24 | t.Error("command json failed") 25 | } 26 | os.Args = []string{"", "-c", "memory", "-o", "tmp/memory.csv", "cases/memory.rdb"} 27 | main() 28 | if f, _ := os.Stat("tmp/memory.csv"); f == nil { 29 | t.Error("command memory failed") 30 | } 31 | os.Args = []string{"", "-c", "aof", "-o", "tmp/memory.aof", "cases/memory.rdb"} 32 | main() 33 | if f, _ := os.Stat("tmp/memory.aof"); f == nil { 34 | t.Error("command memory failed") 35 | } 36 | os.Args = []string{"", "-c", "bigkey", "-o", "tmp/bigkey.csv", "-n", "10", "cases/memory.rdb"} 37 | main() 38 | if f, _ := os.Stat("tmp/bigkey.csv"); f == nil { 39 | t.Error("command bigkey failed") 40 | } 41 | os.Args = []string{"", "-c", "bigkey", "-n", "10", "cases/memory.rdb"} 42 | main() 43 | 44 | os.Args = []string{"", "-c", "memory", "-o", "tmp/memory_regex.csv", "-regex", "^l.*", "cases/memory.rdb"} 45 | main() 46 | if f, _ := os.Stat("tmp/memory_regex.csv"); f == nil { 47 | t.Error("command memory failed") 48 | } 49 | 50 | os.Args = []string{"", "-c", "memory", "-o", "tmp/memory_regex.csv", "-regex", "^l.*", "-no-expired", "cases/memory.rdb"} 51 | main() 52 | if f, _ := os.Stat("tmp/memory_regex.csv"); f == nil { 53 | t.Error("command memory failed") 54 | } 55 | os.Args = []string{"", "-c", "prefix", "-o", "tmp/tree.csv", "cases/tree.rdb"} 56 | main() 57 | if f, _ := os.Stat("tmp/tree.csv"); f == nil { 58 | t.Error("command prefix failed") 59 | } 60 | 61 | // test error command line 62 | os.Args = []string{"", "-c", "json", "-o", "tmp/output", "/none/a"} 63 | main() 64 | os.Args = []string{"", "-c", "aof", "-o", "tmp/output", "/none/a"} 65 | main() 66 | os.Args = []string{"", "-c", "memory", "-o", "tmp/output", "/none/a"} 67 | main() 68 | os.Args = []string{"", "-c", "bigkey", "-o", "tmp/output", "/none/a"} 69 | main() 70 | 71 | os.Args = []string{"", "-c", "bigkey", "-o", "/none/a", "-n", "10", "cases/memory.rdb"} 72 | main() 73 | os.Args = []string{"", "-c", "aof", "-o", "/none/a", "cases/memory.rdb"} 74 | main() 75 | os.Args = []string{"", "-c", "memory", "-o", "/none/a", "cases/memory.rdb"} 76 | main() 77 | os.Args = []string{"", "-c", "json", "-o", "/none/a", "cases/memory.rdb"} 78 | main() 79 | 80 | os.Args = []string{"", "-c", "bigkey", "-o", "tmp/bigkey.csv", "cases/memory.rdb"} 81 | main() 82 | os.Args = []string{"", "-c", "none", "-o", "tmp/memory.aof", "cases/memory.rdb"} 83 | main() 84 | os.Args = []string{""} 85 | main() 86 | os.Args = []string{"", "-c", "aof"} 87 | main() 88 | } 89 | -------------------------------------------------------------------------------- /core/encoder_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "github.com/hdt3213/rdb/model" 6 | "math/rand" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestEncode(t *testing.T) { 12 | auxMap := map[string]string{ 13 | "redis-ver": "4.0.6", 14 | "redis-bits": "64", 15 | "aof-preamble": "0", 16 | } 17 | type valTTLPair struct { 18 | Value string 19 | TTL uint64 20 | } 21 | strMap := map[string]*valTTLPair{ 22 | "a": {Value: "a", TTL: uint64(time.Now().Add(time.Hour).Unix())}, 23 | "b": {Value: "b", TTL: uint64(time.Now().Add(time.Minute).Unix())}, 24 | "c": {Value: "c"}, 25 | "1": {Value: "1"}, 26 | } 27 | for i := 0; i < 1000; i++ { 28 | strMap[RandString(rand.Intn(20))] = &valTTLPair{ 29 | Value: RandString(rand.Intn(50)), 30 | } 31 | } 32 | 33 | buf := bytes.NewBuffer(nil) 34 | enc := NewEncoder(buf) 35 | err := enc.WriteHeader() 36 | if err != nil { 37 | t.Error(err) 38 | return 39 | } 40 | for k, v := range auxMap { 41 | err = enc.WriteAux(k, v) 42 | if err != nil { 43 | t.Error(err) 44 | return 45 | } 46 | } 47 | var ttlCount uint64 48 | for _, v := range strMap { 49 | if v.TTL > 0 { 50 | ttlCount++ 51 | } 52 | } 53 | err = enc.WriteDBHeader(0, uint64(len(strMap)), ttlCount) 54 | if err != nil { 55 | t.Error(err) 56 | return 57 | } 58 | 59 | for k, v := range strMap { 60 | var opts []interface{} 61 | if v.TTL > 0 { 62 | opts = append(opts, WithTTL(v.TTL)) 63 | } 64 | err = enc.WriteStringObject(k, []byte(v.Value), opts...) 65 | if err != nil { 66 | t.Error(err) 67 | return 68 | } 69 | } 70 | err = enc.WriteEnd() 71 | if err != nil { 72 | t.Error(err) 73 | return 74 | } 75 | dec := NewDecoder(buf).WithSpecialOpCode() 76 | err = dec.Parse(func(object model.RedisObject) bool { 77 | switch o := object.(type) { 78 | case *model.StringObject: 79 | expect := strMap[o.GetKey()] 80 | if expect == nil { 81 | t.Errorf("unexpected object: %s", o.GetKey()) 82 | return true 83 | } 84 | if expect.Value != string(o.Value) { 85 | t.Errorf("object: %s with wrong value", o.GetKey()) 86 | return true 87 | } 88 | if o.GetExpiration() == nil { 89 | if expect.TTL > 0 { 90 | t.Errorf("object: %s with wrong ttl", o.GetKey()) 91 | return true 92 | } 93 | } else { 94 | ttl := o.GetExpiration().UnixNano() / int64(time.Millisecond) 95 | if expect.TTL != uint64(ttl) { 96 | t.Errorf("object: %s with wrong ttl", o.GetKey()) 97 | return true 98 | } 99 | } 100 | case *model.AuxObject: 101 | expect := auxMap[o.GetKey()] 102 | if expect == "" { 103 | t.Errorf("unexpected aux: %s", o.GetKey()) 104 | return true 105 | } 106 | if expect != o.Value { 107 | t.Errorf("object: %s with wrong value", o.GetKey()) 108 | return true 109 | } 110 | } 111 | return true 112 | }) 113 | if err != nil { 114 | t.Error(err) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /core/hash.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "github.com/hdt3213/rdb/model" 7 | ) 8 | 9 | /* 10 | if hlen <= ZIPMAP_VALUE_MAX_FREE 11 | */ 12 | 13 | func (dec *Decoder) readHashMap() (map[string][]byte, error) { 14 | size, _, err := dec.readLength() 15 | if err != nil { 16 | return nil, err 17 | } 18 | m := make(map[string][]byte) 19 | for i := 0; i < int(size); i++ { 20 | field, err := dec.readString() 21 | if err != nil { 22 | return nil, err 23 | } 24 | value, err := dec.readString() 25 | if err != nil { 26 | return nil, err 27 | } 28 | m[unsafeBytes2Str(field)] = value 29 | } 30 | return m, nil 31 | } 32 | 33 | func (dec *Decoder) readZipMapHash() (map[string][]byte, error) { 34 | buf, err := dec.readString() 35 | if err != nil { 36 | return nil, err 37 | } 38 | cursor := 0 39 | bLen, err := readByte(buf, &cursor) 40 | if err != nil { 41 | return nil, err 42 | } 43 | length := int(bLen) 44 | if bLen > 254 { 45 | //todo: scan once 46 | cursor0 := cursor // record current cursor 47 | length, err = countZipMapEntries(buf, &cursor) 48 | if err != nil { 49 | return nil, err 50 | } 51 | length /= 2 52 | cursor = cursor0 // recover cursor at begin position of first zip map entry 53 | } 54 | m := make(map[string][]byte) 55 | for i := 0; i < length; i++ { 56 | fieldB, err := readZipMapEntry(buf, &cursor, false) 57 | if err != nil { 58 | return nil, err 59 | } 60 | field := unsafeBytes2Str(fieldB) 61 | value, err := readZipMapEntry(buf, &cursor, true) 62 | if err != nil { 63 | return nil, err 64 | } 65 | m[field] = value 66 | } 67 | return m, nil 68 | } 69 | 70 | // return: len, free, error 71 | func readZipMapEntryLen(buf []byte, cursor *int, readFree bool) (int, int, error) { 72 | b, err := readByte(buf, cursor) 73 | if err != nil { 74 | return 0, 0, err 75 | } 76 | switch b { 77 | case 253: 78 | bs, err := readBytes(buf, cursor, 5) 79 | if err != nil { 80 | return 0, 0, err 81 | } 82 | length := int(binary.BigEndian.Uint32(bs)) 83 | free := int(bs[4]) 84 | return length, free, nil 85 | case 254: 86 | return 0, 0, errors.New("illegal zip map item length") 87 | case 255: 88 | return -1, 0, nil 89 | default: 90 | var free byte 91 | if readFree { 92 | free, err = readByte(buf, cursor) 93 | } 94 | return int(b), int(free), err 95 | } 96 | } 97 | 98 | func readZipMapEntry(buf []byte, cursor *int, readFree bool) ([]byte, error) { 99 | length, free, err := readZipMapEntryLen(buf, cursor, readFree) 100 | if err != nil { 101 | return nil, err 102 | } 103 | if length == -1 { 104 | return nil, nil 105 | } 106 | value, err := readBytes(buf, cursor, length) 107 | if err != nil { 108 | return nil, err 109 | } 110 | *cursor += free // skip free bytes 111 | return value, nil 112 | } 113 | 114 | func countZipMapEntries(buf []byte, cursor *int) (int, error) { 115 | n := 0 116 | for { 117 | readFree := n%2 != 0 118 | length, free, err := readZipMapEntryLen(buf, cursor, readFree) 119 | if err != nil { 120 | return 0, err 121 | } 122 | if length == -1 { 123 | break 124 | } 125 | *cursor += length + free 126 | n++ 127 | } 128 | *cursor = 0 // reset cursor 129 | return n, nil 130 | } 131 | 132 | func (dec *Decoder) readZipListHash() (map[string][]byte, *model.ZiplistDetail, error) { 133 | buf, err := dec.readString() 134 | if err != nil { 135 | return nil, nil, err 136 | } 137 | cursor := 0 138 | size := readZipListLength(buf, &cursor) 139 | m := make(map[string][]byte) 140 | for i := 0; i < size; i += 2 { 141 | key, err := dec.readZipListEntry(buf, &cursor) 142 | if err != nil { 143 | return nil, nil, err 144 | } 145 | val, err := dec.readZipListEntry(buf, &cursor) 146 | if err != nil { 147 | return nil, nil, err 148 | } 149 | m[unsafeBytes2Str(key)] = val 150 | } 151 | detail := &model.ZiplistDetail{ 152 | RawStringSize: len(buf), 153 | } 154 | return m, detail, nil 155 | } 156 | 157 | func (dec *Decoder) readListPackHash() (map[string][]byte, *model.ListpackDetail, error) { 158 | buf, err := dec.readString() 159 | if err != nil { 160 | return nil, nil, err 161 | } 162 | cursor := 0 163 | size := readListPackLength(buf, &cursor) 164 | m := make(map[string][]byte) 165 | for i := 0; i < size; i += 2 { 166 | key, err := dec.readListPackEntryAsString(buf, &cursor) 167 | if err != nil { 168 | return nil, nil, err 169 | } 170 | val, err := dec.readListPackEntryAsString(buf, &cursor) 171 | if err != nil { 172 | return nil, nil, err 173 | } 174 | m[unsafeBytes2Str(key)] = val 175 | } 176 | detail := &model.ListpackDetail{ 177 | RawStringSize: len(buf), 178 | } 179 | return m, detail, nil 180 | } 181 | 182 | func (enc *Encoder) WriteHashMapObject(key string, hash map[string][]byte, options ...interface{}) error { 183 | err := enc.beforeWriteObject(options...) 184 | if err != nil { 185 | return err 186 | } 187 | ok, err := enc.tryWriteZipListHashMap(key, hash, options...) 188 | if err != nil { 189 | return err 190 | } 191 | if !ok { 192 | err = enc.writeHashEncoding(key, hash, options...) 193 | if err != nil { 194 | return err 195 | } 196 | } 197 | enc.state = writtenObjectState 198 | return nil 199 | } 200 | 201 | func (enc *Encoder) writeHashEncoding(key string, hash map[string][]byte, options ...interface{}) error { 202 | err := enc.write([]byte{typeHash}) 203 | if err != nil { 204 | return err 205 | } 206 | err = enc.writeString(key) 207 | if err != nil { 208 | return err 209 | } 210 | err = enc.writeLength(uint64(len(hash))) 211 | if err != nil { 212 | return err 213 | } 214 | for field, value := range hash { 215 | err = enc.writeString(field) 216 | if err != nil { 217 | return err 218 | } 219 | err = enc.writeString(unsafeBytes2Str(value)) 220 | if err != nil { 221 | return err 222 | } 223 | } 224 | return nil 225 | } 226 | 227 | func (enc *Encoder) tryWriteZipListHashMap(key string, hash map[string][]byte, options ...interface{}) (bool, error) { 228 | if len(hash) > enc.hashZipListOpt.getMaxEntries() { 229 | return false, nil 230 | } 231 | maxValue := enc.hashZipListOpt.getMaxValue() 232 | for _, v := range hash { 233 | if len(v) > maxValue { 234 | return false, nil 235 | } 236 | } 237 | err := enc.write([]byte{typeHashZipList}) 238 | if err != nil { 239 | return true, err 240 | } 241 | err = enc.writeString(key) 242 | if err != nil { 243 | return true, err 244 | } 245 | entries := make([]string, 0, len(hash)*2) 246 | for k, v := range hash { 247 | entries = append(entries, k, unsafeBytes2Str(v)) 248 | } 249 | err = enc.writeZipList(entries) 250 | if err != nil { 251 | return true, err 252 | } 253 | return true, nil 254 | } 255 | -------------------------------------------------------------------------------- /core/hash_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "github.com/hdt3213/rdb/model" 6 | "math/rand" 7 | "testing" 8 | ) 9 | 10 | func TestHashEncoding(t *testing.T) { 11 | mapMap := map[string]map[string][]byte{ 12 | "a": { 13 | "a": []byte("foo bar"), 14 | }, 15 | "1": { 16 | "233": []byte("114514"), 17 | }, 18 | "2": { 19 | "n": []byte(RandString(128)), 20 | "o": []byte(RandString(128)), 21 | }, 22 | "001": { 23 | "007": []byte(RandString(128)), 24 | }, 25 | "000": { 26 | "0x11": []byte(RandString(128)), 27 | }, 28 | } 29 | m := make(map[string][]byte) 30 | for i := 0; i < 1024; i++ { 31 | m[RandString(rand.Intn(32))] = []byte(RandString(rand.Intn(128))) 32 | } 33 | mapMap["large"] = m 34 | buf := bytes.NewBuffer(nil) 35 | enc := NewEncoder(buf).SetHashZipListOpt(64, 64) 36 | err := enc.WriteHeader() 37 | if err != nil { 38 | t.Error(err) 39 | return 40 | } 41 | err = enc.WriteDBHeader(0, uint64(len(mapMap)), 0) 42 | if err != nil { 43 | t.Error(err) 44 | return 45 | } 46 | for k, v := range mapMap { 47 | err = enc.WriteHashMapObject(k, v) 48 | if err != nil { 49 | t.Error(err) 50 | return 51 | } 52 | } 53 | err = enc.WriteEnd() 54 | if err != nil { 55 | t.Error(err) 56 | return 57 | } 58 | dec := NewDecoder(buf) 59 | err = dec.Parse(func(object model.RedisObject) bool { 60 | switch o := object.(type) { 61 | case *model.HashObject: 62 | expect := mapMap[o.GetKey()] 63 | if len(expect) != o.GetElemCount() { 64 | t.Errorf("hash %s has wrong element count", o.GetKey()) 65 | return true 66 | } 67 | for field, expectV := range expect { 68 | actualV := o.Hash[field] 69 | if !bytes.Equal(expectV, actualV) { 70 | t.Errorf("hash %s has element at field %s", o.GetKey(), field) 71 | return true 72 | } 73 | } 74 | } 75 | return true 76 | }) 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /core/list.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "github.com/hdt3213/rdb/model" 7 | ) 8 | 9 | const ( 10 | zipStr06B = 0 11 | zipStr14B = 1 12 | zipStr32B = 2 13 | 14 | zipInt04B = 0x0f // high 4 bits of Int 04 encoding 15 | zipInt08B = 0xfe // 11111110 16 | zipInt16B = 0xc0 | 0<<4 // 11000000 17 | zipInt24B = 0xc0 | 3<<4 // 11110000 18 | zipInt32B = 0xc0 | 1<<4 // 11010000 19 | zipInt64B = 0xc0 | 2<<4 //11100000 20 | 21 | zipBigPrevLen = 0xfe 22 | ) 23 | 24 | func (dec *Decoder) readList() ([][]byte, error) { 25 | size64, _, err := dec.readLength() 26 | if err != nil { 27 | return nil, err 28 | } 29 | size := int(size64) 30 | values := make([][]byte, 0, size) 31 | for i := 0; i < size; i++ { 32 | val, err := dec.readString() 33 | if err != nil { 34 | return nil, err 35 | } 36 | values = append(values, val) 37 | } 38 | return values, nil 39 | } 40 | 41 | func (dec *Decoder) readQuickList() ([][]byte, *model.QuicklistDetail, error) { 42 | size, _, err := dec.readLength() 43 | if err != nil { 44 | return nil, nil, err 45 | } 46 | entries := make([][]byte, 0) 47 | detail := &model.QuicklistDetail{} 48 | for i := 0; i < int(size); i++ { 49 | page, err := dec.readZipList() 50 | if err != nil { 51 | return nil, nil, err 52 | } 53 | entries = append(entries, page...) 54 | detail.ZiplistStruct = append(detail.ZiplistStruct, page) 55 | } 56 | return entries, detail, nil 57 | } 58 | 59 | // readQuickList2 returns 60 | func (dec *Decoder) readQuickList2() ([][]byte, *model.Quicklist2Detail, error) { 61 | size, _, err := dec.readLength() 62 | if err != nil { 63 | return nil, nil, err 64 | } 65 | entries := make([][]byte, 0) 66 | detail := &model.Quicklist2Detail{} 67 | for i := 0; i < int(size); i++ { 68 | length, _, err := dec.readLength() 69 | if err != nil { 70 | return nil, nil, err 71 | } 72 | if length == model.QuicklistNodeContainerPlain { 73 | entry, err := dec.readString() 74 | if err != nil { 75 | return nil, nil, err 76 | } 77 | entries = append(entries, entry) 78 | detail.NodeEncodings = append(detail.NodeEncodings, model.QuicklistNodeContainerPlain) 79 | } else if length == model.QuicklistNodeContainerPacked { 80 | page, lengths, err := dec.readListPack() 81 | if err != nil { 82 | return nil, nil, err 83 | } 84 | entries = append(entries, page...) 85 | detail.NodeEncodings = append(detail.NodeEncodings, model.QuicklistNodeContainerPlain) 86 | detail.ListPackEntrySize = append(detail.ListPackEntrySize, lengths) 87 | } else { 88 | return nil, nil, errors.New("unknown quicklist node type") 89 | } 90 | 91 | } 92 | return entries, detail, nil 93 | } 94 | 95 | func (enc *Encoder) WriteListObject(key string, values [][]byte, options ...interface{}) error { 96 | err := enc.beforeWriteObject(options...) 97 | if err != nil { 98 | return err 99 | } 100 | ok, err := enc.tryWriteListZipList(key, values, options...) 101 | if err != nil { 102 | return err 103 | } 104 | if !ok { 105 | err = enc.writeQuickList(key, values, options...) 106 | if err != nil { 107 | return err 108 | } 109 | } 110 | enc.state = writtenObjectState 111 | return nil 112 | } 113 | 114 | func (enc *Encoder) tryWriteListZipList(key string, values [][]byte, options ...interface{}) (bool, error) { 115 | if len(values) > enc.listZipListOpt.getMaxEntries() { 116 | return false, nil 117 | } 118 | strList := make([]string, 0, len(values)) 119 | maxValue := enc.listZipListOpt.getMaxValue() 120 | for _, v := range values { 121 | if len(v) > maxValue { 122 | return false, nil 123 | } 124 | strList = append(strList, unsafeBytes2Str(v)) 125 | } 126 | err := enc.write([]byte{typeListZipList}) 127 | if err != nil { 128 | return true, err 129 | } 130 | err = enc.writeString(key) 131 | if err != nil { 132 | return true, err 133 | } 134 | err = enc.writeZipList(strList) 135 | if err != nil { 136 | return true, err 137 | } 138 | return true, nil 139 | } 140 | 141 | func (enc *Encoder) writeQuickList(key string, values [][]byte, options ...interface{}) error { 142 | var pages [][]string 143 | pageSize := 0 144 | var curPage []string 145 | for _, value := range values { 146 | curPage = append(curPage, unsafeBytes2Str(value)) 147 | pageSize += len(value) 148 | if pageSize >= enc.listZipListSize { 149 | pageSize = 0 150 | pages = append(pages, curPage) 151 | curPage = nil 152 | } 153 | } 154 | if len(curPage) > 0 { 155 | pages = append(pages, curPage) 156 | } 157 | err := enc.write([]byte{typeListQuickList}) 158 | if err != nil { 159 | return err 160 | } 161 | err = enc.writeString(key) 162 | if err != nil { 163 | return err 164 | } 165 | err = enc.writeLength(uint64(len(pages))) 166 | if err != nil { 167 | return err 168 | } 169 | for _, page := range pages { 170 | err = enc.writeZipList(page) 171 | if err != nil { 172 | return err 173 | } 174 | } 175 | return nil 176 | } 177 | 178 | func (enc *Encoder) writeZipList(values []string) error { 179 | buf := make([]byte, 10) // reserve 10 bytes for zip list header 180 | zlBytes := 11 // header(10bytes) + zl end(1byte) 181 | zlTail := 10 182 | var prevLen uint32 183 | for i, value := range values { 184 | entry := encodeZipListEntry(prevLen, value) 185 | buf = append(buf, entry...) 186 | prevLen = uint32(len(entry)) 187 | zlBytes += len(entry) 188 | if i < len(values)-1 { 189 | zlTail += len(entry) 190 | } 191 | } 192 | buf = append(buf, 0xff) 193 | binary.LittleEndian.PutUint32(buf[0:4], uint32(zlBytes)) 194 | binary.LittleEndian.PutUint32(buf[4:8], uint32(zlTail)) 195 | binary.LittleEndian.PutUint16(buf[8:10], uint16(len(values))) 196 | return enc.writeNanString(unsafeBytes2Str(buf)) 197 | } 198 | -------------------------------------------------------------------------------- /core/list_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "github.com/hdt3213/rdb/model" 6 | "math" 7 | "math/rand" 8 | "strconv" 9 | "testing" 10 | ) 11 | 12 | func TestListEncoding(t *testing.T) { 13 | var list [][]byte 14 | for i := 0; i < 1021; i++ { // pick a prime number 15 | list = append(list, []byte(RandString(128))) 16 | } 17 | listMap := map[string][][]byte{ 18 | "a": { 19 | []byte("a"), []byte("b"), []byte("c"), []byte("d"), 20 | }, 21 | "1": { 22 | []byte("1"), []byte("2"), []byte("3"), []byte("4"), 23 | }, 24 | "001": { 25 | []byte("001"), []byte("0x11"), []byte("000"), []byte("0"), 26 | }, 27 | "0": { 28 | []byte("0x11"), []byte("001"), []byte("11111111"), []byte("1"), 29 | }, 30 | "large": list, 31 | } 32 | buf := bytes.NewBuffer(nil) 33 | enc := NewEncoder(buf) 34 | err := enc.WriteHeader() 35 | if err != nil { 36 | t.Error(err) 37 | return 38 | } 39 | err = enc.WriteDBHeader(0, uint64(len(listMap)), 0) 40 | if err != nil { 41 | t.Error(err) 42 | return 43 | } 44 | for k, v := range listMap { 45 | err = enc.WriteListObject(k, v) 46 | if err != nil { 47 | t.Error(err) 48 | return 49 | } 50 | } 51 | err = enc.WriteEnd() 52 | if err != nil { 53 | t.Error(err) 54 | return 55 | } 56 | dec := NewDecoder(buf).WithSpecialOpCode() 57 | err = dec.Parse(func(object model.RedisObject) bool { 58 | switch o := object.(type) { 59 | case *model.ListObject: 60 | expect := listMap[o.GetKey()] 61 | if len(expect) != o.GetElemCount() { 62 | t.Errorf("list %s has wrong element count", o.GetKey()) 63 | return true 64 | } 65 | for i, expectV := range expect { 66 | actualV := o.Values[i] 67 | if !bytes.Equal(expectV, actualV) { 68 | t.Errorf("list %s has element at index %d", o.GetKey(), i) 69 | return true 70 | } 71 | } 72 | } 73 | return true 74 | }) 75 | if err != nil { 76 | t.Error(err) 77 | } 78 | } 79 | 80 | func TestZipListEncoding(t *testing.T) { 81 | buf := bytes.NewBuffer(nil) 82 | enc := NewEncoder(buf).SetListZipListOpt(64, 64) 83 | list := []string{ 84 | "", 85 | "0", 86 | "1", 87 | "13", 88 | "127", 89 | "32766", 90 | "8388607", 91 | "16777216", 92 | "2147483647", 93 | "21474836471", 94 | "a", 95 | "abc", 96 | "007", 97 | "+0", 98 | "-0", 99 | "+1", 100 | "-1", 101 | "0x11", 102 | "0o00", 103 | strconv.Itoa(math.MaxInt8), 104 | strconv.Itoa(math.MinInt8), 105 | strconv.Itoa(math.MaxInt16), 106 | strconv.Itoa(math.MinInt16), 107 | strconv.Itoa(math.MaxInt32), 108 | strconv.Itoa(math.MaxInt32) + "1", 109 | strconv.Itoa(math.MinInt32), 110 | strconv.Itoa(math.MinInt32) + "1", 111 | strconv.Itoa(math.MaxInt64), 112 | strconv.Itoa(math.MaxInt64) + "1", 113 | strconv.Itoa(math.MinInt64), 114 | strconv.Itoa(math.MinInt64) + "1", 115 | RandString(60), 116 | RandString(1638), 117 | RandString(10000), 118 | } 119 | err := enc.writeZipList(list) 120 | if err != nil { 121 | t.Error(err) 122 | return 123 | } 124 | dec := NewDecoder(buf) 125 | actual, err := dec.readZipList() 126 | if err != nil { 127 | t.Error(err) 128 | return 129 | } 130 | if len(list) != len(actual) { 131 | t.Error("wrong result size") 132 | return 133 | } 134 | for i, expectV := range list { 135 | actualV := string(actual[i]) 136 | if expectV != actualV { 137 | t.Errorf("illegal value at %d", i) 138 | } 139 | } 140 | } 141 | 142 | func TestRandomZipListEncoding(t *testing.T) { 143 | size := 32 144 | round := 10000 145 | list := make([]string, size) 146 | for r := 0; r < round; r++ { 147 | for i := 0; i < size; i++ { 148 | list[i] = RandString(rand.Intn(50)) 149 | } 150 | buf := bytes.NewBuffer(nil) 151 | enc := NewEncoder(buf).SetListZipListOpt(64, 64) 152 | err := enc.writeZipList(list) 153 | if err != nil { 154 | t.Error(err) 155 | return 156 | } 157 | dec := NewDecoder(buf) 158 | actual, err := dec.readZipList() 159 | if err != nil { 160 | t.Error(err) 161 | return 162 | } 163 | if len(list) != len(actual) { 164 | t.Error("wrong result size") 165 | return 166 | } 167 | for i, expectV := range list { 168 | actualV := string(actual[i]) 169 | if expectV != actualV { 170 | t.Errorf("illegal value at %d", i) 171 | } 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /core/listpack.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "fmt" 7 | "strconv" 8 | ) 9 | 10 | // readListPack returns: list of entry, list of entry size, error 11 | func (dec *Decoder) readListPack() ([][]byte, []uint32, error) { 12 | buf, err := dec.readString() 13 | if err != nil { 14 | return nil, nil, err 15 | } 16 | cursor := 0 17 | size := readListPackLength(buf, &cursor) 18 | entries := make([][]byte, 0, size) 19 | entrySizes := make([]uint32, 0, size) 20 | for i := 0; i < size; i++ { 21 | str, intval, length, err := dec.readListPackEntry(buf, &cursor) 22 | if err != nil { 23 | return nil, nil, err 24 | } 25 | if str == nil { 26 | str = []byte(strconv.FormatInt(intval, 10)) 27 | } 28 | entries = append(entries, str) 29 | entrySizes = append(entrySizes, length) 30 | } 31 | return entries, entrySizes, nil 32 | } 33 | 34 | func readListPackLength(buf []byte, cursor *int) int { 35 | start := *cursor + 4 36 | end := start + 2 37 | // list pack buf: [0, 4] -> total bytes, [4:6] -> entry count 38 | size := int(binary.LittleEndian.Uint16(buf[start:end])) 39 | *cursor += 6 40 | return size 41 | } 42 | 43 | func getBackLen(elementLen uint32) uint32 { 44 | if elementLen <= 127 { 45 | return 1 46 | } else if elementLen < (1<<14)-1 { 47 | return 2 48 | } else if elementLen < (1<<21)-1 { 49 | return 3 50 | } else if elementLen < (1<<28)-1 { 51 | return 4 52 | } else { 53 | return 5 54 | } 55 | } 56 | 57 | 58 | // readListPackEntry returns: string content, int content, entry length(encoding+content+backlen), error 59 | func (dec *Decoder) readListPackEntry(buf []byte, cursor *int) ([]byte, int64, uint32, error) { 60 | header, err := readByte(buf, cursor) 61 | if err != nil { 62 | return nil, 0, 0, err 63 | } 64 | switch header >> 6 { 65 | case 0, 1: // 0xxxxxxx, uint7 66 | result := int64(int8(header)) 67 | var contentLen uint32 = 1 68 | backlen := getBackLen(contentLen) 69 | *cursor += int(backlen) 70 | return nil, result, contentLen + backlen, nil 71 | case 2: // 10xxxxxx + content, string(len<=63) 72 | strLen := int(header & 0x3f) 73 | result, err := readBytes(buf, cursor, strLen) 74 | if err != nil { 75 | return nil, 0, 0, err 76 | } 77 | var contentLen = uint32(1 + strLen) 78 | backlen := getBackLen(contentLen) 79 | *cursor += int(backlen) 80 | return result, 0, contentLen + backlen, nil 81 | } 82 | // assert header == 11xxxxxx 83 | switch header >> 4 { 84 | case 12, 13: // 110xxxxx yyyyyyyy, int13 85 | // see https://github.com/CN-annotation-team/redis7.0-chinese-annotated/blob/fba43c524524cbdb54955a28af228b513420d78d/src/listpack.c#L586 86 | next, err := readByte(buf, cursor) 87 | if err != nil { 88 | return nil, 0, 0, err 89 | } 90 | val := ((uint(header) & 0x1F) << 8) | uint(next) 91 | if val >= uint(1<<12) { 92 | val = -(8191 - val) - 1 // val is uint, must use -(8191 - val), val - 8191 will cause overflow 93 | } 94 | result := int64(val) 95 | var contentLen uint32 = 2 96 | backlen := getBackLen(contentLen) 97 | *cursor += int(backlen) 98 | return nil, result, contentLen + backlen, nil 99 | case 14: // 1110xxxx yyyyyyyy + content, string(len < 1<<12) 100 | dec.buffer[0] = header & 0x0f 101 | dec.buffer[1], err = readByte(buf, cursor) 102 | if err != nil { 103 | return nil, 0, 0, err 104 | } 105 | strLen := binary.BigEndian.Uint16(dec.buffer[:2]) 106 | result, err := readBytes(buf, cursor, int(strLen)) 107 | if err != nil { 108 | return nil, 0, 0, err 109 | } 110 | var contentLen = uint32(2 + strLen) 111 | backlen := getBackLen(contentLen) 112 | *cursor += int(backlen) 113 | return result, 0, contentLen + backlen, nil 114 | } 115 | // assert header == 1111xxxx 116 | switch header & 0x0f { 117 | case 0: // 11110000 aaaaaaaa bbbbbbbb cccccccc dddddddd + content, string(len < 1<<32) 118 | var lenBytes []byte 119 | lenBytes, err = readBytes(buf, cursor, 4) 120 | if err != nil { 121 | return nil, 0, 0, err 122 | } 123 | strLen := int(binary.LittleEndian.Uint32(lenBytes)) 124 | result, err := readBytes(buf, cursor, strLen) 125 | if err != nil { 126 | return nil, 0, 0, err 127 | } 128 | var contentLen = uint32(1 + 4 + strLen) 129 | backlen := getBackLen(contentLen) 130 | *cursor += int(backlen) 131 | return result, 0, contentLen + backlen, nil 132 | case 1: // 11110001 aaaaaaaa bbbbbbbb, int16 133 | var bs []byte 134 | bs, err = readBytes(buf, cursor, 2) 135 | if err != nil { 136 | return nil, 0, 0, err 137 | } 138 | result := int64(int16(binary.LittleEndian.Uint16(bs))) 139 | var contentLen uint32 = 3 140 | backlen := getBackLen(contentLen) 141 | *cursor += int(backlen) 142 | return nil, result, contentLen + backlen, nil 143 | case 2: // 11110010 aaaaaaaa bbbbbbbb cccccccc, int24 144 | var bs []byte 145 | bs, err = readBytes(buf, cursor, 3) 146 | if err != nil { 147 | return nil, 0, 0, err 148 | } 149 | bs = append([]byte{0}, bs...) 150 | result := int64(int32(binary.LittleEndian.Uint32(bs))>>8) 151 | var contentLen uint32 = 4 152 | backlen := getBackLen(contentLen) 153 | *cursor += int(backlen) 154 | return nil, result, contentLen + backlen, nil 155 | case 3: // 1111 0011 -> int32 156 | var bs []byte 157 | bs, err = readBytes(buf, cursor, 4) 158 | if err != nil { 159 | return nil, 0, 0, err 160 | } 161 | result := int64(int32(binary.LittleEndian.Uint32(bs))) 162 | var contentLen uint32 = 5 163 | backlen := getBackLen(contentLen) 164 | *cursor += int(backlen) 165 | return nil, result, contentLen + backlen, nil 166 | case 4: // 11110100 8Byte -> int64 167 | var bs []byte 168 | bs, err = readBytes(buf, cursor, 8) 169 | if err != nil { 170 | return nil, 0, 0, err 171 | } 172 | result := int64(binary.LittleEndian.Uint64(bs)) 173 | var contentLen uint32 = 9 174 | backlen := getBackLen(contentLen) 175 | *cursor += int(backlen) 176 | return nil, result, contentLen + backlen, nil 177 | case 15: // 11111111 -> end 178 | return nil, 0, 0, errors.New("unexpected end") 179 | } 180 | return nil, 0, 0, fmt.Errorf("unknown entry header") 181 | } 182 | 183 | // readListPackEntryAsString return a string representation of entry 184 | // It means if the entry is a integer, then format it as string 185 | func (dec *Decoder) readListPackEntryAsString(buf []byte, cursor *int) ([]byte, error) { 186 | str, intval, _, err := dec.readListPackEntry(buf, cursor) 187 | if err != nil { 188 | return nil, fmt.Errorf("read from failed: %v", err) 189 | } 190 | if str != nil { 191 | return str, nil 192 | } 193 | str = []byte(strconv.FormatInt(intval, 10)) 194 | return str, nil 195 | } 196 | 197 | func (dec *Decoder) readListPackEntryAsInt(buf []byte, cursor *int) (int64, error) { 198 | str, intval, _, err := dec.readListPackEntry(buf, cursor) 199 | if err != nil { 200 | return 0, fmt.Errorf("read from failed: %v", err) 201 | } 202 | if str != nil { 203 | return 0, fmt.Errorf("%s is not a integer", string(str)) 204 | } 205 | return intval, nil 206 | } 207 | -------------------------------------------------------------------------------- /core/module.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | type Opcode uint8 9 | 10 | const ( 11 | ModuleOpcodeEOF Opcode = iota 12 | ModuleOpcodeSInt 13 | ModuleOpcodeUInt 14 | ModuleOpcodeFloat 15 | ModuleOpcodeDouble 16 | ModuleOpcodeString 17 | ) 18 | 19 | type ModuleTypeHandler interface { 20 | ReadByte() (byte, error) 21 | ReadFull(buf []byte) error 22 | ReadOpcode() (Opcode, error) 23 | ReadUInt() (uint64, error) 24 | ReadSInt() (int64, error) 25 | ReadFloat32() (float32, error) 26 | ReadDouble() (float64, error) 27 | ReadString() ([]byte, error) 28 | ReadLength() (uint64, bool, error) 29 | } 30 | 31 | type moduleTypeHandlerImpl struct { 32 | dec *Decoder 33 | } 34 | 35 | func (m moduleTypeHandlerImpl) ReadByte() (byte, error) { 36 | return m.dec.readByte() 37 | } 38 | 39 | func (m moduleTypeHandlerImpl) ReadFull(buf []byte) error { 40 | return m.dec.readFull(buf) 41 | } 42 | 43 | func (m moduleTypeHandlerImpl) ReadOpcode() (Opcode, error) { 44 | code, _, err := m.dec.readLength() 45 | if err != nil { 46 | return 0, err 47 | } 48 | if code > 5 { 49 | return 0, errors.New("unknown opcode") 50 | } 51 | return Opcode(code), nil 52 | } 53 | 54 | func (m moduleTypeHandlerImpl) ReadUInt() (uint64, error) { 55 | val, _, err := m.dec.readLength() 56 | return val, err 57 | } 58 | 59 | func (m moduleTypeHandlerImpl) ReadSInt() (int64, error) { 60 | val, _, err := m.dec.readLength() 61 | return int64(val), err 62 | } 63 | 64 | func (m moduleTypeHandlerImpl) ReadFloat32() (float32, error) { 65 | return m.dec.readFloat32() 66 | } 67 | func (m moduleTypeHandlerImpl) ReadDouble() (float64, error) { 68 | return m.dec.readFloat() 69 | } 70 | 71 | func (m moduleTypeHandlerImpl) ReadString() ([]byte, error) { 72 | return m.dec.readString() 73 | } 74 | 75 | func (m moduleTypeHandlerImpl) ReadLength() (uint64, bool, error) { 76 | return m.dec.readLength() 77 | } 78 | 79 | type ModuleTypeHandleFunc func(handler ModuleTypeHandler, encVersion int) (interface{}, error) 80 | 81 | func (dec *Decoder) readModuleType() (string, interface{}, error) { 82 | moduleId, _, err := dec.readLength() 83 | if err != nil { 84 | return "", nil, err 85 | } 86 | return dec.handleModuleType(moduleId) 87 | } 88 | 89 | func (dec *Decoder) handleModuleType(moduleId uint64) (string, interface{}, error) { 90 | moduleType := moduleTypeNameByID(moduleId) 91 | handler, found := dec.withSpecialTypes[moduleType] 92 | if !found { 93 | fmt.Printf("unknown module type: %s,will skip\n", moduleType) 94 | handler = skipModuleAuxData 95 | } 96 | encVersion := moduleTypeEncVersionByID(moduleId) 97 | val, err := handler(moduleTypeHandlerImpl{dec: dec}, int(encVersion)) 98 | return moduleType, val, err 99 | } 100 | 101 | func moduleTypeNameByID(moduleId uint64) string { 102 | cset := ModuleTypeNameCharSet 103 | name := make([]byte, 9) 104 | moduleId >>= 10 105 | for j := 0; j < 9; j++ { 106 | name[8-j] = cset[moduleId&63] 107 | moduleId >>= 6 108 | } 109 | return string(name) 110 | } 111 | 112 | func moduleTypeEncVersionByID(moduleId uint64) uint64 { 113 | return moduleId & 1023 114 | } 115 | 116 | // skipModuleAuxData skips module aux data 117 | func skipModuleAuxData(h ModuleTypeHandler, _ int) (interface{}, error) { 118 | opCode, err := h.ReadOpcode() 119 | if err != nil { 120 | return nil, err 121 | } 122 | for opCode != ModuleOpcodeEOF { 123 | switch opCode { 124 | case ModuleOpcodeSInt: 125 | _, err = h.ReadSInt() 126 | case ModuleOpcodeUInt: 127 | _, err = h.ReadUInt() 128 | case ModuleOpcodeFloat: 129 | _, err = h.ReadFloat32() 130 | case ModuleOpcodeDouble: 131 | _, err = h.ReadDouble() 132 | case ModuleOpcodeString: 133 | _, err = h.ReadString() 134 | default: 135 | err = fmt.Errorf("unknown module opcode %d", opCode) 136 | } 137 | if err != nil { 138 | return nil, err 139 | } 140 | opCode, err = h.ReadOpcode() 141 | if err != nil { 142 | return nil, err 143 | } 144 | } 145 | 146 | return nil, nil 147 | } 148 | 149 | const ModuleTypeNameCharSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" 150 | -------------------------------------------------------------------------------- /core/module_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/hdt3213/rdb/model" 7 | "testing" 8 | ) 9 | 10 | const testModuleType = "test-type" 11 | 12 | func TestModuleType(t *testing.T) { 13 | buf := bytes.NewBuffer(nil) 14 | enc := NewEncoder(buf) 15 | 16 | key := "testkey" 17 | expectedModuleEncVersion := uint64(42) 18 | expectedStrData := "testdata123" 19 | expectedUInt := uint64(123) 20 | 21 | err := enc.WriteHeader() 22 | if err != nil { 23 | t.Error(err) 24 | return 25 | } 26 | 27 | err = enc.WriteDBHeader(0, 1, 0) 28 | if err != nil { 29 | t.Error(err) 30 | return 31 | } 32 | err = enc.beforeWriteObject() 33 | if err != nil { 34 | t.Error(err) 35 | return 36 | } 37 | err = enc.write([]byte{typeModule2}) 38 | if err != nil { 39 | t.Error(err) 40 | return 41 | } 42 | err = enc.writeString(key) 43 | if err != nil { 44 | t.Error(err) 45 | return 46 | } 47 | 48 | moduleId := createModuleId(testModuleType, expectedModuleEncVersion) 49 | err = enc.writeLength(moduleId) 50 | if err != nil { 51 | t.Error(err) 52 | return 53 | } 54 | err = enc.writeLength(uint64(ModuleOpcodeString)) 55 | if err != nil { 56 | t.Error(err) 57 | return 58 | } 59 | err = enc.writeString(expectedStrData) 60 | if err != nil { 61 | t.Error(err) 62 | return 63 | } 64 | err = enc.writeLength(uint64(ModuleOpcodeUInt)) 65 | if err != nil { 66 | t.Error(err) 67 | return 68 | } 69 | err = enc.writeLength(expectedUInt) 70 | if err != nil { 71 | t.Error(err) 72 | return 73 | } 74 | 75 | err = enc.writeLength(uint64(ModuleOpcodeEOF)) 76 | if err != nil { 77 | t.Error(err) 78 | return 79 | } 80 | enc.state = writtenObjectState 81 | 82 | err = enc.WriteEnd() 83 | if err != nil { 84 | t.Error(err) 85 | return 86 | } 87 | data := buf.Bytes() 88 | t.Run("with parse", func(t *testing.T) { 89 | 90 | expectedResult := "expected-result" 91 | 92 | dec := NewDecoder(bytes.NewBuffer(data)).WithSpecialType(testModuleType, 93 | func(h ModuleTypeHandler, encVersion int) (interface{}, error) { 94 | if encVersion != 42 { 95 | t.Errorf("invalid encoding version, expected %d, actual %d", 96 | expectedModuleEncVersion, encVersion) 97 | return nil, fmt.Errorf("invalid encoding version, expected %d, actual %d", 98 | expectedModuleEncVersion, encVersion) 99 | } 100 | 101 | opcode, err := h.ReadOpcode() 102 | if err != nil { 103 | return nil, err 104 | } 105 | if opcode != ModuleOpcodeString { 106 | return nil, fmt.Errorf("invalid opcode read, expected %d (string), actual %d", 107 | ModuleOpcodeString, opcode) 108 | } 109 | data, err := h.ReadString() 110 | if err != nil { 111 | return nil, err 112 | } 113 | if !bytes.Equal(data, []byte(expectedStrData)) { 114 | return nil, fmt.Errorf("invalid string data read, expected %s, actual %s", 115 | expectedStrData, string(data)) 116 | } 117 | 118 | opcode, err = h.ReadOpcode() 119 | if err != nil { 120 | return nil, err 121 | } 122 | if opcode != ModuleOpcodeUInt { 123 | return nil, fmt.Errorf("invalid opcode read, expected %d (uint), actual %d", 124 | ModuleOpcodeUInt, opcode) 125 | } 126 | val, err := h.ReadUInt() 127 | if err != nil { 128 | return nil, err 129 | } 130 | if val != expectedUInt { 131 | return nil, fmt.Errorf("invalid unsigned int read, expected %d, actual %d", 132 | expectedUInt, val) 133 | } 134 | opcode, err = h.ReadOpcode() 135 | if err != nil { 136 | return nil, err 137 | } 138 | if opcode != ModuleOpcodeEOF { 139 | return nil, fmt.Errorf("invalid opcode read, expected %d (EOF), actual %d", 140 | ModuleOpcodeEOF, opcode) 141 | } 142 | return expectedResult, nil 143 | }) 144 | 145 | err = dec.Parse(func(o model.RedisObject) bool { 146 | if o.GetKey() != key { 147 | t.Errorf("invalid object key, expected %s, actual %s", key, o.GetKey()) 148 | return false 149 | } 150 | if o.GetType() != testModuleType { 151 | t.Errorf("invalid redis type, expected %s, actual %s", testModuleType, o.GetType()) 152 | return false 153 | } 154 | mtObj, ok := o.(*model.ModuleTypeObject) 155 | if !ok { 156 | t.Errorf("invalid object type, expected model.ModuleTypeObject") 157 | return false 158 | } 159 | 160 | if mtObj.Value != expectedResult { 161 | t.Errorf("invalid return value") 162 | return false 163 | } 164 | 165 | return true 166 | }) 167 | if err != nil { 168 | t.Error(err) 169 | } 170 | }) 171 | t.Run("skip parse", func(t *testing.T) { 172 | dec := NewDecoder(bytes.NewBuffer(data)) 173 | err = dec.Parse(func(o model.RedisObject) bool { 174 | if o.GetKey() != key { 175 | t.Errorf("invalid object key, expected %s, actual %s", key, o.GetKey()) 176 | return false 177 | } 178 | if o.GetType() != testModuleType { 179 | t.Errorf("invalid redis type, expected %s, actual %s", testModuleType, o.GetType()) 180 | return false 181 | } 182 | mtObj, ok := o.(*model.ModuleTypeObject) 183 | if !ok { 184 | t.Errorf("invalid object type, expected model.ModuleTypeObject") 185 | return false 186 | } 187 | 188 | if mtObj.Value != nil { 189 | t.Errorf("invalid return value") 190 | return false 191 | } 192 | 193 | return true 194 | }) 195 | if err != nil { 196 | t.Error(err) 197 | } 198 | }) 199 | } 200 | 201 | func TestCorrectModuleTypeEncodeDecode(t *testing.T) { 202 | moduleId := createModuleId(testModuleType, 42) 203 | name := moduleTypeNameByID(moduleId) 204 | encVersion := moduleTypeEncVersionByID(moduleId) 205 | if name != testModuleType { 206 | t.Errorf("invalid module type name, expected %s, actual %s", testModuleType, name) 207 | } 208 | if encVersion != 42 { 209 | t.Errorf("invalid module encoding version, expected %d, actual %d", 42, encVersion) 210 | } 211 | } 212 | 213 | func createModuleId(moduleType string, encVersion uint64) uint64 { 214 | moduleId := uint64(0) 215 | for i := 0; i < 9; i++ { 216 | moduleId |= uint64(charCode(moduleType[i])) 217 | moduleId <<= 6 218 | } 219 | moduleId <<= 4 220 | moduleId |= encVersion 221 | return moduleId 222 | } 223 | 224 | func charCode(c uint8) uint8 { 225 | for i, csChar := range []byte(ModuleTypeNameCharSet) { 226 | if c == csChar { 227 | return uint8(i) 228 | } 229 | } 230 | panic(fmt.Errorf("unsupported char %c", c)) 231 | } 232 | -------------------------------------------------------------------------------- /core/set.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "math" 7 | "sort" 8 | "strconv" 9 | 10 | "github.com/hdt3213/rdb/model" 11 | ) 12 | 13 | func (dec *Decoder) readSet() ([][]byte, error) { 14 | size64, _, err := dec.readLength() 15 | if err != nil { 16 | return nil, err 17 | } 18 | size := int(size64) 19 | values := make([][]byte, 0, size) 20 | for i := 0; i < size; i++ { 21 | val, err := dec.readString() 22 | if err != nil { 23 | return nil, err 24 | } 25 | values = append(values, val) 26 | } 27 | return values, nil 28 | } 29 | 30 | func (dec *Decoder) readIntSet() (result [][]byte, detail *model.IntsetDetail, err error) { 31 | var buf []byte 32 | buf, err = dec.readString() 33 | if err != nil { 34 | return nil, nil, err 35 | } 36 | sizeBytes := buf[0:4] 37 | intSize := int(binary.LittleEndian.Uint32(sizeBytes)) 38 | if intSize != 2 && intSize != 4 && intSize != 8 { 39 | return nil, nil, fmt.Errorf("unknown intset encoding: %d", intSize) 40 | } 41 | lenBytes := buf[4:8] 42 | cardinality := binary.LittleEndian.Uint32(lenBytes) 43 | cursor := 8 44 | result = make([][]byte, 0, cardinality) 45 | for i := uint32(0); i < cardinality; i++ { 46 | var intBytes []byte 47 | intBytes, err = readBytes(buf, &cursor, intSize) 48 | if err != nil { 49 | return 50 | } 51 | var intString string 52 | switch intSize { 53 | case 2: 54 | intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) 55 | case 4: 56 | intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) 57 | case 8: 58 | intString = strconv.FormatInt(int64(binary.LittleEndian.Uint64(intBytes)), 10) 59 | } 60 | result = append(result, []byte(intString)) 61 | } 62 | detail = &model.IntsetDetail{ 63 | RawStringSize: len(buf), 64 | } 65 | return 66 | } 67 | 68 | func (dec *Decoder) readListPackSet() ([][]byte, *model.ListpackDetail, error) { 69 | buf, err := dec.readString() 70 | if err != nil { 71 | return nil, nil, err 72 | } 73 | cursor := 0 74 | size := readListPackLength(buf, &cursor) 75 | values := make([][]byte, 0, size) 76 | for i := 0; i < size; i += 1 { 77 | member, err := dec.readListPackEntryAsString(buf, &cursor) 78 | if err != nil { 79 | return nil, nil, err 80 | } 81 | values = append(values, member) 82 | } 83 | detail := &model.ListpackDetail{ 84 | RawStringSize: len(buf), 85 | } 86 | return values, detail, nil 87 | } 88 | 89 | func (enc *Encoder) WriteSetObject(key string, values [][]byte, options ...interface{}) error { 90 | err := enc.beforeWriteObject(options...) 91 | if err != nil { 92 | return err 93 | } 94 | ok, err := enc.tryWriteIntSetEncoding(key, values) 95 | if err != nil { 96 | return err 97 | } 98 | if !ok { 99 | err = enc.writeSetEncoding(key, values) 100 | if err != nil { 101 | return err 102 | } 103 | } 104 | enc.state = writtenObjectState 105 | return nil 106 | } 107 | 108 | func (enc *Encoder) writeSetEncoding(key string, values [][]byte) error { 109 | err := enc.write([]byte{typeSet}) 110 | if err != nil { 111 | return err 112 | } 113 | err = enc.writeString(key) 114 | if err != nil { 115 | return err 116 | } 117 | err = enc.writeLength(uint64(len(values))) 118 | if err != nil { 119 | return err 120 | } 121 | for _, value := range values { 122 | err = enc.writeString(unsafeBytes2Str(value)) 123 | if err != nil { 124 | return err 125 | } 126 | } 127 | return nil 128 | } 129 | 130 | func (enc *Encoder) tryWriteIntSetEncoding(key string, values [][]byte) (bool, error) { 131 | max := int64(math.MinInt64) 132 | min := int64(math.MaxInt64) 133 | intList := make([]int64, len(values)) 134 | for i, v := range values { 135 | str := unsafeBytes2Str(v) 136 | intV, ok := isEncodableUint64(str) 137 | if !ok { 138 | return false, nil 139 | } 140 | if intV < min { 141 | min = intV 142 | } else if intV > max { 143 | max = intV 144 | } 145 | intList[i] = intV 146 | } 147 | intSize := uint32(8) 148 | if min >= math.MinInt16 && max <= math.MaxInt16 { 149 | intSize = 2 150 | } else if min >= math.MinInt32 && max <= math.MaxInt32 { 151 | intSize = 4 152 | } 153 | sort.Slice(intList, func(i, j int) bool { 154 | return intList[i] < intList[j] 155 | }) 156 | 157 | err := enc.write([]byte{typeSetIntSet}) 158 | if err != nil { 159 | return true, err 160 | } 161 | err = enc.writeString(key) 162 | if err != nil { 163 | return true, err 164 | } 165 | buf := make([]byte, 8, 8+int(intSize)*len(values)) 166 | binary.LittleEndian.PutUint32(buf[0:4], intSize) 167 | binary.LittleEndian.PutUint32(buf[4:8], uint32(len(values))) 168 | 169 | for _, value := range intList { 170 | switch intSize { 171 | case 2: 172 | binary.LittleEndian.PutUint16(enc.buffer[0:2], uint16(value)) 173 | buf = append(buf, enc.buffer[0:2]...) 174 | case 4: 175 | binary.LittleEndian.PutUint32(enc.buffer[0:4], uint32(value)) 176 | buf = append(buf, enc.buffer[0:4]...) 177 | case 8: 178 | binary.LittleEndian.PutUint64(enc.buffer, uint64(value)) 179 | buf = append(buf, enc.buffer...) 180 | } 181 | } 182 | err = enc.writeNanString(unsafeBytes2Str(buf)) 183 | if err != nil { 184 | return true, err 185 | } 186 | return true, nil 187 | } 188 | -------------------------------------------------------------------------------- /core/set_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "github.com/hdt3213/rdb/model" 6 | "testing" 7 | ) 8 | 9 | func TestSetEncoding(t *testing.T) { 10 | setMap := map[string][][]byte{ 11 | "a": { 12 | []byte("a"), []byte("b"), []byte("c"), []byte("d"), 13 | }, 14 | "1": { 15 | []byte("1"), []byte("2"), []byte("3"), []byte("4"), // int set encoding need ascending order 16 | }, 17 | "2": { 18 | []byte("-32767"), []byte("32767"), 19 | }, 20 | "4": { 21 | []byte("-2147483647"), []byte("2147483647"), 22 | }, 23 | "8": { 24 | []byte("-9222147483647"), []byte("9222147483647"), 25 | }, 26 | "007": { 27 | []byte("-1"), []byte("02"), []byte("0x3"), []byte("0o4"), 28 | }, 29 | } 30 | buf := bytes.NewBuffer(nil) 31 | enc := NewEncoder(buf) 32 | err := enc.WriteHeader() 33 | if err != nil { 34 | t.Error(err) 35 | return 36 | } 37 | err = enc.WriteDBHeader(0, uint64(len(setMap)), 0) 38 | if err != nil { 39 | t.Error(err) 40 | return 41 | } 42 | for k, v := range setMap { 43 | err = enc.WriteSetObject(k, v) 44 | if err != nil { 45 | t.Error(err) 46 | return 47 | } 48 | } 49 | err = enc.WriteEnd() 50 | if err != nil { 51 | t.Error(err) 52 | return 53 | } 54 | dec := NewDecoder(buf).WithSpecialOpCode() 55 | err = dec.Parse(func(object model.RedisObject) bool { 56 | switch o := object.(type) { 57 | case *model.SetObject: 58 | expect := setMap[o.GetKey()] 59 | if len(expect) != o.GetElemCount() { 60 | t.Errorf("set %s has wrong element count", o.GetKey()) 61 | return true 62 | } 63 | for i, expectV := range expect { 64 | actualV := o.Members[i] 65 | if !bytes.Equal(expectV, actualV) { 66 | t.Errorf("set %s has element at index %d", o.GetKey(), i) 67 | return true 68 | } 69 | } 70 | } 71 | return true 72 | }) 73 | if err != nil { 74 | t.Error(err) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /core/specialobj_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/hdt3213/rdb/model" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | ) 9 | 10 | func TestWithSpecialObject(t *testing.T) { 11 | rdbFilename := filepath.Join("../cases", "memory.rdb") 12 | rdbFile, err := os.Open(rdbFilename) 13 | if err != nil { 14 | t.Errorf("open rdb %s failed, %v", rdbFilename, err) 15 | return 16 | } 17 | defer func() { 18 | _ = rdbFile.Close() 19 | }() 20 | expectAux := map[string]string{ 21 | "redis-ver": "6.0.6", 22 | "redis-bits": "64", 23 | "ctime": "1644136130", 24 | "used-mem": "1167584", 25 | "aof-preamble": "0", 26 | } 27 | expectKeyCount := map[int]uint64{ 28 | 0: 7, 29 | } 30 | expectTTLCount := map[int]uint64{ 31 | 0: 1, 32 | } 33 | var auxCount, dbSizeCount int 34 | dec := NewDecoder(rdbFile).WithSpecialOpCode() 35 | err = dec.Parse(func(object model.RedisObject) bool { 36 | switch o := object.(type) { 37 | case *model.AuxObject: 38 | if o.GetType() != model.AuxType { 39 | t.Error("aux obj with wrong type") 40 | } 41 | expectValue := expectAux[o.Key] 42 | if o.Value != expectValue { 43 | t.Errorf("aux %s has wrong value", o.GetKey()) 44 | } 45 | auxCount++ 46 | case *model.DBSizeObject: 47 | dbSizeCount++ 48 | if o.KeyCount != expectKeyCount[o.DB] { 49 | t.Errorf("db %d has wrong key count", o.DB) 50 | } 51 | if o.TTLCount != expectTTLCount[o.DB] { 52 | t.Errorf("db %d has wrong ttl count", o.DB) 53 | } 54 | } 55 | return true 56 | }) 57 | if err != nil { 58 | t.Error(err) 59 | } 60 | if auxCount != len(expectAux) { 61 | t.Error("wrong aux object count") 62 | } 63 | if dbSizeCount != 1 { 64 | t.Error("wrong db size object count") 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /core/string_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "math" 6 | "math/rand" 7 | "strconv" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestLengthEncoding(t *testing.T) { 13 | buf := bytes.NewBuffer(nil) 14 | enc := NewEncoder(buf) 15 | lens := []uint64{1 << 5, 1 << 13, 1 << 31, 1 << 63} 16 | for _, v := range lens { 17 | err := enc.writeLength(v) 18 | if err != nil { 19 | t.Error(err) 20 | return 21 | } 22 | } 23 | dec := NewDecoder(buf) 24 | for _, expect := range lens { 25 | actual, special, err := dec.readLength() 26 | if err != nil { 27 | t.Error(err) 28 | return 29 | } 30 | if special { 31 | t.Error("expect normal actual special") 32 | return 33 | } 34 | if actual != expect { 35 | t.Errorf("expect %d, actual %d", expect, actual) 36 | } 37 | } 38 | } 39 | 40 | func TestStringEncoding(t *testing.T) { 41 | buf := bytes.NewBuffer(nil) 42 | enc := NewEncoder(buf) 43 | strList := []string{ 44 | "", 45 | "abc", 46 | "12", 47 | "32766", 48 | "007", 49 | "0", 50 | "+0", 51 | "-0", 52 | "+1", 53 | "-1", 54 | "0x11", 55 | "0o00", 56 | strconv.Itoa(math.MaxInt8), 57 | strconv.Itoa(math.MinInt8), 58 | strconv.Itoa(math.MaxInt16), 59 | strconv.Itoa(math.MinInt16), 60 | strconv.Itoa(math.MaxInt32), 61 | strconv.Itoa(math.MaxInt32) + "1", 62 | strconv.Itoa(math.MinInt32), 63 | strconv.Itoa(math.MinInt32) + "1", 64 | strconv.Itoa(math.MaxInt64), 65 | strconv.Itoa(math.MaxInt64) + "1", 66 | strconv.Itoa(math.MinInt64), 67 | strconv.Itoa(math.MinInt64) + "1", 68 | RandString(20000), 69 | } 70 | for _, str := range strList { 71 | err := enc.writeString(str) 72 | if err != nil { 73 | t.Error(err) 74 | return 75 | } 76 | } 77 | dec := NewDecoder(buf) 78 | for _, expect := range strList { 79 | actual, err := dec.readString() 80 | if err != nil { 81 | t.Error(err) 82 | continue 83 | } 84 | if string(actual) != expect { 85 | t.Errorf("expect %s, actual %s", expect, string(actual)) 86 | } 87 | } 88 | } 89 | 90 | func TestRandomStringEncoding(t *testing.T) { 91 | size := 1000 92 | round := 2000 93 | strList := make([]string, size) 94 | for r := 0; r < round; r++ { 95 | for i := 0; i < size; i++ { 96 | strList[i] = RandString(rand.Intn(50)) 97 | } 98 | buf := bytes.NewBuffer(nil) 99 | enc := NewEncoder(buf) 100 | for _, str := range strList { 101 | err := enc.writeString(str) 102 | if err != nil { 103 | t.Error(err) 104 | return 105 | } 106 | } 107 | dec := NewDecoder(buf) 108 | for _, expect := range strList { 109 | actual, err := dec.readString() 110 | if err != nil { 111 | t.Error(err) 112 | continue 113 | } 114 | if string(actual) != expect { 115 | t.Errorf("expect %s, actual %s", expect, string(actual)) 116 | } 117 | } 118 | } 119 | } 120 | 121 | func TestLZFString(t *testing.T) { 122 | buf := bytes.NewBuffer(nil) 123 | enc := NewEncoder(buf).EnableCompress() 124 | var strList []string 125 | for i := 0; i < 10; i++ { 126 | strList = append(strList, strings.Repeat(RandString(128), 10)) 127 | } 128 | for _, str := range strList { 129 | err := enc.writeString(str) 130 | if err != nil { 131 | t.Error(err) 132 | return 133 | } 134 | } 135 | dec := NewDecoder(buf) 136 | for _, expect := range strList { 137 | actual, err := dec.readString() 138 | if err != nil { 139 | t.Error(err) 140 | continue 141 | } 142 | if string(actual) != expect { 143 | t.Errorf("expect %s, actual %s", expect, string(actual)) 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /core/utils.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "io" 7 | "math/rand" 8 | "unsafe" 9 | ) 10 | 11 | func readBytes(buf []byte, cursor *int, size int) ([]byte, error) { 12 | if cursor == nil { 13 | return nil, errors.New("cursor is nil") 14 | } 15 | if *cursor+size > len(buf) { 16 | return nil, errors.New("cursor out of range") 17 | } 18 | end := *cursor + size 19 | result := buf[*cursor:end] 20 | *cursor += int(size) 21 | return result, nil 22 | } 23 | 24 | func readByte(buf []byte, cursor *int) (byte, error) { 25 | if cursor == nil { 26 | return 0, errors.New("cursor is nil") 27 | } 28 | if *cursor >= len(buf) { 29 | return 0, errors.New("cursor out of range") 30 | } 31 | b := buf[*cursor] 32 | *cursor++ 33 | return b, nil 34 | } 35 | 36 | func readZipListLength(buf []byte, cursor *int) int { 37 | start := *cursor + 8 38 | end := start + 2 39 | // zip list buf: [0, 4] -> zlbytes, [4:8] -> zltail, [8:10] -> zllen 40 | size := int(binary.LittleEndian.Uint16(buf[start:end])) 41 | *cursor += 10 42 | return size 43 | } 44 | 45 | func (dec *Decoder) readByte() (byte, error) { 46 | b, err := dec.input.ReadByte() 47 | if err != nil { 48 | return 0, err 49 | } 50 | dec.readCount++ 51 | return b, nil 52 | } 53 | 54 | func (dec *Decoder) readFull(buf []byte) error { 55 | n, err := io.ReadFull(dec.input, buf) 56 | if err != nil { 57 | return err 58 | } 59 | dec.readCount += n 60 | return nil 61 | } 62 | 63 | var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 64 | 65 | // RandString create a random string no longer than n 66 | func RandString(n int) string { 67 | b := make([]rune, n) 68 | for i := range b { 69 | b[i] = letters[rand.Intn(len(letters))] 70 | } 71 | return string(b) 72 | } 73 | 74 | func unsafeBytes2Str(b []byte) string { 75 | return *(*string)(unsafe.Pointer(&b)) 76 | } 77 | -------------------------------------------------------------------------------- /core/ziplist.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "math" 8 | "strconv" 9 | "unicode" 10 | ) 11 | 12 | func (dec *Decoder) readZipList() ([][]byte, error) { 13 | buf, err := dec.readString() 14 | if err != nil { 15 | return nil, err 16 | } 17 | cursor := 0 18 | size := readZipListLength(buf, &cursor) 19 | entries := make([][]byte, 0, size) 20 | for i := 0; i < size; i++ { 21 | entry, err := dec.readZipListEntry(buf, &cursor) 22 | if err != nil { 23 | return nil, err 24 | } 25 | entries = append(entries, entry) 26 | } 27 | return entries, nil 28 | } 29 | 30 | func (dec *Decoder) readZipListEntry(buf []byte, cursor *int) (result []byte, err error) { 31 | prevLen := buf[*cursor] 32 | *cursor++ 33 | if prevLen == zipBigPrevLen { 34 | *cursor += 4 35 | } 36 | header := buf[*cursor] 37 | *cursor++ 38 | typ := header >> 6 39 | switch typ { 40 | case zipStr06B: 41 | length := int(header & 0x3f) 42 | result, err = readBytes(buf, cursor, length) 43 | return 44 | case zipStr14B: 45 | b := buf[*cursor] 46 | *cursor++ 47 | length := (int(header&0x3f) << 8) | int(b) 48 | result, err = readBytes(buf, cursor, length) 49 | return 50 | case zipStr32B: 51 | var lenBytes []byte 52 | lenBytes, err = readBytes(buf, cursor, 4) 53 | if err != nil { 54 | return 55 | } 56 | length := int(binary.BigEndian.Uint32(lenBytes)) 57 | result, err = readBytes(buf, cursor, length) 58 | return 59 | } 60 | switch header { 61 | case zipInt08B: 62 | var b byte 63 | b, err = readByte(buf, cursor) 64 | if err != nil { 65 | return 66 | } 67 | result = []byte(strconv.FormatInt(int64(int8(b)), 10)) 68 | return 69 | case zipInt16B: 70 | var bs []byte 71 | bs, err = readBytes(buf, cursor, 2) 72 | if err != nil { 73 | return 74 | } 75 | result = []byte(strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(bs))), 10)) 76 | return 77 | case zipInt32B: 78 | var bs []byte 79 | bs, err = readBytes(buf, cursor, 4) 80 | if err != nil { 81 | return 82 | } 83 | result = []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(bs))), 10)) 84 | return 85 | case zipInt64B: 86 | var bs []byte 87 | bs, err = readBytes(buf, cursor, 8) 88 | if err != nil { 89 | return 90 | } 91 | result = []byte(strconv.FormatInt(int64(binary.LittleEndian.Uint64(bs)), 10)) 92 | return 93 | case zipInt24B: 94 | var bs []byte 95 | bs, err = readBytes(buf, cursor, 3) 96 | if err != nil { 97 | return 98 | } 99 | bs = append([]byte{0}, bs...) 100 | result = []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(bs))>>8), 10)) 101 | return 102 | } 103 | if header>>4 == zipInt04B { 104 | result = []byte(strconv.FormatInt(int64(header&0x0f)-1, 10)) 105 | return 106 | } 107 | return nil, fmt.Errorf("unknown entry header") 108 | } 109 | 110 | func encodeZipListEntry(prevLen uint32, val string) []byte { 111 | buf := bytes.NewBuffer(nil) 112 | // encode prevLen 113 | if prevLen < zipBigPrevLen { 114 | buf.Write([]byte{byte(prevLen)}) 115 | } else { 116 | buf.Write([]byte{0xfe}) 117 | buf0 := make([]byte, 4) 118 | binary.LittleEndian.PutUint32(buf0, prevLen) 119 | buf.Write(buf0) 120 | } 121 | // try int encoding 122 | intVal, ok := isEncodableUint64(val) 123 | if ok { 124 | // use int encoding 125 | if intVal >= 0 && intVal <= 12 { 126 | buf.Write([]byte{0xf0 | byte(intVal+1)}) 127 | } else if intVal >= math.MinInt8 && intVal <= math.MaxInt8 { 128 | // bytes.Buffer never failed 129 | buf.Write([]byte{byte(zipInt08B), byte(intVal)}) 130 | } else if intVal >= minInt24 && intVal <= maxInt24 { 131 | buffer := make([]byte, 4) 132 | binary.LittleEndian.PutUint32(buffer, uint32(intVal)) 133 | buf.Write([]byte{byte(zipInt24B)}) 134 | buf.Write(buffer[0:3]) 135 | } else if intVal >= math.MinInt32 && intVal <= math.MaxInt32 { 136 | buffer := make([]byte, 4) 137 | binary.LittleEndian.PutUint32(buffer, uint32(intVal)) 138 | buf.Write([]byte{byte(zipInt32B)}) 139 | buf.Write(buffer) 140 | } else { 141 | buffer := make([]byte, 8) 142 | binary.LittleEndian.PutUint64(buffer, uint64(intVal)) 143 | buf.Write([]byte{byte(zipInt64B)}) 144 | buf.Write(buffer) 145 | } 146 | return buf.Bytes() 147 | } 148 | // use string encoding 149 | if len(val) <= maxUint6 { 150 | buf.Write([]byte{byte(len(val))}) // 00 + xxxxxx 151 | } else if len(val) <= maxUint14 { 152 | buf.Write([]byte{byte(len(val)>>8) | len14BitMask, byte(len(val))}) 153 | } else if len(val) <= math.MaxUint32 { 154 | buffer := make([]byte, 8) 155 | binary.LittleEndian.PutUint32(buffer, uint32(len(val))) 156 | buf.Write([]byte{0x80}) 157 | buf.Write(buffer) 158 | } else { 159 | panic("too large string") 160 | } 161 | buf.Write([]byte(val)) 162 | return buf.Bytes() 163 | } 164 | 165 | func isEncodableUint64(s string) (int64, bool) { 166 | if s == "" { 167 | return 0, false 168 | } 169 | if s[0] == '0' && len(s) > 1 { 170 | return 0, false 171 | } 172 | for _, r := range s { 173 | if !unicode.IsDigit(r) { 174 | return 0, false 175 | } 176 | } 177 | intVal, err := strconv.ParseInt(s, 10, 64) 178 | if err != nil { 179 | return 0, false 180 | } 181 | return intVal, true 182 | } 183 | -------------------------------------------------------------------------------- /core/zset.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/hdt3213/rdb/model" 7 | ) 8 | 9 | func (dec *Decoder) readZSet(zset2 bool) ([]*model.ZSetEntry, error) { 10 | length, _, err := dec.readLength() 11 | if err != nil { 12 | return nil, err 13 | } 14 | entries := make([]*model.ZSetEntry, 0, int(length)) 15 | for i := uint64(0); i < length; i++ { 16 | member, err := dec.readString() 17 | if err != nil { 18 | return nil, err 19 | } 20 | var score float64 21 | if zset2 { 22 | score, err = dec.readFloat() 23 | } else { 24 | score, err = dec.readLiteralFloat() 25 | } 26 | if err != nil { 27 | return nil, err 28 | } 29 | entries = append(entries, &model.ZSetEntry{ 30 | Member: unsafeBytes2Str(member), 31 | Score: score, 32 | }) 33 | } 34 | return entries, nil 35 | } 36 | 37 | func (dec *Decoder) readZipListZSet() ([]*model.ZSetEntry, *model.ZiplistDetail, error) { 38 | buf, err := dec.readString() 39 | if err != nil { 40 | return nil, nil, err 41 | } 42 | cursor := 0 43 | size := readZipListLength(buf, &cursor) 44 | entries := make([]*model.ZSetEntry, 0, size) 45 | for i := 0; i < size; i += 2 { 46 | member, err := dec.readZipListEntry(buf, &cursor) 47 | if err != nil { 48 | return nil, nil, err 49 | } 50 | scoreLiteral, err := dec.readZipListEntry(buf, &cursor) 51 | if err != nil { 52 | return nil, nil, err 53 | } 54 | score, err := strconv.ParseFloat(unsafeBytes2Str(scoreLiteral), 64) 55 | if err != nil { 56 | return nil, nil, err 57 | } 58 | entries = append(entries, &model.ZSetEntry{ 59 | Member: unsafeBytes2Str(member), 60 | Score: score, 61 | }) 62 | } 63 | detail := &model.ZiplistDetail{ 64 | RawStringSize: len(buf), 65 | } 66 | return entries, detail, nil 67 | } 68 | 69 | func (dec *Decoder) readListPackZSet() ([]*model.ZSetEntry, *model.ListpackDetail, error) { 70 | buf, err := dec.readString() 71 | if err != nil { 72 | return nil, nil, err 73 | } 74 | cursor := 0 75 | size := readListPackLength(buf, &cursor) 76 | entries := make([]*model.ZSetEntry, 0, size) 77 | for i := 0; i < size; i += 2 { 78 | member, err := dec.readListPackEntryAsString(buf, &cursor) 79 | if err != nil { 80 | return nil, nil, err 81 | } 82 | scoreLiteral, err := dec.readListPackEntryAsString(buf, &cursor) 83 | if err != nil { 84 | return nil, nil, err 85 | } 86 | score, err := strconv.ParseFloat(unsafeBytes2Str(scoreLiteral), 64) 87 | if err != nil { 88 | return nil, nil, err 89 | } 90 | entries = append(entries, &model.ZSetEntry{ 91 | Member: unsafeBytes2Str(member), 92 | Score: score, 93 | }) 94 | } 95 | detail := &model.ListpackDetail{ 96 | RawStringSize: len(buf), 97 | } 98 | return entries, detail, nil 99 | } 100 | 101 | func (enc *Encoder) WriteZSetObject(key string, entries []*model.ZSetEntry, options ...interface{}) error { 102 | err := enc.beforeWriteObject(options...) 103 | if err != nil { 104 | return err 105 | } 106 | ok, err := enc.tryWriteZipListZSet(key, entries) 107 | if err != nil { 108 | return err 109 | } 110 | if !ok { 111 | err = enc.writeZSet2Encoding(key, entries) 112 | if err != nil { 113 | return err 114 | } 115 | } 116 | enc.state = writtenObjectState 117 | return nil 118 | } 119 | 120 | func (enc *Encoder) writeZSet2Encoding(key string, entries []*model.ZSetEntry) error { 121 | err := enc.write([]byte{typeZset2}) 122 | if err != nil { 123 | return err 124 | } 125 | err = enc.writeString(key) 126 | if err != nil { 127 | return err 128 | } 129 | err = enc.writeLength(uint64(len(entries))) 130 | if err != nil { 131 | return err 132 | } 133 | for _, entry := range entries { 134 | err = enc.writeString(entry.Member) 135 | if err != nil { 136 | return err 137 | } 138 | err = enc.writeFloat64(entry.Score) 139 | if err != nil { 140 | return err 141 | } 142 | } 143 | return nil 144 | } 145 | 146 | func (enc *Encoder) tryWriteZipListZSet(key string, entries []*model.ZSetEntry) (bool, error) { 147 | if len(entries) > enc.zsetZipListOpt.getMaxEntries() { 148 | return false, nil 149 | } 150 | maxValue := enc.zsetZipListOpt.getMaxValue() 151 | for _, entry := range entries { 152 | if len(entry.Member) > maxValue { 153 | return false, nil 154 | } 155 | } 156 | err := enc.write([]byte{typeZsetZipList}) 157 | if err != nil { 158 | return true, err 159 | } 160 | err = enc.writeString(key) 161 | if err != nil { 162 | return true, err 163 | } 164 | zlElements := make([]string, 0, len(entries)*2) 165 | for _, entry := range entries { 166 | scoreStr := strconv.FormatFloat(entry.Score, 'f', -1, 64) 167 | zlElements = append(zlElements, entry.Member, scoreStr) 168 | } 169 | err = enc.writeZipList(zlElements) 170 | if err != nil { 171 | return true, err 172 | } 173 | return true, nil 174 | } 175 | -------------------------------------------------------------------------------- /core/zset_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "github.com/hdt3213/rdb/model" 6 | "testing" 7 | ) 8 | 9 | func TestZSetEncoding(t *testing.T) { 10 | var entries []*model.ZSetEntry 11 | for i := 0; i < 1024; i++ { 12 | entries = append(entries, &model.ZSetEntry{ 13 | Member: RandString(32), 14 | Score: float64(i), 15 | }) 16 | } 17 | zSetMap := map[string][]*model.ZSetEntry{ 18 | "a": { 19 | { 20 | Member: "1", 21 | Score: 3.14, 22 | }, 23 | { 24 | Member: "b", 25 | Score: 3.15, 26 | }, 27 | }, 28 | "1": { 29 | { 30 | Member: "1", 31 | Score: 2.71828, 32 | }, 33 | { 34 | Member: "zxcv", 35 | Score: 2.8, 36 | }, 37 | }, 38 | "007": { 39 | { 40 | Member: "000", 41 | Score: 3.1, 42 | }, 43 | { 44 | Member: "0x10", 45 | Score: 9.99, 46 | }, 47 | }, 48 | "large": entries, 49 | } 50 | buf := bytes.NewBuffer(nil) 51 | enc := NewEncoder(buf).SetZSetZipListOpt(64, 64) 52 | err := enc.WriteHeader() 53 | if err != nil { 54 | t.Error(err) 55 | return 56 | } 57 | err = enc.WriteDBHeader(0, uint64(len(zSetMap)), 0) 58 | if err != nil { 59 | t.Error(err) 60 | return 61 | } 62 | for k, v := range zSetMap { 63 | err = enc.WriteZSetObject(k, v) 64 | if err != nil { 65 | t.Error(err) 66 | return 67 | } 68 | } 69 | err = enc.WriteEnd() 70 | if err != nil { 71 | t.Error(err) 72 | return 73 | } 74 | dec := NewDecoder(buf).WithSpecialOpCode() 75 | err = dec.Parse(func(object model.RedisObject) bool { 76 | switch o := object.(type) { 77 | case *model.ZSetObject: 78 | expect := zSetMap[o.GetKey()] 79 | if len(expect) != o.GetElemCount() { 80 | t.Errorf("zset %s has wrong element count", o.GetKey()) 81 | return true 82 | } 83 | for i, expectV := range expect { 84 | actualV := o.Entries[i] 85 | if actualV.Member != expectV.Member { 86 | t.Errorf("zset %s has wrong member", o.GetKey()) 87 | return true 88 | } 89 | if actualV.Score != expectV.Score { 90 | t.Errorf("zset %s has wrong score of %s", o.GetKey(), actualV.Member) 91 | return true 92 | } 93 | } 94 | } 95 | return true 96 | }) 97 | if err != nil { 98 | t.Error(err) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /crc64jones/crc64.go: -------------------------------------------------------------------------------- 1 | // Package crc64jones implements a 64-bit cyclic redundancy check, or CRC-64, 2 | // checksum. Specifically the Jones flavour of it which is used by Redis. 3 | // 4 | // Specification of this CRC64 variant follows: 5 | // - Name: crc-64-jones 6 | // - Width: 64 bites 7 | // - Poly: 0xad93d23594c935a9 8 | // - Reflected In: True 9 | // - Xor_In: 0xffffffffffffffff 10 | // - Reflected_Out: True 11 | // - Xor_Out: 0x0 12 | // - Check("123456789"): 0xe9c6d914c4b8d9ca 13 | package crc64jones 14 | 15 | import ( 16 | "hash" 17 | "hash/crc64" 18 | "sync" 19 | ) 20 | 21 | // Predefined polynomials. 22 | const ( 23 | // The Jones polynomial. 24 | Jones = 0xad93d23594c935a9 25 | ) 26 | 27 | var table = crc64.MakeTable(reflect(Jones)) 28 | 29 | // reflect reverses the bit order of the given polynomial. 30 | func reflect(poly uint64) uint64 { 31 | x := poly & 1 32 | for i := 1; i < 64; i++ { 33 | poly >>= 1 34 | x <<= 1 35 | x |= poly & 1 36 | } 37 | return x 38 | } 39 | 40 | var ( 41 | slicing8TablesBuildOnce sync.Once 42 | slicing8TableJones *[8]crc64.Table 43 | ) 44 | 45 | func buildSlicing8TablesOnce() { 46 | slicing8TablesBuildOnce.Do(buildSlicing8Tables) 47 | } 48 | 49 | func buildSlicing8Tables() { 50 | slicing8TableJones = makeSlicingBy8Table(table) 51 | } 52 | 53 | func makeSlicingBy8Table(t *crc64.Table) *[8]crc64.Table { 54 | var helperTable [8]crc64.Table 55 | helperTable[0] = *t 56 | for i := 0; i < 256; i++ { 57 | crc := t[i] 58 | for j := 1; j < 8; j++ { 59 | crc = t[crc&0xff] ^ (crc >> 8) 60 | helperTable[j][i] = crc 61 | } 62 | } 63 | return &helperTable 64 | } 65 | 66 | // digest represents the partial evaluation of a checksum. 67 | type digest struct { 68 | crc uint64 69 | tab *crc64.Table 70 | } 71 | 72 | // New creates a new hash.Hash64 computing the CRC-64 checksum using the 73 | // Jones polynomial. Its Sum method will lay the value out in little-endian 74 | // byte order. 75 | func New() hash.Hash64 { return &digest{0, table} } 76 | 77 | func (d *digest) Size() int { return crc64.Size } 78 | 79 | func (d *digest) BlockSize() int { return 1 } 80 | 81 | func (d *digest) Reset() { d.crc = 0 } 82 | 83 | func update(crc uint64, tab *crc64.Table, p []byte) uint64 { 84 | buildSlicing8TablesOnce() 85 | // Table comparison is somewhat expensive, so avoid it for small sizes 86 | for len(p) >= 64 { 87 | var helperTable *[8]crc64.Table 88 | if *tab == slicing8TableJones[0] { 89 | helperTable = slicing8TableJones 90 | } else if len(p) >= 2048 { 91 | // According to the tests between various x86 and arm CPUs, 2k is a reasonable 92 | // threshold for now. This may change in the future. 93 | helperTable = makeSlicingBy8Table(tab) 94 | } else { 95 | break 96 | } 97 | // Update using slicing-by-8 98 | for len(p) > 8 { 99 | crc ^= uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | 100 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 101 | crc = helperTable[7][crc&0xff] ^ 102 | helperTable[6][(crc>>8)&0xff] ^ 103 | helperTable[5][(crc>>16)&0xff] ^ 104 | helperTable[4][(crc>>24)&0xff] ^ 105 | helperTable[3][(crc>>32)&0xff] ^ 106 | helperTable[2][(crc>>40)&0xff] ^ 107 | helperTable[1][(crc>>48)&0xff] ^ 108 | helperTable[0][crc>>56] 109 | p = p[8:] 110 | } 111 | } 112 | // For reminders or small sizes 113 | for _, v := range p { 114 | crc = tab[byte(crc)^v] ^ (crc >> 8) 115 | } 116 | return crc 117 | } 118 | 119 | // Update returns the result of adding the bytes in p to the crc. 120 | func Update(crc uint64, tab *crc64.Table, p []byte) uint64 { 121 | return update(crc, tab, p) 122 | } 123 | 124 | func (d *digest) Write(p []byte) (n int, err error) { 125 | d.crc = update(d.crc, d.tab, p) 126 | return len(p), nil 127 | } 128 | 129 | func (d *digest) Sum64() uint64 { return d.crc } 130 | 131 | func (d *digest) Sum(in []byte) []byte { 132 | s := d.Sum64() 133 | // Compared to the core hash functions we return in little endian byte order. 134 | return append(in, byte(s), byte(s>>8), byte(s>>16), byte(s>>24), byte(s>>32), byte(s>>40), byte(s>>48), byte(s>>56)) 135 | } 136 | 137 | // Checksum returns the CRC-64 checksum of data 138 | // using the polynomial represented by the [Table]. 139 | func Checksum(data []byte, tab *crc64.Table) uint64 { return update(0, tab, data) } 140 | -------------------------------------------------------------------------------- /crc64jones/crc64_test.go: -------------------------------------------------------------------------------- 1 | package crc64jones 2 | 3 | import ( 4 | "io" 5 | "testing" 6 | ) 7 | 8 | func TestGolden(t *testing.T) { 9 | c := New() 10 | in := "123456789" 11 | io.WriteString(c, in) 12 | s := c.Sum64() 13 | if out := uint64(0xe9c6d914c4b8d9ca); s != out { 14 | t.Fatalf("jones crc64(%s) = 0x%x want 0x%x", in, s, out) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /d3flame/d3-flamegraph.css: -------------------------------------------------------------------------------- 1 | .d3-flame-graph rect { 2 | stroke: #EEEEEE; 3 | fill-opacity: .8; 4 | } 5 | 6 | .d3-flame-graph rect:hover { 7 | stroke: #474747; 8 | stroke-width: 0.5; 9 | cursor: pointer; 10 | } 11 | 12 | .d3-flame-graph-label { 13 | pointer-events: none; 14 | white-space: nowrap; 15 | text-overflow: ellipsis; 16 | overflow: hidden; 17 | font-size: 12px; 18 | font-family: Verdana; 19 | margin-left: 4px; 20 | margin-right: 4px; 21 | line-height: 1.5; 22 | padding: 0 0 0; 23 | font-weight: 400; 24 | color: black; 25 | text-align: left; 26 | } 27 | 28 | .d3-flame-graph .fade { 29 | opacity: 0.6 !important; 30 | } 31 | 32 | .d3-flame-graph .title { 33 | font-size: 20px; 34 | font-family: Verdana; 35 | } 36 | 37 | .d3-flame-graph-tip { 38 | line-height: 1; 39 | font-family: Verdana; 40 | font-size: 12px; 41 | padding: 12px; 42 | background: rgba(0, 0, 0, 0.8); 43 | color: #fff; 44 | border-radius: 2px; 45 | pointer-events: none; 46 | } 47 | 48 | /* Creates a small triangle extender for the tooltip */ 49 | .d3-flame-graph-tip:after { 50 | box-sizing: border-box; 51 | display: inline; 52 | font-size: 10px; 53 | width: 100%; 54 | line-height: 1; 55 | color: rgba(0, 0, 0, 0.8); 56 | position: absolute; 57 | pointer-events: none; 58 | } 59 | 60 | /* Northward tooltips */ 61 | .d3-flame-graph-tip.n:after { 62 | content: "\25BC"; 63 | margin: -1px 0 0 0; 64 | top: 100%; 65 | left: 0; 66 | text-align: center; 67 | } 68 | 69 | /* Eastward tooltips */ 70 | .d3-flame-graph-tip.e:after { 71 | content: "\25C0"; 72 | margin: -4px 0 0 0; 73 | top: 50%; 74 | left: -8px; 75 | } 76 | 77 | /* Southward tooltips */ 78 | .d3-flame-graph-tip.s:after { 79 | content: "\25B2"; 80 | margin: 0 0 1px 0; 81 | top: -8px; 82 | left: 0; 83 | text-align: center; 84 | } 85 | 86 | /* Westward tooltips */ 87 | .d3-flame-graph-tip.w:after { 88 | content: "\25B6"; 89 | margin: -4px 0 0 -1px; 90 | top: 50%; 91 | left: 100%; 92 | } -------------------------------------------------------------------------------- /d3flame/d3-tip.min.js: -------------------------------------------------------------------------------- 1 | !function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e(require("d3-collection"),require("d3-selection")):"function"==typeof define&&define.amd?define(["d3-collection","d3-selection"],e):(t.d3=t.d3||{},t.d3.tip=e(t.d3,t.d3))}(this,function(l,i){"use strict";return function(){var u=function(){return"n"},c=function(){return[0,0]},a=function(){return" "},p=document.body,n=t(),r=null,y=null,d=null;function h(t){var e;e=t.node(),(r=e?"svg"===e.tagName.toLowerCase()?e:e.ownerSVGElement:null)&&(y=r.createSVGPoint(),p.appendChild(n))}h.show=function(){var t=Array.prototype.slice.call(arguments);t[t.length-1]instanceof SVGElement&&(d=t.pop());var e,n=a.apply(this,t),r=c.apply(this,t),o=u.apply(this,t),l=v(),i=x.length,s=document.documentElement.scrollTop||p.scrollTop,f=document.documentElement.scrollLeft||p.scrollLeft;for(l.html(n).style("opacity",1).style("pointer-events","all");i--;)l.classed(x[i],!1);return e=m.get(o).apply(this),l.classed(o,!0).style("top",e.top+r[0]+s+"px").style("left",e.left+r[1]+f+"px"),h},h.hide=function(){return v().style("opacity",0).style("pointer-events","none"),h},h.attr=function(t,e){if(arguments.length<2&&"string"==typeof t)return v().attr(t);var n=Array.prototype.slice.call(arguments);return i.selection.prototype.attr.apply(v(),n),h},h.style=function(t,e){if(arguments.length<2&&"string"==typeof t)return v().style(t);var n=Array.prototype.slice.call(arguments);return i.selection.prototype.style.apply(v(),n),h},h.direction=function(t){return arguments.length?(u=null==t?t:o(t),h):u},h.offset=function(t){return arguments.length?(c=null==t?t:o(t),h):c},h.html=function(t){return arguments.length?(a=null==t?t:o(t),h):a},h.rootElement=function(t){return arguments.length?(p=null==t?t:o(t),h):p},h.destroy=function(){return n&&(v().remove(),n=null),h};var m=l.map({n:function(){var t=e(this);return{top:t.n.y-n.offsetHeight,left:t.n.x-n.offsetWidth/2}},s:function(){var t=e(this);return{top:t.s.y,left:t.s.x-n.offsetWidth/2}},e:function(){var t=e(this);return{top:t.e.y-n.offsetHeight/2,left:t.e.x}},w:function(){var t=e(this);return{top:t.w.y-n.offsetHeight/2,left:t.w.x-n.offsetWidth}},nw:function(){var t=e(this);return{top:t.nw.y-n.offsetHeight,left:t.nw.x-n.offsetWidth}},ne:function(){var t=e(this);return{top:t.ne.y-n.offsetHeight,left:t.ne.x}},sw:function(){var t=e(this);return{top:t.sw.y,left:t.sw.x-n.offsetWidth}},se:function(){var t=e(this);return{top:t.se.y,left:t.se.x}}}),x=m.keys();function t(){var t=i.select(document.createElement("div"));return t.style("position","absolute").style("top",0).style("opacity",0).style("pointer-events","none").style("box-sizing","border-box"),t.node()}function v(){return null==n&&(n=t(),p.appendChild(n)),i.select(n)}function e(t){for(var e=d||t;null==e.getScreenCTM&&null!=e.parentNode;)e=e.parentNode;var n={},r=e.getScreenCTM(),o=e.getBBox(),l=o.width,i=o.height,s=o.x,f=o.y;return y.x=s,y.y=f,n.nw=y.matrixTransform(r),y.x+=l,n.ne=y.matrixTransform(r),y.y+=i,n.se=y.matrixTransform(r),y.x-=l,n.sw=y.matrixTransform(r),y.y-=i/2,n.w=y.matrixTransform(r),y.x+=l,n.e=y.matrixTransform(r),y.x-=l/2,y.y-=i/2,n.n=y.matrixTransform(r),y.y+=i,n.s=y.matrixTransform(r),n}function o(t){return"function"==typeof t?t:function(){return t}}return h}}); -------------------------------------------------------------------------------- /d3flame/template.go: -------------------------------------------------------------------------------- 1 | package d3flame 2 | 3 | import ( 4 | // use go embed to load js and css source file 5 | _ "embed" 6 | ) 7 | 8 | // D3.js is a JavaScript library for manipulating documents based on data. 9 | // https://github.com/d3/d3 10 | 11 | // d3-flame-graph is a D3.js plugin that produces flame graphs from hierarchical data. 12 | // https://github.com/spiermar/d3-flame-graph 13 | 14 | // https://cdn.jsdelivr.net/gh/spiermar/d3-flame-graph@2.0.3/dist/d3-flamegraph.css 15 | //go:embed d3-flamegraph.css 16 | var d3Css string 17 | 18 | // https://d3js.org/d3.v4.min.js 19 | //go:embed d3.v4.min.js 20 | var d3Js string 21 | 22 | // https://cdn.jsdelivr.net/gh/spiermar/d3-flame-graph@2.0.3/dist/d3-flamegraph.min.js 23 | //go:embed d3-flamegraph.min.js 24 | var d3FlameGraphJs string 25 | 26 | // https://cdnjs.cloudflare.com/ajax/libs/d3-tip/0.9.1/d3-tip.min.js 27 | //go:embed d3-tip.min.js 28 | var d3TipJs string 29 | 30 | // https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css 31 | //go:embed bootstrap.min.css 32 | var bootstrapCSS string 33 | 34 | const html = ` 35 | 36 | FlameGraph 37 | 38 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 |
75 |
76 | 88 |

d3-flame-graph

89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 | 178 | 179 | 180 | ` 181 | -------------------------------------------------------------------------------- /d3flame/web.go: -------------------------------------------------------------------------------- 1 | package d3flame 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "html/template" 7 | "net/http" 8 | "strconv" 9 | ) 10 | 11 | var flameTmplData = &struct { 12 | D3Css template.CSS 13 | D3Js template.JS 14 | D3Flame template.JS 15 | D3Tip template.JS 16 | BootstrapCSS template.CSS 17 | }{ 18 | D3Css: template.CSS(d3Css), 19 | D3Js: template.JS(d3Js), 20 | D3Flame: template.JS(d3FlameGraphJs), 21 | D3Tip: template.JS(d3TipJs), 22 | BootstrapCSS: template.CSS(bootstrapCSS), 23 | } 24 | 25 | func flamegraph(w http.ResponseWriter, r *http.Request) { 26 | tmpl := template.Must(template.New("flamegraph").Parse(html)) 27 | err := tmpl.Execute(w, flameTmplData) 28 | if err != nil { 29 | w.WriteHeader(http.StatusInternalServerError) 30 | _, _ = w.Write([]byte("500 - Internal Error")) 31 | } 32 | } 33 | 34 | // FlameItem is an Element in flamegraph 35 | type FlameItem struct { 36 | Name string `json:"n"` 37 | Value int `json:"v"` 38 | Children children `json:"c,omitempty"` 39 | } 40 | 41 | func (ch children) MarshalJSON() ([]byte, error) { 42 | list := make([]*FlameItem, 0, len(ch)) 43 | for _, v := range ch { 44 | list = append(list, v) 45 | } 46 | return json.Marshal(list) 47 | } 48 | 49 | // AddChild add a child node into FlameItem 50 | func (node *FlameItem) AddChild(n *FlameItem) { 51 | node.Children[n.Name] = n 52 | } 53 | 54 | type children map[string]*FlameItem 55 | 56 | // Web starts a web server to render flamegraph 57 | func Web(data []byte, port int) chan<- struct{} { 58 | mux := http.NewServeMux() 59 | mux.HandleFunc("/flamegraph", flamegraph) 60 | mux.HandleFunc("/stacks.json", func(w http.ResponseWriter, r *http.Request) { 61 | w.WriteHeader(http.StatusOK) 62 | _, _ = w.Write(data) 63 | }) 64 | server := &http.Server{ 65 | Addr: ":" + strconv.Itoa(port), 66 | Handler: mux, 67 | } 68 | fmt.Printf("see http://localhost:%d/flamegraph\n", port) 69 | stop := make(chan struct{}) 70 | go func() { 71 | <-stop 72 | _ = server.Close() 73 | }() 74 | go func() { 75 | err := server.ListenAndServe() 76 | if err != nil && err != http.ErrServerClosed { 77 | panic(err) 78 | } 79 | }() 80 | return stop 81 | } 82 | -------------------------------------------------------------------------------- /encoder/portal.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import "github.com/hdt3213/rdb/core" 4 | 5 | // Encoder is used to generate RDB file 6 | type Encoder = core.Encoder 7 | 8 | // NewEncoder creates an encoder instance 9 | var NewEncoder = core.NewEncoder 10 | 11 | // WithTTL specific expiration timestamp for object 12 | var WithTTL = core.WithTTL 13 | -------------------------------------------------------------------------------- /examples/decode/example.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HDT3213/rdb/1ceef5a355b35502cd5a11b531cd03364b6a64af/examples/decode/example.rdb -------------------------------------------------------------------------------- /examples/decode/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/hdt3213/rdb/parser" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | rdbFile, err := os.Open("example.rdb") 10 | if err != nil { 11 | panic("open example.rdb failed") 12 | } 13 | defer func() { 14 | _ = rdbFile.Close() 15 | }() 16 | decoder := parser.NewDecoder(rdbFile) 17 | err = decoder.Parse(func(o parser.RedisObject) bool { 18 | switch o.GetType() { 19 | case parser.StringType: 20 | str := o.(*parser.StringObject) 21 | println(str.Key, str.Value) 22 | case parser.ListType: 23 | list := o.(*parser.ListObject) 24 | println(list.Key, list.Values) 25 | case parser.HashType: 26 | hash := o.(*parser.HashObject) 27 | println(hash.Key, hash.Hash) 28 | case parser.ZSetType: 29 | zset := o.(*parser.ZSetObject) 30 | println(zset.Key, zset.Entries) 31 | case parser.StreamType: 32 | stream := o.(*parser.StreamObject) 33 | println(stream.Entries, stream.Groups) 34 | } 35 | // return true to continue, return false to stop the iteration 36 | return true 37 | }) 38 | if err != nil { 39 | panic(err) 40 | } 41 | } -------------------------------------------------------------------------------- /examples/encode/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/hdt3213/rdb/encoder" 5 | "github.com/hdt3213/rdb/model" 6 | "os" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | rdbFile, err := os.Create("dump.rdb") 12 | if err != nil { 13 | panic(err) 14 | } 15 | defer rdbFile.Close() 16 | enc := encoder.NewEncoder(rdbFile) 17 | err = enc.WriteHeader() 18 | if err != nil { 19 | panic(err) 20 | } 21 | auxMap := map[string]string{ 22 | "redis-ver": "4.0.6", 23 | "redis-bits": "64", 24 | "aof-preamble": "0", 25 | } 26 | for k, v := range auxMap { 27 | err = enc.WriteAux(k, v) 28 | if err != nil { 29 | panic(err) 30 | } 31 | } 32 | 33 | err = enc.WriteDBHeader(0, 5, 1) 34 | if err != nil { 35 | panic(err) 36 | } 37 | expirationMs := uint64(time.Now().Add(time.Hour*8).Unix() * 1000) 38 | err = enc.WriteStringObject("hello", []byte("world"), encoder.WithTTL(expirationMs)) 39 | if err != nil { 40 | panic(err) 41 | } 42 | err = enc.WriteListObject("list", [][]byte{ 43 | []byte("123"), 44 | []byte("abc"), 45 | []byte("la la la"), 46 | }) 47 | if err != nil { 48 | panic(err) 49 | } 50 | err = enc.WriteSetObject("set", [][]byte{ 51 | []byte("123"), 52 | []byte("abc"), 53 | []byte("la la la"), 54 | }) 55 | if err != nil { 56 | panic(err) 57 | } 58 | err = enc.WriteHashMapObject("list", map[string][]byte{ 59 | "1": []byte("123"), 60 | "a": []byte("abc"), 61 | "la": []byte("la la la"), 62 | }) 63 | if err != nil { 64 | panic(err) 65 | } 66 | err = enc.WriteZSetObject("list", []*model.ZSetEntry{ 67 | { 68 | Score: 1.234, 69 | Member: "a", 70 | }, 71 | { 72 | Score: 2.71828, 73 | Member: "b", 74 | }, 75 | }) 76 | if err != nil { 77 | panic(err) 78 | } 79 | err = enc.WriteEnd() 80 | if err != nil { 81 | panic(err) 82 | } 83 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hdt3213/rdb 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/bytedance/sonic v1.12.1 7 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 8 | github.com/klauspost/cpuid/v2 v2.2.8 // indirect 9 | golang.org/x/arch v0.9.0 // indirect 10 | golang.org/x/sys v0.24.0 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= 2 | github.com/bytedance/sonic v1.8.7 h1:d3sry5vGgVq/OpgozRUNP6xBsSo0mtNdwliApw+SAMQ= 3 | github.com/bytedance/sonic v1.8.7/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= 4 | github.com/bytedance/sonic v1.12.1 h1:jWl5Qz1fy7X1ioY74WqO0KjAMtAGQs4sYnjiEBiyX24= 5 | github.com/bytedance/sonic v1.12.1/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= 6 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 7 | github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= 8 | github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 9 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= 10 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= 11 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= 12 | github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= 13 | github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 14 | github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= 15 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 16 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 18 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 20 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 21 | github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= 22 | github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 23 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 24 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 25 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 26 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 27 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 28 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 29 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 30 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 31 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 32 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 33 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 34 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 35 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 36 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= 37 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 38 | golang.org/x/arch v0.9.0 h1:ub9TgUInamJ8mrZIGlBG6/4TqWeMszd4N8lNorbrr6k= 39 | golang.org/x/arch v0.9.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= 40 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 41 | golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= 42 | golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 43 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 44 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 45 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 46 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 47 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 48 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 49 | -------------------------------------------------------------------------------- /helper/bigkey.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "encoding/csv" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "strconv" 9 | 10 | "github.com/hdt3213/rdb/bytefmt" 11 | "github.com/hdt3213/rdb/core" 12 | "github.com/hdt3213/rdb/model" 13 | ) 14 | 15 | // FindBiggestKeys read rdb file and find the largest N keys. 16 | // The invoker owns output, FindBiggestKeys won't close it 17 | func FindBiggestKeys(rdbFilename string, topN int, output *os.File, options ...interface{}) error { 18 | if rdbFilename == "" { 19 | return errors.New("src file path is required") 20 | } 21 | if topN <= 0 { 22 | return errors.New("n must greater than 0") 23 | } 24 | rdbFile, err := os.Open(rdbFilename) 25 | if err != nil { 26 | return fmt.Errorf("open rdb %s failed, %v", rdbFilename, err) 27 | } 28 | defer func() { 29 | _ = rdbFile.Close() 30 | }() 31 | var dec decoder = core.NewDecoder(rdbFile) 32 | if dec, err = wrapDecoder(dec, options...); err != nil { 33 | return err 34 | } 35 | top := newToplist(topN) 36 | err = dec.Parse(func(object model.RedisObject) bool { 37 | top.add(object) 38 | return true 39 | }) 40 | if err != nil { 41 | return err 42 | } 43 | _, err = output.WriteString("database,key,type,size,size_readable,element_count\n") 44 | if err != nil { 45 | return fmt.Errorf("write header failed: %v", err) 46 | } 47 | csvWriter := csv.NewWriter(output) 48 | defer csvWriter.Flush() 49 | for _, o := range top.list { 50 | object := o.(model.RedisObject) 51 | err = csvWriter.Write([]string{ 52 | strconv.Itoa(object.GetDBIndex()), 53 | object.GetKey(), 54 | object.GetType(), 55 | strconv.Itoa(object.GetSize()), 56 | bytefmt.FormatSize(uint64(object.GetSize())), 57 | strconv.Itoa(object.GetElemCount()), 58 | }) 59 | if err != nil { 60 | return fmt.Errorf("csv write failed: %v", err) 61 | } 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /helper/bigkey_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "math/rand" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strconv" 9 | "testing" 10 | 11 | "github.com/hdt3213/rdb/model" 12 | ) 13 | 14 | func TestTopList(t *testing.T) { 15 | topN := 100 16 | n := topN * 10 17 | objects := make([]model.RedisObject, 0) 18 | for i := 0; i < n; i++ { 19 | size := rand.Intn(n * 10) 20 | o := &model.StringObject{ 21 | BaseObject: &model.BaseObject{ 22 | Key: strconv.Itoa(i), 23 | Size: size, 24 | }, 25 | } 26 | objects = append(objects, o) 27 | } 28 | topList := newToplist(topN) 29 | for _, o := range objects { 30 | topList.add(o) 31 | } 32 | sort.Slice(objects, func(i, j int) bool { 33 | return objects[i].GetSize() > objects[j].GetSize() 34 | }) 35 | if len(topList.list) != topN { 36 | t.Error("wrong top list size") 37 | } 38 | for i, actual := range topList.list { 39 | expect := objects[i] 40 | if actual.GetSize() != expect.GetSize() { 41 | t.Error("wrong top list") 42 | return 43 | } 44 | } 45 | } 46 | 47 | func TestFindLargestKeys(t *testing.T) { 48 | err := os.MkdirAll("tmp", os.ModePerm) 49 | if err != nil { 50 | return 51 | } 52 | defer func() { 53 | err := os.RemoveAll("tmp") 54 | if err != nil { 55 | t.Logf("remove tmp directory failed: %v", err) 56 | } 57 | }() 58 | srcRdb := filepath.Join("../cases", "memory.rdb") 59 | expectFile := filepath.Join("../cases", "largest.csv") 60 | outputFilePath := filepath.Join("tmp", "largest.csv") 61 | output, err := os.Create(outputFilePath) 62 | if err != nil { 63 | t.Errorf("create output file failed: %v", err) 64 | return 65 | } 66 | err = FindBiggestKeys(srcRdb, 5, output) 67 | if err != nil { 68 | t.Errorf("FindLargestKeys failed: %v", err) 69 | } 70 | err = output.Close() 71 | if err != nil { 72 | t.Errorf("error occurs during close output %s, err: %v", srcRdb, err) 73 | return 74 | } 75 | equals, err := compareFileByLine(t, outputFilePath, expectFile) 76 | if err != nil { 77 | t.Errorf("error occurs during compare %s, err: %v", srcRdb, err) 78 | return 79 | } 80 | if !equals { 81 | t.Errorf("result is not equal of %s", srcRdb) 82 | return 83 | } 84 | 85 | err = FindBiggestKeys("", 5, os.Stdout) 86 | if err == nil || err.Error() != "src file path is required" { 87 | t.Error("failed when empty output") 88 | } 89 | err = FindBiggestKeys("cases/memory.rdb", 0, os.Stdout) 90 | if err == nil || err.Error() != "n must greater than 0" { 91 | t.Error("failed when empty output") 92 | } 93 | } 94 | 95 | func TestFindBiggestKeyWithRegex(t *testing.T) { 96 | err := os.MkdirAll("tmp", os.ModePerm) 97 | if err != nil { 98 | return 99 | } 100 | defer func() { 101 | err := os.RemoveAll("tmp") 102 | if err != nil { 103 | t.Logf("remove tmp directory failed: %v", err) 104 | } 105 | }() 106 | srcRdb := filepath.Join("../cases", "memory.rdb") 107 | actualFile := filepath.Join("../cases", "memory_regex.biggest.csv") 108 | expectFile := filepath.Join("../cases", "memory_regex.biggest.csv") 109 | output, err := os.Create(actualFile) 110 | if err != nil { 111 | t.Errorf("create output file failed: %v", err) 112 | return 113 | } 114 | err = FindBiggestKeys(srcRdb, 2, output, WithRegexOption("^l.*")) 115 | if err != nil { 116 | t.Errorf("error occurs during parse, err: %v", err) 117 | return 118 | } 119 | equals, err := compareFileByLine(t, actualFile, expectFile) 120 | if err != nil { 121 | t.Errorf("error occurs during compare err: %v", err) 122 | return 123 | } 124 | if !equals { 125 | t.Errorf("result is not equal") 126 | return 127 | } 128 | 129 | err = FindBiggestKeys(srcRdb, 2, output, WithRegexOption(`(i)\1`)) 130 | if err == nil { 131 | t.Error("expect error") 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /helper/converter.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/hdt3213/rdb/core" 7 | "github.com/hdt3213/rdb/model" 8 | "os" 9 | ) 10 | 11 | // ToAOF read rdb file and convert to aof file (Redis Serialization ) 12 | func ToAOF(rdbFilename string, aofFilename string, options ...interface{}) error { 13 | if rdbFilename == "" { 14 | return errors.New("src file path is required") 15 | } 16 | if aofFilename == "" { 17 | return errors.New("output file path is required") 18 | } 19 | rdbFile, err := os.Open(rdbFilename) 20 | if err != nil { 21 | return fmt.Errorf("open rdb %s failed, %v", rdbFilename, err) 22 | } 23 | defer func() { 24 | _ = rdbFile.Close() 25 | }() 26 | aofFile, err := os.Create(aofFilename) 27 | if err != nil { 28 | return fmt.Errorf("create json %s failed, %v", aofFilename, err) 29 | } 30 | defer func() { 31 | _ = aofFile.Close() 32 | }() 33 | 34 | var dec decoder = core.NewDecoder(rdbFile) 35 | if dec, err = wrapDecoder(dec, options...); err != nil { 36 | return err 37 | } 38 | return dec.Parse(func(object model.RedisObject) bool { 39 | cmdLines := ObjectToCmd(object, options...) 40 | data := CmdLinesToResp(cmdLines) 41 | _, err = aofFile.Write(data) 42 | if err != nil { 43 | fmt.Printf("write failed: %v", err) 44 | return true 45 | } 46 | return true 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /helper/converter_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "bufio" 5 | "github.com/bytedance/sonic" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func compareFileByLine(t *testing.T, fn1, fn2 string) (bool, error) { 13 | f1, err := os.Open(fn1) 14 | if err != nil { 15 | return false, err 16 | } 17 | defer func() { 18 | _ = f1.Close() 19 | }() 20 | f2, err := os.Open(fn2) 21 | if err != nil { 22 | return false, err 23 | } 24 | defer func() { 25 | _ = f2.Close() 26 | }() 27 | sc1 := bufio.NewScanner(f1) 28 | sc2 := bufio.NewScanner(f2) 29 | 30 | for { 31 | next1 := sc1.Scan() 32 | next2 := sc2.Scan() 33 | if !next1 && !next2 { 34 | break 35 | } 36 | if next1 != next2 { 37 | // line number is not equals 38 | t.Log("line number is not equal") 39 | return false, nil 40 | } 41 | txt1 := sc1.Text() 42 | txt2 := sc2.Text() 43 | if txt1 != txt2 { 44 | t.Logf("txt1: %s\ntxt2:%s", txt1, txt2) 45 | return false, nil 46 | } 47 | } 48 | return true, nil 49 | } 50 | 51 | func TestToJson(t *testing.T) { 52 | // SortMapKeys will cause performance losses, only enabled during test 53 | jsonEncoder = sonic.ConfigStd 54 | // use same time zone to ensure RedisObject.Expiration has same json value 55 | var cstZone = time.FixedZone("CST", 8*3600) 56 | time.Local = cstZone 57 | 58 | err := os.MkdirAll("tmp", os.ModePerm) 59 | if err != nil { 60 | return 61 | } 62 | defer func() { 63 | err := os.RemoveAll("tmp") 64 | if err != nil { 65 | t.Logf("remove tmp directory failed: %v", err) 66 | } 67 | }() 68 | testCases := []string{ 69 | "stream_listoacks_3", 70 | "issue27", 71 | "set_listpack", 72 | "stream_listpacks_2", 73 | "stream_listpacks_1", 74 | "listpack", 75 | "memory", 76 | "quicklist", 77 | "easily_compressible_string_key", 78 | "empty_database", 79 | "hash", 80 | "hash_as_ziplist", 81 | "integer_keys", 82 | "intset_16", 83 | "intset_32", 84 | "intset_64", 85 | "keys_with_expiry", 86 | "linkedlist", 87 | "multiple_databases", 88 | "non_ascii_values", 89 | "parser_filters", 90 | "rdb_version_5_with_checksum", 91 | "rdb_version_8_with_64b_length_and_scores", 92 | "regular_set", 93 | "regular_sorted_set", 94 | "sorted_set_as_ziplist", 95 | "uncompressible_string_keys", 96 | "ziplist_that_compresses_easily", 97 | "ziplist_that_doesnt_compress", 98 | "ziplist_with_integers", 99 | "zipmap_that_compresses_easily", 100 | "zipmap_that_doesnt_compress", 101 | "zipmap_with_big_values", 102 | "zipmap_big_len", 103 | } 104 | for _, filename := range testCases { 105 | srcRdb := filepath.Join("../cases", filename+".rdb") 106 | actualJSON := filepath.Join("tmp", filename+".json") 107 | expectJSON := filepath.Join("../cases", filename+".json") 108 | err = ToJsons(srcRdb, actualJSON, WithConcurrent(1)) 109 | if err != nil { 110 | t.Errorf("error occurs during parse %s, err: %v", filename, err) 111 | continue 112 | } 113 | equals, err := compareFileByLine(t, actualJSON, expectJSON) 114 | if err != nil { 115 | t.Errorf("error occurs during compare %s, err: %v", filename, err) 116 | continue 117 | } 118 | if !equals { 119 | t.Errorf("result is not equal of %s", filename) 120 | continue 121 | } 122 | } 123 | err = ToJsons("cases/memory.rdb", "") 124 | if err == nil || err.Error() != "output file path is required" { 125 | t.Error("failed when empty output") 126 | } 127 | err = ToJsons("", "tmp/memory.rdb") 128 | if err == nil || err.Error() != "src file path is required" { 129 | t.Error("failed when empty output") 130 | } 131 | } 132 | 133 | func TestToJsonWithRegex(t *testing.T) { 134 | err := os.MkdirAll("tmp", os.ModePerm) 135 | if err != nil { 136 | return 137 | } 138 | defer func() { 139 | err := os.RemoveAll("tmp") 140 | if err != nil { 141 | t.Logf("remove tmp directory failed: %v", err) 142 | } 143 | }() 144 | srcRdb := filepath.Join("../cases", "memory.rdb") 145 | actualJSON := filepath.Join("tmp", "memory_regex.json") 146 | expectJSON := filepath.Join("../cases", "memory_regex.json") 147 | err = ToJsons(srcRdb, actualJSON, WithRegexOption("^l.*")) 148 | if err != nil { 149 | t.Errorf("error occurs during parse, err: %v", err) 150 | return 151 | } 152 | equals, err := compareFileByLine(t, actualJSON, expectJSON) 153 | if err != nil { 154 | t.Errorf("error occurs during compare err: %v", err) 155 | return 156 | } 157 | if !equals { 158 | t.Errorf("result is not equal") 159 | return 160 | } 161 | errJson := filepath.Join("tmp", "memory_regex_err.json") 162 | err = ToJsons(srcRdb, errJson, WithRegexOption(`(i)\1`)) 163 | if err == nil { 164 | t.Error("expect error") 165 | } 166 | } 167 | 168 | func TestToAof(t *testing.T) { 169 | err := os.MkdirAll("tmp", os.ModePerm) 170 | if err != nil { 171 | return 172 | } 173 | defer func() { 174 | err := os.RemoveAll("tmp") 175 | if err != nil { 176 | t.Logf("remove tmp directory failed: %v", err) 177 | } 178 | }() 179 | srcRdb := filepath.Join("../cases", "memory.rdb") 180 | actualFile := filepath.Join("tmp", "memory.aof") 181 | expectFile := filepath.Join("../cases", "memory.aof") 182 | err = ToAOF(srcRdb, actualFile, lexOrder{}) 183 | if err != nil { 184 | t.Errorf("error occurs during parse %s, err: %v", srcRdb, err) 185 | return 186 | } 187 | equals, err := compareFileByLine(t, actualFile, expectFile) 188 | if err != nil { 189 | t.Errorf("error occurs during compare %s, err: %v", srcRdb, err) 190 | return 191 | } 192 | if !equals { 193 | t.Errorf("result is not equal of %s", srcRdb) 194 | return 195 | } 196 | err = ToAOF("cases/memory.rdb", "") 197 | if err == nil || err.Error() != "output file path is required" { 198 | t.Error("failed when empty output") 199 | } 200 | err = ToAOF("", "tmp/err.rdb") 201 | if err == nil || err.Error() != "src file path is required" { 202 | t.Error("failed when empty output") 203 | } 204 | } 205 | 206 | func TestToAofWithRegex(t *testing.T) { 207 | err := os.MkdirAll("tmp", os.ModePerm) 208 | if err != nil { 209 | return 210 | } 211 | defer func() { 212 | err := os.RemoveAll("tmp") 213 | if err != nil { 214 | t.Logf("remove tmp directory failed: %v", err) 215 | } 216 | }() 217 | srcRdb := filepath.Join("../cases", "memory.rdb") 218 | actualFile := filepath.Join("tmp", "memory_regex.aof") 219 | expectFile := filepath.Join("../cases", "memory_regex.aof") 220 | err = ToAOF(srcRdb, actualFile, WithRegexOption("^l.*"), lexOrder{}) 221 | if err != nil { 222 | t.Errorf("error occurs during parse, err: %v", err) 223 | return 224 | } 225 | equals, err := compareFileByLine(t, actualFile, expectFile) 226 | if err != nil { 227 | t.Errorf("error occurs during compare err: %v", err) 228 | return 229 | } 230 | if !equals { 231 | t.Errorf("result is not equal") 232 | return 233 | } 234 | errFile := filepath.Join("tmp", "memory_regex.err.aof") 235 | err = ToAOF(srcRdb, errFile, WithRegexOption(`(i)\1`)) 236 | if err == nil { 237 | t.Error("expect error") 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /helper/flamegraph.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/hdt3213/rdb/core" 8 | "github.com/hdt3213/rdb/d3flame" 9 | "github.com/hdt3213/rdb/model" 10 | "os" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | // TrimThreshold is the min count of keys to enable trim 16 | var TrimThreshold = 1000 17 | 18 | // FlameGraph draws flamegraph in web page to analysis memory usage pattern 19 | func FlameGraph(rdbFilename string, port int, separators []string, options ...interface{}) (chan<- struct{}, error) { 20 | if rdbFilename == "" { 21 | return nil, errors.New("src file path is required") 22 | } 23 | if port == 0 { 24 | port = 16379 // default port 25 | } 26 | rdbFile, err := os.Open(rdbFilename) 27 | if err != nil { 28 | return nil, fmt.Errorf("open rdb %s failed, %v", rdbFilename, err) 29 | } 30 | defer func() { 31 | _ = rdbFile.Close() 32 | }() 33 | 34 | var dec decoder = core.NewDecoder(rdbFile) 35 | if dec, err = wrapDecoder(dec, options...); err != nil { 36 | return nil, err 37 | } 38 | root := &d3flame.FlameItem{ 39 | Name: "root", 40 | Children: make(map[string]*d3flame.FlameItem), 41 | } 42 | var count int 43 | err = dec.Parse(func(object model.RedisObject) bool { 44 | count++ 45 | addObject(root, separators, object) 46 | return true 47 | }) 48 | if err != nil { 49 | return nil, err 50 | } 51 | totalSize := 0 52 | for _, v := range root.Children { 53 | totalSize += v.Value 54 | } 55 | root.Value = totalSize 56 | if count >= TrimThreshold { 57 | trimData(root) 58 | } 59 | data, err := json.Marshal(root) 60 | if err != nil { 61 | return nil, err 62 | } 63 | return d3flame.Web(data, port), nil 64 | } 65 | 66 | func split(s string, separators []string) []string { 67 | sep := ":" 68 | if len(separators) > 0 { 69 | sep = separators[0] 70 | } 71 | for i := 1; i < len(separators); i++ { 72 | s = strings.ReplaceAll(s, separators[i], sep) 73 | } 74 | return strings.Split(s, sep) 75 | } 76 | 77 | func addObject(root *d3flame.FlameItem, separators []string, object model.RedisObject) { 78 | node := root 79 | parts := split(object.GetKey(), separators) 80 | parts = append([]string{"db:" + strconv.Itoa(object.GetDBIndex())}, parts...) 81 | for _, part := range parts { 82 | if node.Children[part] == nil { 83 | n := &d3flame.FlameItem{ 84 | Name: part, 85 | Children: make(map[string]*d3flame.FlameItem), 86 | } 87 | node.AddChild(n) 88 | } 89 | node = node.Children[part] 90 | node.Value += object.GetSize() 91 | } 92 | } 93 | 94 | // bigNodeThreshold is the min size 95 | var bigNodeThreshold = 1024 * 1024 // 1MB 96 | 97 | func trimData(root *d3flame.FlameItem) { 98 | // trim long tail 99 | queue := []*d3flame.FlameItem{ 100 | root, 101 | } 102 | for len(queue) > 0 { 103 | // Aggregate leaf nodes 104 | node := queue[0] 105 | queue = queue[1:] 106 | leafSum := 0 107 | for key, child := range node.Children { 108 | if len(child.Children) == 0 && child.Value < bigNodeThreshold { // child is a leaf node 109 | delete(node.Children, key) // remove small leaf keys 110 | leafSum += child.Value 111 | } 112 | queue = append(queue, child) // reserve big key 113 | } 114 | if leafSum > 0 { 115 | n := &d3flame.FlameItem{ 116 | Name: "others", 117 | Value: leafSum, 118 | } 119 | node.AddChild(n) 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /helper/flamegraph_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "net/http" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestSplit(t *testing.T) { 10 | result := split("a:b:c", nil) 11 | if len(result) != 3 || result[0] != "a" || result[1] != "b" || result[2] != "c" { 12 | t.Errorf("wrong result: %+v", result) 13 | } 14 | result = split("a.b.c", []string{"."}) 15 | if len(result) != 3 || result[0] != "a" || result[1] != "b" || result[2] != "c" { 16 | t.Errorf("wrong result: %+v", result) 17 | } 18 | result = split("a++b--c", []string{"++", "--"}) 19 | if len(result) != 3 || result[0] != "a" || result[1] != "b" || result[2] != "c" { 20 | t.Errorf("wrong result: %+v", result) 21 | } 22 | } 23 | 24 | func TestFlameGraph(t *testing.T) { 25 | TrimThreshold = 1 26 | srcRdb := filepath.Join("../cases", "tree.rdb") 27 | stop, err := FlameGraph(srcRdb, 18888, nil) 28 | if err != nil { 29 | t.Errorf("draw FlameGraph failed: %v", err) 30 | } 31 | resp, err := http.Get("http://localhost:18888/flamegraph") 32 | if err != nil { 33 | t.Error(err) 34 | return 35 | } 36 | if resp.StatusCode != http.StatusOK { 37 | t.Errorf("http %d", resp.StatusCode) 38 | return 39 | } 40 | resp, err = http.Get("http://localhost:18888/stacks.json") 41 | if err != nil { 42 | t.Error(err) 43 | return 44 | } 45 | if resp.StatusCode != http.StatusOK { 46 | t.Errorf("http %d", resp.StatusCode) 47 | return 48 | } 49 | stop <- struct{}{} 50 | 51 | stop, err = FlameGraph(srcRdb, 0, nil) 52 | if err != nil { 53 | t.Errorf("FindLargestKeys failed: %v", err) 54 | } 55 | stop <- struct{}{} 56 | 57 | _, err = FlameGraph("", 0, nil) 58 | if err == nil || err.Error() != "src file path is required" { 59 | t.Error("expect error: src file path is required") 60 | } 61 | } 62 | 63 | func TestFlameGraphWithRegex(t *testing.T) { 64 | srcRdb := filepath.Join("../cases", "tree.rdb") 65 | stop, err := FlameGraph(srcRdb, 18888, nil, WithRegexOption("^l.*")) 66 | if err != nil { 67 | t.Errorf("FindLargestKeys failed: %v", err) 68 | } 69 | stop <- struct{}{} 70 | 71 | _, err = FlameGraph(srcRdb, 18888, nil, WithRegexOption(`(1)\2`)) 72 | if err == nil { 73 | t.Errorf("expect error: %v", err) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /helper/json.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "runtime" 8 | "sync" 9 | 10 | "github.com/bytedance/sonic" 11 | "github.com/hdt3213/rdb/core" 12 | "github.com/hdt3213/rdb/model" 13 | ) 14 | 15 | var jsonEncoder = sonic.ConfigDefault 16 | 17 | // ConcurrentOption sets the number of goroutines for json converter 18 | type ConcurrentOption int 19 | 20 | // WithConcurrent sets the number of goroutines for json converter 21 | func WithConcurrent(c int) ConcurrentOption { 22 | return ConcurrentOption(c) 23 | } 24 | 25 | // ToJsons read rdb file and convert to json file 26 | func ToJsons(rdbFilename string, jsonFilename string, options ...interface{}) error { 27 | if rdbFilename == "" { 28 | return errors.New("src file path is required") 29 | } 30 | if jsonFilename == "" { 31 | return errors.New("output file path is required") 32 | } 33 | // open file 34 | rdbFile, err := os.Open(rdbFilename) 35 | if err != nil { 36 | return fmt.Errorf("open rdb %s failed, %v", rdbFilename, err) 37 | } 38 | defer func() { 39 | _ = rdbFile.Close() 40 | }() 41 | jsonFile, err := os.Create(jsonFilename) 42 | if err != nil { 43 | return fmt.Errorf("create json %s failed, %v", jsonFilename, err) 44 | } 45 | defer func() { 46 | _ = jsonFile.Close() 47 | }() 48 | // create decoder 49 | var dec decoder = core.NewDecoder(rdbFile) 50 | if dec, err = wrapDecoder(dec, options...); err != nil { 51 | return err 52 | } 53 | // parse rdb 54 | _, err = jsonFile.WriteString("[\n") 55 | if err != nil { 56 | return fmt.Errorf("write json failed, %v", err) 57 | } 58 | 59 | // parse options 60 | concurrent := 1 61 | cpuNum := runtime.NumCPU() 62 | if cpuNum > 1 { 63 | concurrent = cpuNum - 1 // leave one core for parser 64 | } 65 | for _, opt := range options { 66 | switch o := opt.(type) { 67 | case ConcurrentOption: 68 | concurrent = int(o) 69 | } 70 | } 71 | 72 | redisObjectBuffer := make(chan model.RedisObject, 1000) 73 | jsonStringBuffer := make(chan []byte, 1000) 74 | 75 | // parser goroutine 76 | empty := true 77 | go func() { 78 | err = dec.Parse(func(object model.RedisObject) bool { 79 | redisObjectBuffer <- object 80 | return true 81 | }) 82 | close(redisObjectBuffer) 83 | }() 84 | // json marshaller goroutine 85 | wg := &sync.WaitGroup{} 86 | wg.Add(concurrent) 87 | for i := 0; i < concurrent; i++ { 88 | go func() { 89 | for object := range redisObjectBuffer { 90 | data, err := jsonEncoder.Marshal(object) // enable SortMapKeys to ensure same result 91 | if err != nil { 92 | fmt.Printf("json marshal failed: %v", err) 93 | continue 94 | } 95 | jsonStringBuffer <- data 96 | } 97 | wg.Done() 98 | }() 99 | } 100 | // write goroutine 101 | wg2 := &sync.WaitGroup{} 102 | wg2.Add(1) 103 | go func() { 104 | for data := range jsonStringBuffer { 105 | data = append(data, ',', '\n') 106 | _, err = jsonFile.Write(data) 107 | if err != nil { 108 | fmt.Printf("write failed: %v", err) 109 | continue 110 | } 111 | empty = false 112 | } 113 | wg2.Done() 114 | }() 115 | 116 | wg.Wait() 117 | close(jsonStringBuffer) 118 | wg2.Wait() // wait writing goroutine 119 | 120 | // finish json 121 | if !empty { 122 | _, err = jsonFile.Seek(-2, 2) 123 | if err != nil { 124 | return fmt.Errorf("error during seek in file: %v", err) 125 | } 126 | } 127 | _, err = jsonFile.WriteString("\n]") 128 | if err != nil { 129 | return fmt.Errorf("error during write in file: %v", err) 130 | } 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /helper/memory.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "encoding/csv" 5 | "errors" 6 | "fmt" 7 | "github.com/hdt3213/rdb/bytefmt" 8 | "github.com/hdt3213/rdb/core" 9 | "github.com/hdt3213/rdb/model" 10 | "os" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | // MemoryProfile read rdb file and analysis memory usage then write result to csv file 16 | func MemoryProfile(rdbFilename string, csvFilename string, options ...interface{}) error { 17 | if rdbFilename == "" { 18 | return errors.New("src file path is required") 19 | } 20 | if csvFilename == "" { 21 | return errors.New("output file path is required") 22 | } 23 | rdbFile, err := os.Open(rdbFilename) 24 | if err != nil { 25 | return fmt.Errorf("open rdb %s failed, %v", rdbFilename, err) 26 | } 27 | defer func() { 28 | _ = rdbFile.Close() 29 | }() 30 | csvFile, err := os.Create(csvFilename) 31 | if err != nil { 32 | return fmt.Errorf("create json %s failed, %v", csvFilename, err) 33 | } 34 | defer func() { 35 | _ = csvFile.Close() 36 | }() 37 | 38 | var dec decoder = core.NewDecoder(rdbFile) 39 | if dec, err = wrapDecoder(dec, options...); err != nil { 40 | return err 41 | } 42 | 43 | _, err = csvFile.WriteString("database,key,type,size,size_readable,element_count,encoding,expiration\n") 44 | if err != nil { 45 | return fmt.Errorf("write csv failed: %v", err) 46 | } 47 | csvWriter := csv.NewWriter(csvFile) 48 | defer csvWriter.Flush() 49 | formatExpiration := func(o model.RedisObject) string { 50 | expiration := o.GetExpiration() 51 | if expiration == nil { 52 | return "" 53 | } 54 | return expiration.Format(time.RFC3339) 55 | } 56 | return dec.Parse(func(object model.RedisObject) bool { 57 | err = csvWriter.Write([]string{ 58 | strconv.Itoa(object.GetDBIndex()), 59 | object.GetKey(), 60 | object.GetType(), 61 | strconv.Itoa(object.GetSize()), 62 | bytefmt.FormatSize(uint64(object.GetSize())), 63 | strconv.Itoa(object.GetElemCount()), 64 | object.GetEncoding(), 65 | formatExpiration(object), 66 | }) 67 | if err != nil { 68 | fmt.Printf("csv write failed: %v", err) 69 | return false 70 | } 71 | return true 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /helper/memory_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestMemoryProfile(t *testing.T) { 10 | err := os.MkdirAll("tmp", os.ModePerm) 11 | if err != nil { 12 | return 13 | } 14 | defer func() { 15 | err := os.RemoveAll("tmp") 16 | if err != nil { 17 | t.Logf("remove tmp directory failed: %v", err) 18 | } 19 | }() 20 | testCases := []string{ 21 | "memory", 22 | "stream_listpacks_1", 23 | "stream_listpacks_2", 24 | "set_listpack", 25 | "listpack", 26 | } 27 | for _, name := range testCases { 28 | srcRdb := filepath.Join("../cases", name+".rdb") 29 | actualFile := filepath.Join("tmp", name+".csv") 30 | expectFile := filepath.Join("../cases", name+".csv") 31 | err = MemoryProfile(srcRdb, actualFile) 32 | if err != nil { 33 | t.Errorf("error occurs during parse %s, err: %v", srcRdb, err) 34 | return 35 | } 36 | equals, err := compareFileByLine(t, actualFile, expectFile) 37 | if err != nil { 38 | t.Errorf("error occurs during compare %s, err: %v", srcRdb, err) 39 | return 40 | } 41 | if !equals { 42 | t.Errorf("result is not equal of %s", srcRdb) 43 | return 44 | } 45 | } 46 | err = MemoryProfile("../cases/memory.rdb", "") 47 | if err == nil || err.Error() != "output file path is required" { 48 | t.Error("failed when empty output") 49 | } 50 | err = MemoryProfile("", "tmp/memory.rdb") 51 | if err == nil || err.Error() != "src file path is required" { 52 | t.Error("failed when empty output") 53 | } 54 | } 55 | 56 | func TestMemoryWithRegex(t *testing.T) { 57 | err := os.MkdirAll("tmp", os.ModePerm) 58 | if err != nil { 59 | return 60 | } 61 | defer func() { 62 | err := os.RemoveAll("tmp") 63 | if err != nil { 64 | t.Logf("remove tmp directory failed: %v", err) 65 | } 66 | }() 67 | srcRdb := filepath.Join("../cases", "memory.rdb") 68 | actualFile := filepath.Join("tmp", "memory_regex.csv") 69 | expectFile := filepath.Join("../cases", "memory_regex.csv") 70 | err = MemoryProfile(srcRdb, actualFile, WithRegexOption("^l.*")) 71 | if err != nil { 72 | t.Errorf("error occurs during parse, err: %v", err) 73 | return 74 | } 75 | equals, err := compareFileByLine(t, actualFile, expectFile) 76 | if err != nil { 77 | t.Errorf("error occurs during compare err: %v", err) 78 | return 79 | } 80 | if !equals { 81 | t.Errorf("result is not equal") 82 | return 83 | } 84 | errFile := filepath.Join("tmp", "memory_regex_err.csv") 85 | err = MemoryProfile(srcRdb, errFile, WithRegexOption(`(i)\1`)) 86 | if err == nil { 87 | t.Error("expect error") 88 | } 89 | } 90 | 91 | func TestMemoryNoExpired(t *testing.T) { 92 | err := os.MkdirAll("tmp", os.ModePerm) 93 | if err != nil { 94 | return 95 | } 96 | defer func() { 97 | err := os.RemoveAll("tmp") 98 | if err != nil { 99 | t.Logf("remove tmp directory failed: %v", err) 100 | } 101 | }() 102 | srcRdb := filepath.Join("../cases", "memory.rdb") 103 | actualFile := filepath.Join("tmp", "memory_expired.csv") 104 | expectFile := filepath.Join("../cases", "memory_expired.csv") 105 | err = MemoryProfile(srcRdb, actualFile, WithNoExpiredOption()) 106 | if err != nil { 107 | t.Errorf("error occurs during parse, err: %v", err) 108 | return 109 | } 110 | equals, err := compareFileByLine(t, actualFile, expectFile) 111 | if err != nil { 112 | t.Errorf("error occurs during compare err: %v", err) 113 | return 114 | } 115 | if !equals { 116 | t.Errorf("result is not equal") 117 | return 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /helper/prefix.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "encoding/csv" 5 | "errors" 6 | "fmt" 7 | "math" 8 | "os" 9 | "strconv" 10 | 11 | "github.com/hdt3213/rdb/bytefmt" 12 | "github.com/hdt3213/rdb/core" 13 | "github.com/hdt3213/rdb/model" 14 | ) 15 | 16 | // PrefixAnalyse read rdb file and find the largest N keys. 17 | // The invoker owns output, FindBiggestKeys won't close it 18 | func PrefixAnalyse(rdbFilename string, topN int, maxDepth int, output *os.File, options ...interface{}) error { 19 | if rdbFilename == "" { 20 | return errors.New("src file path is required") 21 | } 22 | if topN < 0 { 23 | return errors.New("n must greater than 0") 24 | } else if topN == 0 { 25 | topN = math.MaxInt 26 | } 27 | if maxDepth == 0 { 28 | maxDepth = math.MaxInt 29 | } else { 30 | maxDepth += 2 // for root(depth==1) and database root(depth==2) 31 | } 32 | 33 | // decode rdb file 34 | rdbFile, err := os.Open(rdbFilename) 35 | if err != nil { 36 | return fmt.Errorf("open rdb %s failed, %v", rdbFilename, err) 37 | } 38 | defer func() { 39 | _ = rdbFile.Close() 40 | }() 41 | var dec decoder = core.NewDecoder(rdbFile) 42 | if dec, err = wrapDecoder(dec, options...); err != nil { 43 | return err 44 | } 45 | 46 | // prefix tree 47 | tree := newRadixTree() 48 | err = dec.Parse(func(object model.RedisObject) bool { 49 | key := genKey(object.GetDBIndex(), object.GetKey()) 50 | tree.insert(key, object.GetSize()) 51 | return true 52 | }) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | // get top list 58 | toplist := newToplist(topN) 59 | tree.traverse(func(node *radixNode, depth int) bool { 60 | if depth > maxDepth { 61 | return false 62 | } 63 | if depth <= 2 { 64 | // skip root and database root 65 | return true 66 | } 67 | toplist.add(node) 68 | return true 69 | }) 70 | 71 | // write into csv 72 | _, err = output.WriteString("database,prefix,size,size_readable,key_count\n") 73 | if err != nil { 74 | return fmt.Errorf("write header failed: %v", err) 75 | } 76 | csvWriter := csv.NewWriter(output) 77 | defer csvWriter.Flush() 78 | printNode := func(node *radixNode) error { 79 | db, key := parseNodeKey(node.fullpath) 80 | dbStr := strconv.Itoa(db) 81 | return csvWriter.Write([]string{ 82 | dbStr, 83 | key, 84 | strconv.Itoa(node.totalSize), 85 | bytefmt.FormatSize(uint64(node.totalSize)), 86 | strconv.Itoa(node.keyCount), 87 | }) 88 | } 89 | for _, n := range toplist.list { 90 | node := n.(*radixNode) 91 | err := printNode(node) 92 | if err != nil { 93 | return err 94 | } 95 | } 96 | 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /helper/prefix_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestPrefixAnalyse(t *testing.T) { 10 | err := os.MkdirAll("tmp", os.ModePerm) 11 | if err != nil { 12 | return 13 | } 14 | defer func() { 15 | err := os.RemoveAll("tmp") 16 | if err != nil { 17 | t.Logf("remove tmp directory failed: %v", err) 18 | } 19 | }() 20 | srcRdb := filepath.Join("../cases", "tree.rdb") 21 | 22 | // test top3 23 | actualTop3 := filepath.Join("tmp", "tree.csv") 24 | expectTop3 := filepath.Join("../cases", "tree.csv") 25 | actualTop3File, err := os.Create(actualTop3) 26 | if err != nil { 27 | t.Error(err) 28 | return 29 | } 30 | err = PrefixAnalyse(srcRdb, 3, 0, actualTop3File) 31 | if err != nil { 32 | t.Error(err) 33 | return 34 | } 35 | 36 | equals, err := compareFileByLine(t, actualTop3, expectTop3) 37 | if err != nil { 38 | t.Errorf("error occurs during compare top3, err: %v", err) 39 | return 40 | } 41 | if !equals { 42 | t.Error("result is not equal of top3") 43 | return 44 | } 45 | 46 | // test depth=2 47 | actualDepth2 := filepath.Join("tmp", "tree2.csv") 48 | expectDepth2 := filepath.Join("../cases", "tree2.csv") 49 | actualDepth2File, err := os.Create(actualDepth2) 50 | if err != nil { 51 | t.Error(err) 52 | return 53 | } 54 | err = PrefixAnalyse(srcRdb, 0, 2, actualDepth2File) 55 | if err != nil { 56 | t.Error(err) 57 | return 58 | } 59 | 60 | equals, err = compareFileByLine(t, actualDepth2, expectDepth2) 61 | if err != nil { 62 | t.Errorf("error occurs during compare top3, err: %v", err) 63 | return 64 | } 65 | if !equals { 66 | t.Error("result is not equal of top3") 67 | return 68 | } 69 | 70 | } -------------------------------------------------------------------------------- /helper/radix.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "container/list" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | type radixNode struct { 10 | path string 11 | end bool 12 | children []*radixNode 13 | totalSize int // total size of all key-value with this prefix 14 | keyCount int 15 | fullpath string 16 | } 17 | 18 | type radixTree struct{ 19 | root *radixNode 20 | } 21 | 22 | func newRadixTree() *radixTree { 23 | return &radixTree{ 24 | root: &radixNode{}, 25 | } 26 | } 27 | 28 | func commonPrefixLen(wordA, wordB string) int { 29 | var i int 30 | for i < len(wordA) && i < len(wordB) && wordA[i] == wordB[i] { 31 | i++ 32 | } 33 | return i 34 | } 35 | 36 | func (tree *radixTree) insert(word string, size int) { 37 | root := tree.root 38 | fullword := word 39 | node := root 40 | walk: 41 | for { 42 | i := commonPrefixLen(word, node.path) 43 | if i < len(node.path) { 44 | // assert: node == root || i > 0, because it is the first loop or from `for _, child := range node.children` 45 | // split current node `rn` 46 | newChild := &radixNode{ 47 | path: node.path[i:], 48 | children: node.children, 49 | end: node.end, 50 | totalSize: node.totalSize, 51 | keyCount: node.keyCount, 52 | fullpath: node.fullpath, 53 | } 54 | node.children = []*radixNode{newChild} 55 | node.fullpath = node.fullpath[:len(node.fullpath)-(len(node.path)-i)] 56 | node.path = node.path[:i] 57 | node.end = false 58 | } 59 | // word must be a descendants of node 60 | if i > 0 || node == root { 61 | node.totalSize += size 62 | node.keyCount++ 63 | } 64 | if i == len(word) { 65 | // assert node.fullpath == fullword 66 | node.end = true 67 | break 68 | } 69 | if i < len(word) { 70 | word = word[i:] 71 | // word may have common prefix with a child, recurse until no common prefix 72 | c := word[0] 73 | for _, child := range node.children { 74 | if child.path[0] == c { 75 | node = child 76 | continue walk 77 | } 78 | } 79 | 80 | // now, word has no common prefix with child 81 | child := &radixNode{} 82 | child.path = word 83 | child.end = true 84 | child.fullpath = fullword 85 | child.totalSize = size 86 | child.keyCount = 1 87 | node.children = append(node.children, child) 88 | return 89 | } 90 | } 91 | } 92 | 93 | func (tree *radixTree) traverse(cb func(node *radixNode, depth int) bool) { 94 | root := tree.root 95 | type nodeDepth struct { 96 | node *radixNode 97 | depth int 98 | } 99 | queue := list.New() 100 | queue.PushBack(&nodeDepth{root, 1}) 101 | for queue.Len() > 0 { 102 | head := queue.Front() 103 | queue.Remove(head) 104 | node2 := head.Value.(*nodeDepth) 105 | if !cb(node2.node, node2.depth) { 106 | return 107 | } 108 | depth := node2.depth + 1 109 | for _, child := range node2.node.children { 110 | queue.PushBack(&nodeDepth{child, depth}) 111 | } 112 | } 113 | } 114 | 115 | func (n *radixNode) GetSize() int { 116 | return int(n.totalSize) 117 | } 118 | 119 | func genKey(db int, key string) string { 120 | return strconv.Itoa(db) + " " + key 121 | } 122 | 123 | func parseNodeKey(key string) (int, string) { 124 | if key == "" { 125 | return -1, "" 126 | } 127 | index := strings.Index(key, " ") 128 | db, _ := strconv.Atoi(key[:index]) 129 | var realKey string 130 | // if key is db root, index+1 == len(key) 131 | if index+1 < len(key) { 132 | realKey = key[index+1:] 133 | } 134 | return db, realKey 135 | } -------------------------------------------------------------------------------- /helper/radix_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import "testing" 4 | 5 | func TestRadix(t *testing.T) { 6 | words := []string{ 7 | "a", 8 | "b", 9 | "abbd", 10 | "abba", 11 | "abc", 12 | } 13 | tree := newRadixTree() 14 | for i, word := range words { 15 | tree.insert(word, i+1) 16 | } 17 | expectSizeMap := map[string]int { 18 | "": 15, 19 | "a": 13, 20 | "b": 2, 21 | "ab": 12, 22 | "abb": 7, 23 | "abc": 5, 24 | "abbd": 3, 25 | "abba": 4, 26 | } 27 | actualSizeMap := make(map[string]int) 28 | tree.traverse(func(node *radixNode, depth int) bool { 29 | actualSizeMap[node.fullpath] = int(node.totalSize) 30 | return true 31 | }) 32 | for prefix, expectSize := range expectSizeMap { 33 | actualSize := actualSizeMap[prefix] 34 | if expectSize != actualSize { 35 | t.Error("wrong size for " + prefix) 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /helper/regex.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hdt3213/rdb/model" 6 | "regexp" 7 | "time" 8 | ) 9 | 10 | type decoder interface { 11 | Parse(cb func(object model.RedisObject) bool) error 12 | } 13 | 14 | type regexDecoder struct { 15 | reg *regexp.Regexp 16 | dec decoder 17 | } 18 | 19 | func (d *regexDecoder) Parse(cb func(object model.RedisObject) bool) error { 20 | return d.dec.Parse(func(object model.RedisObject) bool { 21 | if d.reg.MatchString(object.GetKey()) { 22 | return cb(object) 23 | } 24 | return true 25 | }) 26 | } 27 | 28 | // regexWrapper returns 29 | func regexWrapper(d decoder, expr string) (*regexDecoder, error) { 30 | reg, err := regexp.Compile(expr) 31 | if err != nil { 32 | return nil, fmt.Errorf("illegal regex expression: %v", expr) 33 | } 34 | return ®exDecoder{ 35 | dec: d, 36 | reg: reg, 37 | }, nil 38 | } 39 | 40 | // RegexOption enable regex filters 41 | type RegexOption *string 42 | 43 | // WithRegexOption creates a WithRegexOption from regex expression 44 | func WithRegexOption(expr string) RegexOption { 45 | return &expr 46 | } 47 | 48 | // noExpiredDecoder filter all expired keys 49 | type noExpiredDecoder struct { 50 | dec decoder 51 | } 52 | 53 | func (d *noExpiredDecoder) Parse(cb func(object model.RedisObject) bool) error { 54 | now := time.Now() 55 | return d.dec.Parse(func(object model.RedisObject) bool { 56 | expiration := object.GetExpiration() 57 | if expiration == nil || expiration.After(now) { 58 | return cb(object) 59 | } 60 | return true 61 | }) 62 | } 63 | 64 | // NoExpiredOption tells decoder to filter all expired keys 65 | type NoExpiredOption bool 66 | 67 | // WithNoExpiredOption tells decoder to filter all expired keys 68 | func WithNoExpiredOption() NoExpiredOption { 69 | return NoExpiredOption(true) 70 | } 71 | 72 | func wrapDecoder(dec decoder, options ...interface{}) (decoder, error) { 73 | var regexOpt RegexOption 74 | var noExpiredOpt NoExpiredOption 75 | for _, opt := range options { 76 | switch o := opt.(type) { 77 | case RegexOption: 78 | regexOpt = o 79 | case NoExpiredOption: 80 | noExpiredOpt = o 81 | } 82 | } 83 | if regexOpt != nil { 84 | var err error 85 | dec, err = regexWrapper(dec, *regexOpt) 86 | if err != nil { 87 | return nil, err 88 | } 89 | } 90 | if noExpiredOpt { 91 | dec = &noExpiredDecoder{ 92 | dec: dec, 93 | } 94 | } 95 | return dec, nil 96 | } 97 | -------------------------------------------------------------------------------- /helper/resp.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "sort" 7 | "strconv" 8 | 9 | "github.com/hdt3213/rdb/model" 10 | ) 11 | 12 | const crlf = "\r\n" 13 | 14 | // CmdLine is alias for [][]byte, represents a command line 15 | type CmdLine = [][]byte 16 | 17 | // lexOrder traversal map in lex order to create 18 | type lexOrder struct{} 19 | 20 | func makeMultiBulkResp(args [][]byte) []byte { 21 | argLen := len(args) 22 | var buf bytes.Buffer 23 | buf.WriteString("*" + strconv.Itoa(argLen) + crlf) 24 | for _, arg := range args { 25 | if arg == nil { 26 | buf.WriteString("$-1" + crlf) 27 | } else { 28 | buf.WriteString("$" + strconv.Itoa(len(arg)) + crlf + string(arg) + crlf) 29 | } 30 | } 31 | return buf.Bytes() 32 | } 33 | 34 | var setCmd = []byte("SET") 35 | 36 | func stringToCmd(obj *model.StringObject) CmdLine { 37 | cmdLine := make([][]byte, 3) 38 | cmdLine[0] = setCmd 39 | cmdLine[1] = []byte(obj.Key) 40 | cmdLine[2] = obj.Value 41 | return cmdLine 42 | } 43 | 44 | var rPushAllCmd = []byte("RPUSH") 45 | 46 | func listToCmd(obj *model.ListObject) CmdLine { 47 | cmdLine := make([][]byte, 2+obj.GetElemCount()) 48 | cmdLine[0] = rPushAllCmd 49 | cmdLine[1] = []byte(obj.Key) 50 | for i, val := range obj.Values { 51 | cmdLine[2+i] = val 52 | } 53 | return cmdLine 54 | } 55 | 56 | var sAddCmd = []byte("SADD") 57 | 58 | func setToCmd(obj *model.SetObject) CmdLine { 59 | cmdLine := make([][]byte, 2+obj.GetElemCount()) 60 | cmdLine[0] = sAddCmd 61 | cmdLine[1] = []byte(obj.GetKey()) 62 | for i, val := range obj.Members { 63 | cmdLine[2+i] = val 64 | } 65 | return cmdLine 66 | } 67 | 68 | var hMSetCmd = []byte("HMSET") 69 | 70 | func hashToCmd(obj *model.HashObject, useLexOrder bool) CmdLine { 71 | cmdLine := make([][]byte, 2+obj.GetElemCount()*2) 72 | cmdLine[0] = hMSetCmd 73 | cmdLine[1] = []byte(obj.GetKey()) 74 | i := 0 75 | if useLexOrder { 76 | entries := make([][2][]byte, 0, 2*len(obj.Hash)) 77 | for field, val := range obj.Hash { 78 | entries = append(entries, [2][]byte{ 79 | []byte(field), val, 80 | }) 81 | } 82 | sort.Slice(entries, func(i, j int) bool { 83 | return string(entries[i][0]) < string(entries[j][0]) 84 | }) 85 | for _, entry := range entries { 86 | cmdLine[2+i*2] = entry[0] 87 | cmdLine[3+i*2] = entry[1] 88 | i++ 89 | } 90 | } else { 91 | for field, val := range obj.Hash { 92 | cmdLine[2+i*2] = []byte(field) 93 | cmdLine[3+i*2] = val 94 | i++ 95 | } 96 | } 97 | return cmdLine 98 | } 99 | 100 | var zAddCmd = []byte("ZADD") 101 | 102 | func zSetToCmd(obj *model.ZSetObject) CmdLine { 103 | cmdLine := make([][]byte, 2+obj.GetElemCount()*2) 104 | cmdLine[0] = zAddCmd 105 | cmdLine[1] = []byte(obj.GetKey()) 106 | for i, e := range obj.Entries { 107 | value := strconv.FormatFloat(e.Score, 'f', -1, 64) 108 | cmdLine[2+i*2] = []byte(value) 109 | cmdLine[3+i*2] = []byte(e.Member) 110 | } 111 | return cmdLine 112 | } 113 | 114 | var pExpireAtBytes = []byte("PEXPIREAT") 115 | 116 | // MakeExpireCmd generates command line to set expiration for the given key 117 | func makeExpireCmd(obj model.RedisObject) CmdLine { 118 | expireAt := obj.GetExpiration() 119 | if expireAt == nil { 120 | return nil 121 | } 122 | args := make([][]byte, 3) 123 | args[0] = pExpireAtBytes 124 | args[1] = []byte(obj.GetKey()) 125 | args[2] = []byte(strconv.FormatInt(expireAt.UnixNano()/1e6, 10)) 126 | return args 127 | } 128 | 129 | var ( 130 | xaddCmd = []byte("XADD") 131 | ) 132 | 133 | func formatStreamID(streamID *model.StreamId) string { 134 | ms := strconv.FormatUint(streamID.Ms, 10) 135 | seq := strconv.FormatUint(streamID.Sequence, 10) 136 | return ms + "-" + seq 137 | } 138 | 139 | func streamToCmd(stream *model.StreamObject) []CmdLine { 140 | commands := make([]CmdLine, 0) 141 | 142 | for _, entry := range stream.Entries { 143 | for _, message := range entry.Msgs { 144 | args := make(CmdLine, 0, 3+len(message.Fields)*2) 145 | args = append( 146 | args, 147 | xaddCmd, 148 | []byte(stream.GetKey()), 149 | []byte(formatStreamID(message.Id)), 150 | ) 151 | 152 | for key, value := range message.Fields { 153 | args = append(args, []byte(key), []byte(value)) 154 | } 155 | 156 | commands = append(commands, args) 157 | 158 | // TODO: groups, consumers, pending 159 | } 160 | } 161 | 162 | return commands 163 | } 164 | 165 | // ObjectToCmd convert redis object to redis command line 166 | func ObjectToCmd(obj model.RedisObject, opts ...interface{}) []CmdLine { 167 | if obj == nil { 168 | return nil 169 | } 170 | useLexOrder := false 171 | for _, o := range opts { 172 | switch o.(type) { 173 | case lexOrder: 174 | useLexOrder = true 175 | } 176 | } 177 | cmdLines := make([]CmdLine, 0) 178 | switch obj.GetType() { 179 | case model.StringType: 180 | strObj := obj.(*model.StringObject) 181 | cmdLines = append(cmdLines, stringToCmd(strObj)) 182 | case model.ListType: 183 | listObj := obj.(*model.ListObject) 184 | cmdLines = append(cmdLines, listToCmd(listObj)) 185 | case model.HashType: 186 | hashObj := obj.(*model.HashObject) 187 | cmdLines = append(cmdLines, hashToCmd(hashObj, useLexOrder)) 188 | case model.SetType: 189 | setObj := obj.(*model.SetObject) 190 | cmdLines = append(cmdLines, setToCmd(setObj)) 191 | case model.ZSetType: 192 | zsetObj := obj.(*model.ZSetObject) 193 | cmdLines = append(cmdLines, zSetToCmd(zsetObj)) 194 | case model.StreamType: 195 | streamObj := obj.(*model.StreamObject) 196 | cmdLines = append(cmdLines, streamToCmd(streamObj)...) 197 | } 198 | if obj.GetExpiration() != nil { 199 | cmdLines = append(cmdLines, makeExpireCmd(obj)) 200 | } 201 | return cmdLines 202 | } 203 | 204 | // CmdLinesToResp convert []CmdLine to RESP bytes 205 | func CmdLinesToResp(cmds []CmdLine) []byte { 206 | buf := bytes.NewBuffer(make([]byte, 0)) 207 | for _, cmdLine := range cmds { 208 | resp := makeMultiBulkResp(cmdLine) 209 | buf.Write(resp) 210 | } 211 | return buf.Bytes() 212 | } 213 | 214 | // WriteObjectToResp convert object to resp and write 215 | func WriteObjectToResp(w io.Writer, obj model.RedisObject) error { 216 | var err error 217 | cmdLines := ObjectToCmd(obj) 218 | 219 | for _, cmdLine := range cmdLines { 220 | argLen := len(cmdLine) 221 | respBytes := make([][]byte, 0) 222 | respBytes = append(respBytes, []byte("*"+strconv.Itoa(argLen)+crlf)) 223 | 224 | for _, arg := range cmdLine { 225 | if arg == nil { 226 | respBytes = append(respBytes, []byte("$-1"+crlf)) 227 | continue 228 | } 229 | respBytes = append(respBytes, []byte("$"+strconv.Itoa(len(arg))+crlf)) 230 | // the arg may be a large key or value 231 | respBytes = append(respBytes, arg) 232 | respBytes = append(respBytes, []byte(crlf)) 233 | } 234 | 235 | for _, bytes := range respBytes { 236 | if _, err = w.Write(bytes); err != nil { 237 | return err 238 | } 239 | } 240 | } 241 | return nil 242 | } 243 | -------------------------------------------------------------------------------- /helper/toplist.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "sort" 5 | ) 6 | 7 | type Sized interface { 8 | GetSize() int 9 | } 10 | 11 | type topList struct { 12 | list []Sized 13 | capacity int 14 | } 15 | 16 | func (tl *topList) add(x Sized) { 17 | index := sort.Search(len(tl.list), func(i int) bool { 18 | return tl.list[i].GetSize() <= x.GetSize() 19 | }) 20 | tl.list = append(tl.list, x) 21 | copy(tl.list[index+1:], tl.list[index:]) 22 | tl.list[index] = x 23 | if len(tl.list) > tl.capacity { 24 | tl.list = tl.list[:tl.capacity] 25 | } 26 | } 27 | 28 | func newToplist(cap int) *topList { 29 | return &topList{ 30 | capacity: cap, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lzf/lzf.go: -------------------------------------------------------------------------------- 1 | package lzf 2 | 3 | import "errors" 4 | 5 | const ( 6 | htabLog uint32 = 14 7 | htabSize uint32 = 1 << htabLog 8 | maxLit = 1 << 5 9 | maxOff = 1 << 13 10 | maxRef = (1 << 8) + (1 << 3) 11 | ) 12 | 13 | var ( 14 | errInsufficientBuffer = errors.New("insufficient buffer") 15 | errDataCorruption = errors.New("data corruption") 16 | ) 17 | 18 | // using https://github.com/zhuyie/golzf according to MIT license 19 | // Decompress decompress lzf compressed data 20 | func Decompress(input []byte, inLen int, outLen int) ([]byte, error) { 21 | input = input[:inLen] 22 | output := make([]byte, outLen) 23 | var inputIndex, outputIndex int 24 | 25 | inputLength := len(input) 26 | outputLength := len(output) 27 | if inputLength == 0 { 28 | return nil, nil 29 | } 30 | 31 | for inputIndex < inputLength { 32 | ctrl := int(input[inputIndex]) 33 | inputIndex++ 34 | 35 | if ctrl < (1 << 5) { /* literal run */ 36 | ctrl++ 37 | 38 | if outputIndex+ctrl > outputLength { 39 | return nil, errInsufficientBuffer 40 | } 41 | 42 | if inputIndex+ctrl > inputLength { 43 | return nil, errDataCorruption 44 | } 45 | 46 | copy(output[outputIndex:outputIndex+ctrl], input[inputIndex:inputIndex+ctrl]) 47 | inputIndex += ctrl 48 | outputIndex += ctrl 49 | 50 | } else { /* back reference */ 51 | length := ctrl >> 5 52 | ref := outputIndex - ((ctrl & 0x1f) << 8) - 1 53 | 54 | if inputIndex >= inputLength { 55 | return nil, errDataCorruption 56 | } 57 | 58 | if length == 7 { 59 | length += int(input[inputIndex]) 60 | inputIndex++ 61 | 62 | if inputIndex >= inputLength { 63 | return nil, errDataCorruption 64 | } 65 | } 66 | 67 | ref -= int(input[inputIndex]) 68 | inputIndex++ 69 | 70 | if outputIndex+length+2 > outputLength { 71 | return nil, errDataCorruption 72 | } 73 | 74 | if ref < 0 { 75 | return nil, errDataCorruption 76 | } 77 | 78 | // Can't use copy(...) here, because it has special handling when source and destination overlap. 79 | for i := 0; i < length+2; i++ { 80 | output[outputIndex+i] = output[ref+i] 81 | } 82 | outputIndex += length + 2 83 | } 84 | } 85 | 86 | return output[:outputIndex], nil 87 | } 88 | 89 | // Compress compress data using lzf algorithm 90 | func Compress(input []byte) ([]byte, error) { 91 | var hval, ref, hslot, off uint32 92 | var inputIndex, outputIndex, lit int 93 | output := make([]byte, len(input)) 94 | 95 | htab := make([]uint32, htabSize) 96 | inputLength := len(input) 97 | if inputLength == 0 { 98 | return nil, nil 99 | } 100 | outputLength := len(output) 101 | if outputLength == 0 { 102 | return nil, errInsufficientBuffer 103 | } 104 | 105 | lit = 0 /* start run */ 106 | outputIndex++ 107 | 108 | hval = uint32(input[inputIndex])<<8 | uint32(input[inputIndex+1]) 109 | for inputIndex < inputLength-2 { 110 | hval = (hval << 8) | uint32(input[inputIndex+2]) 111 | hslot = ((hval >> (3*8 - htabLog)) - hval*5) & (htabSize - 1) 112 | ref = htab[hslot] 113 | htab[hslot] = uint32(inputIndex) 114 | off = uint32(inputIndex) - ref - 1 115 | 116 | if off < maxOff && 117 | (ref > 0) && 118 | (input[ref] == input[inputIndex]) && 119 | (input[ref+1] == input[inputIndex+1]) && 120 | (input[ref+2] == input[inputIndex+2]) { 121 | 122 | /* match found at *ref++ */ 123 | len := 2 124 | maxLen := inputLength - inputIndex - len 125 | if maxLen > maxRef { 126 | maxLen = maxRef 127 | } 128 | 129 | if outputIndex+3+1 >= outputLength { /* first a faster conservative test */ 130 | nlit := 0 131 | if lit == 0 { 132 | nlit = 1 133 | } 134 | if outputIndex-nlit+3+1 >= outputLength { /* second the exact but rare test */ 135 | return nil, errInsufficientBuffer 136 | } 137 | } 138 | 139 | output[outputIndex-lit-1] = byte(lit - 1) /* stop run */ 140 | if lit == 0 { 141 | outputIndex-- /* undo run if length is zero */ 142 | } 143 | 144 | for { 145 | len++ 146 | if (len >= maxLen) || (input[int(ref)+len] != input[inputIndex+len]) { 147 | break 148 | } 149 | } 150 | 151 | len -= 2 /* len is now #octets - 1 */ 152 | inputIndex++ 153 | 154 | if len < 7 { 155 | output[outputIndex] = byte((off >> 8) + uint32(len<<5)) 156 | outputIndex++ 157 | } else { 158 | output[outputIndex] = byte((off >> 8) + (7 << 5)) 159 | output[outputIndex+1] = byte(len - 7) 160 | outputIndex += 2 161 | } 162 | 163 | output[outputIndex] = byte(off) 164 | outputIndex += 2 165 | lit = 0 /* start run */ 166 | 167 | inputIndex += len + 1 168 | 169 | if inputIndex >= inputLength-2 { 170 | break 171 | } 172 | 173 | inputIndex -= 2 174 | 175 | hval = uint32(input[inputIndex])<<8 | uint32(input[inputIndex+1]) 176 | hval = (hval << 8) | uint32(input[inputIndex+2]) 177 | hslot = ((hval >> (3*8 - htabLog)) - (hval * 5)) & (htabSize - 1) 178 | htab[hslot] = uint32(inputIndex) 179 | inputIndex++ 180 | 181 | hval = (hval << 8) | uint32(input[inputIndex+2]) 182 | hslot = ((hval >> (3*8 - htabLog)) - (hval * 5)) & (htabSize - 1) 183 | htab[hslot] = uint32(inputIndex) 184 | inputIndex++ 185 | 186 | } else { 187 | /* one more literal byte we must copy */ 188 | if outputIndex >= outputLength { 189 | return nil, errInsufficientBuffer 190 | } 191 | 192 | lit++ 193 | output[outputIndex] = input[inputIndex] 194 | outputIndex++ 195 | inputIndex++ 196 | 197 | if lit == maxLit { 198 | output[outputIndex-lit-1] = byte(lit - 1) /* stop run */ 199 | lit = 0 /* start run */ 200 | outputIndex++ 201 | } 202 | } 203 | } 204 | 205 | if outputIndex+3 >= outputLength { /* at most 3 bytes can be missing here */ 206 | return nil, errInsufficientBuffer 207 | } 208 | 209 | for inputIndex < inputLength { 210 | lit++ 211 | output[outputIndex] = input[inputIndex] 212 | outputIndex++ 213 | inputIndex++ 214 | 215 | if lit == maxLit { 216 | output[outputIndex-lit-1] = byte(lit - 1) /* stop run */ 217 | lit = 0 /* start run */ 218 | outputIndex++ 219 | } 220 | } 221 | 222 | output[outputIndex-lit-1] = byte(lit - 1) /* end run */ 223 | if lit == 0 { /* undo run if length is zero */ 224 | outputIndex-- 225 | } 226 | 227 | return output[:outputIndex], nil 228 | } 229 | -------------------------------------------------------------------------------- /lzf/lzf_test.go: -------------------------------------------------------------------------------- 1 | package lzf 2 | 3 | import ( 4 | "math/rand" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 10 | 11 | // RandString create a random string no longer than n 12 | func RandString(n int) string { 13 | b := make([]rune, n) 14 | for i := range b { 15 | b[i] = letters[rand.Intn(len(letters))] 16 | } 17 | return string(b) 18 | } 19 | 20 | func TestLzf(t *testing.T) { 21 | for i := 0; i < 10; i++ { 22 | str := strings.Repeat(RandString(128), 10) 23 | compressed, err := Compress([]byte(str)) 24 | if err != nil { 25 | t.Error(err) 26 | return 27 | } 28 | decompressed, err := Decompress(compressed, len(compressed), len(str)) 29 | if err != nil { 30 | t.Error(err) 31 | return 32 | } 33 | if str != string(decompressed) { 34 | t.Error("wrong decompressed") 35 | return 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /memprofiler/common.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import ( 4 | "sort" 5 | "strconv" 6 | "unsafe" 7 | ) 8 | 9 | var jemallocClasses = []int{ 10 | 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 448, 512, 640, 768, 896, 1024, 11 | 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, 20480, 24576, 12 | 28672, 32768, 40960, 49152, 57344, 65536, 81920, 98304, 114688, 131072, 163840, 196608, 229376, 262144, 327680, 13 | 393216, 458752, 524288, 655360, 786432, 917504, 1048576, 1310720, 1572864, 1835008, 2097152, 2621440, 3145728, 14 | 3670016, 4194304, 5242880, 6291456, 7340032, 8388608, 10485760, 12582912, 14680064, 16777216, 20971520, 25165824, 15 | 29360128, 33554432, 41943040, 50331648, 58720256, 67108864, 83886080, 100663296, 117440512, 134217728, 167772160, 16 | 201326592, 234881024, 268435456, 335544320, 402653184, 469762048, 536870912, 671088640, 805306368, 939524096, 17 | 1073741824, 1342177280, 1610612736, 1879048192, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 18 | 5368709120, 6442450944, 7516192768, 8589934592, 10737418240, 12884901888, 15032385536, 17179869184, 21474836480, 19 | 25769803776, 30064771072, 34359738368, 42949672960, 51539607552, 60129542144, 68719476736, 85899345920, 20 | 103079215104, 120259084288, 137438953472, 171798691840, 206158430208, 240518168576, 274877906944, 343597383680, 21 | 412316860416, 481036337152, 549755813888, 687194767360, 824633720832, 962072674304, 1099511627776, 1374389534720, 22 | 1649267441664, 1924145348608, 2199023255552, 2748779069440, 3298534883328, 3848290697216, 4398046511104, 23 | 5497558138880, 6597069766656, 7696581394432, 8796093022208, 10995116277760, 13194139533312, 15393162788864, 24 | 17592186044416, 21990232555520, 26388279066624, 30786325577728, 35184372088832, 43980465111040, 52776558133248, 25 | 61572651155456, 70368744177664, 87960930222080, 105553116266496, 123145302310912, 140737488355328, 175921860444160, 26 | 211106232532992, 246290604621824, 281474976710656, 351843720888320, 422212465065984, 492581209243648, 27 | 562949953421312, 703687441776640, 844424930131968, 985162418487296, 1125899906842624, 1407374883553280, 28 | 1688849860263936, 1970324836974592, 2251799813685248, 2814749767106560, 3377699720527872, 3940649673949184, 29 | 4503599627370496, 5629499534213120, 6755399441055744, 7881299347898368, 9007199254740992, 11258999068426240, 30 | 13510798882111488, 15762598695796736, 18014398509481984, 22517998136852480, 27021597764222976, 31525197391593472, 31 | 36028797018963968, 45035996273704960, 54043195528445952, 63050394783186944, 72057594037927936, 90071992547409920, 32 | 108086391056891904, 126100789566373888, 144115188075855872, 180143985094819840, 216172782113783808, 33 | 252201579132747776, 288230376151711744, 360287970189639680, 432345564227567616, 504403158265495552, 34 | 576460752303423488, 720575940379279360, 864691128455135232, 1008806316530991104, 1152921504606846976, 35 | 1441151880758558720, 1729382256910270464, 2017612633061982208, 2305843009213693952, 2882303761517117440, 36 | 3458764513820540928, 4035225266123964416, 4611686018427387904, 5764607523034234880, 6917529027641081856, 37 | 8070450532247928832, 38 | } 39 | 40 | func getJemallocSize(req int) int { 41 | i := sort.Search(len(jemallocClasses), func(i int) bool { 42 | return jemallocClasses[i] >= req 43 | }) 44 | return jemallocClasses[i] 45 | } 46 | 47 | func sizeOfString(str string) int { 48 | // https://github.com/antirez/redis/blob/unstable/src/sds.h 49 | _, err := strconv.ParseInt(str, 10, 64) 50 | if err == nil { 51 | // REDIS_SHARED_INTEGERS 52 | return 0 53 | } 54 | size := len(str) 55 | if size < 32 { // 2^5 56 | return getJemallocSize(size + 1 + 1) 57 | } else if size < 256 { // 2^8 58 | return getJemallocSize(size + 2 + 1) 59 | } else if size < 25536 { // 2^16 60 | return getJemallocSize(size + 1 + 4 + 1) 61 | } else if size < 4294967296 { // 2^32 62 | return getJemallocSize(size + 1 + 8 + 1) 63 | } 64 | return getJemallocSize(size + 1 + 16 + 1) 65 | } 66 | 67 | func sizeOfPointer() int { 68 | return 8 69 | } 70 | 71 | func sizeOfLong() int { 72 | return 8 73 | } 74 | 75 | func unsafeBytes2Str(b []byte) string { 76 | return *(*string)(unsafe.Pointer(&b)) 77 | } 78 | 79 | func redisObjOverhead() int { 80 | return sizeOfPointer() + 8 81 | } 82 | 83 | func topLevelObjectOverhead(key string, hasTTl bool) int { 84 | size := hashTableEntryOverhead() + sizeOfString(key) + redisObjOverhead() 85 | if !hasTTl { 86 | return size 87 | } 88 | return size + expiryOverhead() 89 | } 90 | 91 | func expiryOverhead() int { 92 | // Key expiry is stored in a hashtable, so we have to pay for the cost of a hashtable entry 93 | // The timestamp itself is stored as an int64, which is a 8 bytes 94 | return hashTableEntryOverhead() + 8 95 | } 96 | 97 | func nextPower(size int) int { 98 | power := 1 99 | for power <= size { 100 | power <<= 1 101 | } 102 | return power 103 | } 104 | -------------------------------------------------------------------------------- /memprofiler/hash.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import "github.com/hdt3213/rdb/model" 4 | 5 | func hashTableEntryOverhead() int { 6 | // See https://github.com/antirez/redis/blob/unstable/src/dict.h 7 | // Each dictEntry has 2 pointers + int64 8 | return 2*sizeOfPointer() + 8 9 | } 10 | 11 | func hashtableOverhead(size int) int { 12 | // See https://github.com/antirez/redis/blob/unstable/src/dict.h 13 | // See the structures dict and dictht 14 | // 2 * (3 unsigned longs + 1 pointer) + int + long + 2 pointers 15 | // 16 | // Additionally, see **table in dictht 17 | // The length of the table is the next nextPower of 2 18 | // When the hashtable is rehashing, another instance of **table is created 19 | // Due to the possibility of rehashing during loading, we calculate the worse 20 | // case in which both tables are allocated, and so multiply 21 | // the size of **table by 1.5 22 | return 4 + 7*sizeOfLong() + 4*sizeOfPointer() + nextPower(size)*sizeOfPointer()*3/2 23 | } 24 | 25 | func sizeOfHashObject(obj *model.HashObject) int { 26 | if obj.GetEncoding() == model.ZipListEncoding { 27 | extra := obj.Extra.(*model.ZiplistDetail) 28 | return extra.RawStringSize 29 | } else if obj.GetEncoding() == model.ListPackEncoding { 30 | extra := obj.Extra.(*model.ListpackDetail) 31 | return extra.RawStringSize 32 | } 33 | size := hashtableOverhead(len(obj.Hash)) 34 | for k, v := range obj.Hash { 35 | size += hashTableEntryOverhead() 36 | size += sizeOfString(k) 37 | size += sizeOfString(unsafeBytes2Str(v)) 38 | } 39 | return size 40 | } 41 | 42 | func sizeOfSetObject(obj *model.SetObject) int { 43 | if obj.GetEncoding() == model.IntSetEncoding { 44 | extra := obj.Extra.(*model.IntsetDetail) 45 | return extra.RawStringSize 46 | } else if obj.GetEncoding() == model.ListPackEncoding { 47 | extra := obj.Extra.(*model.ListpackDetail) 48 | return extra.RawStringSize 49 | } 50 | size := hashtableOverhead(len(obj.Members)) 51 | for _, v := range obj.Members { 52 | size += hashTableEntryOverhead() + sizeOfString(unsafeBytes2Str(v)) 53 | } 54 | return size 55 | } 56 | -------------------------------------------------------------------------------- /memprofiler/list.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import ( 4 | "github.com/hdt3213/rdb/model" 5 | "math" 6 | "strconv" 7 | ) 8 | 9 | func sizeOfListObject(obj *model.ListObject) int { 10 | switch obj.GetEncoding() { 11 | case model.QuickListEncoding: 12 | detail := obj.Extra.(*model.QuicklistDetail) 13 | return sizeOfQuicklist(detail) 14 | case model.ListEncoding: 15 | return sizeOfList(obj.Values) 16 | case model.ZipListEncoding: 17 | return sizeOfZiplist(obj.Values) 18 | case model.QuickList2Encoding: 19 | detail := obj.Extra.(*model.Quicklist2Detail) 20 | return sizeOfQuicklist2(obj.Values, detail) 21 | } 22 | return 0 23 | } 24 | 25 | func zipListIntEntryOverhead(v int) int { 26 | header := 1 27 | size := 0 28 | if v < 12 { 29 | size = 0 30 | } else if v < int(math.Pow(2, 8)) { 31 | size = 1 32 | } else if v < int(math.Pow(2, 16)) { 33 | size = 2 34 | } else if v < int(math.Pow(2, 24)) { 35 | size = 3 36 | } else if v < int(math.Pow(2, 32)) { 37 | size = 4 38 | } else { 39 | size = 8 40 | } 41 | prevLen := 1 42 | if size < 254 { 43 | prevLen = 5 44 | } 45 | return prevLen + header + size 46 | } 47 | 48 | func zipListStrEntryOverhead(v string) int { 49 | size := len(v) 50 | header := 1 51 | if size <= 63 { 52 | header = 1 53 | } else if size <= 1 { 54 | header = 2 55 | } else { 56 | header = 5 57 | } 58 | prevLen := 1 59 | if size < 254 { 60 | prevLen = 5 61 | } 62 | return prevLen + header + size 63 | } 64 | 65 | func sizeOfZiplist(values [][]byte) int { 66 | // See https://github.com/antirez/redis/blob/unstable/src/ziplist.c 67 | // 68 | size := 4 + 4 + 2 + 1 69 | for _, value := range values { 70 | str := unsafeBytes2Str(value) 71 | i, err := strconv.Atoi(str) 72 | if err != nil { // not int 73 | size += zipListStrEntryOverhead(str) 74 | } else { // is int 75 | size += zipListIntEntryOverhead(i) 76 | } 77 | } 78 | return size 79 | } 80 | 81 | func sizeOfQuicklist(detail *model.QuicklistDetail) int { 82 | size := 2*sizeOfPointer() + sizeOfLong() + 2*4 83 | nodeOverhead := 4*sizeOfPointer() + sizeOfLong() + 2*4 84 | size += len(detail.ZiplistStruct) * nodeOverhead 85 | for _, ziplist := range detail.ZiplistStruct { 86 | size += sizeOfZiplist(ziplist) 87 | } 88 | return size 89 | } 90 | 91 | func sizeOfQuicklist2(values [][]byte, detail *model.Quicklist2Detail) int { 92 | size := 2*sizeOfPointer() + 2*sizeOfLong() + 2*4 93 | // https://github.com/CN-annotation-team/redis7.0-chinese-annotated/blob/7.0-cn-annotated/src/quicklist.h#L60 94 | nodeOverhead := 3*sizeOfPointer() + sizeOfLong() + 4 95 | size += nodeOverhead * len(detail.NodeEncodings) 96 | for i, enc := range detail.NodeEncodings { 97 | if enc == model.QuicklistNodeContainerPlain { 98 | size += sizeOfString(unsafeBytes2Str(values[i])) 99 | } else { 100 | // listpack overhead: ... 101 | size += 4 + 2 + 1 102 | for _, s := range detail.ListPackEntrySize[i] { 103 | size += int(s) 104 | } 105 | } 106 | } 107 | return size 108 | } 109 | 110 | func sizeOfList(values [][]byte) int { 111 | // See https://github.com/antirez/redis/blob/unstable/src/adlist.h 112 | // A list has 5 pointers + an unsigned long 113 | size := 5*sizeOfPointer() + sizeOfLong() 114 | // A node has 3 pointers 115 | entryHeadSize := 3 * sizeOfPointer() 116 | size += len(values) * entryHeadSize 117 | // fixme: since redis 4.0, make it compatible with older version 118 | for _, v := range values { 119 | s := unsafeBytes2Str(v) 120 | size += sizeOfString(s) 121 | } 122 | return size 123 | } 124 | -------------------------------------------------------------------------------- /memprofiler/memprofiler.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import ( 4 | "github.com/hdt3213/rdb/model" 5 | ) 6 | 7 | // used to evaluate memory usage 8 | 9 | // RedisMeta stores redis version and architecture 10 | type RedisMeta struct { 11 | Version string 12 | Bits int // 32/64 13 | } 14 | 15 | // SizeOfObject evaluates memory usage of obj 16 | func SizeOfObject(obj model.RedisObject) int { 17 | // todo: memory profile by redis version and architecture 18 | size := topLevelObjectOverhead(obj.GetKey(), obj.GetExpiration() != nil) 19 | switch o := obj.(type) { 20 | case *model.StringObject: 21 | size += sizeOfString(unsafeBytes2Str(o.Value)) 22 | case *model.ListObject: 23 | size += sizeOfListObject(o) 24 | case *model.SetObject: 25 | size += sizeOfSetObject(o) 26 | case *model.HashObject: 27 | size += sizeOfHashObject(o) 28 | case *model.ZSetObject: 29 | size += sizeOfZSetObject(o) 30 | case *model.StreamObject: 31 | size += sizeOfStreamObject(o) 32 | } 33 | return size 34 | } 35 | -------------------------------------------------------------------------------- /memprofiler/stream.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import "github.com/hdt3213/rdb/model" 4 | 5 | func sizeOfStreamObject(obj *model.StreamObject) int { 6 | size := sizeOfPointer()*2 + 8 + 16 + // size of stream struct 7 | sizeOfPointer() + 8*2 + // rax struct 8 | sizeOfStreamRaxTree(len(obj.Entries)) 9 | if obj.Version >= 2 { 10 | size += 16*2 + 8 // size of 2 new streamID and a uint64 11 | } 12 | for _, group := range obj.Groups { 13 | size += sizeOfPointer()*2 + 16 // size of struct streamCG 14 | if obj.Version >= 2 { 15 | size += 8 // size of new field entries_read 16 | } 17 | pendingCount := len(group.Pending) 18 | size += sizeOfStreamRaxTree(pendingCount) + 19 | pendingCount*(sizeOfPointer()+8*2) // streamNACK 20 | for _, consumer := range group.Consumers { 21 | size += sizeOfPointer()*2 + 8 + // streamConsumer 22 | sizeOfString(consumer.Name) + 23 | sizeOfStreamRaxTree(len(consumer.Pending)) 24 | } 25 | } 26 | return size 27 | } 28 | 29 | func sizeOfStreamRaxTree(elementCount int) int { 30 | // This is a very rough estimation. The only alternative to doing an estimation, 31 | // is to fully build a radix tree of similar design, and elementCount the nodes. 32 | // There should be at least as many nodes as there are elements in the radix tree (possibly up to 3 times) 33 | nodeCount := int(float64(elementCount) * 2.5) 34 | // formula for memory estimation copied from Redis's streamRadixTreeMemoryUsage 35 | return 16*elementCount + 4*nodeCount + 30*sizeOfLong()*nodeCount 36 | } 37 | -------------------------------------------------------------------------------- /memprofiler/zset.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import ( 4 | "github.com/hdt3213/rdb/model" 5 | "math" 6 | "math/rand" 7 | ) 8 | 9 | func skipListOverhead(size int) int { 10 | return 2*sizeOfPointer() + hashtableOverhead(size) + (2*sizeOfPointer() + 16) 11 | } 12 | 13 | func skipListEntryOverhead() int { 14 | return hashTableEntryOverhead() + 2*sizeOfPointer() + 8 + int(math.Round(float64(sizeOfPointer()+8)*MathExpectationOfRandomLevel)) 15 | } 16 | 17 | // MathExpectationOfRandomLevel is mathematical expectation of zsetRandomLevel(), used to guarantee the stable results 18 | const MathExpectationOfRandomLevel = 1.33 19 | 20 | func zsetRandomLevel() int { 21 | const maxLevel = 32 22 | const p = 0.25 23 | i := 1 24 | r := rand.Intn(0xFFFF) 25 | for r < 0xFFFF/4 { 26 | i++ 27 | r = rand.Intn(0xFFFF) 28 | if i >= maxLevel { 29 | return maxLevel 30 | } 31 | } 32 | return i 33 | } 34 | 35 | func sizeOfZSetObject(o *model.ZSetObject) int { 36 | if o.GetEncoding() == model.ZipListEncoding { 37 | extra := o.Extra.(*model.ZiplistDetail) 38 | return extra.RawStringSize 39 | } else if o.GetEncoding() == model.ListPackEncoding { 40 | extra := o.Extra.(*model.ListpackDetail) 41 | return extra.RawStringSize 42 | } 43 | size := skipListOverhead(len(o.Entries)) 44 | for _, entry := range o.Entries { 45 | size += sizeOfString(entry.Member) + 8 + skipListEntryOverhead() // size of score is 8 (double) 46 | } 47 | return size 48 | } 49 | -------------------------------------------------------------------------------- /memprofiler/zset_test.go: -------------------------------------------------------------------------------- 1 | package memprofiler 2 | 3 | import "testing" 4 | 5 | func TestRandLevel(t *testing.T) { 6 | size := 100000 7 | sum := 0 8 | for i := 0; i < size; i++ { 9 | sum += zsetRandomLevel() 10 | } 11 | t.Log(float64(sum) / float64(size)) 12 | } 13 | -------------------------------------------------------------------------------- /model/detail.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | // QuicklistDetail stores detail for quicklist 4 | type QuicklistDetail struct { 5 | // ZiplistStruct stores each ziplist within quicklist 6 | ZiplistStruct [][][]byte 7 | } 8 | 9 | // Quicklist2Detail stores detail for quicklist2 10 | type Quicklist2Detail struct { 11 | // NodeEncodings means node of quicklist is QuicklistNodeContainerPlain or QuicklistNodeContainerPacked 12 | NodeEncodings []int 13 | // ListPackEntrySize stores sizes of each listPackEntry is node encoding is QuicklistNodeContainerPacked 14 | ListPackEntrySize [][]uint32 15 | } 16 | 17 | // IntsetDetail stores detail for intset 18 | type IntsetDetail struct { 19 | RawStringSize int 20 | } 21 | 22 | // QuicklistNodeContainerPlain means node of quicklist is normal string 23 | const QuicklistNodeContainerPlain = 1 24 | 25 | // QuicklistNodeContainerPacked means node of quicklist is list pack 26 | const QuicklistNodeContainerPacked = 2 27 | 28 | // ZiplistDetail stores detail for ziplist 29 | type ZiplistDetail struct { 30 | RawStringSize int 31 | } 32 | 33 | // ListpackDetail stores detail for listpack 34 | type ListpackDetail struct { 35 | RawStringSize int 36 | } 37 | -------------------------------------------------------------------------------- /model/stream.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "strconv" 4 | 5 | // StreamObject stores a stream object 6 | type StreamObject struct { 7 | *BaseObject 8 | // Version 2 means is RDB_TYPE_STREAM_LISTPACKS_2, 3 means is RDB_TYPE_STREAM_LISTPACKS_3 9 | Version uint `json:"version,omitempty"` 10 | // Entries stores messages in stream 11 | Entries []*StreamEntry `json:"entries,omitempty"` 12 | // Groups is consumer groups of stream 13 | Groups []*StreamGroup `json:"groups,omitempty"` 14 | // Length is current number of messages inside this stream 15 | Length uint64 `json:"len"` 16 | // LastId is the ID of last entry in stream 17 | LastId *StreamId `json:"lastId"` 18 | // FirstId is the ID of first entry in stream. only valid in V2 19 | FirstId *StreamId `json:"firstId,omitempty"` 20 | // MaxDeletedId is the maximal ID that was deleted in stream. only valid in V2 21 | MaxDeletedId *StreamId `json:"maxDeletedId,omitempty"` 22 | // AddedEntriesCount is count of elements added in all time. only valid in V2 23 | AddedEntriesCount uint64 `json:"addedEntriesCount,omitempty"` 24 | } 25 | 26 | func (obj *StreamObject) GetType() string { 27 | return StreamType 28 | } 29 | 30 | // StreamEntry is a node in the underlying radix tree of redis stream, of type listpacks, which contains several messages. 31 | // There is no need to care about which entry the message belongs to when using it. 32 | type StreamEntry struct { 33 | FirstMsgId *StreamId `json:"firstMsgId"` 34 | Fields []string `json:"fields"` 35 | Msgs []*StreamMessage `json:"msgs"` 36 | } 37 | 38 | // StreamMessage is a message item in stream 39 | type StreamMessage struct { 40 | Id *StreamId `json:"id"` 41 | Fields map[string]string `json:"fields"` 42 | Deleted bool `json:"deleted"` 43 | } 44 | 45 | // StreamId is a 128-bit number composed of a milliseconds time and a sequence counter 46 | type StreamId struct { 47 | Ms uint64 `json:"ms"` 48 | Sequence uint64 `json:"sequence"` 49 | } 50 | 51 | func (id *StreamId) MarshalText() (text []byte, err error) { 52 | txt := strconv.FormatUint(id.Ms, 10) + "-" + strconv.FormatUint(id.Sequence, 10) 53 | return []byte(txt), nil 54 | } 55 | 56 | // StreamGroup is a consumer group 57 | type StreamGroup struct { 58 | Name string `json:"name"` 59 | LastId *StreamId `json:"lastId"` 60 | Pending []*StreamNAck `json:"pending,omitempty"` 61 | Consumers []*StreamConsumer `json:"consumers,omitempty"` 62 | // EntriesRead is only valid in V2. The following comments are from redis to illustrate its use 63 | /*In a perfect world (CG starts at 0-0, no dels, no XGROUP SETID, ...), this is the total number of 64 | group reads. In the real world, the reasoning behind this value is detailed at the top comment of 65 | streamEstimateDistanceFromFirstEverEntry(). */ 66 | EntriesRead uint64 `json:"entriesRead,omitempty"` 67 | } 68 | 69 | // StreamNAck points a pending message 70 | type StreamNAck struct { 71 | Id *StreamId `json:"id"` 72 | DeliveryTime uint64 `json:"deliveryTime"` 73 | DeliveryCount uint64 `json:"deliveryCount"` 74 | } 75 | 76 | // StreamConsumer is a consumer 77 | type StreamConsumer struct { 78 | Name string `json:"name"` 79 | SeenTime uint64 `json:"seenTime"` 80 | Pending []*StreamId `json:"pending,omitempty"` 81 | ActiveTime uint64 `json:"activeTime"` // since rdb11, see https://github.com/redis/redis/pull/11099 82 | } 83 | -------------------------------------------------------------------------------- /parser/portal.go: -------------------------------------------------------------------------------- 1 | // Package parser is interface for parser 2 | package parser 3 | 4 | import ( 5 | "github.com/hdt3213/rdb/core" 6 | "github.com/hdt3213/rdb/model" 7 | ) 8 | 9 | const ( 10 | // StringType is redis string 11 | StringType = model.StringType 12 | // ListType is redis list 13 | ListType = model.ListType 14 | // SetType is redis set 15 | SetType = model.SetType 16 | // HashType is redis hash 17 | HashType = model.HashType 18 | // ZSetType is redis sorted set 19 | ZSetType = model.ZSetType 20 | // AuxType is redis metadata key-value pair 21 | AuxType = model.AuxType 22 | // DBSizeType is for RDB_OPCODE_RESIZEDB 23 | DBSizeType = model.DBSizeType 24 | // StreamType is for redis stream 25 | StreamType = model.StreamType 26 | ) 27 | 28 | type ( 29 | // RedisObject is interface for a redis object 30 | RedisObject = model.RedisObject 31 | // StringObject stores a string object 32 | StringObject = model.StringObject 33 | // ListObject stores a list object 34 | ListObject = model.ListObject 35 | // SetObject stores a set object 36 | SetObject = model.SetObject 37 | // HashObject stores a hash object 38 | HashObject = model.HashObject 39 | // ZSetObject stores a sorted set object 40 | ZSetObject = model.ZSetObject 41 | // StreamObject stores a redis stream object 42 | StreamObject = model.StreamObject 43 | // AuxObject stores redis metadata 44 | AuxObject = model.AuxObject 45 | // DBSizeObject stores db size metadata 46 | DBSizeObject = model.DBSizeObject 47 | ) 48 | 49 | var ( 50 | // NewDecoder creates a new RDB decoder 51 | NewDecoder = core.NewDecoder 52 | ) 53 | -------------------------------------------------------------------------------- /tree.csv: -------------------------------------------------------------------------------- 1 | database,prefix,size,size_readable,element_count 2 | 0,Post:,1170456184,1.1G,701821 3 | 0,Stat:,405483812,386.7M,3759832 4 | 0,Stat:Post:,291081520,277.6M,2775043 5 | 0,User:,241572272,230.4M,265810 6 | 0,Topic:,171146778,163.2M,694498 7 | 0,Topic:Post:,163635096,156.1M,693758 8 | 0,Stat:Post:View,133201208,127M,1387516 9 | 0,Stat:User:,114395916,109.1M,984724 10 | 0,UTL:,87147529,83.1M,43093 11 | 0,Stat:Post:Comment:,80178504,76.5M,693758 12 | 0,Stat:Post:Like:,77701688,74.1M,693768 13 | --------------------------------------------------------------------------------