├── .gitignore ├── go.mod ├── rdb ├── obj_module.go ├── key.go ├── parser_test.go ├── obj_string.go ├── obj_string_test.go ├── obj_list_test.go ├── streamer.go ├── intset.go ├── obj_set_test.go ├── obj_zset_test.go ├── event.go ├── obj_hash_test.go ├── obj_set.go ├── obj_hash.go ├── obj_list.go ├── ziplist.go ├── obj_zset.go ├── listpack.go ├── reader.go ├── parser.go └── obj_stream.go ├── example ├── client │ └── main.go ├── replica │ └── main.go ├── parserdb │ └── main.go └── parsereplica │ └── main.go ├── client ├── client_test.go └── client.go ├── resp ├── write.go └── read.go ├── LICENSE ├── connection └── conn.go ├── go.sum ├── replica ├── replica_test.go └── replica.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vczyh/redis-lib 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 // indirect 7 | github.com/pmezard/go-difflib v1.0.0 // indirect 8 | github.com/stretchr/objx v0.5.0 // indirect 9 | github.com/stretchr/testify v1.8.4 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /rdb/obj_module.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import "fmt" 4 | 5 | type ModuleObjectEvent struct{} 6 | 7 | func parseModule(r *rdbReader, valueType byte) (*ModuleObjectEvent, error) { 8 | // TODO 9 | moduleId, err := r.GetLengthUInt64() 10 | if err != nil { 11 | return nil, err 12 | } 13 | _ = moduleId 14 | 15 | return nil, fmt.Errorf("unsupport module type") 16 | } 17 | -------------------------------------------------------------------------------- /rdb/key.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type RedisKey struct { 9 | DbId int 10 | 11 | Key string 12 | 13 | // millisecond 14 | expireAt int64 15 | } 16 | 17 | func (k RedisKey) debugKey() { 18 | fmt.Printf("DbId: %d\n", k.DbId) 19 | fmt.Printf("Key: %s\n", k.Key) 20 | if k.expireAt != -1 { 21 | fmt.Printf("Expire At: %s\n", time.UnixMicro(k.expireAt*1e3)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /example/client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/vczyh/redis-lib/client" 4 | 5 | func main() { 6 | c, err := client.NewClient(&client.Config{ 7 | Host: "127.0.0.1", 8 | Port: 26379, 9 | Username: "", 10 | Password: "123", 11 | }) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | if err := c.Auth(); err != nil { 17 | panic(err) 18 | } 19 | 20 | if err = c.Ping(); err != nil { 21 | panic(err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "testing" 4 | 5 | func TestClient_Auth(t *testing.T) { 6 | c, err := NewClient(&Config{ 7 | Host: "127.0.0.1", 8 | Port: 26379, 9 | Username: "", 10 | Password: "123", 11 | }) 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | 16 | if err := c.Auth(); err != nil { 17 | t.Fatal(err) 18 | } 19 | 20 | if err = c.Ping(); err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | //if err = c.SyncWithMaster(); err != nil { 25 | // t.Fatal(err) 26 | //} 27 | 28 | } 29 | -------------------------------------------------------------------------------- /rdb/parser_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestParser_Parse(t *testing.T) { 8 | //p, err := NewParser("/tmp/test222.rdb") 9 | p, err := NewParser("/Users/zhangyuheng/workspace/mine/container-images/redis/redis/data/source/dump.rdb") 10 | if err != nil { 11 | t.Fatal(err) 12 | } 13 | 14 | s, err := p.Parse() 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | 19 | for s.HasNext() { 20 | e := s.Next() 21 | e.Event.Debug() 22 | } 23 | 24 | if err := s.Err(); err != nil { 25 | t.Fatal(err) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /example/replica/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/vczyh/redis-lib/replica" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | r, err := replica.NewReplica(&replica.Config{ 10 | MasterIP: "127.0.0.1", 11 | MasterPort: 26379, 12 | MasterUser: "", 13 | MasterPassword: "123", 14 | MasterReplicaOffset: 67528, 15 | RdbWriter: os.Stdout, 16 | AofWriter: os.Stdout, 17 | }) 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | if err := r.SyncWithMaster(); err != nil { 23 | panic(err) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rdb/obj_string.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import "fmt" 4 | 5 | type StringObjectEvent struct { 6 | RedisKey 7 | 8 | Value string 9 | } 10 | 11 | func parseString(key RedisKey, r *rdbReader) (*StringObjectEvent, error) { 12 | value, err := r.GetLengthString() 13 | if err != nil { 14 | return nil, err 15 | } 16 | return &StringObjectEvent{ 17 | RedisKey: key, 18 | Value: value, 19 | }, nil 20 | } 21 | 22 | func (e *StringObjectEvent) Debug() { 23 | fmt.Printf("=== StringObjectEvent ===\n") 24 | e.debugKey() 25 | fmt.Printf("Value: %s\n", e.Value) 26 | fmt.Printf("\n") 27 | } 28 | -------------------------------------------------------------------------------- /rdb/obj_string_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestParseString(t *testing.T) { 10 | b := []byte{0x0A, 0x6B, 0x65, 0x79, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, 0x04, 0x61, 0x61, 0x61, 0x61} 11 | r := newRdbReader(bytes.NewReader(b)) 12 | 13 | key, err := r.GetLengthString() 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | assert.Equal(t, "key_string", key) 18 | 19 | e, err := parseString(RedisKey{}, r) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | assert.Equal(t, "aaaa", e.Value) 24 | } 25 | -------------------------------------------------------------------------------- /example/parserdb/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/vczyh/redis-lib/rdb" 5 | ) 6 | 7 | func main() { 8 | p, err := rdb.NewParser("/tmp/rdb_test.rdb") 9 | if err != nil { 10 | panic(err) 11 | } 12 | 13 | s, err := p.Parse() 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | for s.HasNext() { 19 | e := s.Next() 20 | 21 | switch e.EventType { 22 | case rdb.EventTypeVersion: 23 | e.Event.Debug() 24 | case rdb.EventTypeStringObject: 25 | e.Event.Debug() 26 | case rdb.EventTypeSetObject: 27 | e.Event.Debug() 28 | } 29 | } 30 | 31 | if err := s.Err(); err != nil { 32 | panic(err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /rdb/obj_list_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestParseListWithListQuickList2(t *testing.T) { 10 | b := []byte{0x08, 0x6B, 0x65, 0x79, 0x5F, 0x6C, 0x69, 0x73, 0x74, 11 | 0x01, 0x02, 0x11, 0x11, 0x00, 0x00, 0x00, 0x02, 0x00, 0x83, 0x62, 0x61, 0x72, 0x04, 0x83, 0x66, 0x6F, 0x6F, 0x04, 0xFF, 12 | } 13 | r := newRdbReader(bytes.NewReader(b)) 14 | 15 | key, err := r.GetLengthString() 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | assert.Equal(t, "key_list", key) 20 | 21 | e, err := parseList(RedisKey{}, r, rdbTypeListQuickList2) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | assert.Equal(t, "bar", e.Elements[0]) 26 | assert.Equal(t, "foo", e.Elements[1]) 27 | } 28 | -------------------------------------------------------------------------------- /rdb/streamer.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | type EventStreamer struct { 4 | c chan *eventWrapper 5 | o *RedisRdbEvent 6 | err error 7 | } 8 | 9 | type eventWrapper struct { 10 | e *RedisRdbEvent 11 | err error 12 | } 13 | 14 | func newEventStreamer(c chan *eventWrapper) *EventStreamer { 15 | return &EventStreamer{c: c} 16 | } 17 | 18 | func (s *EventStreamer) HasNext() bool { 19 | if s.err != nil { 20 | return false 21 | } 22 | 23 | w, ok := <-s.c 24 | if !ok { 25 | return false 26 | } 27 | if err := w.err; err != nil { 28 | s.err = err 29 | return false 30 | } 31 | s.o = w.e 32 | return true 33 | } 34 | 35 | func (s *EventStreamer) Next() *RedisRdbEvent { 36 | return s.o 37 | } 38 | 39 | func (s *EventStreamer) Err() error { 40 | return s.err 41 | } 42 | -------------------------------------------------------------------------------- /resp/write.go: -------------------------------------------------------------------------------- 1 | package resp 2 | 3 | import ( 4 | "io" 5 | "strconv" 6 | ) 7 | 8 | func WriteBulkString(w io.Writer, str string) error { 9 | data := []byte{ 10 | DataTypeBulkString, 11 | } 12 | data = append(data, strconv.Itoa(len(str))...) 13 | data = append(data, Separator...) 14 | data = append(data, str...) 15 | data = append(data, Separator...) 16 | if _, err := w.Write(data); err != nil { 17 | return err 18 | } 19 | return nil 20 | } 21 | 22 | func WriteArray(w io.Writer, args ...string) error { 23 | data := []byte{ 24 | DataTypeArray, 25 | } 26 | data = append(data, strconv.Itoa(len(args))...) 27 | data = append(data, Separator...) 28 | if _, err := w.Write(data); err != nil { 29 | return err 30 | } 31 | 32 | for _, arg := range args { 33 | if err := WriteBulkString(w, arg); err != nil { 34 | return err 35 | } 36 | } 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /rdb/intset.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import "bytes" 4 | 5 | func parseIntSet(r *rdbReader) ([]int64, error) { 6 | b, err := r.GetLengthBytes() 7 | if err != nil { 8 | return nil, err 9 | } 10 | r = newRdbReader(bytes.NewReader(b)) 11 | 12 | encoding, err := r.GetLUint32() 13 | if err != nil { 14 | return nil, err 15 | } 16 | length, err := r.GetLUint32() 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | members := make([]int64, length) 22 | for i := 0; i < int(length); i++ { 23 | switch encoding { 24 | case 2: 25 | v, err := r.GetLUint16() 26 | if err != nil { 27 | return nil, err 28 | } 29 | members[i] = int64(v) 30 | case 4: 31 | v, err := r.GetLUint32() 32 | if err != nil { 33 | return nil, err 34 | } 35 | members[i] = int64(v) 36 | case 8: 37 | v, err := r.GetLUint16() 38 | if err != nil { 39 | return nil, err 40 | } 41 | members[i] = int64(v) 42 | } 43 | } 44 | 45 | return members, nil 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Zhang,Yuheng 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /example/parsereplica/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/vczyh/redis-lib/rdb" 5 | "github.com/vczyh/redis-lib/replica" 6 | "io" 7 | "os" 8 | ) 9 | 10 | func main() { 11 | rdbReader, rdbWriter := io.Pipe() 12 | 13 | r, err := replica.NewReplica(&replica.Config{ 14 | MasterIP: "127.0.0.1", 15 | MasterPort: 26379, 16 | MasterUser: "", 17 | MasterPassword: "123", 18 | RdbWriter: rdbWriter, 19 | AofWriter: os.Stdout, 20 | //ContinueAfterFullSync: true, 21 | 22 | MasterReplicaId: "aaa", 23 | MasterReplicaOffset: 22, 24 | ContinueIfPartialFailed: true, 25 | }) 26 | if err != nil { 27 | panic(err) 28 | } 29 | 30 | go func() { 31 | // Parse rbd from master. 32 | if err := parseRdb(rdbReader); err != nil { 33 | panic(err) 34 | } 35 | }() 36 | 37 | if err = r.SyncWithMaster(); err != nil { 38 | panic(err) 39 | } 40 | } 41 | func parseRdb(r io.Reader) error { 42 | p, err := rdb.NewReaderParser(r) 43 | if err != nil { 44 | return err 45 | } 46 | s, err := p.Parse() 47 | if err != nil { 48 | return err 49 | } 50 | for s.HasNext() { 51 | e := s.Next() 52 | e.Event.Debug() 53 | } 54 | return s.Err() 55 | } 56 | -------------------------------------------------------------------------------- /rdb/obj_set_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestParseSetWithListPack(t *testing.T) { 10 | b := []byte{ 11 | 0x07, 0x6B, 0x65, 0x79, 0x5F, 0x73, 0x65, 0x74, 0x0D, 0x0D, 0x00, 0x00, 0x00, 0x02, 0x00, 0x81, 0x61, 0x02, 0x81, 0x62, 0x02, 0xFF, 12 | } 13 | r := newRdbReader(bytes.NewReader(b)) 14 | 15 | key, err := r.GetLengthString() 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | assert.Equal(t, "key_set", key) 20 | 21 | e, err := parseSet(RedisKey{}, r, rdbTypeSetListPack) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | assert.Equal(t, 2, len(e.Members)) 26 | assert.Contains(t, e.Members, "a") 27 | assert.Contains(t, e.Members, "b") 28 | } 29 | 30 | func TestParseSetWithIntSet(t *testing.T) { 31 | b := []byte{0x07, 0x6B, 0x65, 0x79, 0x5F, 0x73, 0x65, 0x74, 0x0C, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x64, 0x00, 0xC8, 0x00} 32 | r := newRdbReader(bytes.NewReader(b)) 33 | 34 | key, err := r.GetLengthString() 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | assert.Equal(t, "key_set", key) 39 | 40 | e, err := parseSet(RedisKey{}, r, rdbTypeIntSet) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | assert.Equal(t, 2, len(e.Members)) 45 | assert.Contains(t, e.Members, "100") 46 | assert.Contains(t, e.Members, "200") 47 | } 48 | -------------------------------------------------------------------------------- /connection/conn.go: -------------------------------------------------------------------------------- 1 | package connection 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/vczyh/redis-lib/resp" 7 | "net" 8 | ) 9 | 10 | type Conn struct { 11 | nc net.Conn 12 | *bufio.Reader 13 | *bufio.Writer 14 | } 15 | 16 | func NewConn(nc net.Conn) (*Conn, error) { 17 | c := &Conn{ 18 | nc: nc, 19 | Reader: bufio.NewReader(nc), 20 | Writer: bufio.NewWriter(nc), 21 | } 22 | return c, nil 23 | } 24 | 25 | func (c *Conn) ReadString() (string, error) { 26 | return resp.ReadString(c.Reader) 27 | } 28 | 29 | func (c *Conn) SkipOk() error { 30 | v, err := c.ReadString() 31 | if err != nil { 32 | return err 33 | } 34 | if v != "OK" { 35 | return fmt.Errorf("response not OK") 36 | } 37 | return nil 38 | } 39 | 40 | func (c *Conn) WriteCommand(command string, args ...string) error { 41 | array := []string{command} 42 | array = append(array, args...) 43 | if err := resp.WriteArray(c, array...); err != nil { 44 | return err 45 | } 46 | return c.Flush() 47 | } 48 | 49 | func (c *Conn) WriteArray(args ...string) error { 50 | if err := resp.WriteArray(c, args...); err != nil { 51 | return err 52 | } 53 | return c.Flush() 54 | } 55 | 56 | func (c *Conn) WriteBulkString(str string) error { 57 | if err := resp.WriteBulkString(c, str); err != nil { 58 | return err 59 | } 60 | return c.Flush() 61 | } 62 | 63 | func (c *Conn) ReadData() ([]byte, error) { 64 | return resp.ReadData(c.Reader) 65 | } 66 | 67 | func (c *Conn) Close() error { 68 | return c.nc.Close() 69 | } 70 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 8 | github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= 9 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 10 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 11 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 12 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 13 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 16 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 17 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 18 | -------------------------------------------------------------------------------- /replica/replica_test.go: -------------------------------------------------------------------------------- 1 | package replica 2 | 3 | import ( 4 | "github.com/vczyh/redis-lib/rdb" 5 | "io" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestNewReplica_FullSync(t *testing.T) { 12 | r, err := NewReplica(&Config{ 13 | MasterIP: "127.0.0.1", 14 | MasterPort: 46379, 15 | MasterUser: "", 16 | MasterPassword: "123", 17 | RdbWriter: os.Stdout, 18 | AofWriter: os.Stdout, 19 | //ContinueAfterFullSync: true, 20 | }) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | if err := r.SyncWithMaster(); err != nil { 25 | t.Fatal(err) 26 | } 27 | } 28 | 29 | func TestReplica_Sync(t *testing.T) { 30 | rdbReader, rdbWriter := io.Pipe() 31 | 32 | r, err := NewReplica(&Config{ 33 | MasterIP: "127.0.0.1", 34 | MasterPort: 46379, 35 | MasterUser: "", 36 | MasterPassword: "123", 37 | RdbWriter: rdbWriter, 38 | AofWriter: os.Stdout, 39 | //ContinueAfterFullSync: true, 40 | }) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | go func() { 46 | if err := parseRdb(rdbReader); err != nil { 47 | t.Error(err) 48 | return 49 | } 50 | }() 51 | 52 | if err = r.SyncWithMaster(); err != nil { 53 | t.Fatal(err) 54 | } 55 | 56 | time.Sleep(1 * time.Second) 57 | } 58 | 59 | func parseRdb(r io.Reader) error { 60 | p, err := rdb.NewReaderParser(r) 61 | if err != nil { 62 | return err 63 | } 64 | s, err := p.Parse() 65 | if err != nil { 66 | return err 67 | } 68 | for s.HasNext() { 69 | e := s.Next() 70 | e.Event.Debug() 71 | } 72 | return s.Err() 73 | } 74 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "github.com/vczyh/redis-lib/connection" 6 | "net" 7 | "strconv" 8 | ) 9 | 10 | type Client struct { 11 | conn *connection.Conn 12 | nc net.Conn 13 | config *Config 14 | } 15 | 16 | type Config struct { 17 | Host string 18 | Port int 19 | Username string 20 | Password string 21 | } 22 | 23 | func NewClient(config *Config) (*Client, error) { 24 | c := &Client{ 25 | config: config, 26 | } 27 | nc, err := net.Dial("tcp", net.JoinHostPort(config.Host, strconv.Itoa(config.Port))) 28 | if err != nil { 29 | return nil, err 30 | } 31 | c.nc = nc 32 | 33 | conn, err := connection.NewConn(nc) 34 | if err != nil { 35 | return nil, err 36 | } 37 | c.conn = conn 38 | 39 | return c, nil 40 | } 41 | 42 | func (c *Client) Auth() error { 43 | var args []string 44 | if c.config.Username != "" { 45 | args = append(args, c.config.Username) 46 | } 47 | args = append(args, c.config.Password) 48 | 49 | if err := c.conn.WriteCommand("AUTH", args...); err != nil { 50 | return err 51 | } 52 | return c.conn.SkipOk() 53 | } 54 | 55 | func (c *Client) Ping() error { 56 | if err := c.conn.WriteCommand("PING"); err != nil { 57 | return err 58 | } 59 | res, err := c.conn.ReadString() 60 | if err != nil { 61 | return err 62 | } 63 | if res != "PONG" { 64 | return fmt.Errorf("PING response not PONG: %s", res) 65 | } 66 | return nil 67 | } 68 | 69 | func (c *Client) Conn() *connection.Conn { 70 | return c.conn 71 | } 72 | 73 | func (c *Client) Close() error { 74 | return c.conn.Close() 75 | } 76 | -------------------------------------------------------------------------------- /rdb/obj_zset_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestParseZSetWithListPack(t *testing.T) { 10 | b := []byte{0x08, 0x6B, 0x65, 0x79, 0x5F, 0x7A, 11 | 0x73, 0x65, 0x74, 0x18, 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x81, 12 | 0x61, 0x02, 0x83, 0x30, 0x2E, 0x32, 0x04, 0x81, 0x62, 0x02, 0x84, 0x30, 0x2E, 13 | 0x34, 0x35, 0x05, 0xFF, 14 | } 15 | r := newRdbReader(bytes.NewReader(b)) 16 | 17 | key, err := r.GetLengthString() 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | assert.Equal(t, "key_zset", key) 22 | 23 | e, err := parseZSet(RedisKey{}, r, rdbTypeZSetListPack) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | assert.Equal(t, 2, len(e.Members)) 28 | 29 | assert.Equal(t, "a", e.Members[0].Value) 30 | assert.Equal(t, 0.2, e.Members[0].Score) 31 | 32 | assert.Equal(t, "b", e.Members[1].Value) 33 | assert.Equal(t, 0.45, e.Members[1].Score) 34 | } 35 | 36 | func TestParseZSetWithZSet2(t *testing.T) { 37 | b := []byte{0x08, 0x6B, 0x65, 0x79, 0x5F, 0x7A, 0x73, 0x65, 0x74, 38 | 0x02, 0x01, 0x62, 0xCD, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xDC, 39 | 0x3F, 0x01, 0x61, 0x9A, 0x99, 0x99, 0x99, 0x99, 0x99, 0xC9, 0x3F, 40 | } 41 | r := newRdbReader(bytes.NewReader(b)) 42 | 43 | key, err := r.GetLengthString() 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | assert.Equal(t, "key_zset", key) 48 | 49 | e, err := parseZSet(RedisKey{}, r, rdbTypeZSet2) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | assert.Equal(t, 2, len(e.Members)) 54 | 55 | assert.Equal(t, "b", e.Members[0].Value) 56 | assert.Equal(t, 0.45, e.Members[0].Score) 57 | 58 | assert.Equal(t, "a", e.Members[1].Value) 59 | assert.Equal(t, 0.2, e.Members[1].Score) 60 | } 61 | -------------------------------------------------------------------------------- /rdb/event.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import "fmt" 4 | 5 | type EventType uint8 6 | 7 | const ( 8 | EventTypeMagicNumber EventType = iota 9 | EventTypeVersion 10 | EventTypeAuxField 11 | EventTypeSelectDb 12 | EventTypeResizeDb 13 | EventTypeStringObject 14 | EventTypeListObject 15 | EventTypeSetObject 16 | EventTypeZSetObject 17 | EventTypeHashObject 18 | EventTypeStreamObject 19 | ) 20 | 21 | type RedisRdbEvent struct { 22 | EventType EventType 23 | Event Event 24 | } 25 | 26 | type Event interface { 27 | Debug() 28 | } 29 | 30 | type MagicNumberEvent struct { 31 | MagicNumber []byte 32 | } 33 | 34 | func (e *MagicNumberEvent) Debug() { 35 | fmt.Printf("=== MagicNumberEvent ===\n") 36 | fmt.Printf("%s\n", string(e.MagicNumber)) 37 | fmt.Printf("\n") 38 | } 39 | 40 | type VersionEvent struct { 41 | Version int 42 | } 43 | 44 | func (e *VersionEvent) Debug() { 45 | fmt.Printf("=== VersionEvent ===\n") 46 | fmt.Printf("%d\n", e.Version) 47 | fmt.Printf("\n") 48 | } 49 | 50 | type AuxFieldEvent struct { 51 | Filed string 52 | Value string 53 | } 54 | 55 | func (e *AuxFieldEvent) Debug() { 56 | fmt.Printf("=== AuxFieldEvent ===\n") 57 | fmt.Printf("%s: %s\n", e.Filed, e.Value) 58 | fmt.Printf("\n") 59 | } 60 | 61 | type SelectDbEvent struct { 62 | Db int 63 | } 64 | 65 | func (e *SelectDbEvent) Debug() { 66 | fmt.Printf("=== SelectDbEvent ===\n") 67 | fmt.Printf("Database: %d\n", e.Db) 68 | fmt.Printf("\n") 69 | } 70 | 71 | type ResizeDbEvent struct { 72 | dbSize int 73 | dbExpireSize int 74 | } 75 | 76 | func (e *ResizeDbEvent) Debug() { 77 | fmt.Printf("=== ResizeDbEvent ===\n") 78 | fmt.Printf("Database size: %d\n", e.dbSize) 79 | fmt.Printf("Database expire size: %d\n", e.dbExpireSize) 80 | fmt.Printf("\n") 81 | } 82 | -------------------------------------------------------------------------------- /rdb/obj_hash_test.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestParseHashWithListPack(t *testing.T) { 10 | b := []byte{0x08, 0x6B, 0x65, 0x79, 0x5F, 0x68, 0x61, 0x73, 0x68, 0x1D, 0x1D, 0x00, 0x00, 0x00, 11 | 0x04, 0x00, 0x84, 0x6B, 0x65, 0x79, 0x31, 0x05, 0x83, 0x66, 12 | 0x6F, 0x6F, 0x04, 0x84, 0x6B, 0x65, 0x79, 0x32, 0x05, 0x83, 0x62, 0x61, 0x72, 0x04, 0xFF, 13 | } 14 | r := newRdbReader(bytes.NewReader(b)) 15 | 16 | key, err := r.GetLengthString() 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | assert.Equal(t, "key_hash", key) 21 | 22 | e, err := parseHash(RedisKey{}, r, rdbTypeHashListPack) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | for _, field := range e.Fields { 27 | switch field.Field { 28 | case "key1": 29 | assert.Equal(t, "foo", field.Value) 30 | case "key2": 31 | assert.Equal(t, "bar", field.Value) 32 | default: 33 | t.Fatal("hash container unexpected field") 34 | } 35 | } 36 | } 37 | 38 | func TestParseHashWithHash(t *testing.T) { 39 | b := []byte{0x08, 0x6B, 0x65, 0x79, 0x5F, 0x68, 0x61, 0x73, 0x68, 0x02, 0x04, 0x6B, 0x65, 0x79, 40 | 0x32, 0x03, 0x62, 0x61, 0x72, 0x04, 0x6B, 0x65, 0x79, 0x31, 0x03, 0x66, 0x6F, 0x6F, 41 | } 42 | r := newRdbReader(bytes.NewReader(b)) 43 | 44 | key, err := r.GetLengthString() 45 | if err != nil { 46 | t.Fatal(err) 47 | } 48 | assert.Equal(t, "key_hash", key) 49 | 50 | e, err := parseHash(RedisKey{}, r, rdbTypeHash) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | for _, field := range e.Fields { 55 | switch field.Field { 56 | case "key1": 57 | assert.Equal(t, "foo", field.Value) 58 | case "key2": 59 | assert.Equal(t, "bar", field.Value) 60 | default: 61 | t.Fatal("hash container unexpected field") 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /rdb/obj_set.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | type SetObjectEvent struct { 9 | RedisKey 10 | 11 | Members []string 12 | } 13 | 14 | func (e *SetObjectEvent) Debug() { 15 | fmt.Printf("=== SetObjectEvent ===\n") 16 | e.debugKey() 17 | fmt.Printf("Size: %d\n", len(e.Members)) 18 | fmt.Printf("Members:\n") 19 | for _, member := range e.Members { 20 | fmt.Printf("\t%s\n", member) 21 | } 22 | fmt.Printf("\n") 23 | } 24 | 25 | func parseSet(key RedisKey, r *rdbReader, valueType byte) (*SetObjectEvent, error) { 26 | set := &SetObjectEvent{ 27 | RedisKey: key, 28 | } 29 | switch valueType { 30 | case rdbTypeSet: 31 | return parseSet0(r, set) 32 | case rdbTypeSetListPack: 33 | return parseSetInListPack(r, set) 34 | case rdbTypeIntSet: 35 | return parseSetInIntSet(r, set) 36 | default: 37 | return nil, fmt.Errorf("unsupported set value type: %x", valueType) 38 | } 39 | } 40 | 41 | func parseSet0(r *rdbReader, set *SetObjectEvent) (*SetObjectEvent, error) { 42 | size, err := r.GetLengthInt() 43 | if err != nil { 44 | return nil, err 45 | } 46 | members := make([]string, size) 47 | for i := 0; i < size; i++ { 48 | item, err := r.GetLengthString() 49 | if err != nil { 50 | return nil, err 51 | } 52 | members[i] = item 53 | } 54 | set.Members = members 55 | 56 | return set, nil 57 | } 58 | 59 | func parseSetInListPack(r *rdbReader, set *SetObjectEvent) (*SetObjectEvent, error) { 60 | list, err := parseListPack(r) 61 | if err != nil { 62 | return nil, err 63 | } 64 | set.Members = append(set.Members, list...) 65 | return set, nil 66 | } 67 | 68 | func parseSetInIntSet(r *rdbReader, set *SetObjectEvent) (*SetObjectEvent, error) { 69 | members, err := parseIntSet(r) 70 | if err != nil { 71 | return nil, err 72 | } 73 | elements := make([]string, len(members)) 74 | for i, member := range members { 75 | elements[i] = strconv.FormatInt(member, 10) 76 | } 77 | set.Members = elements 78 | 79 | return set, nil 80 | } 81 | -------------------------------------------------------------------------------- /resp/read.go: -------------------------------------------------------------------------------- 1 | package resp 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "strconv" 9 | ) 10 | 11 | // https://redis.io/docs/reference/protocol-spec/ 12 | const ( 13 | DataTypeSimpleString = '+' // RESP2 14 | DataTypeSimpleError = '-' // RESP2 15 | DataTypeInteger = ':' // RESP2 16 | DataTypeBulkString = '$' // RESP2 17 | DataTypeArray = '*' // RESP2 18 | DataTypeNull = '_' // RESP3 19 | DataTypeBoolean = '#' // RESP3 20 | DataTypeDouble = ',' // RESP3 21 | DataTypeBigNumbers = '(' // RESP3 22 | DataTypeBulkError = '!' // RESP3 23 | DataTypeVerbatimString = '=' // RESP3 24 | DataTypeMap = '%' // RESP3 25 | DataTypeSet = '~' // RESP3 26 | DataTypePush = '>' // RESP3 27 | ) 28 | 29 | var ( 30 | Separator = []byte{'\r', '\n'} 31 | ) 32 | 33 | func ReadString(r *bufio.Reader) (string, error) { 34 | data, err := ReadData(r) 35 | if err != nil { 36 | return "", err 37 | } 38 | 39 | switch data[0] { 40 | case DataTypeNull: 41 | return "", nil 42 | case DataTypeSimpleString: 43 | return string(data[1:]), nil 44 | case DataTypeBulkString: 45 | size, err := getLen(data) 46 | if err != nil { 47 | return "", err 48 | } 49 | v := make([]byte, size+2) 50 | if _, err := io.ReadFull(r, v); err != nil { 51 | return "", err 52 | } 53 | return string(v[:size]), nil 54 | } 55 | 56 | return "", fmt.Errorf("not string type: %d", data[0]) 57 | } 58 | 59 | func ReadSeparatedBytes(r *bufio.Reader) ([]byte, error) { 60 | data, err := r.ReadBytes('\r') 61 | if err != nil { 62 | return nil, err 63 | } 64 | if _, err := r.Discard(1); err != nil { 65 | return nil, err 66 | } 67 | return data[:len(data)-1], nil 68 | } 69 | 70 | func ReadData(r *bufio.Reader) ([]byte, error) { 71 | line, err := ReadSeparatedBytes(r) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | switch line[0] { 77 | case DataTypeSimpleError: 78 | return nil, errors.New(string(line[1:])) 79 | case DataTypeBulkError: 80 | line, err := ReadSeparatedBytes(r) 81 | l, err := getLen(line) 82 | if err != nil { 83 | return nil, err 84 | } 85 | data := make([]byte, l+2) 86 | if _, err := io.ReadFull(r, data); err != nil { 87 | return nil, err 88 | } 89 | return nil, errors.New(string(data[:l])) 90 | } 91 | 92 | return line, nil 93 | } 94 | 95 | func getLen(data []byte) (int, error) { 96 | return strconv.Atoi(string(data[1:])) 97 | } 98 | -------------------------------------------------------------------------------- /rdb/obj_hash.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type HashObjectEvent struct { 8 | RedisKey 9 | 10 | Fields []HashField 11 | } 12 | 13 | type HashField struct { 14 | Field string 15 | Value string 16 | } 17 | 18 | func (e *HashObjectEvent) Debug() { 19 | fmt.Printf("=== HashObjectEvent ===\n") 20 | e.debugKey() 21 | fmt.Printf("Size: %d\n", len(e.Fields)) 22 | fmt.Printf("Fields:\n") 23 | for i := range e.Fields { 24 | field := e.Fields[i] 25 | fmt.Printf("\t%s = %s\n", field.Field, field.Value) 26 | } 27 | fmt.Printf("\n") 28 | } 29 | 30 | func parseHash(key RedisKey, r *rdbReader, valueType byte) (*HashObjectEvent, error) { 31 | h := &HashObjectEvent{ 32 | RedisKey: key, 33 | } 34 | switch valueType { 35 | case rdbTypeHashZipList: 36 | return parseHashInZipList(r, h) 37 | case rdbTypeHashListPack: 38 | return parseHashInListPack(r, h) 39 | case rdbTypeHash: 40 | return parseHash0(r, h) 41 | default: 42 | return nil, fmt.Errorf("unsupported hash value type: %x", valueType) 43 | } 44 | } 45 | 46 | func parseHash0(r *rdbReader, h *HashObjectEvent) (*HashObjectEvent, error) { 47 | dictSize, err := r.GetLengthInt() 48 | if err != nil { 49 | return nil, err 50 | } 51 | fields := make([]HashField, dictSize) 52 | for i := 0; i < dictSize; i++ { 53 | field, err := r.GetLengthString() 54 | if err != nil { 55 | return nil, err 56 | } 57 | value, err := r.GetLengthString() 58 | if err != nil { 59 | return nil, err 60 | } 61 | fields[i] = HashField{ 62 | Field: field, 63 | Value: value, 64 | } 65 | } 66 | h.Fields = fields 67 | 68 | return h, err 69 | } 70 | 71 | func parseHashInZipList(r *rdbReader, h *HashObjectEvent) (*HashObjectEvent, error) { 72 | list, err := parseZipList(r) 73 | if err != nil { 74 | return nil, err 75 | } 76 | if len(list)%2 != 0 { 77 | return nil, fmt.Errorf("error length for ziplist: %d", len(list)) 78 | } 79 | 80 | fields := make([]HashField, len(list)/2) 81 | for i := 0; i < len(list)/2; i++ { 82 | field := list[i*2] 83 | value := list[i*2+1] 84 | fields[i] = HashField{ 85 | Field: field, 86 | Value: value, 87 | } 88 | } 89 | h.Fields = fields 90 | 91 | return h, nil 92 | } 93 | 94 | func parseHashInListPack(r *rdbReader, h *HashObjectEvent) (*HashObjectEvent, error) { 95 | list, err := parseListPack(r) 96 | if err != nil { 97 | return nil, err 98 | } 99 | if len(list)%2 != 0 { 100 | return nil, fmt.Errorf("error length for ziplist: %d", len(list)) 101 | } 102 | 103 | fields := make([]HashField, len(list)/2) 104 | for i := 0; i < len(list)/2; i++ { 105 | field := list[i*2] 106 | value := list[i*2+1] 107 | fields[i] = HashField{ 108 | Field: field, 109 | Value: value, 110 | } 111 | } 112 | h.Fields = fields 113 | 114 | return h, nil 115 | } 116 | -------------------------------------------------------------------------------- /rdb/obj_list.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type ListObjectEvent struct { 8 | RedisKey 9 | 10 | Elements []string 11 | } 12 | 13 | func (e *ListObjectEvent) Debug() { 14 | fmt.Printf("=== ListObjectEvent ===\n") 15 | e.debugKey() 16 | fmt.Printf("Size: %d\n", len(e.Elements)) 17 | fmt.Printf("Elements:\n") 18 | for _, ele := range e.Elements { 19 | fmt.Printf("\t%s\n", ele) 20 | } 21 | fmt.Printf("\n") 22 | } 23 | 24 | func parseList(key RedisKey, r *rdbReader, valueType byte) (*ListObjectEvent, error) { 25 | list := &ListObjectEvent{ 26 | RedisKey: key, 27 | } 28 | switch valueType { 29 | case rdbTypeList: 30 | // TODO not tested 31 | return parseList0(r, list) 32 | case rdbTypeZipList: 33 | // TODO not tested 34 | return parseListInZipList(r, list) 35 | case rdbTypeListQuickList: 36 | return parseListInQuickList(r, list) 37 | case rdbTypeListQuickList2: 38 | return parseListInQuickList2(r, list) 39 | default: 40 | return nil, fmt.Errorf("unsupported list value type: %x", valueType) 41 | } 42 | } 43 | 44 | func parseList0(r *rdbReader, list *ListObjectEvent) (*ListObjectEvent, error) { 45 | size, err := r.GetLengthInt() 46 | if err != nil { 47 | return nil, err 48 | } 49 | members := make([]string, size) 50 | for i := 0; i < size; i++ { 51 | item, err := r.GetLengthString() 52 | if err != nil { 53 | return nil, err 54 | } 55 | members[i] = item 56 | } 57 | list.Elements = members 58 | return list, nil 59 | } 60 | 61 | func parseListInZipList(r *rdbReader, list *ListObjectEvent) (*ListObjectEvent, error) { 62 | zipList, err := parseZipList(r) 63 | if err != nil { 64 | return nil, err 65 | } 66 | list.Elements = zipList 67 | return list, nil 68 | } 69 | 70 | func parseListInQuickList(r *rdbReader, list *ListObjectEvent) (*ListObjectEvent, error) { 71 | size, err := r.GetLengthInt() 72 | if err != nil { 73 | return nil, err 74 | } 75 | var members []string 76 | for i := 0; i < size; i++ { 77 | zipList, err := parseZipList(r) 78 | if err != nil { 79 | return nil, err 80 | } 81 | members = append(members, zipList...) 82 | } 83 | list.Elements = members 84 | return list, nil 85 | } 86 | 87 | func parseListInQuickList2(r *rdbReader, list *ListObjectEvent) (*ListObjectEvent, error) { 88 | size, err := r.GetLengthInt() 89 | if err != nil { 90 | return nil, err 91 | } 92 | for i := 0; i < size; i++ { 93 | container, err := r.GetLengthInt() 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | switch container { 99 | case 1: 100 | member, err := r.GetLengthString() 101 | if err != nil { 102 | return nil, err 103 | } 104 | list.Elements = append(list.Elements, member) 105 | case 2: 106 | members, err := parseListPack(r) 107 | if err != nil { 108 | return nil, err 109 | } 110 | list.Elements = append(list.Elements, members...) 111 | default: 112 | return nil, fmt.Errorf("quicklist integrity check failed, unsupported listpack container: %d", container) 113 | } 114 | } 115 | 116 | return list, err 117 | } 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Redis lib 2 | 3 | ## Features 4 | 5 | - [Create connection with Redis server](#creating-connection) 6 | - [Parse RDB file](#parsing-rdb) 7 | - [Fake replica, sync RDB and AOF with master](#faking-replica) 8 | 9 | ## Compatibility 10 | 11 | - Support Redis 6 / 7 12 | 13 | ## Installing 14 | 15 | ```shell 16 | go get github.com/vczyh/redis-lib 17 | ``` 18 | 19 | ## Creating Connection 20 | 21 | ```go 22 | c, _ := client.NewClient(&client.Config{ 23 | Host: "127.0.0.1", 24 | Port: 26379, 25 | Username: "", 26 | Password: "123", 27 | }) 28 | 29 | _ = c.Auth() 30 | 31 | _ = c.Ping() 32 | ``` 33 | 34 | ## Parsing RDB 35 | 36 | ```go 37 | p, _ := rdb.NewParser("/tmp/rdb_test.rdb") 38 | 39 | s, _ := p.Parse() 40 | 41 | for s.HasNext() { 42 | e := s.Next() 43 | 44 | switch e.EventType { 45 | case rdb.EventTypeVersion: 46 | e.Event.Debug() 47 | case rdb.EventTypeStringObject: 48 | e.Event.Debug() 49 | case rdb.EventTypeSetObject: 50 | e.Event.Debug() 51 | } 52 | } 53 | 54 | _ = s.Err() 55 | 56 | === VersionEvent === 57 | 9 58 | 59 | === StringObjectEvent === 60 | Key: b 61 | Value: 3 62 | 63 | === SetObjectEvent === 64 | Key: key:set 65 | Size: 4 66 | Members: 67 | s2 68 | s5 69 | s4 70 | s1 71 | 72 | ... 73 | ``` 74 | 75 | ## Faking Replica 76 | 77 | ```go 78 | r, _ := replica.NewReplica(&replica.Config{ 79 | MasterIP: "127.0.0.1", 80 | MasterPort: 26379, 81 | MasterUser: "", 82 | MasterPassword: "123", 83 | MasterReplicaOffset: 67528, 84 | RdbWriter: os.Stdout, 85 | AofWriter: os.Stdout, 86 | }) 87 | _ = r.SyncWithMaster() 88 | ``` 89 | 90 | synchronize data and parse RDB: 91 | 92 | ```go 93 | rdbReader, rdbWriter := io.Pipe() 94 | 95 | r, _ := replica.NewReplica(&replica.Config{ 96 | MasterIP: "127.0.0.1", 97 | MasterPort: 26379, 98 | MasterUser: "", 99 | MasterPassword: "123", 100 | RdbWriter: rdbWriter, 101 | AofWriter: os.Stdout, 102 | }) 103 | 104 | go func() { 105 | _ = parseRdb(rdbReader) 106 | }() 107 | 108 | _ = r.SyncWithMaster() 109 | 110 | func parseRdb(r io.Reader) error { 111 | p, err := rdb.NewReaderParser(r) 112 | if err != nil { 113 | return err 114 | } 115 | s, err := p.Parse() 116 | if err != nil { 117 | return err 118 | } 119 | for s.HasNext() { 120 | e := s.Next() 121 | e.Event.Debug() 122 | } 123 | return s.Err() 124 | } 125 | ``` 126 | 127 | ## Star History 128 | 129 | 130 | 131 | 132 | 133 | Star History Chart 134 | 135 | 136 | -------------------------------------------------------------------------------- /rdb/ziplist.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "strconv" 8 | ) 9 | 10 | const ( 11 | zipStrMask = 0xc0 12 | zipStr06B = 0 << 6 13 | zipStr14B = 1 << 6 14 | zipStr32B = 2 << 6 15 | 16 | zipIntMask = 0x30 17 | zipInt8B = 0xfe 18 | zipInt16B = 0xc0 | 0<<4 19 | zipInt32B = 0xc0 | 1<<4 20 | zipInt64B = 0xc0 | 2<<4 21 | zipInt24B = 0xc0 | 3<<4 22 | zipIntImmMask = 0x0f 23 | zipIntImmMin = 0xf1 24 | zipIntImmMax = 0xfd 25 | ) 26 | 27 | func parseZipList(r *rdbReader) ([]string, error) { 28 | zipBytes, err := r.GetLengthBytes() 29 | if err != nil { 30 | return nil, err 31 | } 32 | r = newRdbReader(bytes.NewReader(zipBytes)) 33 | 34 | _, err = r.GetLUint32() 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | _, err = r.GetLUint32() 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | zlLen, err := r.GetLUint16() 45 | if err != nil { 46 | return nil, err 47 | } 48 | var members []string 49 | for i := 0; i < int(zlLen); i++ { 50 | entryValue, err := parseZipListEntry(r) 51 | if err != nil { 52 | return nil, err 53 | } 54 | members = append(members, entryValue) 55 | } 56 | 57 | b, err := r.ReadByte() 58 | if err != nil { 59 | return nil, err 60 | } 61 | if b != 0xff { 62 | return nil, fmt.Errorf("list should EOF") 63 | } 64 | 65 | return members, nil 66 | } 67 | 68 | // ziplist.c::ZIP_DECODE_LENGTH 69 | func parseZipListEntry(r *rdbReader) (string, error) { 70 | b, err := r.ReadByte() 71 | if err != nil { 72 | return "", err 73 | } 74 | preEntryLen := int(b) 75 | if b == 254 { 76 | u, err := r.GetLUint32() 77 | if err != nil { 78 | return "", err 79 | } 80 | preEntryLen = int(u) 81 | } 82 | _ = preEntryLen 83 | 84 | entryFlag, err := r.ReadByte() 85 | if err != nil { 86 | return "", err 87 | } 88 | 89 | // String: flag < zipStrMask 90 | encoding := entryFlag 91 | if encoding < zipStrMask { 92 | encoding &= zipStrMask 93 | } 94 | switch encoding { 95 | case zipStr06B: 96 | return r.ReadFixedString(int(entryFlag & 0x3F)) 97 | case zipStr14B: 98 | b2, err := r.ReadByte() 99 | if err != nil { 100 | return "", err 101 | } 102 | l := binary.BigEndian.Uint16([]byte{entryFlag & 0x3F, b2}) 103 | return r.ReadFixedString(int(l)) 104 | case zipStr32B: 105 | bUint32, err := r.GetBUint32() 106 | if err != nil { 107 | return "", err 108 | } 109 | return r.ReadFixedString(int(bUint32)) 110 | case zipInt8B: 111 | uInt8, err := r.GetUint8() 112 | if err != nil { 113 | return "", err 114 | } 115 | return strconv.FormatInt(int64(uInt8), 10), nil 116 | case zipInt16B: 117 | uInt16, err := r.GetLUint16() 118 | if err != nil { 119 | return "", err 120 | } 121 | return strconv.FormatInt(int64(uInt16), 10), nil 122 | case zipInt24B: 123 | uInt24, err := r.GetLUint24() 124 | if err != nil { 125 | return "", err 126 | } 127 | return strconv.FormatInt(int64(int32(uInt24)), 10), nil 128 | case zipInt32B: 129 | uInt32, err := r.GetLUint32() 130 | if err != nil { 131 | return "", err 132 | } 133 | return strconv.FormatInt(int64(uInt32), 10), nil 134 | case zipInt64B: 135 | uInt64, err := r.GetLUint64() 136 | if err != nil { 137 | return "", err 138 | } 139 | return strconv.FormatInt(int64(uInt64), 10), nil 140 | default: 141 | if encoding >= zipIntImmMin && encoding <= zipIntImmMax { 142 | v := (encoding & zipIntImmMask) - 1 143 | return strconv.FormatInt(int64(v), 10), nil 144 | } 145 | return "", fmt.Errorf("bad encoding") 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /rdb/obj_zset.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | type ZSetObjectEvent struct { 9 | RedisKey 10 | 11 | Members []ZSetMember 12 | } 13 | 14 | type ZSetMember struct { 15 | Value string 16 | Score float64 17 | } 18 | 19 | func (e *ZSetObjectEvent) Debug() { 20 | fmt.Printf("=== ZSetObjectEvent ===\n") 21 | e.debugKey() 22 | fmt.Printf("Size: %d\n", len(e.Members)) 23 | fmt.Printf("Members:\n") 24 | for _, member := range e.Members { 25 | fmt.Printf("\t%s %f\n", member.Value, member.Score) 26 | } 27 | fmt.Printf("\n") 28 | } 29 | 30 | func parseZSet(key RedisKey, r *rdbReader, valueType byte) (*ZSetObjectEvent, error) { 31 | zSet := &ZSetObjectEvent{ 32 | RedisKey: key, 33 | } 34 | switch valueType { 35 | case rdbTypeZSetZipList: 36 | return parseZSetInZipList(r, zSet) 37 | case rdbTypeZSetListPack: 38 | return parseZSetInListPack(r, zSet) 39 | case rdbTypeZSet: 40 | return parseZSet0(r, zSet) 41 | case rdbTypeZSet2: 42 | return parseZSet2(r, zSet) 43 | default: 44 | return nil, fmt.Errorf("unsupported zset value type: %x", valueType) 45 | } 46 | } 47 | 48 | func parseZSet0(r *rdbReader, set *ZSetObjectEvent) (*ZSetObjectEvent, error) { 49 | length, err := r.GetLengthInt() 50 | if err != nil { 51 | return nil, err 52 | } 53 | members := make([]ZSetMember, length) 54 | for i := 0; i < length; i++ { 55 | value, err := r.GetLengthString() 56 | if err != nil { 57 | return nil, err 58 | } 59 | score, err := r.GetDoubleValue() 60 | if err != nil { 61 | return nil, err 62 | } 63 | members[i].Value = value 64 | members[i].Score = score 65 | } 66 | set.Members = members 67 | 68 | return set, nil 69 | } 70 | 71 | func parseZSet2(r *rdbReader, set *ZSetObjectEvent) (*ZSetObjectEvent, error) { 72 | length, err := r.GetLengthInt() 73 | if err != nil { 74 | return nil, err 75 | } 76 | members := make([]ZSetMember, length) 77 | for i := 0; i < length; i++ { 78 | value, err := r.GetLengthString() 79 | if err != nil { 80 | return nil, err 81 | } 82 | score, err := r.GetLDouble() 83 | if err != nil { 84 | return nil, err 85 | } 86 | members[i].Value = value 87 | members[i].Score = score 88 | } 89 | set.Members = members 90 | 91 | return set, nil 92 | } 93 | 94 | func parseZSetInZipList(r *rdbReader, set *ZSetObjectEvent) (*ZSetObjectEvent, error) { 95 | list, err := parseZipList(r) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | if len(list)%2 != 0 { 101 | return nil, fmt.Errorf("error length for ziplist: %d", len(list)) 102 | } 103 | 104 | members := make([]ZSetMember, len(list)/2) 105 | for i := 0; i < len(list); i += 2 { 106 | value := list[i] 107 | score := list[i+1] 108 | scoreDouble, err := strconv.ParseFloat(score, 10) 109 | if err != nil { 110 | return nil, err 111 | } 112 | members[i/2].Value = value 113 | members[i/2].Score = scoreDouble 114 | } 115 | set.Members = members 116 | 117 | return set, nil 118 | } 119 | 120 | func parseZSetInListPack(r *rdbReader, set *ZSetObjectEvent) (*ZSetObjectEvent, error) { 121 | list, err := parseListPack(r) 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | if len(list)%2 != 0 { 127 | return nil, fmt.Errorf("error length for listpack: %d", len(list)) 128 | } 129 | 130 | members := make([]ZSetMember, len(list)/2) 131 | for i := 0; i < len(list); i += 2 { 132 | value := list[i] 133 | score := list[i+1] 134 | scoreDouble, err := strconv.ParseFloat(score, 10) 135 | if err != nil { 136 | return nil, err 137 | } 138 | members[i/2].Value = value 139 | members[i/2].Score = scoreDouble 140 | } 141 | set.Members = members 142 | 143 | return set, nil 144 | } 145 | -------------------------------------------------------------------------------- /rdb/listpack.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "strconv" 7 | ) 8 | 9 | const ( 10 | lpEncoding7BitUint = 0 11 | lpEncoding7BitUintMask = 0x80 12 | 13 | lpEncoding6BitStr = 0x80 14 | lpEncoding6BitStrMask = 0xC0 15 | 16 | lpEncoding13BitInt = 0xC0 17 | lpEncoding13BitIntMask = 0xE0 18 | 19 | lpEncoding12BitStr = 0xE0 20 | lpEncoding12BitStrMask = 0xF0 21 | 22 | lpEncoding16BitInt = 0xF1 23 | lpEncoding16BitIntMask = 0xFF 24 | 25 | lpEncoding24BitInt = 0xF2 26 | lpEncoding24BitIntMask = 0xFF 27 | 28 | lpEncoding32BitInt = 0xF3 29 | lpEncoding32BitIntMask = 0xFF 30 | 31 | lpEncoding64BitInt = 0xF4 32 | lpEncoding64BitIntMask = 0xFF 33 | 34 | lpEncoding32BitStr = 0xF0 35 | lpEncoding32BitStrMask = 0xFF 36 | ) 37 | 38 | func parseListPack(r *rdbReader) ([]string, error) { 39 | _, err := r.GetLengthInt() 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | // Total length 45 | _, err = r.GetLUint32() 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | // Element size 51 | // TODO exceed 65535 52 | size, err := r.GetLUint16() 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | members := make([]string, size) 58 | for i := 0; i < int(size); i++ { 59 | entry, err := parseListPackEntry(r) 60 | if err != nil { 61 | return nil, err 62 | } 63 | members[i] = entry 64 | } 65 | 66 | eof, err := r.ReadByte() 67 | if err != nil { 68 | return nil, err 69 | } 70 | if eof != 0xff { 71 | return nil, fmt.Errorf("listpack must end of 0xff: %x", eof) 72 | } 73 | 74 | return members, nil 75 | } 76 | 77 | func parseListPackEntry(r *rdbReader) (string, error) { 78 | encoding, err := r.ReadByte() 79 | if err != nil { 80 | return "", err 81 | } 82 | 83 | var val int64 84 | var uVal, negStart, negMax uint64 85 | 86 | switch { 87 | case encoding&lpEncoding7BitUintMask == lpEncoding7BitUint: 88 | negStart = math.MaxUint64 89 | negMax = 0 90 | uVal = uint64(encoding & 0x7f) 91 | // backLen bytes 92 | if _, err := r.ReadFixedBytes(1); err != nil { 93 | return "", err 94 | } 95 | case encoding&lpEncoding6BitStrMask == lpEncoding6BitStr: 96 | count := int(encoding & 0x3f) 97 | s, err := r.ReadFixedString(count) 98 | if err != nil { 99 | return "", err 100 | } 101 | if _, err := r.ReadFixedBytes(lpEncodeBackLen(1 + count)); err != nil { 102 | return "", err 103 | } 104 | return s, nil 105 | case encoding&lpEncoding13BitIntMask == lpEncoding13BitInt: 106 | b2, err := r.ReadByte() 107 | if err != nil { 108 | return "", err 109 | } 110 | uVal = (uint64(encoding&0x1f) << 8) | uint64(b2) 111 | negStart = uint64(1) << 12 112 | negMax = 8191 113 | if _, err := r.ReadFixedBytes(1); err != nil { 114 | return "", err 115 | } 116 | case encoding&lpEncoding16BitIntMask == lpEncoding16BitInt: 117 | lUint16, err := r.GetLUint16() 118 | if err != nil { 119 | return "", err 120 | } 121 | uVal = uint64(lUint16) 122 | negStart = uint64(1) << 15 123 | negMax = math.MaxUint16 124 | if _, err := r.ReadFixedBytes(1); err != nil { 125 | return "", err 126 | } 127 | case encoding&lpEncoding24BitIntMask == lpEncoding24BitInt: 128 | lUint24, err := r.GetLUint24() 129 | if err != nil { 130 | return "", err 131 | } 132 | uVal = uint64(lUint24) 133 | negStart = uint64(1) << 23 134 | negMax = math.MaxUint32 >> 8 135 | if _, err := r.ReadFixedBytes(1); err != nil { 136 | return "", err 137 | } 138 | case encoding&lpEncoding32BitIntMask == lpEncoding32BitInt: 139 | lUint32, err := r.GetLUint32() 140 | if err != nil { 141 | return "", err 142 | } 143 | uVal = uint64(lUint32) 144 | negStart = uint64(1) << 31 145 | negMax = math.MaxUint32 146 | if _, err := r.ReadFixedBytes(1); err != nil { 147 | return "", err 148 | } 149 | case encoding&lpEncoding64BitIntMask == lpEncoding64BitInt: 150 | uVal, err = r.GetLUint64() 151 | if err != nil { 152 | return "", err 153 | } 154 | negStart = uint64(1) << 63 155 | negMax = math.MaxUint64 156 | if _, err := r.ReadFixedBytes(1); err != nil { 157 | return "", err 158 | } 159 | case encoding&lpEncoding12BitStrMask == lpEncoding12BitStr: 160 | b2, err := r.ReadByte() 161 | if err != nil { 162 | return "", err 163 | } 164 | count := int(encoding&0xf)<<8 | int(b2) 165 | s, err := r.ReadFixedString(count) 166 | if err != nil { 167 | return "", err 168 | } 169 | if _, err := r.ReadFixedBytes(lpEncodeBackLen(2 + count)); err != nil { 170 | return "", err 171 | } 172 | return s, nil 173 | case encoding&lpEncoding32BitStrMask == lpEncoding32BitStr: 174 | count, err := r.GetLUint32() 175 | if err != nil { 176 | return "", err 177 | } 178 | s, err := r.ReadFixedString(int(count)) 179 | if err != nil { 180 | return "", err 181 | } 182 | if _, err := r.ReadFixedBytes(lpEncodeBackLen(5 + int(count))); err != nil { 183 | return "", err 184 | } 185 | return s, nil 186 | default: 187 | uVal = 12345678900000000 + uint64(encoding) 188 | negStart = math.MaxUint64 189 | negMax = 0 190 | } 191 | 192 | /* We reach this code path only for integer encodings. 193 | * Convert the unsigned value to the signed one using two's complement 194 | * rule. */ 195 | if uVal >= negStart { 196 | uVal = negMax - uVal 197 | val = int64(uVal) 198 | val = -val - 1 199 | } else { 200 | val = int64(uVal) 201 | } 202 | 203 | return strconv.FormatInt(val, 10), nil 204 | } 205 | 206 | // listpack.c::lpEncodeBacklen 207 | func lpEncodeBackLen(entryLen int) int { 208 | switch { 209 | case entryLen <= 127: 210 | return 1 211 | case entryLen < 16383: 212 | return 2 213 | case entryLen < 2097151: 214 | return 3 215 | case entryLen < 268435455: 216 | return 4 217 | default: 218 | return 5 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /rdb/reader.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "math" 8 | "strconv" 9 | ) 10 | 11 | type rdbReader struct { 12 | r io.Reader 13 | } 14 | 15 | func newRdbReader(r io.Reader) *rdbReader { 16 | return &rdbReader{r: r} 17 | } 18 | 19 | func (r *rdbReader) GetInt8() (int8, error) { 20 | b, err := r.ReadByte() 21 | if err != nil { 22 | return 0, err 23 | } 24 | return int8(b), nil 25 | } 26 | 27 | func (r *rdbReader) GetUint8() (uint8, error) { 28 | return r.ReadByte() 29 | } 30 | 31 | func (r *rdbReader) GetLUint16() (uint16, error) { 32 | b, err := r.ReadFixedBytes(2) 33 | if err != nil { 34 | return 0, err 35 | } 36 | return binary.LittleEndian.Uint16(b), nil 37 | } 38 | 39 | func (r *rdbReader) GetLUint24() (uint32, error) { 40 | b, err := r.ReadFixedBytes(3) 41 | if err != nil { 42 | return 0, err 43 | } 44 | bs := append([]byte{0}, b...) 45 | return binary.LittleEndian.Uint32(bs) >> 8, nil 46 | } 47 | 48 | func (r *rdbReader) GetLUint32() (uint32, error) { 49 | b, err := r.ReadFixedBytes(4) 50 | if err != nil { 51 | return 0, err 52 | } 53 | return binary.LittleEndian.Uint32(b), nil 54 | } 55 | 56 | func (r *rdbReader) GetBUint32() (uint32, error) { 57 | b, err := r.ReadFixedBytes(4) 58 | if err != nil { 59 | return 0, err 60 | } 61 | return binary.BigEndian.Uint32(b), nil 62 | } 63 | 64 | func (r *rdbReader) GetLUint64() (uint64, error) { 65 | b, err := r.ReadFixedBytes(8) 66 | if err != nil { 67 | return 0, err 68 | } 69 | return binary.LittleEndian.Uint64(b), nil 70 | } 71 | 72 | func (r *rdbReader) GetBUint64() (uint64, error) { 73 | b, err := r.ReadFixedBytes(8) 74 | if err != nil { 75 | return 0, err 76 | } 77 | return binary.BigEndian.Uint64(b), nil 78 | } 79 | 80 | func (r *rdbReader) GetLDouble() (float64, error) { 81 | bits, err := r.GetLUint64() 82 | if err != nil { 83 | return 0, err 84 | } 85 | return math.Float64frombits(bits), nil 86 | } 87 | 88 | // GetDoubleValue 89 | // rdb.c:;rdbLoadDoubleValue 90 | func (r *rdbReader) GetDoubleValue() (float64, error) { 91 | length, err := r.ReadByte() 92 | if err != nil { 93 | return 0, err 94 | } 95 | 96 | switch length { 97 | case 255: 98 | return math.Inf(-1), nil 99 | case 254: 100 | return math.Inf(0), nil 101 | case 253: 102 | return math.NaN(), nil 103 | default: 104 | val, err := r.ReadFixedString(int(length)) 105 | if err != nil { 106 | return 0, err 107 | } 108 | return strconv.ParseFloat(val, 64) 109 | } 110 | } 111 | 112 | func (r *rdbReader) GetLengthString() (string, error) { 113 | encoding, n, err := r.GetEncodingLength() 114 | if err != nil { 115 | return "", err 116 | } 117 | switch encoding { 118 | case lengthEncodingLength: 119 | return r.ReadFixedString(int(n)) 120 | case lengthEncodingInteger: 121 | return strconv.Itoa(int(n)), nil 122 | default: 123 | return "", fmt.Errorf("unsupported encoding %d for GetLengthString", encoding) 124 | } 125 | } 126 | 127 | func (r *rdbReader) GetLengthBytes() ([]byte, error) { 128 | encoding, n, err := r.GetEncodingLength() 129 | if err != nil { 130 | return nil, err 131 | } 132 | switch encoding { 133 | case lengthEncodingLength: 134 | return r.ReadFixedBytes(int(n)) 135 | default: 136 | return nil, fmt.Errorf("unsupported encoding %d for GetLengthString", encoding) 137 | } 138 | } 139 | 140 | func (r *rdbReader) GetLengthInt() (int, error) { 141 | encoding, n, err := r.GetEncodingLength() 142 | if err != nil { 143 | return 0, err 144 | } 145 | switch encoding { 146 | case lengthEncodingLength, lengthEncodingInteger: 147 | return int(n), nil 148 | default: 149 | return 0, fmt.Errorf("unsupported encoding %d for GetLengthInt", encoding) 150 | } 151 | } 152 | 153 | func (r *rdbReader) GetLengthUInt64() (uint64, error) { 154 | encoding, n, err := r.GetEncodingLength() 155 | if err != nil { 156 | return 0, err 157 | } 158 | switch encoding { 159 | case lengthEncodingLength, lengthEncodingInteger: 160 | return n, nil 161 | default: 162 | return 0, fmt.Errorf("unsupported encoding %d for GetLengthInt", encoding) 163 | } 164 | } 165 | 166 | func (r *rdbReader) ReadByte() (byte, error) { 167 | bs, err := r.ReadFixedBytes(1) 168 | if err != nil { 169 | return 0, err 170 | } 171 | return bs[0], nil 172 | } 173 | 174 | func (r *rdbReader) Read(p []byte) (n int, err error) { 175 | return r.r.Read(p) 176 | } 177 | 178 | func (r *rdbReader) ReadFixedBytes(size int) ([]byte, error) { 179 | bs := make([]byte, size) 180 | _, err := io.ReadFull(r.r, bs) 181 | return bs, err 182 | } 183 | 184 | func (r *rdbReader) ReadFixedString(size int) (string, error) { 185 | bs, err := r.ReadFixedBytes(size) 186 | if err != nil { 187 | return "", err 188 | } 189 | return string(bs), nil 190 | } 191 | 192 | type lengthEncoding uint8 193 | 194 | const ( 195 | lengthEncodingLength lengthEncoding = iota 196 | lengthEncodingInteger 197 | lengthEncodingCompressed 198 | ) 199 | 200 | func (r *rdbReader) GetEncodingLength() (lengthEncoding, uint64, error) { 201 | b, err := r.ReadByte() 202 | if err != nil { 203 | return 0, 0, err 204 | } 205 | 206 | switch b >> 6 { 207 | case 0: 208 | return lengthEncodingLength, uint64(b & 0x3F), nil 209 | case 1: 210 | b2, err := r.ReadByte() 211 | if err != nil { 212 | return 0, 0, err 213 | } 214 | size := binary.BigEndian.Uint16([]byte{b & 0x3F, b2}) 215 | return lengthEncodingLength, uint64(size), nil 216 | case 2: 217 | b2, err := r.ReadFixedBytes(4) 218 | if err != nil { 219 | return 0, 0, err 220 | } 221 | //bs := []byte{b & 0x3F} 222 | //bs = append(bs, b2...) 223 | size := binary.BigEndian.Uint32(b2) 224 | return lengthEncodingLength, uint64(size), nil 225 | case 3: 226 | switch b & 0x3F { 227 | case 0: 228 | b2, err := r.ReadByte() 229 | if err != nil { 230 | return 0, 0, err 231 | } 232 | return lengthEncodingInteger, uint64(b2), nil 233 | case 1: 234 | b2, err := r.ReadFixedBytes(2) 235 | if err != nil { 236 | return 0, 0, err 237 | } 238 | v := binary.LittleEndian.Uint16(b2) 239 | return lengthEncodingInteger, uint64(v), nil 240 | case 2: 241 | b2, err := r.ReadFixedBytes(4) 242 | if err != nil { 243 | return 0, 0, err 244 | } 245 | v := binary.LittleEndian.Uint32(b2) 246 | return lengthEncodingInteger, uint64(v), nil 247 | case 3: 248 | // TODO comparessed 249 | panic("compressed") 250 | default: 251 | return 0, 0, fmt.Errorf("unsupported 6 bits: %x", b&0x3F) 252 | } 253 | default: 254 | return 0, 0, fmt.Errorf("unsupported bit prefix: %x", b) 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /rdb/parser.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strconv" 9 | ) 10 | 11 | const ( 12 | opCodeFunction2 = 245 13 | opcodeFunction = 246 14 | opCodeModuleAux = 247 15 | opCodeIdle = 248 16 | opCodeFreq = 249 17 | opCodeAux = 250 18 | opCodeResizeDb = 251 19 | opExpireTimeMs = 252 20 | opExpireTime = 253 21 | opCodeSelectDb = 254 22 | opCodeEOF = 255 23 | 24 | rdbTypeString = 0 25 | rdbTypeList = 1 26 | rdbTypeSet = 2 27 | rdbTypeZSet = 3 28 | rdbTypeHash = 4 29 | rdbTypeZSet2 = 5 // ZSET version 2 with doubles stored in binary. 30 | rdbTypeModulePreGA = 6 31 | rdbTypeModule2 = 7 // Module value with annotations for parsing without the generating module being loaded. 32 | rdbTypeHashZipMap = 9 33 | rdbTypeZipList = 10 34 | rdbTypeIntSet = 11 35 | rdbTypeZSetZipList = 12 36 | rdbTypeHashZipList = 13 37 | rdbTypeListQuickList = 14 38 | rdbTypeStreamListPacks = 15 39 | rdbTypeHashListPack = 16 40 | rdbTypeZSetListPack = 17 41 | rdbTypeListQuickList2 = 18 42 | rdbTypeStreamListPacks2 = 19 43 | rdbTypeSetListPack = 20 44 | rdbTypeStreamListPacks3 = 21 45 | ) 46 | 47 | type Parser struct { 48 | file string 49 | fd *os.File 50 | r *rdbReader 51 | 52 | version int 53 | } 54 | 55 | func NewParser(name string) (*Parser, error) { 56 | p := &Parser{ 57 | file: name, 58 | } 59 | return p, nil 60 | } 61 | 62 | func NewReaderParser(r io.Reader) (*Parser, error) { 63 | p := &Parser{ 64 | r: newRdbReader(r), 65 | } 66 | return p, nil 67 | } 68 | 69 | func (p *Parser) Parse() (*EventStreamer, error) { 70 | if p.file != "" { 71 | f, err := os.Open(p.file) 72 | if err != nil { 73 | return nil, err 74 | } 75 | p.fd = f 76 | r := newRdbReader(f) 77 | p.r = r 78 | } 79 | 80 | eventC := make(chan *eventWrapper) 81 | es := newEventStreamer(eventC) 82 | 83 | go func() { 84 | defer func() { 85 | close(eventC) 86 | p.close() 87 | }() 88 | 89 | if err := p.parse(eventC); err != nil { 90 | eventC <- &eventWrapper{e: nil, err: err} 91 | } 92 | }() 93 | 94 | return es, nil 95 | } 96 | 97 | func (p *Parser) close() { 98 | if p.fd != nil { 99 | p.fd.Close() 100 | } 101 | } 102 | 103 | func (p *Parser) parse(eventC chan *eventWrapper) error { 104 | // magic number 105 | magic, err := p.r.ReadFixedBytes(5) 106 | if err != nil { 107 | return err 108 | } 109 | magicEvent := &MagicNumberEvent{MagicNumber: magic} 110 | eventC <- &eventWrapper{ 111 | e: &RedisRdbEvent{ 112 | EventType: EventTypeMagicNumber, 113 | Event: magicEvent, 114 | }, 115 | } 116 | 117 | // version 118 | version, err := p.r.ReadFixedBytes(4) 119 | if err != nil { 120 | return err 121 | } 122 | versionNumber, err := strconv.Atoi(string(version)) 123 | if err != nil { 124 | return err 125 | } 126 | p.version = versionNumber 127 | versionEvent := &VersionEvent{Version: versionNumber} 128 | eventC <- &eventWrapper{ 129 | e: &RedisRdbEvent{ 130 | EventType: EventTypeVersion, 131 | Event: versionEvent, 132 | }, 133 | } 134 | 135 | dbId := 0 136 | var expireTime int64 = -1 137 | 138 | var isEnd bool 139 | for !isEnd { 140 | rdbType, err := p.r.ReadByte() 141 | if err != nil { 142 | return err 143 | } 144 | switch rdbType { 145 | case opExpireTime: 146 | expireTime, err = p.parseExpireTime() 147 | if err != nil { 148 | return err 149 | } 150 | continue 151 | case opExpireTimeMs: 152 | expireTime, err = p.parseExpireTimeMs() 153 | if err != nil { 154 | return err 155 | } 156 | continue 157 | case opCodeFreq: 158 | // LFU frequency. 159 | if err := p.parseFreq(); err != nil { 160 | return err 161 | } 162 | continue 163 | case opCodeIdle: 164 | // LRU idle time. 165 | _, err := p.parseIdle() 166 | if err != nil { 167 | return err 168 | } 169 | continue 170 | case opCodeEOF: 171 | isEnd = true 172 | continue 173 | case opCodeSelectDb: 174 | e, err := p.parseSelectDb() 175 | if err != nil { 176 | return err 177 | } 178 | dbId = e.Db 179 | eventC <- &eventWrapper{ 180 | e: &RedisRdbEvent{ 181 | EventType: EventTypeSelectDb, 182 | Event: e, 183 | }, 184 | } 185 | continue 186 | case opCodeResizeDb: 187 | // RESIZEDB: Hint about the size of the keys in the currently 188 | // selected data base, in order to avoid useless rehashing. 189 | e, err := p.parseResizeDb() 190 | if err != nil { 191 | return err 192 | } 193 | eventC <- &eventWrapper{ 194 | e: &RedisRdbEvent{ 195 | EventType: EventTypeResizeDb, 196 | Event: e, 197 | }, 198 | } 199 | continue 200 | case opCodeAux: 201 | // AUX: generic string-string fields. Use to add state to RDB 202 | // which is backward compatible. Implementations of RDB loading 203 | // are required to skip AUX fields they don't understand. 204 | // 205 | // An AUX field is composed of two strings: key and value. 206 | e, err := p.parseAuxiliaryFields() 207 | if err != nil { 208 | return err 209 | } 210 | eventC <- &eventWrapper{ 211 | e: &RedisRdbEvent{ 212 | EventType: EventTypeAuxField, 213 | Event: e, 214 | }, 215 | } 216 | continue 217 | case opCodeModuleAux: 218 | // Load module data that is not related to the Redis key space. 219 | // Such data can be potentially be stored both before and after the 220 | // RDB keys-values section. 221 | // TODO 222 | continue 223 | case opcodeFunction: 224 | return fmt.Errorf("pre-release function format not supported") 225 | case opCodeFunction2: 226 | // TODO 227 | return fmt.Errorf("function not suuported") 228 | } 229 | 230 | // Load object key. 231 | key, err := p.parseKey() 232 | if err != nil { 233 | return err 234 | } 235 | // Load object value. 236 | e, err := p.parseEntryWithValueType(rdbType, key, dbId, expireTime) 237 | if err != nil { 238 | return err 239 | } 240 | eventC <- &eventWrapper{e: e} 241 | 242 | // Reset state. 243 | expireTime = -1 244 | } 245 | 246 | if p.version >= 5 { 247 | // TODO compare? 248 | if _, err := p.r.ReadFixedBytes(8); err != nil { 249 | return err 250 | } 251 | } 252 | 253 | return nil 254 | } 255 | 256 | func (p *Parser) parseFreq() error { 257 | _, err := p.r.ReadByte() 258 | return err 259 | } 260 | 261 | func (p *Parser) parseIdle() (uint64, error) { 262 | return p.r.GetLengthUInt64() 263 | } 264 | 265 | func (p *Parser) parseAuxiliaryFields() (*AuxFieldEvent, error) { 266 | field, err := p.r.GetLengthString() 267 | if err != nil { 268 | return nil, err 269 | } 270 | value, err := p.r.GetLengthString() 271 | if err != nil { 272 | return nil, err 273 | } 274 | return &AuxFieldEvent{ 275 | Filed: field, 276 | Value: value, 277 | }, nil 278 | } 279 | 280 | func (p *Parser) parseSelectDb() (*SelectDbEvent, error) { 281 | // DB number 282 | dbNumber, err := p.r.GetLengthInt() 283 | if err != nil { 284 | return nil, err 285 | } 286 | //fmt.Printf("Db number: %d\n", dbNumber) 287 | return &SelectDbEvent{Db: dbNumber}, nil 288 | } 289 | 290 | func (p *Parser) parseResizeDb() (*ResizeDbEvent, error) { 291 | dbSize, err := p.r.GetLengthInt() 292 | if err != nil { 293 | return nil, err 294 | } 295 | dbExpiresSize, err := p.r.GetLengthInt() 296 | if err != nil { 297 | return nil, err 298 | } 299 | return &ResizeDbEvent{ 300 | dbSize: dbSize, 301 | dbExpireSize: dbExpiresSize, 302 | }, nil 303 | } 304 | 305 | func (p *Parser) parseExpireTime() (int64, error) { 306 | b, err := p.r.ReadFixedBytes(4) 307 | if err != nil { 308 | return 0, nil 309 | } 310 | expireAt := binary.LittleEndian.Uint32(b) 311 | return int64(expireAt) * 1000, nil 312 | } 313 | 314 | func (p *Parser) parseExpireTimeMs() (int64, error) { 315 | b, err := p.r.ReadFixedBytes(8) 316 | if err != nil { 317 | return 0, nil 318 | } 319 | expireAt := binary.LittleEndian.Uint64(b) 320 | return int64(expireAt), nil 321 | } 322 | 323 | func (p *Parser) parseKey() (string, error) { 324 | return p.r.GetLengthString() 325 | } 326 | 327 | func (p *Parser) parseEntryWithValueType(valueType byte, key string, DbId int, expireAt int64) (*RedisRdbEvent, error) { 328 | redisKey := RedisKey{ 329 | DbId: DbId, 330 | Key: key, 331 | expireAt: expireAt, 332 | } 333 | 334 | switch valueType { 335 | case rdbTypeString: 336 | event, err := parseString(redisKey, p.r) 337 | if err != nil { 338 | return nil, err 339 | } 340 | return &RedisRdbEvent{EventType: EventTypeStringObject, Event: event}, nil 341 | case rdbTypeList, rdbTypeZipList, rdbTypeListQuickList, rdbTypeListQuickList2: 342 | event, err := parseList(redisKey, p.r, valueType) 343 | if err != nil { 344 | return nil, err 345 | } 346 | return &RedisRdbEvent{EventType: EventTypeListObject, Event: event}, nil 347 | case rdbTypeSet, rdbTypeSetListPack, rdbTypeIntSet: 348 | event, err := parseSet(redisKey, p.r, valueType) 349 | if err != nil { 350 | return nil, err 351 | } 352 | return &RedisRdbEvent{EventType: EventTypeSetObject, Event: event}, nil 353 | case rdbTypeZSetZipList, rdbTypeZSetListPack, rdbTypeZSet, rdbTypeZSet2: 354 | event, err := parseZSet(redisKey, p.r, valueType) 355 | if err != nil { 356 | return nil, err 357 | } 358 | return &RedisRdbEvent{EventType: EventTypeZSetObject, Event: event}, nil 359 | case rdbTypeHashZipList, rdbTypeHashListPack, rdbTypeHash: 360 | event, err := parseHash(redisKey, p.r, valueType) 361 | if err != nil { 362 | return nil, err 363 | } 364 | return &RedisRdbEvent{EventType: EventTypeHashObject, Event: event}, nil 365 | case rdbTypeStreamListPacks, rdbTypeStreamListPacks2: 366 | event, err := parseStream(redisKey, p.r, valueType) 367 | if err != nil { 368 | return nil, err 369 | } 370 | return &RedisRdbEvent{EventType: EventTypeStreamObject, Event: event}, nil 371 | default: 372 | return nil, fmt.Errorf("unsupported rdb value type: 0x%x", valueType) 373 | } 374 | } 375 | -------------------------------------------------------------------------------- /replica/replica.go: -------------------------------------------------------------------------------- 1 | package replica 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/vczyh/redis-lib/client" 7 | "io" 8 | "math" 9 | "strconv" 10 | "strings" 11 | "sync/atomic" 12 | "time" 13 | ) 14 | 15 | type Replica struct { 16 | config *Config 17 | client *client.Client 18 | 19 | replicaId string 20 | replicaOffset atomic.Int64 21 | } 22 | 23 | type Config struct { 24 | MasterIP string 25 | MasterPort int 26 | MasterUser string 27 | MasterPassword string 28 | 29 | AnnounceIP string 30 | AnnouncePort int 31 | 32 | // Send command (PSYNC MasterReplicaId MasterReplicaOffset) to master. 33 | // Replica will start a partial synchronization if replicaId and offset exist in master. 34 | // Set ContinueIfPartialFailed true if you hope do a full synchronized after partial synchronization failed. 35 | MasterReplicaId string 36 | MasterReplicaOffset int 37 | 38 | // Whether to do a full synchronized after partial synchronization failed. 39 | ContinueIfPartialFailed bool 40 | 41 | // Receive RDB from master in full synchronization. 42 | RdbWriter io.Writer 43 | 44 | // Whether to continue incremental synchronization(AOF) after full synchronization. 45 | ContinueAfterFullSync bool 46 | 47 | // Receive AOF bytes stream after full synchronization if ContinueAfterFullSync is true. 48 | // Receive all AOF bytes stream in partial synchronization. 49 | AofWriter io.Writer 50 | 51 | // Whether to support diskless replication. 52 | // https://redis.io/docs/management/replication/#:~:text=Normally%20a%20full,as%20intermediate%20storage. 53 | // Change the default of repl-diskless-sync to yes in Redis 7.0. Replica has to wait repl-diskless-sync-delay 54 | // to receive the rdb. So instead of waiting, default not support diskless replication. 55 | // With slow disks and fast (large bandwidth) networks, diskless replication works better. 56 | SupportDisklessReplication bool 57 | } 58 | 59 | func NewReplica(config *Config) (*Replica, error) { 60 | r := &Replica{ 61 | config: config, 62 | } 63 | return r, nil 64 | } 65 | 66 | // SyncWithMaster establish a connection with the master, and sync data from master. 67 | // replication.c::syncWithMaster 68 | func (r *Replica) SyncWithMaster() error { 69 | // Create connection with master. 70 | if err := r.createClient(); err != nil { 71 | return err 72 | } 73 | 74 | if err := r.client.Auth(); err != nil { 75 | return err 76 | } 77 | 78 | // Check for errors in the socket: after a non blocking connect() we may find that the socket is in error state. 79 | if err := r.client.Ping(); err != nil { 80 | return err 81 | } 82 | 83 | conn := r.client.Conn() 84 | if port := r.config.AnnouncePort; port != 0 { 85 | if err := conn.WriteCommand("REPLCONF", "listening-port", strconv.Itoa(port)); err != nil { 86 | return err 87 | } 88 | if err := conn.SkipOk(); err != nil { 89 | return err 90 | } 91 | } 92 | if ip := r.config.AnnounceIP; ip != "" { 93 | if err := conn.WriteCommand("REPLCONF", "ip-address", ip); err != nil { 94 | return err 95 | } 96 | if err := conn.SkipOk(); err != nil { 97 | return err 98 | } 99 | } 100 | 101 | // Inform the master of our (slave) capabilities. 102 | // 103 | // EOF: supports EOF-style RDB transfer for diskless replication. 104 | // PSYNC2: supports PSYNC v2, so understands +CONTINUE . 105 | // 106 | // The master will ignore capabilities it does not understand. 107 | args := []string{ 108 | "capa", "psync2", 109 | } 110 | if r.config.SupportDisklessReplication { 111 | args = append(args, "capa", "eof") 112 | } 113 | if err := conn.WriteCommand("REPLCONF", args...); err != nil { 114 | return err 115 | } 116 | if err := conn.SkipOk(); err != nil { 117 | return err 118 | } 119 | 120 | // Send PSYNC command. 121 | // 122 | // Full sync: PSYNC ? 01 123 | // Partial sync: PSYNC replicaId offset 124 | replicaId := "?" 125 | offset := -1 126 | partial := r.config.MasterReplicaId != "" && r.config.MasterReplicaOffset > 0 127 | if masterReplicaId := r.config.MasterReplicaId; masterReplicaId != "" { 128 | replicaId = masterReplicaId 129 | } 130 | if masterOffset := r.config.MasterReplicaOffset; masterOffset != 0 { 131 | offset = masterOffset 132 | } 133 | if err := conn.WriteCommand("PSYNC", replicaId, strconv.Itoa(offset)); err != nil { 134 | return err 135 | } 136 | reply, err := r.receiveSynchronousResponse() 137 | if err != nil { 138 | return err 139 | } 140 | 141 | // Write offset ack and keepalive. 142 | go func() { 143 | if err := r.sendOffsetAckTicker(); err != nil { 144 | fmt.Printf("fail send ack to master: %s\n", err) 145 | return 146 | } 147 | }() 148 | 149 | switch { 150 | case strings.HasPrefix(reply, "FULLRESYNC"): 151 | split := strings.Split(reply, " ") 152 | if len(split) != 3 { 153 | return fmt.Errorf("PSYNC FULLRESYNC response format invalid: %s", reply) 154 | } 155 | r.replicaId = split[1] 156 | offsetInt, err := strconv.Atoi(split[2]) 157 | if err != nil { 158 | return err 159 | } 160 | 161 | if partial && !r.config.ContinueIfPartialFailed { 162 | return fmt.Errorf("master tells you that you need do a full synchroinzation") 163 | } 164 | 165 | if err := r.fullSync(offsetInt); err != nil { 166 | return err 167 | } 168 | case strings.HasPrefix(reply, "CONTINUE"): 169 | r.replicaId = replicaId 170 | split := strings.Split(reply, " ") 171 | if len(split) >= 2 { 172 | r.replicaId = split[1] 173 | } 174 | r.replicaOffset.Store(int64(offset)) 175 | if err := r.partialSync(); err != nil { 176 | return err 177 | } 178 | default: 179 | return fmt.Errorf("unsupported PSYNC commadn response: %s", reply) 180 | } 181 | 182 | return nil 183 | } 184 | 185 | // replication.c::receiveSynchronousResponse 186 | func (r *Replica) receiveSynchronousResponse() (string, error) { 187 | // Read the reply from the server. 188 | conn := r.client.Conn() 189 | for { 190 | bytes, err := conn.Peek(1) 191 | if err != nil { 192 | return "", err 193 | } 194 | if bytes[0] != '\n' { 195 | break 196 | } else { 197 | if _, err := conn.Discard(1); err != nil { 198 | return "", err 199 | } 200 | } 201 | } 202 | return conn.ReadString() 203 | } 204 | 205 | func (r *Replica) Close() error { 206 | return r.client.Close() 207 | } 208 | 209 | func (r *Replica) partialSync() error { 210 | buf := make([]byte, 10*1024) 211 | for { 212 | n, err := r.client.Conn().Read(buf) 213 | if err != nil { 214 | return err 215 | } 216 | if _, err = r.config.AofWriter.Write(buf[:n]); err != nil { 217 | return err 218 | } 219 | r.replicaOffset.Add(int64(n)) 220 | } 221 | } 222 | 223 | func (r *Replica) fullSync(offset int) error { 224 | conn := r.client.Conn() 225 | 226 | for { 227 | bs, err := conn.Peek(1) 228 | if err != nil { 229 | return err 230 | } 231 | if bs[0] != '\n' { 232 | break 233 | } else { 234 | if _, err := conn.Discard(1); err != nil { 235 | return err 236 | } 237 | } 238 | } 239 | replayData, err := conn.ReadData() 240 | if err != nil { 241 | return err 242 | } 243 | if len(replayData) == 0 { 244 | /* At this stage just a newline works as a PING in order to take 245 | * the connection live. So we refresh our last interaction 246 | * timestamp. */ 247 | return nil 248 | } else if replayData[0] != '$' { 249 | return fmt.Errorf("bad protocol from MASTER, the first byte is not '$' (we received '%s'), are you sure the host and port are right", replayData) 250 | } 251 | reply := string(replayData[1:]) 252 | 253 | // Static vars used to hold the EOF mark, and the last bytes received 254 | // from the server: when they match, we reached the end of the transfer. 255 | eofMark := make([]byte, 40) 256 | lastBytes := make([]byte, 40) 257 | lastBytesSize := 0 258 | transferSize := 0 259 | var useMark bool 260 | if strings.HasPrefix(reply, "EOF") { 261 | useMark = true 262 | copy(eofMark, reply[4:]) 263 | transferSize = 0 264 | } else { 265 | useMark = false 266 | transferSize, err = strconv.Atoi(reply) 267 | if err != nil { 268 | return err 269 | } 270 | } 271 | 272 | bufSize := 10 * 1024 * 1024 273 | buf := make([]byte, bufSize) 274 | 275 | if useMark { 276 | for { 277 | n, err := conn.Read(buf) 278 | if err != nil { 279 | return err 280 | } 281 | 282 | if n > 40 { 283 | if _, err = r.config.RdbWriter.Write(lastBytes[:lastBytesSize]); err != nil { 284 | return err 285 | } 286 | copy(lastBytes, buf[n-40:]) 287 | lastBytesSize = 40 288 | if _, err = r.config.RdbWriter.Write(buf[:n-40]); err != nil { 289 | return err 290 | } 291 | } else { 292 | if lastBytesSize+n > 40 { 293 | if _, err = r.config.RdbWriter.Write(lastBytes[:lastBytesSize+n-40]); err != nil { 294 | return err 295 | } 296 | } 297 | copy(lastBytes, lastBytes[n:]) 298 | copy(lastBytes[40-n:], buf[:n]) 299 | lastBytesSize = int(math.Min(float64(n+lastBytesSize), 40)) 300 | } 301 | 302 | if bytes.Equal(lastBytes, eofMark) { 303 | break 304 | } 305 | } 306 | } else { 307 | unReadSize := transferSize 308 | for unReadSize > 0 { 309 | needSize := bufSize 310 | if unReadSize < needSize { 311 | needSize = unReadSize 312 | } 313 | n, err := conn.Read(buf[:needSize]) 314 | if err != nil { 315 | return err 316 | } 317 | unReadSize -= n 318 | if _, err = r.config.RdbWriter.Write(buf[:n]); err != nil { 319 | return err 320 | } 321 | } 322 | } 323 | 324 | r.replicaOffset.Store(int64(offset)) 325 | 326 | if r.config.ContinueAfterFullSync { 327 | if err = r.syncAOF(); err != nil { 328 | return err 329 | } 330 | } 331 | 332 | return nil 333 | } 334 | 335 | func (r *Replica) syncAOF() error { 336 | buf := make([]byte, 1024*100) 337 | for { 338 | n, err := r.client.Conn().Read(buf) 339 | if err != nil { 340 | return err 341 | } 342 | if _, err = r.config.AofWriter.Write(buf[:n]); err != nil { 343 | return err 344 | } 345 | r.replicaOffset.Add(int64(n)) 346 | } 347 | } 348 | 349 | func (r *Replica) sendOffsetAckTicker() error { 350 | t := time.Tick(1 * time.Second) 351 | for range t { 352 | offset := r.replicaOffset.Load() 353 | if offset > 0 { 354 | if err := r.client.Conn().WriteCommand("REPLCONF", "ACK", strconv.FormatInt(offset, 10)); err != nil { 355 | return err 356 | } 357 | } 358 | } 359 | return nil 360 | } 361 | 362 | func (r *Replica) createClient() error { 363 | c, err := client.NewClient(&client.Config{ 364 | Host: r.config.MasterIP, 365 | Port: r.config.MasterPort, 366 | Username: r.config.MasterUser, 367 | Password: r.config.MasterPassword, 368 | }) 369 | if err != nil { 370 | return err 371 | } 372 | r.client = c 373 | return nil 374 | } 375 | -------------------------------------------------------------------------------- /rdb/obj_stream.go: -------------------------------------------------------------------------------- 1 | package rdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | streamItemFlagDeleted = 1 << 0 12 | streamItemFlagSameFields = 1 << 1 13 | ) 14 | 15 | type StreamObjectEvent struct { 16 | RedisKey 17 | 18 | Entries []*StreamEntry 19 | Groups []*StreamConsumerGroup 20 | } 21 | 22 | func (e *StreamObjectEvent) Debug() { 23 | fmt.Printf("=== StreamObjectEvent ===\n") 24 | e.debugKey() 25 | 26 | fmt.Printf("Entry size: %d\n", len(e.Entries)) 27 | fmt.Printf("Entries:\n") 28 | for _, e := range e.Entries { 29 | id := fmt.Sprintf("%d-%d", e.Id.Ms, e.Id.Seq) 30 | var fields []string 31 | for field, value := range e.Fields { 32 | fields = append(fields, fmt.Sprintf("%s=%s", field, value)) 33 | } 34 | fmt.Printf("\tid=%s fields=%s\n", id, strings.Join(fields, ",")) 35 | } 36 | 37 | fmt.Printf("Group size: %d\n", len(e.Groups)) 38 | if len(e.Groups) > 0 { 39 | fmt.Printf("Groups:\n") 40 | for _, g := range e.Groups { 41 | fmt.Printf("\tname=%s\n", g.Name) 42 | } 43 | } 44 | 45 | fmt.Printf("\n") 46 | } 47 | 48 | type StreamEntry struct { 49 | Id StreamId 50 | Fields map[string]string 51 | } 52 | 53 | type StreamConsumerGroup struct { 54 | Name string 55 | 56 | // Last delivered (not acknowledged) ID for this group. 57 | // Consumers that will just ask for more messages will served with IDs > than this. 58 | LastId StreamId 59 | 60 | PEL []*StreamNAck 61 | Consumers []*StreamConsumer 62 | } 63 | 64 | type StreamConsumer struct { 65 | // Last time this consumer was active. 66 | SeenTime uint64 67 | 68 | Name string 69 | 70 | // Consumer specific pending entries list: all the pending messages delivered to this 71 | // consumer not yet acknowledged. 72 | PEL []*StreamNAck 73 | } 74 | 75 | type StreamNAck struct { 76 | Id StreamId 77 | 78 | // Last time this message was delivered. 79 | DeliveryTime uint64 80 | 81 | // Number of times this message was delivered. 82 | DeliveryCount uint64 83 | 84 | // The consumer this message was delivered to in the last delivery. 85 | Consumer *StreamConsumer 86 | } 87 | 88 | type StreamId struct { 89 | Ms uint64 90 | Seq uint64 91 | } 92 | 93 | func parseStream(key RedisKey, r *rdbReader, valueType byte) (*StreamObjectEvent, error) { 94 | stream := &StreamObjectEvent{ 95 | RedisKey: key, 96 | } 97 | switch valueType { 98 | case rdbTypeStreamListPacks, rdbTypeListQuickList2: 99 | return parseStream0(r, valueType, stream) 100 | default: 101 | return nil, fmt.Errorf("unsupported stream rdb type: 0x%x", valueType) 102 | } 103 | } 104 | 105 | // t_stream.c::streamAppendItem 106 | func parseStream0(r *rdbReader, valueType byte, stream *StreamObjectEvent) (*StreamObjectEvent, error) { 107 | listPackSize, err := r.GetLengthInt() 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | for i := 0; i < listPackSize; i++ { 113 | /* Get the master ID, the one we'll use as key of the radix tree 114 | * node: the entries inside the listpack itself are delta-encoded 115 | * relatively to this ID. */ 116 | nodeKey, err := r.GetLengthBytes() 117 | if err != nil { 118 | return nil, err 119 | } 120 | masterMs := binary.BigEndian.Uint64(nodeKey[:8]) 121 | masterSeq := binary.BigEndian.Uint64(nodeKey[8:]) 122 | 123 | // Parse list pack 124 | members, err := parseListPack(r) 125 | if err != nil { 126 | return nil, err 127 | } 128 | 129 | // Parse master entry from list pack members. 130 | // 131 | // +-------+---------+------------+---------+--/--+---------+---------+-+ 132 | // | count | deleted | num-fields | field_1 | field_2 | ... | field_N |0| 133 | // +-------+---------+------------+---------+--/--+---------+---------+-+ 134 | // 135 | mIndex := 0 136 | validCount, err := strconv.ParseInt(members[mIndex], 10, 64) 137 | if err != nil { 138 | return nil, err 139 | } 140 | mIndex++ 141 | 142 | deletedCount, err := strconv.ParseInt(members[mIndex], 10, 64) 143 | if err != nil { 144 | return nil, err 145 | } 146 | mIndex++ 147 | 148 | masterNumFields, err := strconv.ParseInt(members[mIndex], 10, 64) 149 | if err != nil { 150 | return nil, err 151 | } 152 | mIndex++ 153 | 154 | masterFields := members[mIndex : mIndex+int(masterNumFields)] 155 | mIndex += int(masterNumFields) 156 | 157 | if last := members[mIndex]; last != "0" { 158 | return nil, fmt.Errorf("stream master entry must end of '0', not %s", last) 159 | } 160 | mIndex++ 161 | 162 | for i := int64(0); i < validCount+deletedCount; i++ { 163 | // 164 | // +-----+--------+----------+-------+-------+-/-+-------+-------+--------+ 165 | // |flags|entry-id|num-fields|field-1|value-1|...|field-N|value-N|lp-count| 166 | // +-----+--------+----------+-------+-------+-/-+-------+-------+--------+ 167 | // 168 | // However if the SAMEFIELD flag is set, we have just to populate 169 | // the entry with the values, so it becomes: 170 | // 171 | // +-----+--------+-------+-/-+-------+--------+ 172 | // |flags|entry-id|value-1|...|value-N|lp-count| 173 | // +-----+--------+-------+-/-+-------+--------+ 174 | 175 | flags, err := strconv.ParseUint(members[mIndex], 10, 64) 176 | if err != nil { 177 | return nil, err 178 | } 179 | mIndex++ 180 | 181 | entryIdMs, err := strconv.ParseUint(members[mIndex], 10, 64) 182 | if err != nil { 183 | return nil, err 184 | } 185 | mIndex++ 186 | 187 | entryIdSeq, err := strconv.ParseUint(members[mIndex], 10, 64) 188 | if err != nil { 189 | return nil, err 190 | } 191 | mIndex++ 192 | 193 | entry := &StreamEntry{ 194 | Id: StreamId{ 195 | Ms: entryIdMs + masterMs, 196 | Seq: entryIdSeq + masterSeq, 197 | }, 198 | } 199 | 200 | if flags&streamItemFlagSameFields == 0 { 201 | nFields, err := strconv.ParseInt(members[mIndex], 10, 64) 202 | if err != nil { 203 | return nil, err 204 | } 205 | mIndex++ 206 | entry.Fields = make(map[string]string, int(nFields)) 207 | for i := 0; i < int(nFields); i++ { 208 | field := members[mIndex] 209 | value := members[mIndex+1] 210 | mIndex += 2 211 | entry.Fields[field] = value 212 | } 213 | } else { 214 | entry.Fields = make(map[string]string, int(masterNumFields)) 215 | for i := 0; i < int(masterNumFields); i++ { 216 | field := masterFields[i] 217 | value := members[mIndex] 218 | mIndex++ 219 | entry.Fields[field] = value 220 | } 221 | } 222 | 223 | _ = members[mIndex] 224 | mIndex++ 225 | 226 | if flags&streamItemFlagDeleted == 0 { 227 | stream.Entries = append(stream.Entries, entry) 228 | } 229 | } 230 | } 231 | 232 | // Load total number of items inside the stream. 233 | // Current number of elements inside this stream. 234 | streamSize, err := r.GetLengthInt() 235 | if err != nil { 236 | return nil, err 237 | } 238 | _ = streamSize 239 | 240 | // Load the last entry ID. 241 | // Zero if there are yet no items. 242 | lastIdMs, err := r.GetLengthUInt64() 243 | if err != nil { 244 | return nil, err 245 | } 246 | lastIdSeq, err := r.GetLengthUInt64() 247 | if err != nil { 248 | return nil, err 249 | } 250 | _ = lastIdMs 251 | _ = lastIdSeq 252 | 253 | if valueType == rdbTypeStreamListPacks2 { 254 | // Load the first entry ID. 255 | // The first non-tombstone entry, zero if empty. 256 | firstIdMs, err := r.GetLengthUInt64() 257 | if err != nil { 258 | return nil, err 259 | } 260 | firstIdSeq, err := r.GetLengthUInt64() 261 | if err != nil { 262 | return nil, err 263 | } 264 | _ = firstIdMs 265 | _ = firstIdSeq 266 | 267 | // Load the maximal deleted entry ID. 268 | // The maximal ID that was deleted. 269 | maxDeletedEntryIdMs, err := r.GetLengthUInt64() 270 | if err != nil { 271 | return nil, err 272 | } 273 | maxDeletedEntryIdSeq, err := r.GetLengthUInt64() 274 | if err != nil { 275 | return nil, err 276 | } 277 | _ = maxDeletedEntryIdMs 278 | _ = maxDeletedEntryIdSeq 279 | 280 | // Load the offset. 281 | // All time count of elements added. 282 | entriesAdded, err := r.GetLengthUInt64() 283 | if err != nil { 284 | return nil, err 285 | } 286 | _ = entriesAdded 287 | } else { 288 | // During migration the offset can be initialized to the stream's 289 | // length. At this point, we also don't care about tombstones 290 | // because CG offsets will be later initialized as well. 291 | 292 | // Not load from reader, so do nothing. 293 | } 294 | 295 | // Consumer groups loading 296 | consumerGroupCount, err := r.GetLengthInt() 297 | if err != nil { 298 | return nil, err 299 | } 300 | for i := 0; i < consumerGroupCount; i++ { 301 | // Get the consumer group name and ID. We can then create the 302 | // consumer group ASAP and populate its structure as 303 | // we read more data. 304 | cg := new(StreamConsumerGroup) 305 | groupName, err := r.GetLengthString() 306 | if err != nil { 307 | return nil, err 308 | } 309 | cg.Name = groupName 310 | 311 | groupLastIdMs, err := r.GetLengthUInt64() 312 | if err != nil { 313 | return nil, err 314 | } 315 | groupLastIdSeq, err := r.GetLengthUInt64() 316 | if err != nil { 317 | return nil, err 318 | } 319 | cg.LastId = StreamId{ 320 | Ms: groupLastIdMs, 321 | Seq: groupLastIdSeq, 322 | } 323 | 324 | // Load group offset. 325 | var groupOffset uint64 326 | if valueType == rdbTypeStreamListPacks2 { 327 | groupOffset, err = r.GetLengthUInt64() 328 | if err != nil { 329 | return nil, err 330 | } 331 | } else { 332 | // Not load offset from reader, so do nothing. 333 | } 334 | _ = groupOffset 335 | 336 | // Load the global PEL for this consumer group, however we'll 337 | // not yet populate the NACK structures with the message 338 | // owner, since consumers for this group and their messages will 339 | // be read as a next step. So for now leave them not resolved 340 | // and later populate it. 341 | pelSize, err := r.GetLengthInt() 342 | if err != nil { 343 | return nil, err 344 | } 345 | //pelMap := make(map[StreamId]*StreamNAck, pelSize) 346 | pelMap := make(map[string]*StreamNAck, pelSize) 347 | for i := 0; i < pelSize; i++ { 348 | pelIdMs, err := r.GetBUint64() 349 | if err != nil { 350 | return nil, err 351 | } 352 | pelIdSeq, err := r.GetBUint64() 353 | if err != nil { 354 | return nil, err 355 | } 356 | pelDeliveryTime, err := r.GetLUint64() 357 | if err != nil { 358 | return nil, err 359 | } 360 | pelDeliveryCount, err := r.GetLengthUInt64() 361 | if err != nil { 362 | return nil, err 363 | } 364 | pel := &StreamNAck{ 365 | Id: StreamId{ 366 | Ms: pelIdMs, 367 | Seq: pelIdSeq, 368 | }, 369 | DeliveryTime: pelDeliveryTime, 370 | DeliveryCount: pelDeliveryCount, 371 | Consumer: nil, 372 | } 373 | cg.PEL = append(cg.PEL, pel) 374 | pelMap[fmt.Sprintf("%d-%d", pelIdMs, pelIdSeq)] = pel 375 | } 376 | 377 | // Now that we loaded our global PEL, we need to load the 378 | // consumers and their local PELs. 379 | consumerNum, err := r.GetLengthUInt64() 380 | if err != nil { 381 | return nil, err 382 | } 383 | for i := 0; i < int(consumerNum); i++ { 384 | c := new(StreamConsumer) 385 | name, err := r.GetLengthString() 386 | if err != nil { 387 | return nil, err 388 | } 389 | c.Name = name 390 | 391 | // Last time this consumer was active 392 | seenTime, err := r.GetLUint64() 393 | if err != nil { 394 | return nil, err 395 | } 396 | c.SeenTime = seenTime 397 | 398 | // Load the PEL about entries owned by this specific consumer. 399 | pelSize, err := r.GetLengthUInt64() 400 | if err != nil { 401 | return nil, err 402 | } 403 | 404 | for i := 0; i < int(pelSize); i++ { 405 | consumerPelIdMs, err := r.GetBUint64() 406 | if err != nil { 407 | return nil, err 408 | } 409 | consumerPelIdSeq, err := r.GetBUint64() 410 | if err != nil { 411 | return nil, err 412 | } 413 | messageId := fmt.Sprintf("%d-%d", consumerPelIdMs, consumerPelIdSeq) 414 | pel, ok := pelMap[messageId] 415 | if !ok { 416 | return nil, fmt.Errorf("consumer pel not found in global pel") 417 | } 418 | pel.Consumer = c 419 | c.PEL = append(c.PEL, pel) 420 | } 421 | cg.Consumers = append(cg.Consumers, c) 422 | } 423 | stream.Groups = append(stream.Groups, cg) 424 | } 425 | 426 | return stream, nil 427 | } 428 | --------------------------------------------------------------------------------