├── .gitignore ├── CNAME ├── LICENSE ├── README.md ├── _config.yml ├── siphon ├── main.go ├── perf.go └── sync.go └── ssdb ├── binlog.go ├── const.go ├── salve.go └── ssdbclient.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.rdb 2 | *.swp 3 | *.log 4 | *.tmp 5 | *.out 6 | /bin/* 7 | .idea/ 8 | *.iml -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | siphon.neov.im -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 reborndb 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Siphon 2 | =========== 3 | 4 | Siphon is a hot data sync tools. 5 | 6 | Mock slave server, sync data between ssdb master and other server. 7 | 8 | Siphon now supports [ssdb](https://github.com/ideawu/ssdb) to [redis](https://github.com/antirez/redis)/[pika](https://github.com/Qihoo360/pika). 9 | 10 | * **SYNC** data from master to slave 11 | 12 | ```sh 13 | siphon sync [--pprof=0.0.0.0:6060] [--ncpu=N] --f=MASTER --t=TARGET [-F masterpassword] [-T targetpassword] 14 | ``` 15 | 16 | Usage: 17 | siphon sync [--ncpu=N] [--parallel=M] --from=MASTER --target=TARGET [--frompassword=MASTERPASSWORD] [--targetpassword=SLAVEPASSWORD] 18 | 19 | Options: 20 | --pprof Set pprof addr and port,like 0.0.0.0:6060 . 21 | 22 | 23 | Options 24 | ------- 25 | + -n _N_, --ncpu=_N_ 26 | 27 | > set runtime.GOMAXPROCS to _N_ 28 | 29 | + -p _P_, --parallel=_P_ 30 | 31 | > set redis/pika maximum number of connections to _P_, default is runtime.GOMAXPROCS 32 | 33 | + --pprof=ip:port 34 | 35 | > binding pprof on ip:port 36 | 37 | Builder 38 | ------- 39 | ``` 40 | go build github.com/imneov/siphon/siphon 41 | ``` 42 | 43 | Example 44 | ------- 45 | ``` 46 | siphon sync -n 4 -p 4 -f 127.0.0.1:8888 -t 127.0.0.1:9221 -T Stip1234 --pprof=0.0.0.0:6060 47 | ``` 48 | 49 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /siphon/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Reborndb Org. All Rights Reserved. 2 | // Licensed under the MIT (MIT-LICENSE.txt) license. 3 | 4 | package main 5 | 6 | import ( 7 | "runtime" 8 | "strconv" 9 | "github.com/docopt/docopt-go" 10 | "github.com/reborndb/go/errors" 11 | "github.com/reborndb/go/log" 12 | "fmt" 13 | "encoding/binary" 14 | ) 15 | 16 | 17 | var args struct { 18 | parallel int 19 | 20 | from string 21 | target string 22 | 23 | fromAuth string 24 | targetAuth string 25 | } 26 | 27 | func parseIntFromString(s string, min, max int) (int, error) { 28 | n, err := strconv.Atoi(s) 29 | if err != nil { 30 | return 0, err 31 | } 32 | if n >= min && n <= max { 33 | return n, nil 34 | } 35 | return 0, errors.Errorf("out of range [%d,%d], got %d", min, max, n) 36 | } 37 | 38 | 39 | func main() { 40 | usage := ` 41 | Usage: 42 | siphon sync [--pprof=0.0.0.0:6060] [--ncpu=N] [--parallel=M] --from=MASTER --target=TARGET [--frompassword=MASTERPASSWORD] [--targetpassword=SLAVEPASSWORD] 43 | 44 | Options: 45 | --pprof Set pprof addr and port,like 0.0.0.0:6060 . 46 | -n N, --ncpu=N Set runtime.GOMAXPROCS to N . 47 | -p M, --parallel=M Set the number of parallel routines to M . 48 | -f MASTER, --from=MASTER Set host:port of master . 49 | -t TARGET, --target=TARGET Set host:port of target . 50 | -F MASTERPASSWORD, --frompassword Set password of master . 51 | -T SLAVEPASSWORD, --targetpassword Set password of target . 52 | ` 53 | 54 | i := int64(-3) 55 | 56 | b := make([]byte, 8) 57 | binary.BigEndian.PutUint64(b, uint64(i)) 58 | fmt.Println(b) 59 | 60 | d, err := docopt.Parse(usage, nil, true, "", false) 61 | if err != nil { 62 | log.PanicError(err, "parse arguments failed") 63 | } 64 | 65 | if s, ok := d["--ncpu"].(string); ok && s != "" { 66 | n, err := parseIntFromString(s, 1, 1024) 67 | if err != nil { 68 | log.PanicErrorf(err, "parse --ncpu failed") 69 | } 70 | runtime.GOMAXPROCS(n) 71 | } 72 | ncpu := runtime.GOMAXPROCS(0) 73 | 74 | if s, ok := d["--parallel"].(string); ok && s != "" { 75 | n, err := parseIntFromString(s, 1, 1024) 76 | if err != nil { 77 | log.PanicErrorf(err, "parse --parallel failed") 78 | } 79 | args.parallel = n 80 | } 81 | if ncpu > args.parallel { 82 | args.parallel = ncpu 83 | } 84 | if args.parallel == 0 { 85 | args.parallel = 4 86 | } 87 | 88 | args.target, _ = d["--target"].(string) 89 | args.from, _ = d["--from"].(string) 90 | 91 | args.fromAuth, _ = d["--frompassword"].(string) 92 | args.targetAuth, _ = d["--targetpassword"].(string) 93 | 94 | log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel) 95 | 96 | 97 | 98 | pprofAddr, _ := d["--pprof"].(string) 99 | if pprofAddr != ""{ 100 | log.Infof("init pprof on %s\n", pprofAddr) 101 | InitPerf([]string{pprofAddr}) 102 | } 103 | 104 | switch { 105 | case d["sync"].(bool): 106 | new(cmdSync).Main() 107 | } 108 | } 109 | 110 | 111 | -------------------------------------------------------------------------------- /siphon/perf.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "net/http/pprof" 6 | 7 | log "github.com/thinkboy/log4go" 8 | ) 9 | 10 | // StartPprof start http pprof. 11 | func InitPerf(pprofBind []string) { 12 | pprofServeMux := http.NewServeMux() 13 | pprofServeMux.HandleFunc("/debug/pprof/", pprof.Index) 14 | pprofServeMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 15 | pprofServeMux.HandleFunc("/debug/pprof/profile", pprof.Profile) 16 | pprofServeMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 17 | for _, addr := range pprofBind { 18 | go func() { 19 | if err := http.ListenAndServe(addr, pprofServeMux); err != nil { 20 | log.Error("http.ListenAndServe(\"%s\", pprofServeMux) error(%v)", addr, err) 21 | panic(err) 22 | } 23 | }() 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /siphon/sync.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Reborndb Org. All Rights Reserved. 2 | // Licensed under the MIT (MIT-LICENSE.txt) license. 3 | 4 | package main 5 | 6 | import ( 7 | "time" 8 | 9 | "github.com/reborndb/go/atomic2" 10 | "github.com/reborndb/go/log" 11 | redis "github.com/garyburd/redigo/redis" 12 | ssdb "github.com/imneov/siphon/ssdb" 13 | "fmt" 14 | "runtime" 15 | ) 16 | 17 | var ( 18 | pool *redis.Pool 19 | ) 20 | 21 | type cmdSync struct { 22 | nread, nrecv, nobjs atomic2.Int64 23 | } 24 | 25 | func (cmd *cmdSync) Main() { 26 | from, fromAuth, target, targetAuth := args.from, args.fromAuth, args.target, args.targetAuth 27 | 28 | if len(from) == 0 { 29 | log.Panic("invalid argument: from") 30 | } 31 | if len(target) == 0 { 32 | log.Panic("invalid argument: target") 33 | } 34 | 35 | log.Infof("sync from '%s' to '%s'\n", from, target) 36 | 37 | cmdsQueue := make(chan []string, 1000) 38 | listcmdsQueue := make(chan []string, 1000) 39 | 40 | pool = newPool(target, targetAuth) 41 | for i:=0;i< args.parallel;i++ { 42 | go func(){ 43 | for{ 44 | cmd := <- cmdsQueue 45 | //log.Info("cmd: (%v)(%v)", cmd, len(cmdsQueue)) 46 | sendCmd(pool, cmd) 47 | } 48 | }() 49 | } 50 | 51 | go func(){ 52 | for{ 53 | cmd := <- listcmdsQueue 54 | //log.Info("list cmd: (%v)(%v)", cmd, len(listcmdsQueue)) 55 | sendCmd(pool, cmd) 56 | } 57 | }() 58 | 59 | if server, err := ssdb.NewSSDBSalve(from, fromAuth, &cmdsQueue, &listcmdsQueue); err == nil { 60 | server.Start() 61 | } 62 | 63 | runtime.Gosched() 64 | 65 | return 66 | } 67 | 68 | 69 | //初始化一个pool 70 | func newPool(target, password string) *redis.Pool { 71 | return &redis.Pool{ 72 | MaxIdle: 4, 73 | MaxActive: 1024, 74 | IdleTimeout: 240 * time.Second, 75 | Dial: func() (redis.Conn, error) { 76 | c, err := redis.Dial("tcp", target, 77 | redis.DialConnectTimeout(5 * time.Second), 78 | redis.DialReadTimeout(5 * time.Second), 79 | redis.DialWriteTimeout(5 * time.Second), 80 | ) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | if password != "" { 86 | status, err := c.Do("AUTH", password) 87 | if err != nil { 88 | c.Close() 89 | return nil, err 90 | } 91 | fmt.Println("AUTH",status) 92 | } 93 | 94 | return c, err 95 | }, 96 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 97 | if time.Since(t) < 20 * time.Second { 98 | return nil 99 | } 100 | _, err := c.Do("PING") 101 | return err 102 | }, 103 | } 104 | } 105 | 106 | func sendCmd(pool *redis.Pool, cmd []string) (err error){ 107 | 108 | if len(cmd) < 1 { 109 | log.Error("cmd is empty(%v)", cmd) 110 | return 111 | } 112 | conn := pool.Get() 113 | defer conn.Close() 114 | //redis操作 115 | commandName := cmd[0] 116 | arguments := []interface{}{} 117 | for _, command := range cmd[1:] { 118 | arguments = append(arguments, command) 119 | } 120 | reply, err := conn.Do(commandName,arguments...) 121 | if err != nil { 122 | log.Error("conn.Do(%v,%v) error(%v)", commandName, arguments, err) 123 | fmt.Println(err, reply) 124 | return err 125 | } 126 | return err 127 | } 128 | -------------------------------------------------------------------------------- /ssdb/binlog.go: -------------------------------------------------------------------------------- 1 | package ssdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | // "strconv" 7 | ) 8 | 9 | var ( 10 | SEQ_SIZE = 8 11 | DATATYPE_SIZE = 1 12 | CMD_SIZE = 1 13 | ) 14 | 15 | type Binlog struct { 16 | seq uint64 17 | datatype uint8 18 | cmdtype uint8 19 | cmd []string 20 | body [][]byte 21 | } 22 | 23 | func LoadBinlog(bytes [][]byte) (binlog *Binlog, err error) { 24 | binlog = &Binlog{} 25 | OFFSET := 0 26 | binlog.seq = binary.LittleEndian.Uint64(bytes[0][OFFSET:SEQ_SIZE]) 27 | OFFSET += SEQ_SIZE 28 | binlog.datatype = (uint8)(bytes[0][OFFSET]) 29 | OFFSET += DATATYPE_SIZE 30 | binlog.cmdtype = (uint8)(bytes[0][OFFSET]) 31 | OFFSET += CMD_SIZE 32 | body := bytes[0][OFFSET:] 33 | binlog.body = bytes 34 | 35 | if len(body) > 1 { 36 | offset := 0 37 | cmdtag := body[offset] 38 | offset += 1 //first byte is cmd tag,second byte is split byte 39 | switch binlog.cmdtype { 40 | case BINLOGCOMMAND_KSET: 41 | if len(bytes) != 2 { 42 | break 43 | } 44 | if cmdtag != DATATYPE_KV { 45 | break 46 | } 47 | binlog.cmd = []string{ 48 | "set", 49 | string(body[offset:]), 50 | string(bytes[1]), 51 | } 52 | case BINLOGCOMMAND_KDEL: 53 | if cmdtag != DATATYPE_KV { 54 | break 55 | } 56 | binlog.cmd = []string{ 57 | "del", 58 | string(body[offset:]), 59 | } 60 | case BINLOGCOMMAND_HSET: 61 | if len(bytes) != 2 { 62 | break 63 | } 64 | if cmdtag != DATATYPE_HASH { 65 | break 66 | } 67 | size := int(body[offset]) 68 | offset += 1 69 | name := string(body[offset : offset+size]) 70 | offset = offset + size 71 | offset += 1 72 | key := string(body[offset:]) 73 | val := string(bytes[1]) 74 | binlog.cmd = []string{ 75 | "hset", 76 | name, 77 | key, 78 | val, 79 | } 80 | case BINLOGCOMMAND_HDEL: 81 | if cmdtag != DATATYPE_HASH { 82 | break 83 | } 84 | size := int(body[offset]) 85 | offset += 1 86 | name := string(body[offset : offset+size]) 87 | offset = offset + size 88 | offset += 1 89 | key := string(body[offset:]) 90 | binlog.cmd = []string{ 91 | "hdel", 92 | name, 93 | key, 94 | } 95 | case BINLOGCOMMAND_ZSET: 96 | if len(bytes) != 2 { 97 | break 98 | } 99 | if cmdtag != DATATYPE_ZSET { 100 | break 101 | } 102 | size := int(body[offset]) 103 | offset += 1 104 | name := string(body[offset : offset+size]) 105 | offset = offset + size 106 | sign := "" 107 | if body[offset] == '-' { 108 | sign = "-" 109 | } 110 | offset += 1 111 | key := string(body[offset:]) 112 | val := sign + string(bytes[1]) //add number's sign 113 | if name != SSDB_EXPIRATION_LIST_KEY { 114 | binlog.cmd = []string{ 115 | "zadd", 116 | name, 117 | val, 118 | key, 119 | } 120 | } else { 121 | // set ttl key 122 | binlog.cmd = []string{ 123 | "expireat", 124 | key, 125 | val[0:10], 126 | } 127 | } 128 | case BINLOGCOMMAND_ZDEL: 129 | if cmdtag != DATATYPE_ZSET { 130 | break 131 | } 132 | size := int(body[offset]) 133 | offset += 1 134 | name := string(body[offset : offset+size]) 135 | offset = offset + size 136 | offset += 1 137 | key := string(body[offset:]) 138 | if name != SSDB_EXPIRATION_LIST_KEY { 139 | binlog.cmd = []string{ 140 | "zrem", 141 | name, 142 | key, 143 | } 144 | } 145 | case BINLOGCOMMAND_QSET, BINLOGCOMMAND_QPUSH_BACK, BINLOGCOMMAND_QPUSH_FRONT: 146 | if len(bytes) != 2 { 147 | break 148 | } 149 | size := int(body[offset]) 150 | offset += 1 151 | name := string(body[offset : offset+size]) 152 | offset = offset + size 153 | seq := binary.BigEndian.Uint64(body[offset:]) 154 | if seq < SSDB_QITEM_MIN_SEQ || seq > SSDB_QITEM_MAX_SEQ { 155 | break 156 | } 157 | val := string(bytes[1]) //list val 158 | switch binlog.cmdtype{ 159 | case BINLOGCOMMAND_QPUSH_BACK: 160 | binlog.cmd = []string{ 161 | "rpush", 162 | name, 163 | val, 164 | } 165 | case BINLOGCOMMAND_QPUSH_FRONT: 166 | binlog.cmd = []string{ 167 | "lpush", 168 | name, 169 | val, 170 | } 171 | case BINLOGCOMMAND_QSET: 172 | fmt.Println("unsuported qset binlog:", name, seq, val) 173 | default: 174 | fmt.Println("unknown qet/push binlog:", binlog) 175 | } 176 | case BINLOGCOMMAND_QPOP_BACK,BINLOGCOMMAND_QPOP_FRONT: 177 | name := string(body[:]) 178 | switch binlog.cmdtype{ 179 | case BINLOGCOMMAND_QPOP_BACK: 180 | binlog.cmd = []string{ 181 | "rpop", 182 | name, 183 | } 184 | case BINLOGCOMMAND_QPOP_FRONT: 185 | binlog.cmd = []string{ 186 | "lpop", 187 | name, 188 | } 189 | default: 190 | fmt.Println("unknown pop binlog:", binlog) 191 | } 192 | default: 193 | fmt.Println("unknown binlog:", binlog) 194 | } 195 | 196 | } 197 | return 198 | } 199 | -------------------------------------------------------------------------------- /ssdb/const.go: -------------------------------------------------------------------------------- 1 | package ssdb 2 | 3 | 4 | const ( 5 | SALVESTATUS_DISCONNECTED = 0 6 | SALVESTATUS_INIT = 1 7 | SALVESTATUS_COPY = 2 8 | SALVESTATUS_SYNC = 4 9 | SALVESTATUS_OUT_OF_SYNC = 8 10 | 11 | 12 | DATATYPE_SYNCLOG = 1 13 | DATATYPE_KV = 'k' 14 | DATATYPE_HASH = 'h' 15 | DATATYPE_HSIZE = 'H' 16 | DATATYPE_ZSET = 's' // key => score 17 | DATATYPE_ZSCORE = 'z' // key|score => "" 18 | DATATYPE_ZSIZE = 'Z' 19 | DATATYPE_QUEUE = 'q' 20 | DATATYPE_QSIZE = 'Q' 21 | DATATYPE_MIN_PREFIX = DATATYPE_HASH 22 | DATATYPE_MAX_PREFIX = DATATYPE_ZSET 23 | 24 | BINLOGTYPE_NOOP = 0 25 | BINLOGTYPE_SYNC = 1 26 | BINLOGTYPE_MIRROR = 2 27 | BINLOGTYPE_COPY = 3 28 | BINLOGTYPE_CTRL = 4 29 | 30 | BINLOGCOMMAND_NONE = 0 31 | BINLOGCOMMAND_KSET = 1 32 | BINLOGCOMMAND_KDEL = 2 33 | BINLOGCOMMAND_HSET = 3 34 | BINLOGCOMMAND_HDEL = 4 35 | BINLOGCOMMAND_ZSET = 5 36 | BINLOGCOMMAND_ZDEL = 6 37 | 38 | BINLOGCOMMAND_QPUSH_BACK = 10 39 | BINLOGCOMMAND_QPUSH_FRONT = 11 40 | BINLOGCOMMAND_QPOP_BACK = 12 41 | BINLOGCOMMAND_QPOP_FRONT = 13 42 | BINLOGCOMMAND_QSET = 14 43 | 44 | BINLOGCOMMAND_BEGIN = 7 45 | BINLOGCOMMAND_END = 8 46 | 47 | SSDB_EXPIRATION_LIST_KEY = "\xff\xff\xff\xff\xff|EXPIRE_LIST|KV" 48 | SSDB_QFRONT_SEQ = uint64(2) 49 | SSDB_QBACK_SEQ = uint64(3) 50 | SSDB_QITEM_MIN_SEQ = uint64(10000) 51 | SSDB_QITEM_MAX_SEQ = uint64(9223372036854775807) 52 | SSDB_QITEM_SEQ_INIT = SSDB_QITEM_MAX_SEQ/2 53 | 54 | ) 55 | 56 | 57 | var ( 58 | STATUS = map[uint32]string{ 59 | 0: "DISCONNECTED", 60 | 1: "INIT", 61 | 2: "COPY", 62 | 4: "SYNC", 63 | 8: "OUT_OF_SYNC", 64 | } 65 | 66 | DATATYPE = map[byte]string{ 67 | byte('k'): "set", 68 | byte('h'): "hset", 69 | byte('s'): "zadd", 70 | } 71 | 72 | DATATRUECOMMAND = map[byte]string{ 73 | byte(1): "set", 74 | byte(2): "del", 75 | byte(3): "hset", 76 | byte(4): "hdel", 77 | byte(5): "zset", 78 | byte(6): "zdel", 79 | } 80 | ) -------------------------------------------------------------------------------- /ssdb/salve.go: -------------------------------------------------------------------------------- 1 | package ssdb 2 | 3 | import ( 4 | log "github.com/thinkboy/log4go" 5 | "fmt" 6 | "time" 7 | ) 8 | var ( 9 | count int32 10 | lastBinlogType string 11 | binlogCmdMetrics map[string]int32 12 | ) 13 | 14 | type SSDBSalve struct { 15 | id string 16 | c *SSDBClient 17 | from string 18 | auth string 19 | status uint32 20 | connectRetry uint32 21 | cmdsQueue *chan []string 22 | listcmdsQueue *chan []string 23 | } 24 | 25 | 26 | func NewSSDBSalve(from string, fromAuth string, cmdsQueue *chan []string, listcmdsQueue *chan []string) (*SSDBSalve, error) { 27 | server := &SSDBSalve{id: fmt.Sprintf("id-%s-%d", from, time.Now().Unix()), 28 | from: from, 29 | auth: fromAuth, 30 | cmdsQueue: cmdsQueue, 31 | listcmdsQueue: listcmdsQueue, 32 | } 33 | return server, nil 34 | 35 | } 36 | 37 | 38 | func (s *SSDBSalve) Start() (err error) { 39 | 40 | s.status = SALVESTATUS_DISCONNECTED; 41 | 42 | binlogCmdMetrics = make(map[string]int32, 1000) 43 | binlogCmdMetrics[string(BINLOGCOMMAND_NONE)] = 0 44 | binlogCmdMetrics[string(BINLOGCOMMAND_KSET)] = 0 45 | binlogCmdMetrics[string(BINLOGCOMMAND_KDEL)] = 0 46 | binlogCmdMetrics[string(BINLOGCOMMAND_HSET)] = 0 47 | binlogCmdMetrics[string(BINLOGCOMMAND_HDEL)] = 0 48 | binlogCmdMetrics[string(BINLOGCOMMAND_ZSET)] = 0 49 | binlogCmdMetrics[string(BINLOGCOMMAND_ZDEL)] = 0 50 | binlogCmdMetrics[string(BINLOGCOMMAND_QPUSH_BACK)] = 0 51 | binlogCmdMetrics[string(BINLOGCOMMAND_QPUSH_FRONT)] = 0 52 | binlogCmdMetrics[string(BINLOGCOMMAND_QPOP_BACK)] = 0 53 | binlogCmdMetrics[string(BINLOGCOMMAND_QPOP_FRONT)] = 0 54 | binlogCmdMetrics[string(BINLOGCOMMAND_QSET)] = 0 55 | binlogCmdMetrics[string(BINLOGCOMMAND_BEGIN)] = 0 56 | binlogCmdMetrics[string(BINLOGCOMMAND_END)] = 0 57 | 58 | reconnect := true 59 | for { 60 | if reconnect { 61 | s.status = SALVESTATUS_DISCONNECTED 62 | reconnect = false; 63 | if s.c != nil { 64 | s.c.Close() 65 | } 66 | s.c = nil 67 | time.Sleep(time.Second * 2) 68 | } 69 | if s.c == nil { 70 | if err = s.connectToMaster();err != nil { 71 | time.Sleep(time.Second * 100) 72 | continue 73 | } 74 | } 75 | 76 | for { 77 | if resp, err := s.c.RecvBinlog();err == nil { 78 | s.handleRecv(resp) 79 | continue 80 | } 81 | fmt.Println("sync error", err) 82 | return err 83 | } 84 | } 85 | } 86 | 87 | 88 | func (s *SSDBSalve) handleRecv(binlog *Binlog) (err error) { 89 | 90 | binlogCmdMetrics[string(binlog.cmdtype)]++ 91 | count++ 92 | if count % 500000 == 1 { 93 | s.dumpStatus() 94 | log.Info("[%s] cmd: %v", STATUS[s.status], binlog.cmd) 95 | } 96 | 97 | switch binlog.datatype { 98 | case BINLOGTYPE_NOOP: 99 | s.handleNoopRecv(binlog) 100 | case BINLOGTYPE_COPY: 101 | s.handleCopyRecv(binlog) 102 | case BINLOGTYPE_SYNC: 103 | s.handleSyncRecv(binlog) 104 | case BINLOGTYPE_CTRL: 105 | s.handleCtrlRecv(binlog) 106 | case BINLOGTYPE_MIRROR: 107 | s.handleMirrorRecv(binlog) 108 | default: 109 | log.Error("[%s] unknown datatype (%v)", STATUS[s.status], binlog) 110 | } 111 | return 112 | } 113 | 114 | 115 | func (s *SSDBSalve) dumpStatus() (err error) { 116 | log.Info("[%s] count is %d", STATUS[s.status], count) 117 | log.Info("binlogCmdMetrics[none:%d begin:%d end:%d ks:%d kd:%d hs:%d hd:%d zs:%d zd:%d qpushb:%d qpushf:%d qpb:%d qpf:%d qset:%d]", 118 | binlogCmdMetrics[string(BINLOGCOMMAND_NONE)], 119 | binlogCmdMetrics[string(BINLOGCOMMAND_BEGIN)], 120 | binlogCmdMetrics[string(BINLOGCOMMAND_END)], 121 | binlogCmdMetrics[string(BINLOGCOMMAND_KSET)], 122 | binlogCmdMetrics[string(BINLOGCOMMAND_KDEL)], 123 | binlogCmdMetrics[string(BINLOGCOMMAND_HSET)], 124 | binlogCmdMetrics[string(BINLOGCOMMAND_HDEL)], 125 | binlogCmdMetrics[string(BINLOGCOMMAND_ZSET)], 126 | binlogCmdMetrics[string(BINLOGCOMMAND_ZDEL)], 127 | binlogCmdMetrics[string(BINLOGCOMMAND_QPUSH_BACK)], 128 | binlogCmdMetrics[string(BINLOGCOMMAND_QPUSH_FRONT)], 129 | binlogCmdMetrics[string(BINLOGCOMMAND_QPOP_BACK)], 130 | binlogCmdMetrics[string(BINLOGCOMMAND_QPOP_FRONT)], 131 | binlogCmdMetrics[string(BINLOGCOMMAND_QSET)], 132 | ) 133 | return nil 134 | } 135 | 136 | 137 | func (s *SSDBSalve) handleNoopRecv(binlog *Binlog) (err error) { 138 | //log.Info("handleNoopRecv:(%v)", binlog) 139 | return nil 140 | } 141 | 142 | func (s *SSDBSalve) handleCopyRecv(binlog *Binlog) (err error) { 143 | //log.Info("handleCopyRecv:(%v)", binlog) 144 | if binlog.cmdtype == BINLOGCOMMAND_BEGIN{ 145 | log.Info("Copy Start,Recv:(%v)", binlog) 146 | s.status = SALVESTATUS_COPY 147 | return 148 | } 149 | if binlog.cmdtype == BINLOGCOMMAND_END{ 150 | log.Info("Copy Stop,Recv:(%v)", binlog) 151 | s.status = SALVESTATUS_SYNC 152 | return 153 | } 154 | s.handleRecvCmd(binlog) 155 | return nil 156 | } 157 | 158 | func (s *SSDBSalve) handleSyncRecv(binlog *Binlog) (err error) { 159 | //log.Info("handleSyncRecv:(%v)", binlog) 160 | s.handleRecvCmd(binlog) 161 | return nil 162 | } 163 | 164 | func (s *SSDBSalve) handleCtrlRecv(binlog *Binlog) (err error) { 165 | log.Info("[%s]handleCtrlRecv:(%v)", STATUS[s.status], binlog) 166 | return nil 167 | } 168 | 169 | func (s *SSDBSalve) handleMirrorRecv(binlog *Binlog) (err error) { 170 | log.Info("[%s]handleMirrorRecv:(%v)", STATUS[s.status], binlog) 171 | return nil 172 | } 173 | 174 | func (s *SSDBSalve) handleRecvCmd(binlog *Binlog) (err error) { 175 | //log.Info("handleRecvCmd: (%v)(%v)", binlog.cmd, binlog) 176 | if len(binlog.cmd) < 1 { 177 | log.Info("[%s]handleRecvCmd: empty cmd in (%v)", STATUS[s.status], binlog.cmd, binlog) 178 | return 179 | } 180 | switch binlog.cmdtype{ 181 | case BINLOGCOMMAND_QSET, BINLOGCOMMAND_QPUSH_BACK, BINLOGCOMMAND_QPUSH_FRONT, BINLOGCOMMAND_QPOP_BACK,BINLOGCOMMAND_QPOP_FRONT: 182 | *s.listcmdsQueue <- binlog.cmd 183 | default: 184 | *s.cmdsQueue <- binlog.cmd 185 | } 186 | return nil 187 | } 188 | 189 | func (s *SSDBSalve) loadStatus() (err error) { 190 | return nil 191 | } 192 | 193 | func (s *SSDBSalve) saveStatus() (err error) { 194 | return nil 195 | } 196 | 197 | func (s *SSDBSalve) connectToMaster() (err error) { 198 | if s.connectRetry % 50 == 1 { 199 | log.Info("[%s][%d] connecting to master at %s...", s.id, s.connectRetry, s.from) 200 | } 201 | 202 | if s.c, err = Connect(s.from); err != nil{ 203 | log.Error("[%s] failed to connect to master:%s (%v)", s.id, s.from, err) 204 | return 205 | } 206 | s.status = SALVESTATUS_INIT 207 | s.connectRetry = 0 208 | 209 | if s.auth != "" { 210 | //开始同步命令 211 | if err = s.c.Send("auth", s.auth); err != nil{ 212 | log.Error("Auth error(%v)", err) 213 | return 214 | } 215 | } 216 | 217 | //开始同步命令 218 | if err = s.c.Send("sync140","0","","sync"); err != nil{ 219 | log.Error("Send sync error(%v)", err) 220 | return 221 | } 222 | log.Info("[%s] ready to receive binlogs", s.id) 223 | 224 | return 225 | } 226 | -------------------------------------------------------------------------------- /ssdb/ssdbclient.go: -------------------------------------------------------------------------------- 1 | package ssdb 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "github.com/seefan/goerr" 8 | "net" 9 | "strconv" 10 | log "github.com/thinkboy/log4go" 11 | ) 12 | 13 | type SSDBClient struct { 14 | isOpen bool 15 | Password string 16 | addr string 17 | sock *net.TCPConn 18 | buf *bufio.ReadWriter 19 | packetBuf bytes.Buffer 20 | } 21 | 22 | var ( 23 | byt = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9} 24 | maxByteSize byte = 58 25 | ) 26 | 27 | func ToNum(bs []byte) int { 28 | re := 0 29 | for _, v := range bs { 30 | if v >= maxByteSize { 31 | return 0 32 | } 33 | re = re*10 + byt[v] 34 | } 35 | return re 36 | } 37 | 38 | func Connect(addr string) (*SSDBClient, error) { 39 | l := &SSDBClient{addr: addr} 40 | l.Start() 41 | return l, nil 42 | 43 | } 44 | 45 | 46 | //打开连接 47 | func (s *SSDBClient) Start() error { 48 | addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s", s.addr)) 49 | if err != nil { 50 | log.Error("Can't connect to (\"%s\")", addr) 51 | return err 52 | } 53 | sock, err := net.DialTCP("tcp", nil, addr) 54 | if err != nil { 55 | return err 56 | } 57 | sock.SetReadBuffer(8192) 58 | //sock.SetWriteBuffer(1024) 59 | s.buf = bufio.NewReadWriter(bufio.NewReader(sock), bufio.NewWriter(sock)) 60 | s.sock = sock 61 | 62 | s.isOpen = true 63 | return nil 64 | } 65 | func (s *SSDBClient) Close() error { 66 | s.isOpen = false 67 | s.buf = nil 68 | s.packetBuf.Reset() 69 | return s.sock.Close() 70 | } 71 | func (s *SSDBClient) IsOpen() bool { 72 | return s.isOpen 73 | } 74 | func (s *SSDBClient) Ping() bool { 75 | _, err := s.Do("info") 76 | return err == nil 77 | } 78 | func (s *SSDBClient) do(args ...interface{}) ([]string, error) { 79 | err := s.send(args) 80 | if err != nil { 81 | return nil, err 82 | } 83 | resp, err := s.Recv() 84 | return resp, err 85 | } 86 | 87 | //通用调用方法，如果有需要在所有方法前执行的，可以在这里执行 88 | func (s *SSDBClient) Do(args ...interface{}) ([]string, error) { 89 | if s.Password != "" { 90 | resp, err := s.do("auth", []string{s.Password}) 91 | if err != nil { 92 | s.sock.Close() 93 | s.isOpen = false 94 | return nil, goerr.NewError(err, "authentication failed") 95 | } 96 | if len(resp) > 0 && resp[0] == "ok" { 97 | //验证成功 98 | s.Password = "" 99 | } else { 100 | return nil, goerr.New("Authentication failed,password is wrong") 101 | } 102 | } 103 | resp, err := s.do(args...) 104 | if err != nil { 105 | s.sock.Close() 106 | s.isOpen = false 107 | } 108 | return resp, err 109 | } 110 | 111 | func (s *SSDBClient) Send(args ...interface{}) error { 112 | return s.send(args) 113 | } 114 | 115 | func (s *SSDBClient) send(args []interface{}) error { 116 | 117 | for _, arg := range args { 118 | switch arg := arg.(type) { 119 | case string: 120 | s.buf.Write(strconv.AppendInt(nil, int64(len(arg)), 10)) 121 | s.buf.WriteByte('\n') 122 | s.buf.WriteString(arg) 123 | case []byte: 124 | s.buf.Write(strconv.AppendInt(nil, int64(len(arg)), 10)) 125 | s.buf.WriteByte('\n') 126 | s.buf.Write(arg) 127 | case int: 128 | bs := strconv.AppendInt(nil, int64(arg), 10) 129 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 130 | s.buf.WriteByte('\n') 131 | s.buf.Write(bs) 132 | case int8: 133 | bs := strconv.AppendInt(nil, int64(arg), 10) 134 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 135 | s.buf.WriteByte('\n') 136 | s.buf.Write(bs) 137 | case int16: 138 | bs := strconv.AppendInt(nil, int64(arg), 10) 139 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 140 | s.buf.WriteByte('\n') 141 | s.buf.Write(bs) 142 | case int32: 143 | bs := strconv.AppendInt(nil, int64(arg), 10) 144 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 145 | s.buf.WriteByte('\n') 146 | s.buf.Write(bs) 147 | case int64: 148 | bs := strconv.AppendInt(nil, arg, 10) 149 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 150 | s.buf.WriteByte('\n') 151 | s.buf.Write(bs) 152 | case uint8: 153 | bs := strconv.AppendUint(nil, uint64(arg), 10) 154 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 155 | s.buf.WriteByte('\n') 156 | s.buf.Write(bs) 157 | case uint16: 158 | bs := strconv.AppendUint(nil, uint64(arg), 10) 159 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 160 | s.buf.WriteByte('\n') 161 | s.buf.Write(bs) 162 | case uint32: 163 | bs := strconv.AppendUint(nil, uint64(arg), 10) 164 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 165 | s.buf.WriteByte('\n') 166 | s.buf.Write(bs) 167 | case uint64: 168 | bs := strconv.AppendUint(nil, uint64(arg), 10) 169 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 170 | s.buf.WriteByte('\n') 171 | s.buf.Write(bs) 172 | case float32: 173 | bs := strconv.AppendFloat(nil, float64(arg), 'g', -1, 32) 174 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 175 | s.buf.WriteByte('\n') 176 | s.buf.Write(bs) 177 | case float64: 178 | bs := strconv.AppendFloat(nil, arg, 'g', -1, 64) 179 | s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10)) 180 | s.buf.WriteByte('\n') 181 | s.buf.Write(bs) 182 | case bool: 183 | s.buf.WriteByte(1) 184 | s.buf.WriteByte('\n') 185 | if arg { 186 | s.buf.WriteByte(1) 187 | } else { 188 | s.buf.WriteByte(0) 189 | } 190 | case nil: 191 | s.buf.WriteByte(0) 192 | s.buf.WriteByte('\n') 193 | s.buf.WriteString("") 194 | default: 195 | fmt.Errorf("bad arguments type,can not json marshal") 196 | } 197 | s.buf.WriteByte('\n') 198 | } 199 | s.buf.WriteByte('\n') 200 | return s.buf.Flush() 201 | } 202 | 203 | func (s *SSDBClient) Recv() (resp []string, err error) { 204 | packetSize := -1 205 | drop := 1 206 | bufSize := 0 207 | s.packetBuf.Reset() 208 | for { 209 | buf, err := s.buf.ReadBytes('\n') 210 | if err != nil { 211 | return nil, err 212 | } 213 | bufSize = len(buf) 214 | if packetSize == -1 && (bufSize == 1 || bufSize == 2 && buf[0] == '\r') { //空行，说明一个数据包结束 215 | return resp, nil 216 | } 217 | if bufSize > 2 && buf[bufSize-2] == '\r' { // drop end 218 | drop = 2 219 | } else { 220 | drop = 1 221 | } 222 | if packetSize == -1 { 223 | packetSize = ToNum(buf[:(bufSize - drop)]) 224 | } else { 225 | //data endwith '\n' or '\r\n' 226 | drop = 0 227 | if s.packetBuf.Len()+bufSize == packetSize+1 { // 228 | drop = 1 229 | } 230 | if s.packetBuf.Len()+bufSize == packetSize+2 { 231 | drop = 2 232 | } 233 | if drop > 0 { 234 | s.packetBuf.Write(buf[:bufSize-drop]) 235 | resp = append(resp, s.packetBuf.String()) 236 | s.packetBuf.Reset() 237 | packetSize = -1 238 | } else { 239 | s.packetBuf.Write(buf) 240 | } 241 | } 242 | } 243 | return resp, nil 244 | } 245 | 246 | 247 | func (s *SSDBClient) RecvBinlog() (resp *Binlog, err error) { 248 | packetSize := -1 249 | drop := 1 250 | bufSize := 0 251 | s.packetBuf.Reset() 252 | var body [][]byte 253 | for { 254 | buf, err := s.buf.ReadBytes('\n') 255 | if err != nil { 256 | return nil, err 257 | } 258 | bufSize = len(buf) 259 | //fmt.Printf("RecvBinlog:packetSize(%d),packetSize(%d),packetSize(%d)\n", packetSize, bufSize, buf[0]) 260 | if packetSize == -1 && (bufSize == 1 || (bufSize == 2 && buf[0] == '\r')) { //空行，说明一个数据包结束 261 | resp,err = LoadBinlog(body) 262 | return resp, nil 263 | } 264 | 265 | if bufSize > 2 && buf[bufSize-2] == '\r' { // drop end 266 | drop = 2 //end with \n\n 267 | } else { 268 | drop = 1 //end with \n 269 | } 270 | if packetSize == -1 { 271 | packetSize = ToNum(buf[:(bufSize - drop)]) 272 | } else { 273 | //data endwith '\n' or '\r\n' 274 | drop = 0 275 | if s.packetBuf.Len()+bufSize == packetSize+1 { // 276 | drop = 1 277 | } 278 | if s.packetBuf.Len()+bufSize == packetSize+2 { 279 | drop = 2 280 | } 281 | if drop > 0 { 282 | s.packetBuf.Write(buf[:bufSize-drop]) 283 | packet := make([]byte, s.packetBuf.Len()) 284 | copy(packet, s.packetBuf.Bytes()) 285 | body = append(body, packet) 286 | s.packetBuf.Reset() 287 | packetSize = -1 288 | }else { 289 | s.packetBuf.Write(buf) 290 | } 291 | } 292 | } 293 | return 294 | } --------------------------------------------------------------------------------