├── db └── data.db ├── .idea ├── vcs.xml ├── modules.xml ├── goMysqlSync.iml └── workspace.xml ├── util ├── dateutil │ ├── date_format_test.go │ ├── date_util.go │ └── date_format.go ├── netutil │ ├── net_util_test.go │ └── net_util.go ├── tools_test.go ├── httputil │ ├── respond.go │ ├── http_util.go │ └── perform.go ├── byteutil │ ├── byteutil_test.go │ └── byteutil.go ├── logutil │ ├── elastic_log_agent.go │ ├── zk_log_agent.go │ ├── metrics_log_agent.go │ ├── log_config.go │ ├── lumberjack_log.go │ ├── zap_log.go │ ├── rocketmq_log_agent.go │ ├── global_log.go │ └── etcd_log_agent.go ├── stringutil │ ├── string_util_test.go │ └── string_util.go ├── zkutil │ └── zk_util.go ├── fileutil │ └── file_util.go ├── etcdutil │ └── etcd_util.go └── tools.go ├── restart.sh ├── .gitignore ├── global ├── debug.go ├── rate_counter.go ├── model.go └── metrics.go ├── storage ├── election_storage.go ├── position_storage.go ├── etcd_position_storage.go ├── zk_position_storage.go ├── bolt_row_storage.go ├── storage.go └── bolt_position_storage.go ├── service ├── cluster │ ├── election.go │ ├── zk_election.go │ └── etcd_election.go ├── cluster_service.go ├── luaengine │ ├── mq_actuator.go │ ├── db_actuator.go │ ├── es_actuator.go │ ├── mongo_actuator.go │ ├── http_actuator.go │ ├── redis_actuator.go │ └── actuator.go ├── application_service.go └── endpoint │ ├── mysql_test.go │ ├── rabbit.go │ ├── kafka.go │ └── rocket.go ├── app.yml ├── go.mod ├── README.md └── main.go /db/data.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j262965682/goMysqlSync/HEAD/db/data.db -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /util/dateutil/date_format_test.go: -------------------------------------------------------------------------------- 1 | package dateutil 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestConvertGoFormat(t *testing.T) { 9 | 10 | fmt.Println(ConvertGoFormat("yyyy-MM-dd HH:mm:ss")) 11 | 12 | fmt.Println(ConvertGoFormat("yyyy-MM-dd")) 13 | 14 | } 15 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | while : #循环，为了让脚本一直运行监控 3 | do 4 | COUNT=`ps -ef | grep ./MysqlSync |wc -l` 5 | if [ "$COUNT" -gt 1 ]; 6 | then 7 | echo "server service is ok" 8 | else 9 | echo "server servicie not exist" 10 | nohup ./MysqlSync > reStart.log 2>&1 & 11 | fi 12 | sleep 60 13 | done -------------------------------------------------------------------------------- /.idea/goMysqlSync.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | *.yml 8 | 9 | system.log 10 | # Test binary, built with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Dependency directories (remove the comment below to include it) 17 | # vendor/ 18 | 19 | #the Configuration files are not uploaded 20 | -------------------------------------------------------------------------------- /util/netutil/net_util_test.go: -------------------------------------------------------------------------------- 1 | package netutil 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestGetFreePort(t *testing.T) { 9 | fmt.Println(GetFreePort("127.0.0.1")) 10 | } 11 | 12 | func TestGetIp(t *testing.T) { 13 | fmt.Println(GetIpList()) 14 | } 15 | 16 | func TestCheckIp(t *testing.T) { 17 | fmt.Println(CheckIp("192.168.56.1")) 18 | fmt.Println(CheckIp("192.168.56")) 19 | fmt.Println(CheckIp("192.168.56.300")) 20 | } 21 | -------------------------------------------------------------------------------- /util/tools_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestDdlChangeTableName(t *testing.T) { 9 | sql := "ALTER TABLE `site_page`.`t12` \n\tMODIFY COLUMN `tab_logo` varchar(256) NULL COMMENT '页签logo' AFTER `tab_name`;" 10 | newTable := "t1232" 11 | 12 | newSQL, err := DDLChangeTableName(sql, newTable) 13 | if err != nil { 14 | fmt.Println(err) 15 | } else { 16 | fmt.Println(len(newSQL)) 17 | fmt.Println(newSQL) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /util/httputil/respond.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | ) 7 | 8 | // 应答实体 9 | type RespondEntity struct { 10 | statusCode int 11 | data []byte 12 | } 13 | 14 | func (t *RespondEntity) StatusCode() int { 15 | return t.statusCode 16 | } 17 | 18 | func (t *RespondEntity) StatusText() string { 19 | return http.StatusText(t.statusCode) 20 | } 21 | 22 | func (t *RespondEntity) Data() []byte { 23 | return t.data 24 | } 25 | 26 | func (t *RespondEntity) AsString() string { 27 | return string(t.data) 28 | } 29 | 30 | func (t *RespondEntity) Unmarshal(entity interface{}) error { 31 | err := json.Unmarshal(t.data, entity) 32 | if err != nil { 33 | return err 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /util/byteutil/byteutil_test.go: -------------------------------------------------------------------------------- 1 | package byteutil 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/gob" 7 | "fmt" 8 | "testing" 9 | ) 10 | 11 | func TestGetBytes(t *testing.T) { 12 | var buf bytes.Buffer 13 | enc := gob.NewEncoder(&buf) 14 | enc.Encode(true) 15 | println(buf.Bytes()) 16 | println(len(buf.Bytes())) 17 | println(string(buf.Bytes())) 18 | 19 | buf2 := bytes.NewBuffer([]byte{}) 20 | binary.Write(buf2, binary.BigEndian, true) 21 | println(buf2.Bytes()) 22 | println(len(buf2.Bytes())) 23 | } 24 | 25 | func TestCopyBytes(t *testing.T) { 26 | buf2 := bytes.NewBuffer([]byte{}) 27 | binary.Write(buf2, binary.BigEndian, true) 28 | a := buf2.Bytes() 29 | fmt.Println(a) 30 | b := a 31 | fmt.Println(b) 32 | a = nil 33 | fmt.Println(a) 34 | fmt.Println(b) 35 | } 36 | -------------------------------------------------------------------------------- /global/debug.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | *

17 | */ 18 | package global 19 | 20 | import ( 21 | "fmt" 22 | "go-mysql-sync/util/stringutil" 23 | ) 24 | 25 | func Debug(msg string, data interface{}) { 26 | fmt.Println(msg, " :: \n", stringutil.ToJsonString(data)) 27 | } 28 | -------------------------------------------------------------------------------- /util/logutil/elastic_log_agent.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | type ElsLoggerAgent struct { 21 | } 22 | 23 | func NewElsLoggerAgent() *ElsLoggerAgent { 24 | return &ElsLoggerAgent{} 25 | } 26 | 27 | func (s *ElsLoggerAgent) Printf(format string, v ...interface{}) { 28 | Infof(format, v) 29 | } 30 | -------------------------------------------------------------------------------- /util/logutil/zk_log_agent.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | type ZkLoggerAgent struct { 21 | } 22 | 23 | func NewZkLoggerAgent() *ZkLoggerAgent { 24 | return &ZkLoggerAgent{} 25 | } 26 | 27 | func (s *ZkLoggerAgent) Printf(template string, args ...interface{}) { 28 | Infof(template, args...) 29 | } 30 | -------------------------------------------------------------------------------- /util/stringutil/string_util_test.go: -------------------------------------------------------------------------------- 1 | package stringutil 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | "unsafe" 8 | ) 9 | 10 | func TestIsChineseChar(t *testing.T) { 11 | println(IsChineseChar("a")) 12 | println(IsChineseChar(",")) 13 | println(IsChineseChar("a我b")) 14 | println(IsChineseChar("，")) 15 | } 16 | 17 | type RowRequest struct { 18 | RuleKey string 19 | Action int 20 | Row []interface{} 21 | } 22 | 23 | //func TestStringCopy(t *testing.T) { 24 | // copies := stringu.Copy("ssssss") 25 | // fmt.Println(copies) 26 | //} 27 | 28 | func TestIndexof(t *testing.T) { 29 | str := "ss_sss_s" 30 | index := strings.Index(str, "_") 31 | fmt.Println(index) 32 | fmt.Println(str[index+1 : len(str)]) 33 | } 34 | 35 | func TestToUint32(t *testing.T) { 36 | str := "964063387" 37 | fmt.Println(ToUint32(str)) 38 | 39 | str2 := "a964063387" 40 | fmt.Println(ToUint32(str2)) 41 | fmt.Println(ToUint32Safe(str2)) 42 | } 43 | 44 | func TestInt(t *testing.T) { 45 | var ti int 46 | fmt.Println(unsafe.Sizeof(ti)) 47 | } 48 | -------------------------------------------------------------------------------- /util/logutil/metrics_log_agent.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | type MetricsLoggerAgent struct { 21 | } 22 | 23 | func NewMetricsLoggerAgent() *MetricsLoggerAgent { 24 | return &MetricsLoggerAgent{} 25 | } 26 | 27 | // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. 28 | func (s *MetricsLoggerAgent) Printf(format string, v ...interface{}) { 29 | Infof(format, v) 30 | } 31 | -------------------------------------------------------------------------------- /storage/election_storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import "go-mysql-sync/global" 21 | 22 | type ElectionStorage interface { 23 | Elect() error 24 | } 25 | 26 | func NewElectionStorage(conf *global.Config) PositionStorage { 27 | if conf.IsCluster() { 28 | if conf.IsZk() { 29 | return &zkPositionStorage{ 30 | Conf: conf, 31 | } 32 | } 33 | if conf.IsEtcd() { 34 | 35 | } 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /service/cluster/election.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package cluster 19 | 20 | import ( 21 | "go-mysql-sync/global" 22 | ) 23 | 24 | type Election interface { 25 | Elect() error 26 | IsLeader() bool 27 | Leader() string 28 | } 29 | 30 | func NewElection(_informCh chan bool, cfg *global.Config) Election { 31 | if cfg.IsZk() { 32 | return newZkElection(_informCh, cfg) 33 | } 34 | if cfg.IsEtcd() { 35 | return newEtcdElection(_informCh, cfg) 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /util/zkutil/zk_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package zkutil 19 | 20 | import ( 21 | "github.com/samuel/go-zookeeper/zk" 22 | ) 23 | 24 | func CreateDirIfNecessary(dir string, conn *zk.Conn) error { 25 | exist, _, err := conn.Exists(dir) 26 | if err != nil { 27 | return err 28 | } 29 | if !exist { 30 | _, err := conn.Create(dir, nil, 0, zk.WorldACL(zk.PermAll)) 31 | if err != nil { 32 | return err 33 | } 34 | } 35 | 36 | return nil 37 | } 38 | 39 | func DeleteDir(dir string, conn *zk.Conn) error { 40 | _, stat, err := conn.Get(dir) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | return conn.Delete(dir, stat.Version) 46 | } 47 | -------------------------------------------------------------------------------- /util/logutil/log_config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | const ( 21 | _logFileName = "system.log" 22 | _logLevelInfo = "info" 23 | _logLevelWarn = "warn" 24 | _logLevelError = "error" 25 | _logMaxSize = 500 26 | _logMaxAge = 30 27 | _logEncodingJson = "json" 28 | ) 29 | 30 | // logger 配置 31 | type LoggerConfig struct { 32 | Level string `yaml:"level"` //日志级别 debug|info|warn|error 33 | Store string `yaml:"store"` //日志目录 34 | FileName string `yaml:"file_name"` //日志文件名称 35 | MaxSize int `yaml:"max_size"` //日志文件最大M字节 36 | MaxAge int `yaml:"max_age"` //日志文件最大存活的天数 37 | Compress bool `yaml:"compress"` //是否启用压缩 38 | Encoding string `yaml:"encoding"` //日志编码 console|json 39 | } 40 | -------------------------------------------------------------------------------- /util/logutil/lumberjack_log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | import ( 21 | "path/filepath" 22 | 23 | "gopkg.in/natefinch/lumberjack.v2" 24 | ) 25 | 26 | func NewLumberjackLogger(config *LoggerConfig) *lumberjack.Logger { 27 | if config.MaxSize <= 0 { 28 | config.MaxSize = _logMaxSize 29 | } 30 | if config.MaxSize <= 0 { 31 | config.MaxAge = _logMaxAge 32 | } 33 | if config.FileName == "" { 34 | config.FileName = _logFileName 35 | } 36 | 37 | return &lumberjack.Logger{ // 定义日志分割器 38 | Filename: filepath.Join(config.Store, config.FileName), // 日志文件全路径 39 | MaxSize: config.MaxSize, // 文件最大M字节 40 | MaxAge: config.MaxAge, // 最多保留几天 41 | Compress: config.Compress, // 是否压缩 42 | LocalTime: true, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /storage/position_storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import ( 21 | "github.com/siddontang/go-mysql/mysql" 22 | 23 | "go-mysql-sync/global" 24 | ) 25 | 26 | type PositionStorage interface { 27 | Initialize() error 28 | Save(pos mysql.Position) error 29 | Get() (mysql.Position, error) 30 | 31 | //重写上面三个接口方法 32 | Init() error 33 | RecordPosition(pos global.PosRequest) error 34 | AcquirePosition() (pos mysql.Position, err error) 35 | AcquirePositionBySecond(second uint32) (pos mysql.Position, err error) 36 | DeletePositionBySecond(second uint32) (err error) 37 | } 38 | 39 | func NewPositionStorage(conf *global.Config) PositionStorage { 40 | if conf.IsCluster() { 41 | if conf.IsZk() { 42 | return &zkPositionStorage{ 43 | Conf: conf, 44 | } 45 | } 46 | if conf.IsEtcd() { 47 | return &etcdPositionStorage{ 48 | Conf: conf, 49 | } 50 | } 51 | } 52 | 53 | return &boltPositionStorage{} 54 | } 55 | -------------------------------------------------------------------------------- /global/rate_counter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package global 19 | 20 | import ( 21 | "go-mysql-sync/util/logutil" 22 | "sync" 23 | "time" 24 | ) 25 | 26 | var max int64 27 | var buckets = make(map[int64]int64) 28 | var lock sync.Mutex 29 | 30 | func Mark(n int) { 31 | curr := time.Now().Unix() 32 | 33 | nnn := int64(n) 34 | nn, ok := buckets[curr] 35 | if ok { 36 | buckets[curr] = nnn + nn 37 | } else { 38 | buckets[curr] = nnn 39 | } 40 | } 41 | 42 | func InitRateCounter() { 43 | ticker := time.NewTicker(5 * time.Second) 44 | go func() { 45 | for { 46 | <-ticker.C 47 | if len(buckets) == 0 { 48 | continue 49 | } 50 | 51 | var sum int64 52 | for k, v := range buckets { 53 | sum = sum + v 54 | logutil.Infof("时间桶：%d ，处理条数：%d", k, v) 55 | } 56 | rate := sum / int64(len(buckets)) 57 | if rate > max { 58 | max = rate 59 | } 60 | logutil.Infof("近60s平均速率(TPS)：%d", rate) 61 | logutil.Infof("历史最高速率(TPS)：%d", max) 62 | 63 | curr := time.Now().Unix() 64 | for k, _ := range buckets { 65 | if curr-k > 60 { 66 | delete(buckets, k) 67 | } 68 | } 69 | } 70 | }() 71 | } 72 | -------------------------------------------------------------------------------- /app.yml: -------------------------------------------------------------------------------- 1 | # 源mysql配置 2 | addr: 10.110.2.87:3306 3 | user: user_sync 4 | pass: user_sync 5 | charset : utf8 6 | slave_id: 520 #slave ID 7 | flavor: mysql #mysql or mariadb,默认mysql 8 | mysqldump: # /usr/local/mysql-5.7.31/bin/mysqldump 9 | 10 | #系统相关配置 11 | db_days: 30 12 | data_dir: D:\GOPROJECT1\goMysqlSync #/usr/local/goMysqlSync #应用产生的数据存放地址，包括日志、缓存数据等，默认当前运行目录下store文件夹 13 | logger: 14 | file_name: system.log 15 | level: debug #日志级别；支持：debug|info|warn|error，默认info 16 | store: D:\GOPROJECT1\goMysqlSync #/usr/local/goMysqlSync 17 | 18 | #prometheus相关配置 19 | label: test #prometheus exporter的tag 20 | enable_exporter: true #是否启用prometheus exporter，默认false 21 | exporter_addr: 9595 #prometheus exporter端口，默认9595 22 | 23 | #目标类型 24 | target: mysql 25 | #mysql连接配置 26 | mysql_addrs: 10.110.2.90:3306 #mysql地址，多个用逗号分隔 27 | mysql_username: user_sync #mysql用户名 28 | mysql_pass: user_sync #mysql密码 29 | threads: 20 #增量数据回放多线程数量 30 | record_rows: 255 #增量数据回放每批次大小 31 | dump_threads: 40 #全量同步线程数 32 | dump_record_rows: 1000 #全量同步每批次大小 33 | 34 | #规则配置 35 | rule: 36 | - 37 | schema: db_expense #数据库名称 38 | table: table_all_in #表名称 # table_all_in 表示全部表 39 | column_underscore_to_camel: true #列名称下划线转驼峰,默认为false 40 | value_encoder: json #值编码，支持json、kv-commas、v-commas；默认为json；json形如：{"id":123,"name":"wangjie"} 、kv-commas形如：id=123,name="wangjie"、v-commas形如：123,wangjie 41 | # - 42 | # schema: test_bi_data_sync #数据库名称 43 | # table: table_all_in #表名称 # table_all_in 表示全部表 44 | # column_underscore_to_camel: true #列名称下划线转驼峰,默认为false 45 | # value_encoder: json #值编码，支持json、kv-commas、v-commas；默认为json；json形如：{"id":123,"name":"wangjie"} 、kv-commas形如：id=123,name="wangjie"、v-commas形如：123,wangjie 46 | # 47 | # 48 | # 49 | -------------------------------------------------------------------------------- /service/cluster_service.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package service 19 | 20 | import ( 21 | "go-mysql-sync/global" 22 | "go-mysql-sync/service/cluster" 23 | "go-mysql-sync/util/logutil" 24 | ) 25 | 26 | type ClusterService struct { 27 | electionCh chan bool 28 | election cluster.Election 29 | } 30 | 31 | func (s *ClusterService) boot(cfg *global.Config) error { 32 | s.electionCh = make(chan bool, 1) 33 | s.election = cluster.NewElection(s.electionCh, cfg) 34 | logutil.BothInfof("Start master election...") 35 | 36 | err := s.election.Elect() 37 | if err != nil { 38 | return err 39 | } 40 | 41 | go s.startElectListener() 42 | 43 | return nil 44 | } 45 | 46 | func (s *ClusterService) startElectListener() { 47 | for { 48 | select { 49 | case flag := <-s.electionCh: 50 | if flag { 51 | global.SetLeaderState(global.MetricsStateOK) 52 | TransferServiceIns().Restart() 53 | logutil.BothInfof("The current node is the master") 54 | } else { 55 | global.SetLeaderState(global.MetricsStateNO) 56 | TransferServiceIns().Pause() 57 | logutil.BothInfof("The current node is the follower, master node is : %s", s.election.Leader()) 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go-mysql-sync 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.27.2 7 | github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect 8 | github.com/apache/rocketmq-client-go/v2 v2.0.0 9 | github.com/coreos/etcd v3.3.25+incompatible // indirect 10 | github.com/go-echarts/statsview v0.3.4 11 | github.com/go-ole/go-ole v1.2.5 // indirect 12 | github.com/go-redis/redis v6.15.9+incompatible 13 | github.com/juju/errors v0.0.0-20200330140219-3fe23663418f 14 | github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf 15 | github.com/mattn/go-colorable v0.1.8 16 | github.com/olivere/elastic v6.2.35+incompatible 17 | github.com/olivere/elastic/v7 v7.0.22 18 | github.com/panjf2000/ants/v2 v2.5.0 // indirect 19 | github.com/pingcap/errors v0.11.0 20 | github.com/pingcap/tidb v2.0.11+incompatible // indirect 21 | github.com/pkg/errors v0.9.1 22 | github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 23 | github.com/prometheus/client_golang v1.8.0 24 | github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 25 | github.com/satori/go.uuid v1.2.0 26 | github.com/shirou/gopsutil v3.21.3+incompatible 27 | github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 28 | github.com/siddontang/go-mysql v1.1.0 29 | github.com/streadway/amqp v1.0.0 30 | github.com/tklauser/go-sysconf v0.3.5 // indirect 31 | github.com/vmihailenco/msgpack v4.0.4+incompatible 32 | github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da 33 | go.etcd.io/bbolt v1.3.5 34 | go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 35 | go.mongodb.org/mongo-driver v1.4.4 36 | go.uber.org/atomic v1.7.0 37 | go.uber.org/zap v1.16.0 38 | google.golang.org/appengine v1.6.7 // indirect 39 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 40 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 41 | gopkg.in/yaml.v2 v2.4.0 42 | gorm.io/driver/mysql v1.0.3 43 | gorm.io/gorm v1.20.8 44 | ) 45 | -------------------------------------------------------------------------------- /util/fileutil/file_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package fileutil 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | "strings" 25 | ) 26 | 27 | // 获取程序运行路径 28 | func GetCurrentDirectory() string { 29 | dir, err := filepath.Abs(filepath.Dir(os.Args[0])) 30 | if err != nil { 31 | fmt.Errorf(err.Error()) 32 | } 33 | return strings.Replace(dir, "\\", "/", -1) 34 | } 35 | 36 | // 判断给定的文件路径是否存在 37 | func IsExist(path string) bool { 38 | _, err := os.Stat(path) 39 | if err == nil { 40 | return true 41 | } 42 | if os.IsNotExist(err) { 43 | return false 44 | } 45 | return false 46 | } 47 | 48 | // 判断给定的路径是否是文件夹 49 | func IsDir(path string) bool { 50 | if stat, err := os.Stat(path); err == nil { 51 | return stat.IsDir() 52 | } 53 | return false 54 | } 55 | 56 | // 给定的文件不存在则创建 57 | func CreateFileIfNecessary(path string) bool { 58 | _, err := os.Stat(path) 59 | if err != nil && os.IsNotExist(err) { 60 | if file, err := os.Create(path); err == nil { 61 | file.Close() 62 | } 63 | } 64 | exist := IsExist(path) 65 | return exist 66 | } 67 | 68 | // 给定的目录不存在则创建 69 | func MkdirIfNecessary(path string) error { 70 | _, err := os.Stat(path) 71 | if err != nil && os.IsNotExist(err) { 72 | if err := os.MkdirAll(path, os.ModePerm); err != nil { 73 | // os.Chmod(path, 0777) 74 | return err 75 | } 76 | } 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /service/luaengine/mq_actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | lua "github.com/yuin/gopher-lua" 22 | 23 | "go-mysql-sync/global" 24 | ) 25 | 26 | func mqModule(L *lua.LState) int { 27 | t := L.NewTable() 28 | L.SetFuncs(t, _mqModuleApi) 29 | L.Push(t) 30 | return 1 31 | } 32 | 33 | var _mqModuleApi = map[string]lua.LGFunction{ 34 | "rawRow": rawRow, 35 | "rawAction": rawAction, 36 | 37 | "SEND": msgSend, 38 | } 39 | 40 | func msgSend(L *lua.LState) int { 41 | topic := L.CheckAny(1) 42 | msg := L.CheckAny(2) 43 | 44 | ret := L.GetGlobal(_globalRET) 45 | L.SetTable(ret, msg, topic) 46 | return 0 47 | } 48 | 49 | func DoMQOps(input map[string]interface{}, action string, rule *global.Rule) ([]*global.MQRespond, error) { 50 | L := _pool.Get() 51 | defer _pool.Put(L) 52 | 53 | row := L.NewTable() 54 | paddingTable(L, row, input) 55 | ret := L.NewTable() 56 | L.SetGlobal(_globalRET, ret) 57 | L.SetGlobal(_globalROW, row) 58 | L.SetGlobal(_globalACT, lua.LString(action)) 59 | 60 | funcFromProto := L.NewFunctionFromProto(rule.LuaProto) 61 | L.Push(funcFromProto) 62 | err := L.PCall(0, lua.MultRet, nil) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | list := make([]*global.MQRespond, 0, ret.Len()) 68 | ret.ForEach(func(k lua.LValue, v lua.LValue) { 69 | resp := global.MQRespondPool.Get().(*global.MQRespond) 70 | resp.ByteArray = lvToByteArray(k) 71 | resp.Topic = lvToString(v) 72 | list = append(list, resp) 73 | }) 74 | 75 | return list, nil 76 | } 77 | -------------------------------------------------------------------------------- /util/logutil/zap_log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | import ( 21 | "errors" 22 | "fmt" 23 | "io" 24 | "time" 25 | 26 | "go-mysql-sync/util/fileutil" 27 | "go.uber.org/zap" 28 | "go.uber.org/zap/zapcore" 29 | ) 30 | 31 | func NewZapLogger(config *LoggerConfig, writer io.Writer, options ...zap.Option) (*zap.Logger, error) { 32 | if config.MaxSize <= 0 { 33 | config.MaxSize = _logMaxSize 34 | } 35 | if config.MaxSize <= 0 { 36 | config.MaxAge = _logMaxAge 37 | } 38 | 39 | if err := fileutil.MkdirIfNecessary(config.Store); err != nil { 40 | return nil, errors.New(fmt.Sprintf("create log store : %s", err.Error())) 41 | } 42 | 43 | encoderConfig := newEncoderConfig() 44 | var encoder zapcore.Encoder 45 | if config.Encoding == _logEncodingJson { 46 | encoder = zapcore.NewJSONEncoder(encoderConfig) 47 | } else { 48 | encoder = zapcore.NewConsoleEncoder(encoderConfig) 49 | } 50 | core := zapcore.NewCore( 51 | encoder, 52 | zapcore.AddSync(writer), 53 | getZapLevel(config.Level), 54 | ) 55 | return zap.New(core), nil 56 | } 57 | 58 | func getZapLevel(level string) zapcore.Level { 59 | var zapLevel zapcore.Level 60 | switch level { 61 | case _logLevelInfo: 62 | zapLevel = zap.InfoLevel 63 | case _logLevelWarn: 64 | zapLevel = zap.WarnLevel 65 | case _logLevelError: 66 | zapLevel = zap.ErrorLevel 67 | default: 68 | zapLevel = zap.DebugLevel 69 | } 70 | return zapLevel 71 | } 72 | 73 | func newEncoderConfig() zapcore.EncoderConfig { 74 | encoderConfig := zap.NewProductionEncoderConfig() 75 | encoderConfig.EncodeTime = func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { 76 | enc.AppendString(t.Format("2006-01-02 15:04:05")) 77 | } 78 | encoderConfig.CallerKey = "" 79 | return encoderConfig 80 | } 81 | -------------------------------------------------------------------------------- /storage/etcd_position_storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import ( 21 | "encoding/json" 22 | 23 | "github.com/siddontang/go-mysql/mysql" 24 | 25 | "go-mysql-sync/global" 26 | "go-mysql-sync/util/etcdutil" 27 | ) 28 | 29 | type etcdPositionStorage struct { 30 | Conf *global.Config 31 | } 32 | 33 | func (s *etcdPositionStorage) Initialize() error { 34 | data, err := json.Marshal(mysql.Position{}) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | err = etcdutil.CreateIfNecessary(s.Conf.ZePositionDir(), string(data), _etcdOps) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func (s *etcdPositionStorage) Save(pos mysql.Position) error { 48 | data, err := json.Marshal(pos) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | return etcdutil.Save(s.Conf.ZePositionDir(), string(data), _etcdOps) 54 | } 55 | 56 | func (s *etcdPositionStorage) Get() (mysql.Position, error) { 57 | var entity mysql.Position 58 | 59 | data, _, err := etcdutil.Get(s.Conf.ZePositionDir(), _etcdOps) 60 | if err != nil { 61 | return entity, err 62 | } 63 | 64 | err = json.Unmarshal(data, &entity) 65 | 66 | return entity, err 67 | } 68 | 69 | func (s *etcdPositionStorage) Init() error { 70 | //TODO 71 | return nil 72 | } 73 | 74 | func (s *etcdPositionStorage) RecordPosition(pos global.PosRequest) error { 75 | //TODO 76 | return nil 77 | } 78 | 79 | func (s *etcdPositionStorage) AcquirePosition() (pos mysql.Position, err error) { 80 | //TODO 81 | return pos, err 82 | } 83 | 84 | func (s *etcdPositionStorage) AcquirePositionBySecond(second uint32) (pos mysql.Position, err error) { 85 | return pos, err 86 | } 87 | 88 | func (s *etcdPositionStorage) DeletePositionBySecond(second uint32) (err error) { 89 | //TODO 90 | return nil 91 | } 92 | -------------------------------------------------------------------------------- /util/dateutil/date_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package dateutil 19 | 20 | import ( 21 | "time" 22 | ) 23 | 24 | const ( 25 | // 日期格式 26 | DayFormatter = "2006-01-02" 27 | // 日期时间格式--分 28 | DayTimeMinuteFormatter = "2006-01-02 15:04" 29 | // 日期时间格式--秒 30 | DayTimeSecondFormatter = "2006-01-02 15:04:05" 31 | // 日期时间格式--毫秒 32 | DayTimeMillisecondFormatter = "2006-01-02 15:04:05.sss" 33 | // 时间格式--秒 34 | TimeSecondFormatter = "15:04:05" 35 | ) 36 | 37 | // 默认格式2006-01-02 15:04:05 38 | func NowFormatted() string { 39 | return time.Now().Format(DayTimeSecondFormatter) 40 | } 41 | 42 | func NowLayout(layout string) string { 43 | return time.Now().Format(layout) 44 | } 45 | 46 | func Layout(date time.Time, layout string) string { 47 | return date.Format(layout) 48 | } 49 | 50 | func DefaultLayout(time time.Time) string { 51 | return time.Format(DayTimeSecondFormatter) 52 | } 53 | 54 | func FromDefaultLayout(str string) time.Time { 55 | loc, _ := time.LoadLocation("Local") 56 | theTime, _ := time.ParseInLocation(DayTimeSecondFormatter, str, loc) 57 | return theTime 58 | } 59 | 60 | // 当前的毫秒时间戳 61 | func NowMillisecond() int64 { 62 | return time.Now().UnixNano() / 1e6 63 | } 64 | 65 | func PastDayDate(pastDay int) time.Time { 66 | return time.Now().AddDate(0, 0, -pastDay) 67 | } 68 | 69 | func FutureDayDate(futureDay int) time.Time { 70 | return time.Now().AddDate(0, 0, futureDay) 71 | } 72 | 73 | func WeekStartDayDate() time.Time { 74 | now := time.Now() 75 | offset := int(time.Monday - now.Weekday()) 76 | if offset > 0 { 77 | offset = -6 78 | } 79 | weekStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) 80 | return weekStart 81 | } 82 | 83 | func MonthStartDayDate() time.Time { 84 | year, month, _ := time.Now().Date() 85 | monthStart := time.Date(year, month, 1, 0, 0, 0, 0, time.Local) 86 | return monthStart 87 | } 88 | -------------------------------------------------------------------------------- /util/logutil/rocketmq_log_agent.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | import ( 21 | "fmt" 22 | "go.uber.org/zap/zapcore" 23 | ) 24 | 25 | type RocketmqLoggerAgent struct { 26 | } 27 | 28 | func NewRocketmqLoggerAgent() *RocketmqLoggerAgent { 29 | return &RocketmqLoggerAgent{} 30 | } 31 | 32 | func (s *RocketmqLoggerAgent) Debug(msg string, fields map[string]interface{}) { 33 | zapFields := make([]zapcore.Field, 0, len(fields)) 34 | for k, v := range fields { 35 | zapFields = append(zapFields, zapcore.Field{ 36 | Key: k, 37 | Type: zapcore.StringType, 38 | String: fmt.Sprintf("%v", v), 39 | }) 40 | } 41 | Debug(msg, zapFields...) 42 | } 43 | 44 | func (s *RocketmqLoggerAgent) Info(msg string, fields map[string]interface{}) { 45 | zapFields := make([]zapcore.Field, 0, len(fields)) 46 | for k, v := range fields { 47 | zapFields = append(zapFields, zapcore.Field{ 48 | Key: k, 49 | Type: zapcore.StringType, 50 | String: fmt.Sprintf("%v", v), 51 | }) 52 | } 53 | Info(msg, zapFields...) 54 | } 55 | 56 | func (s *RocketmqLoggerAgent) Warning(msg string, fields map[string]interface{}) { 57 | zapFields := make([]zapcore.Field, 0, len(fields)) 58 | for k, v := range fields { 59 | zapFields = append(zapFields, zapcore.Field{ 60 | Key: k, 61 | Type: zapcore.StringType, 62 | String: fmt.Sprintf("%v", v), 63 | }) 64 | } 65 | Warn(msg, zapFields...) 66 | } 67 | 68 | func (s *RocketmqLoggerAgent) Error(msg string, fields map[string]interface{}) { 69 | zapFields := make([]zapcore.Field, 0, len(fields)) 70 | for k, v := range fields { 71 | zapFields = append(zapFields, zapcore.Field{ 72 | Key: k, 73 | Type: zapcore.StringType, 74 | String: fmt.Sprintf("%v", v), 75 | }) 76 | } 77 | Error(msg, zapFields...) 78 | } 79 | 80 | func (s *RocketmqLoggerAgent) Fatal(msg string, fields map[string]interface{}) { 81 | s.Error(msg, fields) 82 | } 83 | -------------------------------------------------------------------------------- /service/application_service.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package service 19 | 20 | import ( 21 | sidlog "github.com/siddontang/go-log/log" 22 | 23 | "go-mysql-sync/global" 24 | "go-mysql-sync/storage" 25 | "go-mysql-sync/util/logutil" 26 | ) 27 | 28 | var ( 29 | _transferServiceIns *TransferService 30 | _clusterServiceIns *ClusterService 31 | ) 32 | 33 | func InitApplication(cfgPath string) error { 34 | //解析配置文件 35 | cfg, err := global.NewConfigWithFile(cfgPath) 36 | if err != nil { 37 | return err 38 | } 39 | //配置日志 40 | err = logutil.InitGlobalLogger(cfg.LoggerConfig) 41 | if err != nil { 42 | return err 43 | } 44 | //print tranfer two side 45 | logutil.BothInfof("source %s(%s)", cfg.Flavor, cfg.Addr) 46 | logutil.BothInfof("destination %s", cfg.Destination()) 47 | 48 | var streamHandler *sidlog.StreamHandler 49 | streamHandler, err = sidlog.NewStreamHandler(logutil.GlobalLogWriter()) 50 | if err != nil { 51 | return err 52 | } 53 | agent := sidlog.New(streamHandler, sidlog.Ltime|sidlog.Lfile|sidlog.Llevel) 54 | sidlog.SetDefaultLogger(agent) 55 | 56 | err = storage.InitStorage(cfg) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | transferService := &TransferService{ 62 | config: cfg, 63 | } 64 | err = transferService.initialize() 65 | if err != nil { 66 | return err 67 | } 68 | _transferServiceIns = transferService 69 | 70 | _clusterServiceIns = &ClusterService{} 71 | 72 | return nil 73 | } 74 | 75 | func CtxDone() <-chan struct{} { 76 | return _transferServiceIns.ctx.Done() 77 | } 78 | 79 | func CtxErr() error { 80 | return _transferServiceIns.ctx.Err() 81 | } 82 | 83 | func TransferServiceIns() *TransferService { 84 | return _transferServiceIns 85 | } 86 | 87 | func StartApplication() { 88 | go _transferServiceIns.run() 89 | } 90 | 91 | func CloseApplication() { 92 | _transferServiceIns.close() 93 | storage.CloseStorage() 94 | } 95 | 96 | func BootCluster() error { 97 | return _clusterServiceIns.boot(global.Cfg()) 98 | } 99 | -------------------------------------------------------------------------------- /util/byteutil/byteutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package byteutil 19 | 20 | import ( 21 | "bytes" 22 | "encoding/binary" 23 | "github.com/pquerna/ffjson/ffjson" 24 | "math" 25 | ) 26 | 27 | func Uint32ToBytes(u uint32) []byte { 28 | buf := make([]byte, 4) 29 | binary.BigEndian.PutUint32(buf, u) 30 | return buf 31 | } 32 | 33 | func BytesToUint32(b []byte) uint32 { 34 | return binary.BigEndian.Uint32(b) 35 | } 36 | 37 | func Uint64ToBytes(u uint64) []byte { 38 | buf := make([]byte, 8) 39 | binary.BigEndian.PutUint64(buf, u) 40 | return buf 41 | } 42 | 43 | func BytesToUint64(b []byte) uint64 { 44 | return binary.BigEndian.Uint64(b) 45 | } 46 | 47 | func Uint8ToBytes(u uint8) ([]byte, error) { 48 | bytesBuffer := bytes.NewBuffer([]byte{}) 49 | err := binary.Write(bytesBuffer, binary.BigEndian, &u) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return bytesBuffer.Bytes(), nil 54 | } 55 | 56 | func BytesToUint8(b []byte) (uint8, error) { 57 | bytesBuffer := bytes.NewBuffer(b) 58 | var tmp uint8 59 | err := binary.Read(bytesBuffer, binary.BigEndian, &tmp) 60 | if err != nil { 61 | return 0, err 62 | } 63 | return tmp, nil 64 | } 65 | 66 | func Float32ToBytes(float float32) []byte { 67 | bits := math.Float32bits(float) 68 | bytes := make([]byte, 4) 69 | binary.LittleEndian.PutUint32(bytes, bits) 70 | 71 | return bytes 72 | } 73 | 74 | func ByteToFloat32(bytes []byte) float32 { 75 | bits := binary.LittleEndian.Uint32(bytes) 76 | 77 | return math.Float32frombits(bits) 78 | } 79 | 80 | func Float64ToByte(float float64) []byte { 81 | bits := math.Float64bits(float) 82 | bytes := make([]byte, 8) 83 | binary.LittleEndian.PutUint64(bytes, bits) 84 | 85 | return bytes 86 | } 87 | 88 | func ByteToFloat64(bytes []byte) float64 { 89 | bits := binary.LittleEndian.Uint64(bytes) 90 | 91 | return math.Float64frombits(bits) 92 | } 93 | 94 | func JsonBytes(v interface{}) []byte { 95 | bytes, err := ffjson.Marshal(v) 96 | if nil != err { 97 | return nil 98 | } 99 | return bytes 100 | } 101 | -------------------------------------------------------------------------------- /service/cluster/zk_election.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package cluster 19 | 20 | import ( 21 | "sync" 22 | 23 | "github.com/samuel/go-zookeeper/zk" 24 | "go.uber.org/atomic" 25 | 26 | "go-mysql-sync/global" 27 | "go-mysql-sync/storage" 28 | "go-mysql-sync/util/logutil" 29 | ) 30 | 31 | type zkElection struct { 32 | once sync.Once 33 | 34 | cfg *global.Config 35 | informCh chan bool 36 | 37 | flag atomic.Bool 38 | leader atomic.String 39 | } 40 | 41 | func newZkElection(_informCh chan bool, _cfg *global.Config) *zkElection { 42 | return &zkElection{ 43 | informCh: _informCh, 44 | cfg: _cfg, 45 | } 46 | } 47 | 48 | func (s *zkElection) Elect() error { 49 | data := []byte(s.cfg.Cluster.CurrentNode) 50 | 51 | acl := zk.WorldACL(zk.PermAll) 52 | _, err := storage.ZKConn().Create(s.cfg.ZeElectionDir(), data, zk.FlagEphemeral, acl) 53 | if err == nil { 54 | s.flag.Store(true) 55 | } else { 56 | s.flag.Store(false) 57 | v, _, err := storage.ZKConn().Get(s.cfg.ZeElectionDir()) 58 | if err != nil { 59 | return err 60 | } 61 | s.leader.Store(string(v)) 62 | } 63 | 64 | s.inform() 65 | 66 | s.once.Do(func() { 67 | go s.startWatchTask() 68 | }) 69 | 70 | return nil 71 | } 72 | 73 | func (s *zkElection) IsLeader() bool { 74 | return s.flag.Load() 75 | } 76 | 77 | func (s *zkElection) Leader() string { 78 | if s.flag.Load() { 79 | return s.cfg.Cluster.CurrentNode 80 | } 81 | return s.leader.Load() 82 | } 83 | 84 | func (s *zkElection) inform() { 85 | s.informCh <- s.flag.Load() 86 | } 87 | 88 | func (s *zkElection) startWatchTask() { 89 | logutil.Info("Start Zookeeper watch task") 90 | _, _, ch, _ := storage.ZKConn().ChildrenW(s.cfg.ZeElectionDir()) 91 | for { 92 | select { 93 | case childEvent := <-ch: 94 | if childEvent.Type == zk.EventNodeDeleted { 95 | logutil.Info("Start elect new master ...") 96 | err := s.Elect() 97 | if err != nil { 98 | logutil.Errorf("elect new master error %s ", err.Error()) 99 | } 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /storage/zk_position_storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import ( 21 | "encoding/json" 22 | 23 | "github.com/samuel/go-zookeeper/zk" 24 | "github.com/siddontang/go-mysql/mysql" 25 | 26 | "go-mysql-sync/global" 27 | ) 28 | 29 | type zkPositionStorage struct { 30 | Conf *global.Config 31 | } 32 | 33 | func (s *zkPositionStorage) Initialize() error { 34 | exist, _, err := _zkConn.Exists(s.Conf.ZePositionDir()) 35 | if err != nil { 36 | return err 37 | } 38 | if !exist { 39 | data, err := json.Marshal(mysql.Position{}) 40 | if err != nil { 41 | return err 42 | } 43 | _, err = _zkConn.Create(s.Conf.ZePositionDir(), data, 0, zk.WorldACL(zk.PermAll)) 44 | if err != nil { 45 | return err 46 | } 47 | } 48 | 49 | return nil 50 | } 51 | 52 | func (s *zkPositionStorage) Save(pos mysql.Position) error { 53 | _, stat, err := _zkConn.Get(s.Conf.ZePositionDir()) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | data, err := json.Marshal(pos) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | _, err = _zkConn.Set(s.Conf.ZePositionDir(), data, stat.Version) 64 | 65 | return err 66 | } 67 | 68 | func (s *zkPositionStorage) Get() (mysql.Position, error) { 69 | var entity mysql.Position 70 | 71 | data, _, err := _zkConn.Get(s.Conf.ZePositionDir()) 72 | if err != nil { 73 | return entity, err 74 | } 75 | 76 | err = json.Unmarshal(data, &entity) 77 | 78 | return entity, err 79 | } 80 | 81 | func (s *zkPositionStorage) Init() error { 82 | //TODO 83 | return nil 84 | } 85 | 86 | func (s *zkPositionStorage) RecordPosition(pos global.PosRequest) error { 87 | //TODO 88 | return nil 89 | } 90 | 91 | func (s *zkPositionStorage) AcquirePosition() (pos mysql.Position, err error) { 92 | //TODO 93 | return pos, err 94 | } 95 | 96 | func (s *zkPositionStorage) AcquirePositionBySecond(second uint32) (pos mysql.Position, err error) { 97 | return pos, err 98 | } 99 | 100 | func (s *zkPositionStorage) DeletePositionBySecond(second uint32) (err error) { 101 | //TODO 102 | return nil 103 | } 104 | -------------------------------------------------------------------------------- /storage/bolt_row_storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import ( 21 | "github.com/juju/errors" 22 | "go.etcd.io/bbolt" 23 | 24 | "go-mysql-sync/util/byteutil" 25 | ) 26 | 27 | type BoltRowStorage struct { 28 | } 29 | 30 | func (s *BoltRowStorage) Size() int { 31 | var size int 32 | _bolt.View(func(tx *bbolt.Tx) error { 33 | bt := tx.Bucket(_rowRequestBucket) 34 | size = bt.Stats().KeyN 35 | return nil 36 | }) 37 | 38 | return size 39 | } 40 | 41 | func (s *BoltRowStorage) Add(data []byte) { 42 | _bolt.Update(func(tx *bbolt.Tx) error { 43 | bt := tx.Bucket(_rowRequestBucket) 44 | seq, _ := bt.NextSequence() 45 | return bt.Put(byteutil.Uint64ToBytes(seq), data) 46 | }) 47 | } 48 | 49 | func (s *BoltRowStorage) BatchAdd(list [][]byte) { 50 | _bolt.Batch(func(tx *bbolt.Tx) error { 51 | bt := tx.Bucket(_rowRequestBucket) 52 | for _, data := range list { 53 | seq, _ := bt.NextSequence() 54 | bt.Put(byteutil.Uint64ToBytes(seq), data) 55 | } 56 | return nil 57 | }) 58 | } 59 | 60 | func (s *BoltRowStorage) IdList() []uint64 { 61 | ls := make([]uint64, 0) 62 | _bolt.View(func(tx *bbolt.Tx) error { 63 | bt := tx.Bucket(_rowRequestBucket) 64 | cursor := bt.Cursor() 65 | for k, _ := cursor.First(); k != nil; k, _ = cursor.Next() { 66 | ls = append(ls, byteutil.BytesToUint64(k)) 67 | } 68 | return nil 69 | }) 70 | 71 | return ls 72 | } 73 | 74 | func (s *BoltRowStorage) Get(key uint64) ([]byte, error) { 75 | var entity []byte 76 | err := _bolt.View(func(tx *bbolt.Tx) error { 77 | bt := tx.Bucket(_rowRequestBucket) 78 | data := bt.Get(byteutil.Uint64ToBytes(key)) 79 | if data == nil { 80 | return errors.NotFoundf("Row") 81 | } 82 | 83 | entity = data 84 | return nil 85 | }) 86 | 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | return entity, nil 92 | } 93 | 94 | func (s *BoltRowStorage) Delete(key uint64) error { 95 | return _bolt.Update(func(tx *bbolt.Tx) error { 96 | bt := tx.Bucket(_rowRequestBucket) 97 | return bt.Delete(byteutil.Uint64ToBytes(key)) 98 | }) 99 | } 100 | -------------------------------------------------------------------------------- /util/netutil/net_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package netutil 19 | 20 | import ( 21 | "net" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | ) 26 | 27 | // 检测是否为 IP 格式 28 | func CheckIp(addr string) bool { 29 | if "" == addr { 30 | return false 31 | } 32 | 33 | a := net.ParseIP(addr) 34 | if a == nil { 35 | return false 36 | } 37 | 38 | return true 39 | } 40 | 41 | // 检测地址是否为 IP:端口格式 42 | func CheckHostAddr(addr string) bool { 43 | if "" == addr { 44 | return false 45 | } 46 | 47 | items := strings.Split(addr, ":") 48 | if items == nil || len(items) != 2 { 49 | return false 50 | } 51 | 52 | a := net.ParseIP(items[0]) 53 | if a == nil { 54 | return false 55 | } 56 | 57 | match, err := regexp.MatchString("^[0-9]*$", items[1]) 58 | if err != nil { 59 | return false 60 | } 61 | 62 | i, err := strconv.Atoi(items[1]) 63 | if err != nil { 64 | return false 65 | } 66 | if i < 0 || i > 65535 { 67 | return false 68 | } 69 | 70 | if match == false { 71 | return false 72 | } 73 | return true 74 | } 75 | 76 | // 获取一个空闲的TCP端口 77 | func GetFreePort(bind string) int { 78 | ip := ":" 79 | if "" != bind { 80 | ip = bind + ":" 81 | } 82 | 83 | var port int 84 | for i := 17070; i < 65536; i++ { 85 | addr, _ := net.ResolveTCPAddr("tcp", ip+strconv.Itoa(i)) 86 | listener, err := net.ListenTCP("tcp", addr) 87 | if err == nil { 88 | listener.Close() 89 | port = i 90 | break 91 | } 92 | } 93 | return port 94 | } 95 | 96 | func GetIpList() ([]string, error) { 97 | var ips []string 98 | netInterfaces, err := net.Interfaces() 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | for i := 0; i < len(netInterfaces); i++ { 104 | if (netInterfaces[i].Flags & net.FlagUp) != 0 { 105 | ls, _ := netInterfaces[i].Addrs() 106 | for _, address := range ls { 107 | if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { 108 | if ipNet.IP.To4() != nil { 109 | ips = append(ips, ipNet.IP.String()) 110 | } 111 | } 112 | } 113 | } 114 | } 115 | return ips, nil 116 | } 117 | -------------------------------------------------------------------------------- /service/endpoint/mysql_test.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | gormMysql "gorm.io/driver/mysql" 6 | "gorm.io/gorm" 7 | "testing" 8 | ) 9 | 10 | func TestTimeConsuming(t *testing.T) { 11 | dsn := "yangtuojia001:yangtuojia001@tcp(172.16.50.221:3306)/test?charset=utf8&parseTime=True&loc=Local" 12 | db, _ := gorm.Open(gormMysql.New(gormMysql.Config{ 13 | DSN: dsn, // DSN data source name 14 | DefaultStringSize: 256, // string 类型字段的默认长度 15 | DisableDatetimePrecision: true, // 禁用 datetime 精度，MySQL 5.6 之前的数据库不支持 16 | DontSupportRenameIndex: true, // 重命名索引时采用删除并新建的方式，MySQL 5.7 之前的数据库和 MariaDB 不支持重命名索引 17 | DontSupportRenameColumn: true, // 用 `change` 重命名列，MySQL 8 之前的数据库和 MariaDB 不支持重命名列 18 | SkipInitializeWithVersion: false, // 根据当前 MySQL 版本自动配置 19 | }), &gorm.Config{}) 20 | 21 | type Tables struct { 22 | TableName string 23 | } 24 | 25 | type TableDesc struct { 26 | Table string 27 | CreateTable string `gorm:"column:Create Table"` 28 | } 29 | 30 | //sql := `select table_name from information_schema.tables where TABLE_SCHEMA = "yt_otter";` 31 | //tableList := make([]Tables,10) 32 | //db.Exec(sql).Scan(&tableList) 33 | //db.Table("information_schema.tables").Select("table_name").Where("TABLE_SCHEMA = ?", "yt_otter").Scan(&tableList) 34 | //db.Table("tables").Where("TABLE_SCHEMA = ?","yt_otter").Select("TABLE_NAME") 35 | //var tableDesc TableDesc 36 | var err error 37 | var val []interface{} 38 | val = append(val, 1) 39 | val = append(val, "1") 40 | val = append(val, 1) 41 | err = db.Exec("insert into test.t_voucher (ID, vono,dc_id) values (?, ?, ?)", val...).Error 42 | fmt.Println(err) 43 | 44 | } 45 | 46 | func TestIsContain(t *testing.T) { 47 | var items = make([]string, 0) 48 | var item = "sa" 49 | schema := IsContain(items, item) 50 | fmt.Println(schema) 51 | } 52 | 53 | func TestChangeTable(t *testing.T) { 54 | sql := "CREATE TABLE `budget_dict` (\n `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键',\n `config_code` varchar(20) NOT NULL COMMENT '数据字典-code',\n `config_value` varchar(50) NOT NULL COMMENT '数据字典-value',\n `config_seq` int(11) DEFAULT '1' COMMENT '数据字典-顺序',\n `type_code` varchar(50) NOT NULL COMMENT '数据字典类型编码',\n `type_name` varchar(50) NOT NULL COMMENT '数据字典类型名称',\n `enable` tinyint(1) NOT NULL DEFAULT '1' COMMENT '是否启用(1-启用; 0-停用)',\n `add_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '数据行创建时间',\n `modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '数据行最后修改时间',\n PRIMARY KEY (`id`) USING BTREE,\n KEY `idx_add_time` (`add_time`) USING BTREE,\n KEY `idx_type_code` (`type_code`) USING BTREE,\n KEY `idx_config_code` (`config_code`) USING BTREE\n) ENGINE=InnoDB AUTO_INCREMENT=35 DEFAULT CHARSET=utf8 COMMENT='预算字典表'" 55 | oldTable := "budget_dict" 56 | newTable := "t2" 57 | 58 | newSQL := createTableSqlChangeTableName(sql, oldTable, newTable) 59 | fmt.Println(len(newSQL)) 60 | fmt.Println(newSQL) 61 | } 62 | -------------------------------------------------------------------------------- /service/luaengine/db_actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | json "github.com/layeh/gopher-json" 22 | lua "github.com/yuin/gopher-lua" 23 | "go-mysql-sync/util/logutil" 24 | ) 25 | 26 | func dbModule(L *lua.LState) int { 27 | t := L.NewTable() 28 | L.SetFuncs(t, _dbModuleApi) 29 | L.Push(t) 30 | return 1 31 | } 32 | 33 | var _dbModuleApi = map[string]lua.LGFunction{ 34 | "selectOne": selectOne, 35 | "select": selectList, 36 | } 37 | 38 | func selectOne(L *lua.LState) int { 39 | sql := L.CheckString(1) 40 | 41 | logutil.Infof("lua db module execute sql: %s", sql) 42 | 43 | rs, err := _ds.Execute(sql) 44 | if err != nil { 45 | logutil.Error(err.Error()) 46 | L.Push(lua.LNil) 47 | L.Push(lua.LString(err.Error())) 48 | return 2 49 | } 50 | rowNumber := rs.RowNumber() 51 | 52 | table := L.NewTable() 53 | if rowNumber > 1 { 54 | logutil.Error("return more than 1 row") 55 | L.Push(lua.LNil) 56 | L.Push(lua.LString("return more than 1 row")) 57 | return 2 58 | } 59 | 60 | if rowNumber == 1 { 61 | for field, index := range rs.FieldNames { 62 | v, err := rs.GetValue(0, index) 63 | if err != nil { 64 | logutil.Error(err.Error()) 65 | L.Push(lua.LNil) 66 | L.Push(lua.LString(err.Error())) 67 | return 2 68 | } 69 | val := interfaceToLv(v) 70 | L.SetTable(table, lua.LString(field), val) 71 | } 72 | } 73 | 74 | if data, err := json.Encode(table); err == nil { 75 | logutil.Infof("lua db module result: %s", string(data)) 76 | } 77 | 78 | L.Push(table) 79 | return 1 80 | } 81 | 82 | func selectList(L *lua.LState) int { 83 | sql := L.CheckString(1) 84 | 85 | logutil.Infof("lua db module execute sql: %s", sql) 86 | 87 | rs, err := _ds.Execute(sql) 88 | if err != nil { 89 | logutil.Error(err.Error()) 90 | L.Push(lua.LNil) 91 | L.Push(lua.LString(err.Error())) 92 | return 2 93 | } 94 | rowNumber := rs.RowNumber() 95 | 96 | ret := L.NewTable() 97 | if rowNumber > 0 { 98 | for i := 0; i < rowNumber; i++ { 99 | table := L.NewTable() 100 | for field, index := range rs.FieldNames { 101 | v, err := rs.GetValue(i, index) 102 | if err != nil { 103 | logutil.Error(err.Error()) 104 | L.Push(lua.LNil) 105 | L.Push(lua.LString(err.Error())) 106 | return 2 107 | } 108 | val := interfaceToLv(v) 109 | L.SetTable(table, lua.LString(field), val) 110 | } 111 | L.SetTable(ret, lua.LNumber(i+1), table) 112 | } 113 | } 114 | 115 | if data, err := json.Encode(ret); err == nil { 116 | logutil.Infof("lua db module result: %s", string(data)) 117 | } else { 118 | logutil.Error(err.Error()) 119 | } 120 | 121 | L.Push(ret) 122 | return 1 123 | } 124 | -------------------------------------------------------------------------------- /util/logutil/global_log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | import ( 21 | "errors" 22 | "fmt" 23 | "io" 24 | "log" 25 | "os" 26 | "path/filepath" 27 | 28 | "go.uber.org/zap" 29 | "go.uber.org/zap/zapcore" 30 | "gopkg.in/natefinch/lumberjack.v2" 31 | 32 | "go-mysql-sync/util/fileutil" 33 | ) 34 | 35 | var _globalLogWriter io.Writer 36 | var _globalLogger *zap.Logger 37 | 38 | func InitGlobalLogger(config *LoggerConfig, options ...zap.Option) error { 39 | if nil == config { 40 | _globalLogger = zap.NewExample() 41 | return nil 42 | } 43 | 44 | if config.FileName == "" { 45 | config.FileName = _logFileName 46 | } 47 | 48 | logFile := filepath.Join(config.Store, config.FileName) 49 | if succeed := fileutil.CreateFileIfNecessary(logFile); !succeed { 50 | return errors.New(fmt.Sprintf("create log file : %s error", logFile)) 51 | } 52 | _globalLogWriter = &lumberjack.Logger{ //定义日志分割器 53 | Filename: logFile, // 日志文件路径 54 | MaxSize: config.MaxSize, // 文件最大M字节 55 | MaxAge: config.MaxAge, // 最多保留几天 56 | Compress: config.Compress, // 是否压缩 57 | LocalTime: true, 58 | } 59 | 60 | zap, err := NewZapLogger(config, _globalLogWriter, options...) 61 | if err == nil { 62 | _globalLogger = zap 63 | } 64 | 65 | return err 66 | } 67 | 68 | func GlobalLogger() *zap.Logger { 69 | if nil == _globalLogger { 70 | _globalLogger = zap.NewExample() 71 | } 72 | return _globalLogger 73 | } 74 | 75 | func GlobalLogWriter() io.Writer { 76 | if nil == _globalLogWriter { 77 | _globalLogWriter = os.Stdout 78 | } 79 | return _globalLogWriter 80 | } 81 | 82 | func GlobalSugar() *zap.SugaredLogger { 83 | return GlobalLogger().Sugar() 84 | } 85 | 86 | func Debug(msg string, fields ...zapcore.Field) { 87 | _globalLogger.Debug(msg, fields...) 88 | } 89 | 90 | func Debugf(template string, args ...interface{}) { 91 | _globalLogger.Sugar().Debugf(template, args...) 92 | } 93 | 94 | func Info(msg string, fields ...zapcore.Field) { 95 | _globalLogger.Info(msg, fields...) 96 | } 97 | 98 | func Infof(template string, args ...interface{}) { 99 | _globalLogger.Sugar().Infof(template, args...) 100 | } 101 | 102 | func Warn(msg string, fields ...zapcore.Field) { 103 | _globalLogger.Warn(msg, fields...) 104 | } 105 | 106 | func Warnf(template string, args ...interface{}) { 107 | _globalLogger.Sugar().Warnf(template, args...) 108 | } 109 | 110 | func Error(msg string, fields ...zapcore.Field) { 111 | _globalLogger.Error(msg, fields...) 112 | } 113 | 114 | func Errorf(template string, args ...interface{}) { 115 | _globalLogger.Sugar().Errorf(template, args...) 116 | } 117 | 118 | func BothInfof(template string, args ...interface{}) { 119 | log.Println(fmt.Sprintf(template, args...)) 120 | if _globalLogger != nil { 121 | _globalLogger.Sugar().Infof(template, args...) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /service/luaengine/es_actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | "github.com/siddontang/go-mysql/canal" 22 | lua "github.com/yuin/gopher-lua" 23 | 24 | "go-mysql-sync/global" 25 | "go-mysql-sync/util/stringutil" 26 | ) 27 | 28 | func esModule(L *lua.LState) int { 29 | t := L.NewTable() 30 | L.SetFuncs(t, _esModuleApi) 31 | L.Push(t) 32 | return 1 33 | } 34 | 35 | var _esModuleApi = map[string]lua.LGFunction{ 36 | "rawRow": rawRow, 37 | "rawAction": rawAction, 38 | 39 | "INSERT": esInsert, 40 | "UPDATE": esUpdate, 41 | "DELETE": esDelete, 42 | } 43 | 44 | func esInsert(L *lua.LState) int { 45 | index := L.CheckAny(1) 46 | id := L.CheckAny(2) 47 | body := L.CheckAny(3) 48 | 49 | data := L.NewTable() 50 | L.SetTable(data, lua.LString("index"), index) 51 | L.SetTable(data, lua.LString("action"), lua.LString(canal.InsertAction)) 52 | L.SetTable(data, lua.LString("id"), id) 53 | L.SetTable(data, lua.LString("body"), body) 54 | 55 | ret := L.GetGlobal(_globalRET) 56 | L.SetTable(ret, lua.LString(stringutil.UUID()), data) 57 | return 0 58 | } 59 | 60 | func esUpdate(L *lua.LState) int { 61 | index := L.CheckAny(1) 62 | id := L.CheckAny(2) 63 | body := L.CheckAny(3) 64 | 65 | data := L.NewTable() 66 | L.SetTable(data, lua.LString("index"), index) 67 | L.SetTable(data, lua.LString("action"), lua.LString(canal.UpdateAction)) 68 | L.SetTable(data, lua.LString("id"), id) 69 | L.SetTable(data, lua.LString("body"), body) 70 | 71 | ret := L.GetGlobal(_globalRET) 72 | L.SetTable(ret, lua.LString(stringutil.UUID()), data) 73 | return 0 74 | } 75 | 76 | func esDelete(L *lua.LState) int { 77 | index := L.CheckAny(1) 78 | id := L.CheckAny(2) 79 | 80 | data := L.NewTable() 81 | L.SetTable(data, lua.LString("index"), index) 82 | L.SetTable(data, lua.LString("action"), lua.LString(canal.DeleteAction)) 83 | L.SetTable(data, lua.LString("id"), id) 84 | 85 | ret := L.GetGlobal(_globalRET) 86 | L.SetTable(ret, lua.LString(stringutil.UUID()), data) 87 | return 0 88 | } 89 | 90 | func DoESOps(input map[string]interface{}, action string, rule *global.Rule) ([]*global.ESRespond, error) { 91 | L := _pool.Get() 92 | defer _pool.Put(L) 93 | 94 | row := L.NewTable() 95 | paddingTable(L, row, input) 96 | ret := L.NewTable() 97 | L.SetGlobal(_globalRET, ret) 98 | L.SetGlobal(_globalROW, row) 99 | L.SetGlobal(_globalACT, lua.LString(action)) 100 | 101 | funcFromProto := L.NewFunctionFromProto(rule.LuaProto) 102 | L.Push(funcFromProto) 103 | err := L.PCall(0, lua.MultRet, nil) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | responds := make([]*global.ESRespond, 0, ret.Len()) 109 | ret.ForEach(func(k lua.LValue, v lua.LValue) { 110 | resp := new(global.ESRespond) 111 | resp.Index = lvToString(L.GetTable(v, lua.LString("index"))) 112 | resp.Id = lvToString(L.GetTable(v, lua.LString("id"))) 113 | resp.Action = lvToString(L.GetTable(v, lua.LString("action"))) 114 | 115 | var data string 116 | body := L.GetTable(v, lua.LString("body")) 117 | switch body.Type() { 118 | case lua.LTNumber: 119 | data = lua.LVAsString(body) 120 | case lua.LTString: 121 | data = lua.LVAsString(body) 122 | case lua.LTTable: 123 | mm, _ := lvToMap(body) 124 | data = stringutil.ToJsonString(mm) 125 | default: 126 | data = stringutil.ToJsonString(body) 127 | } 128 | resp.Date = data 129 | responds = append(responds, resp) 130 | }) 131 | 132 | return responds, nil 133 | } 134 | -------------------------------------------------------------------------------- /service/cluster/etcd_election.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package cluster 19 | 20 | import ( 21 | "context" 22 | "sync" 23 | "time" 24 | 25 | "github.com/juju/errors" 26 | "go.etcd.io/etcd/clientv3/concurrency" 27 | "go.uber.org/atomic" 28 | 29 | "go-mysql-sync/global" 30 | "go-mysql-sync/storage" 31 | "go-mysql-sync/util/etcdutil" 32 | "go-mysql-sync/util/logutil" 33 | ) 34 | 35 | const _electionNodeTTL = 5 36 | 37 | type etcdElection struct { 38 | once sync.Once 39 | 40 | cfg *global.Config 41 | informCh chan bool 42 | 43 | flag atomic.Bool 44 | leader atomic.String 45 | } 46 | 47 | func newEtcdElection(_informCh chan bool, _cfg *global.Config) *etcdElection { 48 | return &etcdElection{ 49 | informCh: _informCh, 50 | cfg: _cfg, 51 | } 52 | } 53 | 54 | func (s *etcdElection) Elect() error { 55 | go s.doElect() 56 | 57 | go func() { 58 | for { 59 | if s.flag.Load() { 60 | break 61 | } 62 | err := s.ensureLeader() 63 | if err == nil { 64 | s.informFalse() 65 | break 66 | } 67 | } 68 | logutil.BothInfof("End master election") 69 | }() 70 | 71 | return nil 72 | } 73 | 74 | func (s *etcdElection) doElect() error { 75 | for { 76 | session, err := concurrency.NewSession(storage.EtcdConn(), concurrency.WithTTL(_electionNodeTTL)) 77 | if err != nil { 78 | return errors.Trace(err) 79 | } 80 | 81 | election := concurrency.NewElection(session, s.cfg.ZeElectionDir()) 82 | ctx := context.Background() 83 | 84 | if err = election.Campaign(ctx, s.cfg.Cluster.CurrentNode); err != nil { 85 | logutil.Error(errors.ErrorStack(err)) 86 | session.Close() 87 | s.informFalse() 88 | continue 89 | } 90 | 91 | logutil.BothInfof("elected key : %s", election.Key()) 92 | s.elected(election.Key(), s.cfg) 93 | s.informTrue() 94 | 95 | shouldBreak := false 96 | for !shouldBreak { 97 | select { 98 | case <-session.Done(): 99 | logutil.Warn("etcd session has done") 100 | shouldBreak = true 101 | s.informFalse() 102 | break 103 | case <-ctx.Done(): 104 | ctxTmp, _ := context.WithTimeout(context.Background(), time.Second*_electionNodeTTL) 105 | election.Resign(ctxTmp) 106 | session.Close() 107 | s.informFalse() 108 | return nil 109 | } 110 | } 111 | } 112 | } 113 | 114 | func (s *etcdElection) IsLeader() bool { 115 | return s.flag.Load() 116 | } 117 | 118 | func (s *etcdElection) Leader() string { 119 | if s.flag.Load() { 120 | return s.cfg.Cluster.CurrentNode 121 | } 122 | return s.leader.Load() 123 | } 124 | 125 | func (s *etcdElection) ensureLeader() error { 126 | elected, _, err := etcdutil.Get(s.cfg.ZeElectedDir(), storage.EtcdOps()) 127 | if err != nil { 128 | return err 129 | } 130 | 131 | data, _, err := etcdutil.Get(string(elected), storage.EtcdOps()) 132 | if err != nil { 133 | return err 134 | } 135 | 136 | current := string(data) 137 | s.leader.Store(current) 138 | 139 | return nil 140 | } 141 | 142 | func (s *etcdElection) informTrue() { 143 | s.flag.Store(true) 144 | s.informCh <- s.flag.Load() 145 | } 146 | 147 | func (s *etcdElection) informFalse() { 148 | s.flag.Store(false) 149 | s.informCh <- s.flag.Load() 150 | } 151 | 152 | func (s *etcdElection) elected(key string, cfg *global.Config) error { 153 | err := etcdutil.UpdateOrCreate(cfg.ZeElectedDir(), key, storage.EtcdOps()) 154 | if err != nil { 155 | logutil.Error(errors.ErrorStack(err)) 156 | return err 157 | } 158 | 159 | return nil 160 | } 161 | -------------------------------------------------------------------------------- /util/dateutil/date_format.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package dateutil 19 | 20 | import ( 21 | "strings" 22 | ) 23 | 24 | const ( 25 | yyyy = "2006" 26 | yy = "06" 27 | mmmm = "January" 28 | mmm = "Jan" 29 | mm = "01" 30 | m = "1" 31 | 32 | dddd = "Monday" 33 | ddd = "Mon" 34 | dd = "02" 35 | 36 | HHT = "03" 37 | HH = "15" 38 | MM = "04" 39 | SS = "05" 40 | ss = "05" 41 | tt = "PM" 42 | Z = "MST" 43 | ZZZ = "MST" 44 | 45 | o = "Z07:00" 46 | ) 47 | 48 | func ConvertGoFormat(format string) string { 49 | var goFormate = format 50 | if strings.Contains(goFormate, "YYYY") { 51 | goFormate = strings.Replace(goFormate, "YYYY", yyyy, -1) 52 | } else if strings.Contains(goFormate, "yyyy") { 53 | goFormate = strings.Replace(goFormate, "yyyy", yyyy, -1) 54 | } else if strings.Contains(goFormate, "YY") { 55 | goFormate = strings.Replace(goFormate, "YY", yy, -1) 56 | } else if strings.Contains(goFormate, "yy") { 57 | goFormate = strings.Replace(goFormate, "yy", yy, -1) 58 | } 59 | 60 | if strings.Contains(goFormate, "MMMM") { 61 | goFormate = strings.Replace(goFormate, "MMMM", mmmm, -1) 62 | } else if strings.Contains(goFormate, "mmmm") { 63 | goFormate = strings.Replace(goFormate, "mmmm", mmmm, -1) 64 | } else if strings.Contains(goFormate, "MMM") { 65 | goFormate = strings.Replace(goFormate, "MMM", mmm, -1) 66 | } else if strings.Contains(goFormate, "mmm") { 67 | goFormate = strings.Replace(goFormate, "mmm", mmm, -1) 68 | } else if strings.Contains(goFormate, "mm") { 69 | goFormate = strings.Replace(goFormate, "mm", mm, -1) 70 | } 71 | 72 | if strings.Contains(goFormate, "dddd") { 73 | goFormate = strings.Replace(goFormate, "dddd", dddd, -1) 74 | } else if strings.Contains(goFormate, "ddd") { 75 | goFormate = strings.Replace(goFormate, "ddd", ddd, -1) 76 | } else if strings.Contains(goFormate, "dd") { 77 | goFormate = strings.Replace(goFormate, "dd", dd, -1) 78 | } 79 | 80 | if strings.Contains(goFormate, "tt") { 81 | if strings.Contains(goFormate, "HH") { 82 | goFormate = strings.Replace(goFormate, "HH", HHT, -1) 83 | } else if strings.Contains(goFormate, "hh") { 84 | goFormate = strings.Replace(goFormate, "hh", HHT, -1) 85 | } 86 | goFormate = strings.Replace(goFormate, "tt", tt, -1) 87 | } else { 88 | if strings.Contains(goFormate, "HH") { 89 | goFormate = strings.Replace(goFormate, "HH", HH, -1) 90 | } else if strings.Contains(goFormate, "hh") { 91 | goFormate = strings.Replace(goFormate, "hh", HH, -1) 92 | } 93 | goFormate = strings.Replace(goFormate, "tt", "", -1) 94 | } 95 | 96 | if strings.Contains(goFormate, "MM") { 97 | goFormate = strings.Replace(goFormate, "MM", MM, -1) 98 | } 99 | 100 | if strings.Contains(goFormate, "SS") { 101 | goFormate = strings.Replace(goFormate, "SS", SS, -1) 102 | } else if strings.Contains(goFormate, "ss") { 103 | goFormate = strings.Replace(goFormate, "ss", SS, -1) 104 | } 105 | 106 | if strings.Contains(goFormate, "ZZZ") { 107 | goFormate = strings.Replace(goFormate, "ZZZ", ZZZ, -1) 108 | } else if strings.Contains(goFormate, "zzz") { 109 | goFormate = strings.Replace(goFormate, "zzz", ZZZ, -1) 110 | } else if strings.Contains(goFormate, "Z") { 111 | goFormate = strings.Replace(goFormate, "Z", Z, -1) 112 | } else if strings.Contains(goFormate, "z") { 113 | goFormate = strings.Replace(goFormate, "z", Z, -1) 114 | } 115 | 116 | if strings.Contains(goFormate, "tt") { 117 | goFormate = strings.Replace(goFormate, "tt", tt, -1) 118 | } 119 | if strings.Contains(goFormate, "o") { 120 | goFormate = strings.Replace(goFormate, "o", o, -1) 121 | } 122 | 123 | return goFormate 124 | } 125 | -------------------------------------------------------------------------------- /service/luaengine/mongo_actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | "github.com/juju/errors" 22 | "github.com/siddontang/go-mysql/canal" 23 | lua "github.com/yuin/gopher-lua" 24 | 25 | "go-mysql-sync/global" 26 | "go-mysql-sync/util/stringutil" 27 | ) 28 | 29 | func mongoModule(L *lua.LState) int { 30 | t := L.NewTable() 31 | L.SetFuncs(t, _mongoModuleApi) 32 | L.Push(t) 33 | return 1 34 | } 35 | 36 | var _mongoModuleApi = map[string]lua.LGFunction{ 37 | "rawRow": rawRow, 38 | "rawAction": rawAction, 39 | 40 | "INSERT": mongoInsert, 41 | "UPDATE": mongoUpdate, 42 | "DELETE": mongoDelete, 43 | } 44 | 45 | func mongoInsert(L *lua.LState) int { 46 | collection := L.CheckAny(1) 47 | table := L.CheckAny(2) 48 | 49 | data := L.NewTable() 50 | L.SetTable(data, lua.LString("collection"), collection) 51 | L.SetTable(data, lua.LString("action"), lua.LString(canal.InsertAction)) 52 | L.SetTable(data, lua.LString("table"), table) 53 | 54 | ret := L.GetGlobal(_globalRET) 55 | L.SetTable(ret, lua.LString(stringutil.UUID()), data) 56 | return 0 57 | } 58 | 59 | func mongoUpdate(L *lua.LState) int { 60 | collection := L.CheckAny(1) 61 | id := L.CheckAny(2) 62 | table := L.CheckAny(3) 63 | 64 | data := L.NewTable() 65 | L.SetTable(data, lua.LString("collection"), collection) 66 | L.SetTable(data, lua.LString("action"), lua.LString(canal.UpdateAction)) 67 | L.SetTable(data, lua.LString("id"), id) 68 | L.SetTable(data, lua.LString("table"), table) 69 | 70 | ret := L.GetGlobal(_globalRET) 71 | L.SetTable(ret, lua.LString(stringutil.UUID()), data) 72 | return 0 73 | } 74 | 75 | func mongoDelete(L *lua.LState) int { 76 | collection := L.CheckAny(1) 77 | id := L.CheckAny(2) 78 | 79 | data := L.NewTable() 80 | L.SetTable(data, lua.LString("collection"), collection) 81 | L.SetTable(data, lua.LString("action"), lua.LString(canal.DeleteAction)) 82 | L.SetTable(data, lua.LString("id"), id) 83 | 84 | ret := L.GetGlobal(_globalRET) 85 | L.SetTable(ret, lua.LString(stringutil.UUID()), data) 86 | return 0 87 | } 88 | 89 | func DoMongoOps(input map[string]interface{}, action string, rule *global.Rule) ([]*global.MongoRespond, error) { 90 | L := _pool.Get() 91 | defer _pool.Put(L) 92 | 93 | row := L.NewTable() 94 | paddingTable(L, row, input) 95 | ret := L.NewTable() 96 | L.SetGlobal(_globalRET, ret) 97 | L.SetGlobal(_globalROW, row) 98 | L.SetGlobal(_globalACT, lua.LString(action)) 99 | 100 | funcFromProto := L.NewFunctionFromProto(rule.LuaProto) 101 | L.Push(funcFromProto) 102 | err := L.PCall(0, lua.MultRet, nil) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | asserted := true 108 | responds := make([]*global.MongoRespond, 0, ret.Len()) 109 | ret.ForEach(func(k lua.LValue, v lua.LValue) { 110 | resp := new(global.MongoRespond) 111 | resp.Collection = lvToString(L.GetTable(v, lua.LString("collection"))) 112 | resp.Action = lvToString(L.GetTable(v, lua.LString("action"))) 113 | resp.Id = lvToInterface(L.GetTable(v, lua.LString("id")), true) 114 | lvTable := L.GetTable(v, lua.LString("table")) 115 | 116 | var table map[string]interface{} 117 | if action != canal.DeleteAction { 118 | table, asserted = lvToMap(lvTable) 119 | if !asserted { 120 | return 121 | } 122 | resp.Table = table 123 | } 124 | 125 | if action == canal.InsertAction { 126 | _id, ok := table["_id"] 127 | if !ok { 128 | resp.Id = stringutil.UUID() 129 | table["_id"] = resp.Id 130 | } else { 131 | resp.Id = _id 132 | } 133 | } 134 | 135 | responds = append(responds, resp) 136 | }) 137 | 138 | if !asserted { 139 | return nil, errors.New("The parameter must be of table type") 140 | } 141 | 142 | return responds, nil 143 | } 144 | -------------------------------------------------------------------------------- /service/luaengine/http_actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | "fmt" 22 | lua "github.com/yuin/gopher-lua" 23 | 24 | "go-mysql-sync/util/httputil" 25 | "go-mysql-sync/util/logutil" 26 | ) 27 | 28 | var _httpClient = httputil.NewClient() 29 | 30 | func httpModule(L *lua.LState) int { 31 | t := L.NewTable() 32 | L.SetFuncs(t, _httpModuleApi) 33 | L.Push(t) 34 | return 1 35 | } 36 | 37 | var _httpModuleApi = map[string]lua.LGFunction{ 38 | "get": doGet, 39 | "delete": doDelete, 40 | "post": doPost, 41 | "put": doPut, 42 | } 43 | 44 | func doGet(L *lua.LState) int { 45 | ret := L.NewTable() 46 | paramUrl := L.CheckString(1) 47 | paramOps := L.CheckTable(2) 48 | 49 | cli := _httpClient.GET(paramUrl) 50 | if headers, ok := lvToMap(paramOps); ok { 51 | cli.SetHeaders(headers) 52 | } 53 | 54 | entity, err := cli.DoForEntity() 55 | if err != nil { 56 | logutil.Error(err.Error()) 57 | L.Push(lua.LNil) 58 | L.Push(lua.LString(err.Error())) 59 | return 2 60 | } 61 | 62 | ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) 63 | ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) 64 | 65 | L.Push(ret) 66 | return 1 67 | } 68 | 69 | func doDelete(L *lua.LState) int { 70 | ret := L.NewTable() 71 | paramUrl := L.CheckString(1) 72 | paramOps := L.CheckTable(2) 73 | 74 | cli := _httpClient.DELETE(paramUrl) 75 | if headers, ok := lvToMap(paramOps); ok { 76 | cli.SetHeaders(headers) 77 | } 78 | 79 | entity, err := cli.DoForEntity() 80 | if err != nil { 81 | logutil.Error(err.Error()) 82 | L.Push(lua.LNil) 83 | L.Push(lua.LString(err.Error())) 84 | return 2 85 | } 86 | 87 | ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) 88 | ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) 89 | 90 | L.Push(ret) 91 | return 1 92 | } 93 | 94 | func doPost(L *lua.LState) int { 95 | ret := L.NewTable() 96 | paramUrl := L.CheckString(1) 97 | paramHeaders := L.CheckTable(2) 98 | paramContents := L.CheckTable(3) 99 | 100 | cli := _httpClient.POST(paramUrl) 101 | if headers, ok := lvToMap(paramHeaders); ok { 102 | cli.SetHeaders(headers) 103 | } 104 | 105 | contents, ok := lvToMap(paramContents) 106 | fmt.Println("contents: ", contents) 107 | if !ok { 108 | logutil.Error("The argument must Table") 109 | L.Push(lua.LNil) 110 | L.Push(lua.LString("The argument must Table")) 111 | return 2 112 | } 113 | 114 | entity, err := cli.SetForm(contents).DoForEntity() 115 | if err != nil { 116 | logutil.Error(err.Error()) 117 | L.Push(lua.LNil) 118 | L.Push(lua.LString(err.Error())) 119 | return 2 120 | } 121 | 122 | ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) 123 | ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) 124 | 125 | L.Push(ret) 126 | return 1 127 | } 128 | 129 | func doPut(L *lua.LState) int { 130 | ret := L.NewTable() 131 | paramUrl := L.CheckString(1) 132 | paramHeaders := L.CheckTable(2) 133 | paramContents := L.CheckTable(3) 134 | 135 | cli := _httpClient.PUT(paramUrl) 136 | if headers, ok := lvToMap(paramHeaders); ok { 137 | cli.SetHeaders(headers) 138 | } 139 | 140 | contents, ok := lvToMap(paramContents) 141 | if !ok { 142 | logutil.Error("The argument must Table") 143 | L.Push(lua.LNil) 144 | L.Push(lua.LString("The argument must Table")) 145 | return 2 146 | } 147 | 148 | entity, err := cli.SetForm(contents).DoForEntity() 149 | if err != nil { 150 | logutil.Error(err.Error()) 151 | L.Push(lua.LNil) 152 | L.Push(lua.LString(err.Error())) 153 | return 2 154 | } 155 | 156 | ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) 157 | ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) 158 | 159 | L.Push(ret) 160 | return 1 161 | } 162 | -------------------------------------------------------------------------------- /util/etcdutil/etcd_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package etcdutil 19 | 20 | import ( 21 | "context" 22 | "time" 23 | 24 | "github.com/juju/errors" 25 | "go.etcd.io/etcd/clientv3" 26 | ) 27 | 28 | const _etcdOpsTimeout = 1 * time.Second 29 | 30 | type Node struct { 31 | Key string 32 | Value []byte 33 | Revision int64 34 | } 35 | 36 | func CreateIfNecessary(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { 37 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 38 | defer cancel() 39 | 40 | _, err := ops.Txn(ctx).If( 41 | clientv3.Compare(clientv3.ModRevision(key), "=", 0), 42 | ).Then( 43 | clientv3.OpPut(key, val, opts...), 44 | ).Commit() 45 | 46 | if err != nil { 47 | return errors.Trace(err) 48 | } 49 | 50 | //if !resp.Succeeded { 51 | // return errors.AlreadyExistsf("key %s in etcd", key) 52 | //} 53 | 54 | return nil 55 | } 56 | 57 | func Get(key string, ops clientv3.KV) ([]byte, int64, error) { 58 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 59 | defer cancel() 60 | 61 | resp, err := ops.Get(ctx, key) 62 | if err != nil { 63 | return nil, -1, errors.Trace(err) 64 | } 65 | 66 | if len(resp.Kvs) == 0 { 67 | return nil, -1, errors.NotFoundf("key %s in etcd", key) 68 | } 69 | 70 | return resp.Kvs[0].Value, resp.Header.Revision, nil 71 | } 72 | 73 | func HasChildren(key string, ops clientv3.KV) (bool, error) { 74 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 75 | defer cancel() 76 | 77 | resp, err := ops.Get(ctx, key, clientv3.WithPrefix()) 78 | if err != nil { 79 | return false, errors.Trace(err) 80 | } 81 | 82 | length := len(key) 83 | for _, kv := range resp.Kvs { 84 | key := string(kv.Key) 85 | if len(key) > length { 86 | return true, nil 87 | } 88 | } 89 | 90 | return false, nil 91 | } 92 | 93 | func List(key string, ops clientv3.KV) (map[string]*Node, error) { 94 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 95 | defer cancel() 96 | 97 | ret := make(map[string]*Node) 98 | 99 | resp, err := ops.Get(ctx, key, clientv3.WithPrefix()) 100 | if err != nil { 101 | return ret, errors.Trace(err) 102 | } 103 | 104 | length := len(key) 105 | for _, kv := range resp.Kvs { 106 | key := string(kv.Key) 107 | if len(key) <= length { 108 | continue 109 | } 110 | node := &Node{ 111 | Key: key, 112 | Value: kv.Value, 113 | Revision: kv.Version, 114 | } 115 | ret[key] = node 116 | } 117 | 118 | return ret, nil 119 | } 120 | 121 | func Save(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { 122 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 123 | defer cancel() 124 | 125 | resp, err := ops.Txn(ctx).If( 126 | clientv3.Compare(clientv3.ModRevision(key), ">", 0), 127 | ).Then( 128 | clientv3.OpPut(key, val, opts...), 129 | ).Commit() 130 | 131 | if err != nil { 132 | return errors.Trace(err) 133 | } 134 | 135 | if !resp.Succeeded { 136 | return errors.NotFoundf("key %s in etcd", key) 137 | } 138 | 139 | return nil 140 | } 141 | 142 | // UpdateOrCreate updates a key/value, if the key does not exist then create, or update 143 | func UpdateOrCreate(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { 144 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 145 | defer cancel() 146 | 147 | _, err := ops.Do(ctx, clientv3.OpPut(key, val, opts...)) 148 | if err != nil { 149 | return errors.Trace(err) 150 | } 151 | 152 | return nil 153 | } 154 | 155 | func Delete(key string, ops clientv3.KV, opts ...clientv3.OpOption) error { 156 | ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) 157 | defer cancel() 158 | 159 | _, err := ops.Delete(ctx, key, opts...) 160 | 161 | if err != nil { 162 | return errors.Trace(err) 163 | } 164 | 165 | return nil 166 | } 167 | -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import ( 21 | "errors" 22 | "fmt" 23 | "go-mysql-sync/util/byteutil" 24 | "path/filepath" 25 | "strings" 26 | "time" 27 | 28 | "github.com/samuel/go-zookeeper/zk" 29 | "go.etcd.io/bbolt" 30 | "go.etcd.io/etcd/clientv3" 31 | etcdlog "go.etcd.io/etcd/pkg/logutil" 32 | 33 | "go-mysql-sync/global" 34 | "go-mysql-sync/util/fileutil" 35 | "go-mysql-sync/util/logutil" 36 | "go-mysql-sync/util/zkutil" 37 | ) 38 | 39 | const ( 40 | _boltFilePath = "db" 41 | _boltFileName = "data.db" 42 | _boltFileMode = 0600 43 | ) 44 | 45 | var ( 46 | _rowRequestBucket = []byte("RowRequest") 47 | _positionBucket = []byte("Position") 48 | _fixPositionId = byteutil.Uint64ToBytes(uint64(1)) 49 | //_firstKey = byteutil.Uint32ToBytes(uint32(1)) 50 | //file 6位 pos 14位 51 | _firstKey = []byte("00000100000000000001") 52 | 53 | _bolt *bbolt.DB 54 | _zkConn *zk.Conn 55 | _etcdConn *clientv3.Client 56 | _etcdOps clientv3.KV 57 | ) 58 | 59 | func InitStorage(conf *global.Config) error { 60 | if err := initBolt(conf); err != nil { 61 | return err 62 | } 63 | 64 | if conf.IsZk() { 65 | if err := initZk(conf); err != nil { 66 | return err 67 | } 68 | } 69 | 70 | if conf.IsEtcd() { 71 | if err := initEtcd(conf); err != nil { 72 | return err 73 | } 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func initBolt(conf *global.Config) error { 80 | blotStorePath := filepath.Join(conf.DataDir, _boltFilePath) 81 | if err := fileutil.MkdirIfNecessary(blotStorePath); err != nil { 82 | return errors.New(fmt.Sprintf("create boltdb store : %s", err.Error())) 83 | } 84 | 85 | boltFilePath := filepath.Join(blotStorePath, _boltFileName) 86 | bolt, err := bbolt.Open(boltFilePath, _boltFileMode, bbolt.DefaultOptions) 87 | if err != nil { 88 | return errors.New(fmt.Sprintf("open boltdb: %s", err.Error())) 89 | } 90 | 91 | err = bolt.Update(func(tx *bbolt.Tx) error { 92 | tx.CreateBucketIfNotExists(_rowRequestBucket) 93 | tx.CreateBucketIfNotExists(_positionBucket) 94 | return nil 95 | }) 96 | 97 | _bolt = bolt 98 | 99 | return err 100 | } 101 | 102 | func initZk(conf *global.Config) error { 103 | option := zk.WithLogger(logutil.NewZkLoggerAgent()) 104 | list := strings.Split(conf.Cluster.ZkAddrs, ",") 105 | conn, _, err := zk.Connect(list, time.Second, option) //*10) 106 | 107 | if err != nil { 108 | return err 109 | } 110 | 111 | if conf.Cluster.ZkAuthentication != "" { 112 | err = conn.AddAuth("digest", []byte(conf.Cluster.ZkAuthentication)) 113 | if err != nil { 114 | return err 115 | } 116 | } 117 | 118 | err = zkutil.CreateDirIfNecessary(conf.ZeRootDir(), conn) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | err = zkutil.CreateDirIfNecessary(conf.ZeClusterDir(), conn) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | _zkConn = conn 129 | 130 | return nil 131 | } 132 | 133 | func initEtcd(conf *global.Config) error { 134 | etcdlog.DefaultZapLoggerConfig = logutil.EtcdZapLoggerConfig() 135 | clientv3.SetLogger(logutil.NewEtcdLoggerAgent()) 136 | 137 | list := strings.Split(conf.Cluster.EtcdAddrs, ",") 138 | config := clientv3.Config{ 139 | Endpoints: list, 140 | Username: conf.Cluster.EtcdUser, 141 | Password: conf.Cluster.EtcdPassword, 142 | DialTimeout: 10 * time.Second, 143 | } 144 | 145 | client, err := clientv3.New(config) 146 | if err != nil { 147 | return err 148 | } 149 | _etcdConn = client 150 | _etcdOps = clientv3.NewKV(_etcdConn) 151 | 152 | return nil 153 | } 154 | 155 | func ZKConn() *zk.Conn { 156 | return _zkConn 157 | } 158 | 159 | func EtcdConn() *clientv3.Client { 160 | return _etcdConn 161 | } 162 | 163 | func EtcdOps() clientv3.KV { 164 | return _etcdOps 165 | } 166 | 167 | func CloseStorage() { 168 | if _bolt != nil { 169 | _bolt.Close() 170 | } 171 | if _zkConn != nil { 172 | _zkConn.Close() 173 | } 174 | if _etcdConn != nil { 175 | _etcdConn.Close() 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /global/model.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package global 19 | 20 | import ( 21 | "fmt" 22 | "github.com/siddontang/go-mysql/schema" 23 | "sync" 24 | ) 25 | 26 | //type DDLRequest struct { 27 | // RuleKey string 28 | // Action string 29 | // Query string 30 | // Pos PosRequest 31 | //} 32 | 33 | var GlobalChangeChan ChangeChan 34 | var GlobalInstance string 35 | 36 | //DDL线程控制器 37 | type ChangeChan struct { 38 | DdlControl bool 39 | Mutex sync.Mutex 40 | DdlControlChan chan struct{} 41 | } 42 | 43 | type Tables struct { 44 | TableName string 45 | } 46 | 47 | type TableDesc struct { 48 | Table string 49 | CreateTable string `gorm:"column:Create Table"` 50 | } 51 | 52 | type DatabaseDesc struct { 53 | Database string 54 | CreateDatabase string `gorm:"column:Create Database"` 55 | } 56 | 57 | type ConsumeQueue struct { 58 | Number int 59 | Queue chan []*RowRequest 60 | } 61 | 62 | type RowRequest struct { 63 | IdNum interface{} 64 | IdString string 65 | RuleKey string 66 | Action string 67 | OldRow []interface{} 68 | Row []interface{} 69 | Query string 70 | Table *schema.Table 71 | Hash int 72 | SchemaStart string 73 | SchemaEnd string 74 | Schema string 75 | Timestamp uint32 76 | } 77 | 78 | type PosRequest struct { 79 | Name string 80 | Pos uint32 81 | Force bool 82 | Timestamp uint32 83 | } 84 | 85 | func (p PosRequest) String() string { 86 | return fmt.Sprintf("(%d, %s, %d)", p.Timestamp, p.Name, p.Pos) 87 | } 88 | 89 | type MysqlRespond struct { 90 | RuleKey string 91 | Schema string 92 | Table string 93 | Action string 94 | Sql string 95 | Id interface{} 96 | Field string 97 | Val []interface{} 98 | ManyVal []interface{} 99 | ManyId []interface{} 100 | KeyVal map[string]interface{} 101 | SchemaStart string 102 | SchemaEnd string 103 | LastTableName string 104 | LastTableSchema string 105 | NumOfSql int 106 | Timestamp uint32 107 | } 108 | 109 | type RedisRespond struct { 110 | Action string 111 | Structure string 112 | Key string 113 | Field string 114 | Score float64 115 | OldVal interface{} 116 | Val interface{} 117 | } 118 | 119 | type MQRespond struct { 120 | Topic string `json:"-"` 121 | Action string `json:"action"` 122 | Date interface{} `json:"date"` 123 | ByteArray []byte `json:"-"` 124 | } 125 | 126 | type ESRespond struct { 127 | Index string 128 | Id string 129 | Action string 130 | Date string 131 | } 132 | 133 | type MongoRespond struct { 134 | RuleKey string 135 | Collection string 136 | Action string 137 | Id interface{} 138 | Table map[string]interface{} 139 | } 140 | 141 | type Padding struct { 142 | WrapName string 143 | 144 | ColumnName string 145 | ColumnIndex int 146 | ColumnType int 147 | ColumnMetadata *schema.TableColumn 148 | } 149 | 150 | var MysqlRespondPool = sync.Pool{ 151 | New: func() interface{} { 152 | return new(MysqlRespond) 153 | }, 154 | } 155 | 156 | var RedisRespondPool = sync.Pool{ 157 | New: func() interface{} { 158 | return new(RedisRespond) 159 | }, 160 | } 161 | 162 | var MQRespondPool = sync.Pool{ 163 | New: func() interface{} { 164 | return new(MQRespond) 165 | }, 166 | } 167 | 168 | var RowRequestPool = sync.Pool{ 169 | New: func() interface{} { 170 | return new(RowRequest) 171 | }, 172 | } 173 | 174 | type HashMap struct { 175 | Array []chan *RowRequest //chan list 176 | ChanLen int //线程数量 177 | Lock sync.Mutex //线程锁 178 | } 179 | 180 | func NewHashMap(len int, bulkSize int) (hashMap *HashMap) { 181 | list := make([]chan *RowRequest, len) 182 | for i := 0; i < len; i++ { 183 | list[i] = make(chan *RowRequest, bulkSize) 184 | } 185 | hashMap = &HashMap{ 186 | Array: list, 187 | ChanLen: len, 188 | Lock: sync.Mutex{}, 189 | } 190 | return 191 | } 192 | -------------------------------------------------------------------------------- /util/logutil/etcd_log_agent.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package logutil 19 | 20 | import ( 21 | "go.uber.org/zap" 22 | "go.uber.org/zap/zapcore" 23 | ) 24 | 25 | func EtcdZapLoggerConfig() zap.Config { 26 | return zap.Config{ 27 | Level: zap.NewAtomicLevelAt(zap.InfoLevel), 28 | Development: false, 29 | Sampling: &zap.SamplingConfig{ 30 | Initial: 100, 31 | Thereafter: 100, 32 | }, 33 | Encoding: "console", 34 | // copied from "zap.NewProductionEncoderConfig" with some updates 35 | EncoderConfig: zapcore.EncoderConfig{ 36 | TimeKey: "ts", 37 | LevelKey: "level", 38 | NameKey: "logger", 39 | CallerKey: "caller", 40 | MessageKey: "msg", 41 | StacktraceKey: "stacktrace", 42 | LineEnding: zapcore.DefaultLineEnding, 43 | EncodeLevel: zapcore.LowercaseLevelEncoder, 44 | EncodeTime: zapcore.ISO8601TimeEncoder, 45 | EncodeDuration: zapcore.StringDurationEncoder, 46 | EncodeCaller: zapcore.ShortCallerEncoder, 47 | }, 48 | 49 | // Use "/dev/null" to discard all 50 | OutputPaths: []string{"stderr"}, 51 | ErrorOutputPaths: []string{"stderr"}, 52 | } 53 | } 54 | 55 | type EtcdLoggerAgent struct { 56 | } 57 | 58 | func NewEtcdLoggerAgent() *EtcdLoggerAgent { 59 | return &EtcdLoggerAgent{} 60 | } 61 | 62 | // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. 63 | func (s *EtcdLoggerAgent) Info(args ...interface{}) { 64 | for _, arg := range args { 65 | Infof("%v", arg) 66 | } 67 | } 68 | 69 | // Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. 70 | func (s *EtcdLoggerAgent) Infoln(args ...interface{}) { 71 | for _, arg := range args { 72 | Infof("%v", arg) 73 | } 74 | } 75 | 76 | // Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. 77 | func (s *EtcdLoggerAgent) Infof(format string, args ...interface{}) { 78 | Infof(format, args...) 79 | } 80 | 81 | // Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. 82 | func (s *EtcdLoggerAgent) Warning(args ...interface{}) { 83 | for _, arg := range args { 84 | Warnf("%v", arg) 85 | } 86 | } 87 | 88 | // Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. 89 | func (s *EtcdLoggerAgent) Warningln(args ...interface{}) { 90 | for _, arg := range args { 91 | Warnf("%v", arg) 92 | } 93 | } 94 | 95 | // Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. 96 | func (s *EtcdLoggerAgent) Warningf(format string, args ...interface{}) { 97 | Warnf(format, args...) 98 | } 99 | 100 | // Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. 101 | func (s *EtcdLoggerAgent) Error(args ...interface{}) { 102 | for _, arg := range args { 103 | Errorf("%v", arg) 104 | } 105 | } 106 | 107 | // Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. 108 | func (s *EtcdLoggerAgent) Errorln(args ...interface{}) { 109 | for _, arg := range args { 110 | Errorf("%v", arg) 111 | } 112 | } 113 | 114 | // Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. 115 | func (s *EtcdLoggerAgent) Errorf(format string, args ...interface{}) { 116 | Errorf(format, args) 117 | } 118 | 119 | // Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. 120 | // gRPC ensures that all Fatal logs will exit with os.Exit(1). 121 | // Implementations may also call os.Exit() with a non-zero exit code. 122 | func (s *EtcdLoggerAgent) Fatal(args ...interface{}) { 123 | for _, arg := range args { 124 | Errorf("%v", arg) 125 | } 126 | } 127 | 128 | // Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. 129 | // gRPC ensures that all Fatal logs will exit with os.Exit(1). 130 | // Implementations may also call os.Exit() with a non-zero exit code. 131 | func (s *EtcdLoggerAgent) Fatalln(args ...interface{}) { 132 | for _, arg := range args { 133 | Errorf("%v", arg) 134 | } 135 | } 136 | 137 | // Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. 138 | // gRPC ensures that all Fatal logs will exit with os.Exit(1). 139 | // Implementations may also call os.Exit() with a non-zero exit code. 140 | func (s *EtcdLoggerAgent) Fatalf(format string, args ...interface{}) { 141 | Errorf(format, args) 142 | } 143 | 144 | // V reports whether verbosity level l is at least the requested verbose level. 145 | func (s *EtcdLoggerAgent) V(l int) bool { return false } 146 | -------------------------------------------------------------------------------- /util/tools.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "go-mysql-sync/global" 6 | "hash/crc32" 7 | "reflect" 8 | "regexp" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | var GIncrFlag bool 14 | var GSchemaFlag bool 15 | var GStockFlag bool 16 | 17 | var G_pos global.PosRequest 18 | 19 | func IsNil(i interface{}) bool { 20 | vi := reflect.ValueOf(i) 21 | if vi.Kind() == reflect.Ptr { 22 | return vi.IsNil() 23 | } 24 | return false 25 | } 26 | 27 | func SetSchemaFlag(val bool) { 28 | GSchemaFlag = val 29 | } 30 | 31 | func SetStockFlag(val bool) { 32 | GStockFlag = val 33 | } 34 | 35 | func SetIncrFlag(val bool) { 36 | GIncrFlag = val 37 | } 38 | 39 | func StoreNewPos(pos global.PosRequest) { 40 | G_pos = pos 41 | } 42 | 43 | func Hash(str string) int { 44 | v := int(crc32.ChecksumIEEE([]byte(str))) 45 | if v >= 0 { 46 | return v 47 | } 48 | if -v >= 0 { 49 | return -v 50 | } 51 | // v == MinInt 52 | return 0 53 | } 54 | 55 | func TimestampToDatetime(Timestamp int64) string { 56 | timeLayout := "2006-01-02 15:04:05" 57 | return time.Unix(Timestamp, 0).Format(timeLayout) 58 | } 59 | 60 | func CaptureTableName(sql string) (tableInfo []string, err error) { 61 | var indexOfTable int 62 | var indexOfOn int 63 | 64 | sql = strings.Replace(sql, `"`, "", -1) 65 | sql = strings.Replace(sql, `'`, "", -1) 66 | sql = strings.Replace(sql, "`", "", -1) 67 | reg := regexp.MustCompile("\\s+") 68 | sqlStr := reg.ReplaceAllString(sql, " ") 69 | //fmt.Println(sqlStr) 70 | indexOfTable = -1 71 | indexOfOn = -1 72 | //取 table关键字后面一个字符串，如果有‘.’,则取‘.’后的字符 73 | array := strings.Fields(sqlStr) 74 | for index, value := range array { 75 | if strings.ToLower(value) == "on" { 76 | indexOfOn = index 77 | } 78 | if strings.ToLower(value) == "table" { 79 | indexOfTable = index 80 | break 81 | } 82 | } 83 | 84 | //fmt.Println("indexOfTable:",indexOfTable) 85 | //fmt.Println("indexOfOn:",indexOfOn) 86 | //fmt.Println("array:",array) 87 | 88 | //如果没找到 table 关键字，则找 ON 关键字 89 | if indexOfTable > -1 { 90 | tableInfo = indexToTableName(indexOfTable, array) 91 | //tableName = append(tableName,indexToTableName(indexOfTable,array)) 92 | } else if indexOfTable == -1 && indexOfOn > -1 { 93 | tableInfo = indexToTableName(indexOfOn, array) 94 | } else { 95 | return nil, errors.New("DDL解析表名报错") 96 | } 97 | return tableInfo, nil 98 | } 99 | 100 | func indexToTableName(index int, array []string) (tableInfo []string) { 101 | var tableName, schema, tableAndSchema string 102 | tableAndSchema = array[index+1] 103 | tableName = tableAndSchema 104 | //fmt.Println(tableAndSchema) 105 | indexOfPoint := strings.Index(tableAndSchema, ".") 106 | if indexOfPoint >= 0 { 107 | tableName = tableAndSchema[indexOfPoint+1:] 108 | schema = tableAndSchema[:indexOfPoint] 109 | } 110 | if tableName[len(tableName)-1:] == ";" { 111 | tableName = tableName[:len(tableName)-1] 112 | } 113 | tableInfo = append(tableInfo, tableName) 114 | tableInfo = append(tableInfo, schema) 115 | return tableInfo 116 | } 117 | 118 | //func (s *MysqlEndpoint) Consume(n int,message chan []*global.RowRequest) { 119 | // var err error 120 | // for rows := range message { 121 | // //失败的sql重试 122 | // //只把执行失败的sql 保存到bolt 每五分钟重试一次如果重试的第一条sql就报错则不往下重试等待下一次重试 123 | // //-------------------------------------- 124 | // //if err = s.doRetryTask(); err != nil { 125 | // // logutil.Error(err.Error()) 126 | // // pushFailedRows(rows, s.cached) 127 | // // return 128 | // //} 129 | // //-------------------------------------- 130 | // expect := true 131 | // for _, row := range rows { 132 | // exportActionNum(row.Action, row.RuleKey) 133 | // resp := s.toBeRespond(row) 134 | // err = s.Exec(resp) 135 | // if err != nil { 136 | // logutil.Error(errors.ErrorStack(err)) 137 | // expect = false 138 | // break 139 | // } 140 | // } 141 | // if !expect { 142 | // pushFailedRows(rows, s.cached) 143 | // } else { 144 | // logutil.Infof("%d号线程，处理完成 %d 条数据", n, len(rows)) 145 | // } 146 | // } 147 | //} 148 | 149 | //func pushFailedRows(rs []*global.RowRequest, cached *storage.BoltRowStorage) { 150 | // logutil.Infof("%d 条数据处理失败，插入重试队列", len(rs)) 151 | // 152 | // list := make([][]byte, 0, len(rs)) 153 | // for _, r := range rs { 154 | // if data, err := msgpack.Marshal(r); err == nil { 155 | // list = append(list, data) 156 | // } 157 | // } 158 | // 159 | // cached.BatchAdd(list) 160 | //} 161 | 162 | func DDLChangeTableName(sql, newTableName string) (string, error) { 163 | // table的位置 164 | indexOfTable := -1 165 | indexOfOn := -1 166 | array := strings.Split(sql, " ") 167 | for index, value := range array { 168 | if strings.ToLower(value) == "on" { 169 | indexOfOn = index 170 | } 171 | if strings.ToLower(value) == "table" { 172 | indexOfTable = index 173 | break 174 | } 175 | } 176 | if indexOfTable > -1 { 177 | array[indexOfTable+1] = "`" + newTableName + "`" 178 | } else if indexOfTable == -1 && indexOfOn > -1 { 179 | tableName := array[indexOfOn+1] 180 | if tableName[len(tableName)-1:] == ";" { 181 | array[indexOfOn+1] = "`" + newTableName + "`" + ";" 182 | } else { 183 | array[indexOfOn+1] = "`" + newTableName + "`" 184 | } 185 | } else { 186 | return "", errors.New("DDL解析表名报错") 187 | } 188 | return strings.Join(array, " "), nil 189 | } 190 | -------------------------------------------------------------------------------- /util/httputil/http_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | 19 | package httputil 20 | 21 | import ( 22 | "net/http" 23 | "time" 24 | ) 25 | 26 | var DefaultClient = NewClient() 27 | 28 | // 参数 29 | type H map[string]interface{} 30 | 31 | // 重试条件 32 | type RetryConditionFunc func(*http.Response) bool 33 | 34 | // 先决条件 35 | type httpRequisite struct { 36 | timeout int 37 | retryCount int 38 | retryInterval int 39 | retryConditions []RetryConditionFunc 40 | headers H 41 | } 42 | 43 | func newHttpRequisite() *httpRequisite { 44 | return &httpRequisite{ 45 | headers: make(H), 46 | } 47 | } 48 | 49 | // 设置请求头 50 | func (c *httpRequisite) AddHeader(name string, value interface{}) { 51 | c.headers[name] = value 52 | } 53 | 54 | // 设置请求头 55 | func (c *httpRequisite) SetHeaders(values H) { 56 | c.headers = values 57 | } 58 | 59 | // 设置超时时间，单位为秒 60 | func (c *httpRequisite) SetTimeout(_timeout int) { 61 | c.timeout = _timeout 62 | } 63 | 64 | // 设置重试次数 65 | func (c *httpRequisite) SetRetryCount(_retryCount int) { 66 | c.retryCount = _retryCount 67 | } 68 | 69 | // 设置重试间隔时间，单位为秒 70 | func (c *httpRequisite) SetRetryInterval(_retryInterval int) { 71 | c.retryInterval = _retryInterval 72 | } 73 | 74 | // 添加重试条件 75 | func (c *httpRequisite) AddRetryCondition(_retryCondition RetryConditionFunc) { 76 | if _retryCondition != nil { 77 | c.retryConditions = append(c.retryConditions, _retryCondition) 78 | } 79 | } 80 | 81 | // 判断是否需要重试 82 | func (c *httpRequisite) retryNecessary(res *http.Response) bool { 83 | for _, condition := range c.retryConditions { 84 | if condition(res) { 85 | return true 86 | } 87 | } 88 | return false 89 | } 90 | 91 | // 重构 92 | func (c *httpRequisite) refactorIfNecessary(global *httpRequisite) { 93 | if c.retryCount == 0 { 94 | c.retryCount = global.retryCount 95 | } 96 | 97 | if c.retryInterval == 0 { 98 | c.retryInterval = global.retryInterval 99 | } 100 | 101 | for _, retryCondition := range global.retryConditions { 102 | c.retryConditions = append(c.retryConditions, retryCondition) 103 | } 104 | 105 | for k, v := range global.headers { 106 | if _, exist := c.headers[k]; !exist { 107 | c.headers[k] = v 108 | } 109 | } 110 | } 111 | 112 | type HttpClient struct { 113 | cli *http.Client 114 | globals *httpRequisite 115 | } 116 | 117 | // 创建Client 118 | func NewClient() *HttpClient { 119 | return &HttpClient{ 120 | cli: &http.Client{}, 121 | globals: newHttpRequisite(), 122 | } 123 | } 124 | 125 | // 设置超时时间，单位为秒 126 | func (c *HttpClient) SetTimeout(timeout int) *HttpClient { 127 | if timeout > 0 { 128 | c.globals.SetTimeout(timeout) 129 | c.cli.Timeout = time.Duration(timeout) * time.Second 130 | } 131 | return c 132 | } 133 | 134 | // 获取超时时间 135 | func (c *HttpClient) Timeout() int { 136 | return c.globals.timeout 137 | } 138 | 139 | // 设置重试次数 140 | func (c *HttpClient) SetRetryCount(retryCount int) *HttpClient { 141 | c.globals.SetRetryCount(retryCount) 142 | return c 143 | } 144 | 145 | // 获取重试次数 146 | func (c *HttpClient) RetryCount() int { 147 | return c.globals.retryCount 148 | } 149 | 150 | // 设置重试间隔时间，单位为秒 151 | func (c *HttpClient) SetRetryInterval(retryInterval int) *HttpClient { 152 | c.globals.SetRetryInterval(retryInterval) 153 | return c 154 | } 155 | 156 | // 获取重试间隔时间 157 | func (c *HttpClient) RetryInterval() int { 158 | return c.globals.retryInterval 159 | } 160 | 161 | // 添加重试条件 162 | func (c *HttpClient) AddRetryCondition(retryCondition RetryConditionFunc) *HttpClient { 163 | c.globals.AddRetryCondition(retryCondition) 164 | return c 165 | } 166 | 167 | // 设置Transport （用于确定HTTP请求的创建机制） 168 | // 如果为空，将会使用DefaultTransport 169 | func (c *HttpClient) SetTransport(transport http.RoundTripper) *HttpClient { 170 | if transport != nil { 171 | c.cli.Transport = transport 172 | } 173 | return c 174 | } 175 | 176 | // 添加请求头 177 | func (c *HttpClient) AddHeader(key string, val string) *HttpClient { 178 | c.globals.AddHeader(key, val) 179 | return c 180 | } 181 | 182 | // 添加请求头 183 | func (c *HttpClient) AddHeaders(values H) *HttpClient { 184 | c.globals.SetHeaders(values) 185 | return c 186 | } 187 | 188 | // Get请求 189 | func (c *HttpClient) GET(url string) *NobodyPerform { 190 | return newNobodyPerform(url, http.MethodGet, c) 191 | } 192 | 193 | // Delete请求 194 | func (c *HttpClient) DELETE(url string) *NobodyPerform { 195 | return newNobodyPerform(url, http.MethodDelete, c) 196 | } 197 | 198 | // Post请求 199 | func (c *HttpClient) POST(url string) *BodyPerform { 200 | return newBodyPerform(url, http.MethodPost, c) 201 | } 202 | 203 | // Put请求 204 | func (c *HttpClient) PUT(url string) *BodyPerform { 205 | return newBodyPerform(url, http.MethodPut, c) 206 | } 207 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # goMysqlSync 2 | golang mysql to mysql 轻量级多线程库表级数据同步 3 | 4 | ### 测试运行 5 | 1. 设置当前binlog位置并且开始运行 6 | 7 | `go run main.go -position mysql-bin.000001 1 1619431429` 8 | 9 | 2. 查询当前binlog位置，参数n为秒数，查询结果为n秒前的binlog位置 10 | 11 | `go run main.go -status n` 12 | 13 | 3. 开始同步，会先进行一次全量同步，增量同步也会自动执行 14 | 15 | `go run main.go -full` 16 | 17 | 4. 若程序掉线，可以继续执行，回滚到掉线时 900秒之前的binlog位置重新开始同步 18 | 19 | `go run main.go` 20 | 21 | ### 编译运行 22 | 23 | set GOOS=linux 24 | 25 | set GOARCH=amd64 26 | 27 | go build -o "goMysqlSync" 28 | 29 | chmod +x goMysqlSync 30 | 31 | 编辑配置文件 app.yml 32 | ```yaml 33 | #源 mysql 配置 34 | addr: 127.0.0.1:3306 35 | user: test_sync 36 | pass: test_sync 37 | charset : utf8 38 | #伪装id 39 | slave_id: 520 #slave ID 40 | 41 | #系统相关配置 42 | db_days: 30 43 | data_dir: #/usr/local/goMysqlSync #应用产生的数据存放地址，包括日志、缓存数据等，默认当前运行目录下store文件夹 44 | logger: 45 | file_name: system.log 46 | level: debug #日志级别；支持：debug|info|warn|error，默认info 47 | store: #/usr/local/goMysqlSync 48 | 49 | #prometheus相关配置 50 | label: goMysqlSync #prometheus exporter的tag 51 | enable_exporter: true #是否启用prometheus exporter，默认false 52 | exporter_addr: 9595 #prometheus exporter端口，默认9595 53 | 54 | #目标类型 55 | target: mysql 56 | mysql_addrs: 127.0.0.2:3306 #mysql地址，多个用逗号分隔 57 | mysql_username: test_sync #mysql用户名 58 | mysql_pass: test_sync #mysql密码 59 | threads: 20 #增量数据回放多线程数量 60 | record_rows: 255 #增量数据回放每批次大小 61 | dump_threads: 40 #全量同步线程数 62 | dump_record_rows: 1000 #全量同步每批次大小 63 | 64 | #规则配置 65 | rule: 66 | - 67 | schema: db_student #源库名 68 | target_schema: db_student_bak #目标库名 69 | table: student #源表名 70 | target_table: student_bak #目标表名 71 | - 72 | schema: db_student #源库名 73 | target_schema: db_student_bak #目标库名 74 | table: budget_dict #源表名 75 | target_table: budget_dict #目标表名 76 | - 77 | schema: db_student #源库名 78 | target_schema: db_student_bak #目标库名 79 | table: economy_section #源表名 80 | target_table: economy_section #目标表名 81 | - 82 | schema: db_student #源库名 83 | target_schema: db_student_bak #目标库名 84 | table: funds_nature #源表名 85 | target_table: funds_nature #目标表名 86 | ``` 87 | 启动 88 | ```shell 89 | nohup ./goMysqlSync --sync-mode=6 >/dev/null 2>nohup.goMysqlSync.log & 90 | ``` 91 | 92 | 93 | ### 监控 94 | 95 | http://localhost:9595/metrics 96 | 97 | 监控项包括机器的CPU、磁盘情况和每秒网络流量值，以及目标库的连接状态和程序每秒同步速度 98 | 99 | ### 注意事项: 100 | 101 | 1. 需要使github.com/siddontang/go-mysql项目的Position结构体支持时间戳 102 | 103 | 修改 "github.com/siddontang/go-mysql/mysql/position.go"文件 104 | ``` 105 | type Position struct { 106 | Name string 107 | Pos uint32 108 | Timestamp uint32 #新加 109 | } 110 | 111 | func (p Position) String() string { 112 | return fmt.Sprintf("(%s, %d, %d)", p.Name, p.Pos,p.Timestamp) #修改 113 | } 114 | ``` 115 | 修改 "github.com/siddontang/go-mysql/canal/sync.go"库文件 116 | ``` 117 | curPos := pos.Pos 118 | // next binlog pos 119 | pos.Pos = ev.Header.LogPos 120 | pos.Timestamp = ev.Header.Timestamp #新加 121 | ``` 122 | 123 | 2. 由于项目基于mysqldump，必须要有mysqldump的工具文件，并且在配置文件中指定路径，本项目 linux和windows 都支持 124 | 125 | 3. mysql 的binlog格式必须是 row 模式，不支持外键约束，数据表必须有id字段类型为整型并且为主键 126 | 127 | 4. 修改go-mysql项目的 github.com/siddontang/go-mysql/canal/sync.go 128 | 129 | 支持 CREATE INDEX INDEX_1NAME3 ON `courses12` (score) 和 DROP INDEX INDEX_1NAME3 ON `courses12`;形式的sql 130 | 131 | ``` 132 | func parseStmt(stmt ast.StmtNode) (ns []*node) { 133 | switch t := stmt.(type) { 134 | case *ast.RenameTableStmt: 135 | for _, tableInfo := range t.TableToTables { 136 | n := &node{ 137 | db: tableInfo.OldTable.Schema.String(), 138 | table: tableInfo.OldTable.Name.String(), 139 | } 140 | ns = append(ns, n) 141 | } 142 | case *ast.AlterTableStmt: 143 | n := &node{ 144 | db: t.Table.Schema.String(), 145 | table: t.Table.Name.String(), 146 | } 147 | ns = []*node{n} 148 | case *ast.DropTableStmt: 149 | for _, table := range t.Tables { 150 | n := &node{ 151 | db: table.Schema.String(), 152 | table: table.Name.String(), 153 | } 154 | ns = append(ns, n) 155 | } 156 | case *ast.CreateTableStmt: 157 | n := &node{ 158 | db: t.Table.Schema.String(), 159 | table: t.Table.Name.String(), 160 | } 161 | ns = []*node{n} 162 | case *ast.TruncateTableStmt: 163 | n := &node{ 164 | db: t.Table.Schema.String(), 165 | table: t.Table.Schema.String(), 166 | } 167 | ns = []*node{n} 168 | case *ast.CreateIndexStmt: #新加 169 | n := &node{ #新加 170 | db: t.Table.Schema.String(), #新加 171 | table: t.Table.Schema.String(), #新加 172 | } #新加 173 | ns = []*node{n} #新加 174 | case *ast.DropIndexStmt: #新加 175 | n := &node{ #新加 176 | db: t.Table.Schema.String(), #新加 177 | table: t.Table.Schema.String(), #新加 178 | } #新加 179 | ns = []*node{n} #新加 180 | } 181 | return 182 | } 183 | ``` 184 | 185 | 5. 项目必须有 test 库，用来做执行跳板库，项目不会对 test 库的数据做任何操作。 186 | 187 | 6. 源库需要的账号权限 188 | 189 | ``` 190 | GRANT SELECT, PROCESS, REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO 'xxx'@'%' IDENTIFIED BY 'xxx'; 191 | 192 | FLUSH PRIVILEGES; 193 | ``` 194 | 195 | ### 本项目基于二次开发: 196 | github.com/siddontang/go-mysql 197 | 198 | github.com/wj596/go-mysql-sync 199 | 200 | -------------------------------------------------------------------------------- /service/luaengine/redis_actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | "github.com/siddontang/go-mysql/canal" 22 | lua "github.com/yuin/gopher-lua" 23 | 24 | "go-mysql-sync/global" 25 | "go-mysql-sync/util/stringutil" 26 | ) 27 | 28 | const _globalOLDROW = "___OLDROW___" 29 | 30 | func redisModule(L *lua.LState) int { 31 | t := L.NewTable() 32 | L.SetFuncs(t, _redisModuleApi) 33 | L.Push(t) 34 | return 1 35 | } 36 | 37 | var _redisModuleApi = map[string]lua.LGFunction{ 38 | "rawRow": rawRow, 39 | "rawOldRow": rawOldRow, 40 | "rawAction": rawAction, 41 | 42 | "SET": redisSet, 43 | "DEL": redisDel, 44 | 45 | "HSET": redisHSet, 46 | "HDEL": redisHDel, 47 | 48 | "RPUSH": redisRPush, 49 | "LREM": redisLRem, 50 | 51 | "SADD": redisSAdd, 52 | "SREM": redisSRem, 53 | 54 | "ZADD": redisZAdd, 55 | "ZREM": redisZRem, 56 | } 57 | 58 | func rawOldRow(L *lua.LState) int { 59 | row := L.GetGlobal(_globalOLDROW) 60 | L.Push(row) 61 | return 1 62 | } 63 | 64 | func redisSet(L *lua.LState) int { 65 | key := L.CheckString(1) 66 | val := L.CheckAny(2) 67 | ret := L.GetGlobal(_globalRET) 68 | L.SetTable(ret, lua.LString("insert_1_"+key), val) 69 | return 0 70 | } 71 | 72 | func redisDel(L *lua.LState) int { 73 | key := L.CheckString(1) 74 | ret := L.GetGlobal(_globalRET) 75 | L.SetTable(ret, lua.LString("delete_1_"+key), lua.LBool(true)) 76 | return 0 77 | } 78 | 79 | func redisHSet(L *lua.LState) int { 80 | key := L.CheckString(1) 81 | field := L.CheckAny(2) 82 | val := L.CheckAny(3) 83 | 84 | hash := L.NewTable() 85 | L.SetTable(hash, lua.LString("key"), lua.LString(key)) 86 | L.SetTable(hash, lua.LString("field"), field) 87 | L.SetTable(hash, lua.LString("val"), val) 88 | 89 | ret := L.GetGlobal(_globalRET) 90 | L.SetTable(ret, lua.LString("insert_2_"+stringutil.UUID()), hash) 91 | return 0 92 | } 93 | 94 | func redisHDel(L *lua.LState) int { 95 | key := L.CheckAny(1) 96 | field := L.CheckAny(2) 97 | 98 | hash := L.NewTable() 99 | L.SetTable(hash, lua.LString("key"), key) 100 | L.SetTable(hash, lua.LString("field"), field) 101 | L.SetTable(hash, lua.LString("val"), lua.LNumber(1)) 102 | 103 | ret := L.GetGlobal(_globalRET) 104 | L.SetTable(ret, lua.LString("delete_2_"+stringutil.UUID()), hash) 105 | return 0 106 | } 107 | 108 | func redisRPush(L *lua.LState) int { 109 | key := L.CheckString(1) 110 | val := L.CheckAny(2) 111 | 112 | ret := L.GetGlobal(_globalRET) 113 | L.SetTable(ret, lua.LString("insert_3_"+key), val) 114 | return 0 115 | } 116 | 117 | func redisLRem(L *lua.LState) int { 118 | key := L.CheckString(1) 119 | val := L.CheckAny(2) 120 | 121 | ret := L.GetGlobal(_globalRET) 122 | L.SetTable(ret, lua.LString("delete_3_"+key), val) 123 | return 0 124 | } 125 | 126 | func redisSAdd(L *lua.LState) int { 127 | key := L.CheckString(1) 128 | val := L.CheckAny(2) 129 | 130 | ret := L.GetGlobal(_globalRET) 131 | L.SetTable(ret, lua.LString("insert_4_"+key), val) 132 | return 0 133 | } 134 | 135 | func redisSRem(L *lua.LState) int { 136 | key := L.CheckString(1) 137 | val := L.CheckAny(2) 138 | 139 | ret := L.GetGlobal(_globalRET) 140 | L.SetTable(ret, lua.LString("delete_4_"+key), val) 141 | return 0 142 | } 143 | 144 | func redisZAdd(L *lua.LState) int { 145 | key := L.CheckString(1) 146 | score := L.CheckAny(2) 147 | val := L.CheckAny(3) 148 | 149 | hash := L.NewTable() 150 | L.SetTable(hash, lua.LString("key"), lua.LString(key)) 151 | L.SetTable(hash, lua.LString("score"), score) 152 | L.SetTable(hash, lua.LString("val"), val) 153 | 154 | ret := L.GetGlobal(_globalRET) 155 | L.SetTable(ret, lua.LString("insert_5_"+stringutil.UUID()), hash) 156 | return 0 157 | } 158 | 159 | func redisZRem(L *lua.LState) int { 160 | key := L.CheckString(1) 161 | val := L.CheckAny(2) 162 | 163 | ret := L.GetGlobal(_globalRET) 164 | L.SetTable(ret, lua.LString("delete_5_"+key), val) 165 | return 0 166 | } 167 | 168 | func DoRedisOps(input map[string]interface{}, previous map[string]interface{}, action string, rule *global.Rule) ([]*global.RedisRespond, error) { 169 | L := _pool.Get() 170 | defer _pool.Put(L) 171 | 172 | row := L.NewTable() 173 | paddingTable(L, row, input) 174 | ret := L.NewTable() 175 | L.SetGlobal(_globalRET, ret) 176 | L.SetGlobal(_globalROW, row) 177 | L.SetGlobal(_globalACT, lua.LString(action)) 178 | 179 | if action == canal.UpdateAction { 180 | oldRow := L.NewTable() 181 | paddingTable(L, oldRow, previous) 182 | L.SetGlobal(_globalOLDROW, oldRow) 183 | } 184 | 185 | funcFromProto := L.NewFunctionFromProto(rule.LuaProto) 186 | L.Push(funcFromProto) 187 | err := L.PCall(0, lua.MultRet, nil) 188 | if err != nil { 189 | return nil, err 190 | } 191 | 192 | ls := make([]*global.RedisRespond, 0, ret.Len()) 193 | ret.ForEach(func(k lua.LValue, v lua.LValue) { 194 | resp := global.RedisRespondPool.Get().(*global.RedisRespond) 195 | kk := lvToString(k) 196 | resp.Action = kk[0:6] 197 | resp.Structure = structureName(kk[7:8]) 198 | if resp.Action == canal.DeleteAction { 199 | resp.Key = kk[9:len(kk)] 200 | resp.Val = lvToInterface(v, true) 201 | } else { 202 | if resp.Structure == global.RedisStructureHash { 203 | key := L.GetTable(v, lua.LString("key")) 204 | field := L.GetTable(v, lua.LString("field")) 205 | val := L.GetTable(v, lua.LString("val")) 206 | resp.Key = key.String() 207 | resp.Field = lvToString(field) 208 | resp.Val = lvToInterface(val, true) 209 | } else if resp.Structure == global.RedisStructureSortedSet { 210 | key := L.GetTable(v, lua.LString("key")) 211 | score := L.GetTable(v, lua.LString("score")) 212 | val := L.GetTable(v, lua.LString("val")) 213 | resp.Key = key.String() 214 | scoreTemp := lvToString(score) 215 | resp.Score = stringutil.ToFloat64Safe(scoreTemp) 216 | resp.Val = lvToInterface(val, true) 217 | } else { 218 | resp.Key = kk[9:len(kk)] 219 | resp.Val = lvToInterface(v, true) 220 | } 221 | } 222 | 223 | ls = append(ls, resp) 224 | }) 225 | 226 | return ls, nil 227 | } 228 | 229 | func structureName(code string) string { 230 | switch code { 231 | case "1": 232 | return global.RedisStructureString 233 | case "2": 234 | return global.RedisStructureHash 235 | case "3": 236 | return global.RedisStructureList 237 | case "4": 238 | return global.RedisStructureSet 239 | case "5": 240 | return global.RedisStructureSortedSet 241 | } 242 | 243 | return "" 244 | } 245 | -------------------------------------------------------------------------------- /service/endpoint/rabbit.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package endpoint 19 | 20 | import ( 21 | "github.com/juju/errors" 22 | "github.com/pquerna/ffjson/ffjson" 23 | "github.com/streadway/amqp" 24 | "github.com/vmihailenco/msgpack" 25 | "go-mysql-sync/global" 26 | "go-mysql-sync/service/luaengine" 27 | "go-mysql-sync/storage" 28 | "go-mysql-sync/util/logutil" 29 | ) 30 | 31 | type RabbitEndpoint struct { 32 | config *global.Config 33 | cached *storage.BoltRowStorage 34 | 35 | rabCon *amqp.Connection 36 | rabChl *amqp.Channel 37 | queues map[string]bool 38 | } 39 | 40 | func newRabbitEndpoint(c *global.Config) *RabbitEndpoint { 41 | r := &RabbitEndpoint{} 42 | r.config = c 43 | r.cached = &storage.BoltRowStorage{} 44 | r.queues = make(map[string]bool) 45 | return r 46 | } 47 | 48 | func (s *RabbitEndpoint) Start() error { 49 | con, err := amqp.Dial(s.config.RabbitmqAddr) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | var chl *amqp.Channel 55 | chl, err = con.Channel() 56 | if err != nil { 57 | return err 58 | } 59 | 60 | s.rabCon = con 61 | s.rabChl = chl 62 | 63 | for _, rule := range global.RuleInsList() { 64 | _, err := s.rabChl.QueueDeclare( 65 | rule.RabbitmqQueue, false, false, false, false, nil, 66 | ) 67 | if err != nil { 68 | return err 69 | } 70 | s.queues[rule.RabbitmqQueue] = true 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func (s *RabbitEndpoint) Ping() error { 77 | return nil 78 | } 79 | 80 | func (s *RabbitEndpoint) mergeQueue(name string) { 81 | _, ok := s.queues[name] 82 | if ok { 83 | return 84 | } 85 | 86 | s.rabChl.QueueDeclare(name, false, false, false, false, nil) 87 | s.queues[name] = true 88 | } 89 | 90 | func (s *RabbitEndpoint) Consume(n int, message chan []*global.RowRequest, changeChan global.ChangeChan) { 91 | for rows := range message { 92 | if err := s.doRetryTask(); err != nil { 93 | logutil.Error(err.Error()) 94 | pushFailedRows(rows, s.cached) 95 | return 96 | } 97 | 98 | expect := true 99 | for _, row := range rows { 100 | rule, _ := global.RuleIns(row.RuleKey) 101 | if rule.TableColumnSize != len(row.Row) { 102 | logutil.Warnf("%s schema mismatching", row.RuleKey) 103 | continue 104 | } 105 | 106 | exportActionNum(row.Action, row.RuleKey) 107 | 108 | if rule.LuaNecessary() { 109 | err := s.doLuaConsume(row, rule) 110 | if err != nil { 111 | logutil.Errorf(errors.ErrorStack(err)) 112 | expect = false 113 | break 114 | } 115 | } else { 116 | err := s.doRuleConsume(row, rule) 117 | if err != nil { 118 | logutil.Errorf(errors.ErrorStack(err)) 119 | expect = false 120 | break 121 | } 122 | } 123 | } 124 | 125 | if !expect { 126 | pushFailedRows(rows, s.cached) 127 | } else { 128 | logutil.Infof("处理完成 %d 条数据", len(rows)) 129 | } 130 | } 131 | } 132 | 133 | func (s *RabbitEndpoint) Stock(rows []*global.RowRequest) int64 { 134 | var sum int64 135 | for _, row := range rows { 136 | rule, _ := global.RuleIns(row.RuleKey) 137 | if rule.TableColumnSize != len(row.Row) { 138 | logutil.Warnf("%s schema mismatching", row.RuleKey) 139 | continue 140 | } 141 | 142 | if rule.LuaNecessary() { 143 | err := s.doLuaConsume(row, rule) 144 | if err != nil { 145 | logutil.Errorf(errors.ErrorStack(err)) 146 | break 147 | } 148 | } else { 149 | err := s.doRuleConsume(row, rule) 150 | if err != nil { 151 | logutil.Errorf(errors.ErrorStack(err)) 152 | break 153 | } 154 | } 155 | sum++ 156 | } 157 | 158 | return sum 159 | } 160 | 161 | func (s *RabbitEndpoint) doLuaConsume(row *global.RowRequest, rule *global.Rule) error { 162 | kvm := keyValueMap(row, rule, true) 163 | ls, err := luaengine.DoMQOps(kvm, row.Action, rule) 164 | if err != nil { 165 | return errors.Errorf("lua 脚本执行失败 : %s ", err) 166 | } 167 | 168 | for _, resp := range ls { 169 | s.mergeQueue(resp.Topic) 170 | err := s.rabChl.Publish("", resp.Topic, false, false, 171 | amqp.Publishing{ 172 | ContentType: "text/plain", 173 | Body: resp.ByteArray, 174 | }) 175 | 176 | logutil.Infof("topic: %s, message: %s", resp.Topic, string(resp.ByteArray)) 177 | 178 | global.MQRespondPool.Put(resp) 179 | if err != nil { 180 | return err 181 | } 182 | } 183 | 184 | return nil 185 | } 186 | 187 | func (s *RabbitEndpoint) doRuleConsume(row *global.RowRequest, rule *global.Rule) error { 188 | kvm := keyValueMap(row, rule, false) 189 | 190 | resp := global.MQRespondPool.Get().(*global.MQRespond) 191 | resp.Action = row.Action 192 | if rule.ValueEncoder == global.ValEncoderJson { 193 | resp.Date = kvm 194 | } else { 195 | resp.Date = encodeStringValue(rule, kvm) 196 | } 197 | 198 | body, err := ffjson.Marshal(resp) 199 | global.MQRespondPool.Put(resp) 200 | if err != nil { 201 | return err 202 | } 203 | err = s.rabChl.Publish("", rule.RabbitmqQueue, false, false, 204 | amqp.Publishing{ 205 | ContentType: "text/plain", 206 | Body: body, 207 | }) 208 | 209 | logutil.Infof("topic: %s, message: %s", rule.RabbitmqQueue, string(body)) 210 | 211 | return err 212 | } 213 | 214 | func (s *RabbitEndpoint) DoRetryRow() { 215 | //TODO 216 | } 217 | 218 | func (s *RabbitEndpoint) doRetryTask() error { 219 | if s.cached.Size() == 0 { 220 | return nil 221 | } 222 | 223 | if err := s.Ping(); err != nil { 224 | return err 225 | } 226 | 227 | logutil.Infof("当前重试队列有%d 条数据", s.cached.Size()) 228 | 229 | var data []byte 230 | ids := s.cached.IdList() 231 | for _, id := range ids { 232 | var err error 233 | data, err = s.cached.Get(id) 234 | if err != nil { 235 | logutil.Warn(err.Error()) 236 | s.cached.Delete(id) 237 | continue 238 | } 239 | 240 | var row global.RowRequest 241 | err = msgpack.Unmarshal(data, row) 242 | if err != nil { 243 | logutil.Errorf(err.Error()) 244 | s.cached.Delete(id) 245 | continue 246 | } 247 | 248 | rule, _ := global.RuleIns(row.RuleKey) 249 | if rule.LuaNecessary() { 250 | err := s.doLuaConsume(&row, rule) 251 | if err != nil { 252 | return err 253 | } 254 | 255 | } else { 256 | err := s.doRuleConsume(&row, rule) 257 | if err != nil { 258 | return err 259 | } 260 | } 261 | 262 | logutil.Infof("cached id :%d , 数据重试成功", id) 263 | s.cached.Delete(id) 264 | } 265 | 266 | return nil 267 | } 268 | 269 | func (s *RabbitEndpoint) Close() { 270 | if s.rabChl != nil { 271 | s.rabChl.Close() 272 | } 273 | if s.rabCon != nil { 274 | s.rabCon.Close() 275 | } 276 | } 277 | 278 | func (s *RabbitEndpoint) StockExecSql(sql string, valuesList []interface{}) (int64, error) { 279 | // TODO 280 | return 0, nil 281 | } 282 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package main 19 | 20 | import ( 21 | "flag" 22 | "fmt" 23 | "go-mysql-sync/util" 24 | "log" 25 | "net/http" 26 | "os" 27 | "os/signal" 28 | "runtime" 29 | "strconv" 30 | "strings" 31 | "syscall" 32 | 33 | "github.com/juju/errors" 34 | "go-mysql-sync/global" 35 | "go-mysql-sync/service" 36 | "go-mysql-sync/storage" 37 | "go-mysql-sync/util/logutil" 38 | "go-mysql-sync/util/stringutil" 39 | _ "net/http/pprof" 40 | ) 41 | 42 | var ( 43 | helpFlag bool 44 | cfgPath string 45 | positionFlag bool 46 | startPosition bool 47 | statusFlag bool 48 | 49 | deletePosFlag bool 50 | 51 | full bool 52 | incrFlag bool 53 | schemaFlag bool 54 | stockFlag bool 55 | syncMode int 56 | ) 57 | 58 | func init() { 59 | flag.BoolVar(&helpFlag, "help", false, "this help") 60 | flag.StringVar(&cfgPath, "config", "app.yml", "application config file") 61 | 62 | //// go run main.go -full 63 | // 64 | //// 只同步一次全量数据 65 | //flag.BoolVar(&stockFlag, "only-stock", false, "One-off stock,no increment sync") 66 | // 67 | //// 只增量数据 68 | //flag.BoolVar(&incrFlag, "only-increment", false, "only increment sync,no stock") 69 | // 70 | //// 全量+增量同步---默认值 71 | //flag.BoolVar(&full, "stock-increment", false, "stock and increment sync") 72 | 73 | //// 只同步表结构 74 | //flag.BoolVar(&schemaFlag, "only-schema", false, "only sync schema") 75 | 76 | //设置当前binlog位置并且开始运行 go run main.go -start-position mysql-bin.000469 324783 1619431429 77 | flag.BoolVar(&startPosition, "start-position", false, "start increment sync with that position") 78 | 79 | //查询当前binlog位置 go run main.go -status 900 80 | flag.BoolVar(&statusFlag, "current-position", false, "Query the position of many seconds age,This position is the first one greater than yours seconds.") 81 | 82 | //position data 日志过大，清理data go run main.go -delete pos 83 | flag.BoolVar(&deletePosFlag, "delete-pos", false, "sync table struct or not") 84 | 85 | // sync-mode 同步模式 go run .\main.go --sync-mode=6 86 | flag.IntVar(&syncMode, "sync-mode", 0, `数值型 :一般为 6 87 | 同步模式表结构全量数据增量数据 88 | 1 是否否只同步表结构 89 | 2 否是否只同步全量数据 90 | 3 否否是只同步增量数据 91 | 4 是是否同步表结构和全量数据 92 | 5 否是是同步全量数据和增量数据，忽略表结构 93 | 6 是是是先同步表结构，再同步全量数据和增量数据`) 94 | 95 | flag.Usage = usage 96 | } 97 | 98 | func main() { 99 | //第一次需要同步表结构 100 | //如果掉线了重启则直接跑 go run main.go，会沿着记录的binlog点继续跑 101 | flag.Parse() 102 | if helpFlag { 103 | flag.Usage() 104 | return 105 | } 106 | // sync-mode 同步模式 107 | // 表结构全量数据增量数据 108 | // 1 是否否只同步表结构 109 | // 2 否是否只同步全量数据 110 | // 3 否否是只同步增量数据 111 | // 4 是是否同步表结构和全量数据 112 | // 5 否是是同步全量数据和增量数据，忽略表结构 113 | // 6 是是是先同步表结构，再同步全量数据和增量数据 114 | 115 | others := make([]string, 2) 116 | if syncMode != 0 { 117 | switch syncMode { 118 | case 1: 119 | util.SetSchemaFlag(true) 120 | case 2: 121 | util.SetStockFlag(true) 122 | case 3: 123 | util.SetIncrFlag(true) 124 | case 4: 125 | util.SetSchemaFlag(true) 126 | util.SetStockFlag(true) 127 | case 5: 128 | util.SetStockFlag(true) 129 | util.SetIncrFlag(true) 130 | case 6: 131 | util.SetSchemaFlag(true) 132 | util.SetStockFlag(true) 133 | util.SetIncrFlag(true) 134 | fmt.Println("全量加增量") 135 | default: 136 | fmt.Println("Unidentifiable number,only 1 to 6.") 137 | return 138 | } 139 | } 140 | 141 | //----------------------------------- 142 | go http.ListenAndServe(":9999", nil) 143 | //----------------------------------- 144 | n := runtime.GOMAXPROCS(runtime.NumCPU()) 145 | 146 | err := service.InitApplication(cfgPath) 147 | if err != nil { 148 | println( 149 | errors.ErrorStack(err)) 150 | return 151 | } 152 | 153 | logutil.Infof("GOMAXPROCS :%d", n) 154 | 155 | if statusFlag { 156 | var err error 157 | var secondInt int 158 | others = flag.Args() 159 | second := others[0] 160 | if secondInt, err = strconv.Atoi(second); err != nil { 161 | println("error: The parameter second must be number") 162 | return 163 | } 164 | ps := storage.NewPositionStorage(global.Cfg()) 165 | pos, _ := ps.AcquirePositionBySecond(uint32(secondInt)) 166 | fmt.Printf("The current dump position is : %s, %d, %d \n", pos.Name, pos.Pos, pos.Timestamp) 167 | return 168 | } 169 | 170 | if positionFlag { 171 | //go run main.go -position mysql-bin.000469 324783 1619431429 172 | var err error 173 | var pp, timestamp uint32 174 | others = flag.Args() 175 | if len(others) != 3 { 176 | println("error: please input the binlog's File and Position") 177 | return 178 | } 179 | f := others[0] 180 | p := others[1] 181 | timestampStr := others[2] 182 | 183 | if !strings.HasPrefix(f, "mysql-bin.") { 184 | println("error: The parameter File must be like: mysql-bin.000001") 185 | return 186 | } 187 | 188 | pp, err = stringutil.ToUint32(p) 189 | if nil != err { 190 | println("error: The parameter Position must be number") 191 | return 192 | } 193 | 194 | timestamp, err = stringutil.ToUint32(timestampStr) 195 | if nil != err { 196 | println("error: The parameter timestamp must be number") 197 | return 198 | } 199 | 200 | //ps := storage.NewPositionStorage(global.Cfg()) 201 | pos := global.PosRequest{ 202 | Name: f, 203 | Pos: pp, 204 | Timestamp: timestamp, 205 | } 206 | 207 | util.StoreNewPos(pos) 208 | fmt.Printf("The current dump position is : %s, %d, %d \n", f, pp, timestamp) 209 | } 210 | 211 | if !util.GIncrFlag { 212 | logutil.Infof("OK.") 213 | } 214 | //if stockFlag { 215 | // transfer := service.TransferServiceIns() 216 | // stock := service.NewStockService(transfer) 217 | // err = stock.Run() 218 | // if err != nil { 219 | // println(errors.ErrorStack(err)) 220 | // } 221 | // stock.Close() 222 | // return 223 | //} 224 | //开始上传 promethus 监控数据 225 | global.StartMonitor() 226 | 227 | signalChan := make(chan os.Signal, 1) 228 | signal.Notify(signalChan, 229 | os.Kill, 230 | os.Interrupt, 231 | syscall.SIGINT, 232 | syscall.SIGTERM, 233 | syscall.SIGQUIT) 234 | 235 | if global.Cfg().NotCluster() { 236 | service.StartApplication() 237 | } else { 238 | service.BootCluster() 239 | } 240 | 241 | select { 242 | case sig := <-signalChan: 243 | log.Printf("Application Stop，Signal: %s \n", sig.String()) 244 | case <-service.CtxDone(): 245 | log.Printf("context is done with %v, closing", service.CtxErr()) 246 | } 247 | 248 | service.CloseApplication() 249 | } 250 | 251 | func usage() { 252 | fmt.Fprintf(os.Stderr, `version: 1.0.0 253 | Usage: transfer [-c filename] [-s stock] 254 | 255 | Options: 256 | `) 257 | flag.PrintDefaults() 258 | } 259 | -------------------------------------------------------------------------------- /service/luaengine/actuator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package luaengine 19 | 20 | import ( 21 | "encoding/json" 22 | luaJson "github.com/layeh/gopher-json" 23 | "github.com/siddontang/go-mysql/canal" 24 | lua "github.com/yuin/gopher-lua" 25 | "sync" 26 | 27 | "go-mysql-sync/util/byteutil" 28 | "go-mysql-sync/util/stringutil" 29 | ) 30 | 31 | const ( 32 | _globalRET = "___RET___" 33 | _globalROW = "___ROW___" 34 | _globalACT = "___ACT___" 35 | ) 36 | 37 | var ( 38 | _pool *luaStatePool 39 | _ds *canal.Canal 40 | ) 41 | 42 | type luaStatePool struct { 43 | lock sync.Mutex 44 | saved []*lua.LState 45 | } 46 | 47 | func InitActuator(ds *canal.Canal) { 48 | _ds = ds 49 | _pool = &luaStatePool{ 50 | saved: make([]*lua.LState, 0, 3), 51 | } 52 | } 53 | 54 | func (p *luaStatePool) Get() *lua.LState { 55 | p.lock.Lock() 56 | defer p.lock.Unlock() 57 | 58 | n := len(p.saved) 59 | if n == 0 { 60 | return p.New() 61 | } 62 | x := p.saved[n-1] 63 | p.saved = p.saved[0 : n-1] 64 | return x 65 | } 66 | 67 | func (p *luaStatePool) New() *lua.LState { 68 | L := lua.NewState() 69 | 70 | luaJson.Preload(L) 71 | 72 | L.PreloadModule("dbOps", dbModule) 73 | L.PreloadModule("httpOps", httpModule) 74 | 75 | L.PreloadModule("redisOps", redisModule) 76 | L.PreloadModule("mqOps", mqModule) 77 | L.PreloadModule("mongodbOps", mongoModule) 78 | L.PreloadModule("esOps", esModule) 79 | 80 | return L 81 | } 82 | 83 | func (p *luaStatePool) Put(L *lua.LState) { 84 | p.lock.Lock() 85 | defer p.lock.Unlock() 86 | 87 | p.saved = append(p.saved, L) 88 | } 89 | 90 | func (p *luaStatePool) Shutdown() { 91 | for _, L := range p.saved { 92 | L.Close() 93 | } 94 | } 95 | 96 | func rawRow(L *lua.LState) int { 97 | row := L.GetGlobal(_globalROW) 98 | L.Push(row) 99 | return 1 100 | } 101 | 102 | func rawAction(L *lua.LState) int { 103 | act := L.GetGlobal(_globalACT) 104 | L.Push(act) 105 | return 1 106 | } 107 | 108 | func paddingTable(l *lua.LState, table *lua.LTable, kv map[string]interface{}) { 109 | for k, v := range kv { 110 | switch v.(type) { 111 | case float64: 112 | ft := v.(float64) 113 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 114 | case float32: 115 | ft := v.(float32) 116 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 117 | case int: 118 | ft := v.(int) 119 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 120 | case uint: 121 | ft := v.(uint) 122 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 123 | case int8: 124 | ft := v.(int8) 125 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 126 | case uint8: 127 | ft := v.(uint8) 128 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 129 | case int16: 130 | ft := v.(int16) 131 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 132 | case uint16: 133 | ft := v.(uint16) 134 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 135 | case int32: 136 | ft := v.(int32) 137 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 138 | case uint32: 139 | ft := v.(uint32) 140 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 141 | case int64: 142 | ft := v.(int64) 143 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 144 | case uint64: 145 | ft := v.(uint64) 146 | l.SetTable(table, lua.LString(k), lua.LNumber(ft)) 147 | case string: 148 | ft := v.(string) 149 | l.SetTable(table, lua.LString(k), lua.LString(ft)) 150 | case []byte: 151 | ft := string(v.([]byte)) 152 | l.SetTable(table, lua.LString(k), lua.LString(ft)) 153 | case nil: 154 | l.SetTable(table, lua.LString(k), lua.LNil) 155 | default: 156 | jsonValue, _ := json.Marshal(v) 157 | l.SetTable(table, lua.LString(k), lua.LString(jsonValue)) 158 | } 159 | } 160 | } 161 | 162 | func lvToString(lv lua.LValue) string { 163 | if lua.LVCanConvToString(lv) { 164 | return lua.LVAsString(lv) 165 | } 166 | 167 | return lv.String() 168 | } 169 | 170 | func lvToByteArray(lv lua.LValue) []byte { 171 | switch lv.Type() { 172 | case lua.LTNil: 173 | return nil 174 | case lua.LTBool: 175 | return byteutil.JsonBytes(lua.LVAsBool(lv)) 176 | case lua.LTNumber: 177 | return []byte(lv.String()) 178 | case lua.LTString: 179 | return []byte(lua.LVAsString(lv)) 180 | case lua.LTTable: 181 | ret := lvToInterface(lv, false) 182 | return byteutil.JsonBytes(ret) 183 | default: 184 | return byteutil.JsonBytes(lv) 185 | } 186 | } 187 | 188 | func lvToInterface(lv lua.LValue, tableToJson bool) interface{} { 189 | switch lv.Type() { 190 | case lua.LTNil: 191 | return nil 192 | case lua.LTBool: 193 | return lua.LVAsBool(lv) 194 | case lua.LTNumber: 195 | return float64(lua.LVAsNumber(lv)) 196 | case lua.LTString: 197 | return lua.LVAsString(lv) 198 | case lua.LTTable: 199 | t, _ := lv.(*lua.LTable) 200 | len := t.MaxN() 201 | if len == 0 { // table 202 | ret := make(map[string]interface{}) 203 | t.ForEach(func(key, value lua.LValue) { 204 | ret[lvToString(key)] = lvToInterface(value, false) 205 | }) 206 | if tableToJson { 207 | return stringutil.ToJsonString(ret) 208 | } 209 | return ret 210 | } else { // array 211 | ret := make([]interface{}, 0, len) 212 | for i := 1; i <= len; i++ { 213 | ret = append(ret, lvToInterface(t.RawGetInt(i), false)) 214 | } 215 | if tableToJson { 216 | return stringutil.ToJsonString(ret) 217 | } 218 | return ret 219 | } 220 | default: 221 | return lv 222 | } 223 | } 224 | 225 | func lvToMap(lv lua.LValue) (map[string]interface{}, bool) { 226 | switch lv.Type() { 227 | case lua.LTTable: 228 | t := lvToInterface(lv, false) 229 | ret := t.(map[string]interface{}) 230 | return ret, true 231 | default: 232 | return nil, false 233 | } 234 | } 235 | 236 | func interfaceToLv(v interface{}) lua.LValue { 237 | switch v.(type) { 238 | case float64: 239 | ft := v.(float64) 240 | return lua.LNumber(ft) 241 | case float32: 242 | ft := v.(float32) 243 | return lua.LNumber(ft) 244 | case int: 245 | ft := v.(int) 246 | return lua.LNumber(ft) 247 | case uint: 248 | ft := v.(uint) 249 | return lua.LNumber(ft) 250 | case int8: 251 | ft := v.(int8) 252 | return lua.LNumber(ft) 253 | case uint8: 254 | ft := v.(uint8) 255 | return lua.LNumber(ft) 256 | case int16: 257 | ft := v.(int16) 258 | return lua.LNumber(ft) 259 | case uint16: 260 | ft := v.(uint16) 261 | return lua.LNumber(ft) 262 | case int32: 263 | ft := v.(int32) 264 | return lua.LNumber(ft) 265 | case uint32: 266 | ft := v.(uint32) 267 | return lua.LNumber(ft) 268 | case int64: 269 | ft := v.(int64) 270 | return lua.LNumber(ft) 271 | case uint64: 272 | ft := v.(uint64) 273 | return lua.LNumber(ft) 274 | case string: 275 | ft := v.(string) 276 | return lua.LString(ft) 277 | case []byte: 278 | ft := string(v.([]byte)) 279 | return lua.LString(ft) 280 | case nil: 281 | return lua.LNil 282 | default: 283 | jsonValue, _ := json.Marshal(v) 284 | return lua.LString(jsonValue) 285 | } 286 | 287 | } 288 | -------------------------------------------------------------------------------- /storage/bolt_position_storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package storage 19 | 20 | import ( 21 | "fmt" 22 | "github.com/juju/errors" 23 | "github.com/siddontang/go-mysql/mysql" 24 | "github.com/vmihailenco/msgpack" 25 | "go-mysql-sync/global" 26 | "go-mysql-sync/util/logutil" 27 | "go.etcd.io/bbolt" 28 | "strconv" 29 | "strings" 30 | ) 31 | 32 | type boltPositionStorage struct { 33 | } 34 | 35 | func (s *boltPositionStorage) Initialize() error { 36 | return _bolt.Update(func(tx *bbolt.Tx) error { 37 | bt := tx.Bucket(_positionBucket) 38 | data := bt.Get(_fixPositionId) 39 | if data != nil { 40 | return nil 41 | } 42 | 43 | bytes, err := msgpack.Marshal(mysql.Position{}) 44 | if err != nil { 45 | return err 46 | } 47 | return bt.Put(_fixPositionId, bytes) 48 | }) 49 | } 50 | 51 | func (s *boltPositionStorage) Save(pos mysql.Position) error { 52 | fmt.Println(pos.String()) 53 | return _bolt.Update(func(tx *bbolt.Tx) error { 54 | bt := tx.Bucket(_positionBucket) 55 | data, err := msgpack.Marshal(pos) 56 | if err != nil { 57 | return err 58 | } 59 | return bt.Put(_fixPositionId, data) 60 | }) 61 | } 62 | 63 | func (s *boltPositionStorage) Get() (mysql.Position, error) { 64 | var entity mysql.Position 65 | err := _bolt.View(func(tx *bbolt.Tx) error { 66 | bt := tx.Bucket(_positionBucket) 67 | data := bt.Get(_fixPositionId) 68 | if data == nil { 69 | return errors.NotFoundf("PositionStorage") 70 | } 71 | 72 | return msgpack.Unmarshal(data, &entity) 73 | }) 74 | 75 | return entity, err 76 | } 77 | 78 | func (s *boltPositionStorage) Init() error { 79 | return _bolt.Update(func(tx *bbolt.Tx) error { 80 | //_positionBucket = []byte("Position") 81 | //_fixPositionId = byteutil.Uint64ToBytes(uint64(1)) 82 | bt := tx.Bucket(_positionBucket) 83 | value := bt.Get(_firstKey) 84 | if value != nil { 85 | return nil 86 | } 87 | bytes, err := msgpack.Marshal(global.PosRequest{}) 88 | if err != nil { 89 | return err 90 | } 91 | return bt.Put(_firstKey, bytes) 92 | }) 93 | } 94 | 95 | //记录 position 96 | func (s *boltPositionStorage) RecordPosition(pos global.PosRequest) error { 97 | var err error 98 | var value []byte 99 | var posKey string 100 | fmt.Println(pos.String()) 101 | return _bolt.Update(func(tx *bbolt.Tx) error { 102 | bt := tx.Bucket(_positionBucket) 103 | //key := byteutil.Uint32ToBytes(pos.Timestamp) 104 | if posKey, err = PosToKey(pos); err != nil { 105 | return err 106 | } 107 | value, err = msgpack.Marshal(pos) 108 | if err != nil { 109 | return err 110 | } 111 | return bt.Put([]byte(posKey), value) 112 | }) 113 | } 114 | 115 | func (s *boltPositionStorage) AcquirePosition() (pos mysql.Position, err error) { 116 | return s.AcquirePositionBySecond(900) 117 | } 118 | 119 | // 按当前的binlog时间戳往前推n秒取binlog位置 120 | func (s *boltPositionStorage) AcquirePositionBySecond(second uint32) (pos mysql.Position, err error) { 121 | var lastKeyByte, lastValueByte, keyByte, valueByte []byte 122 | //var lastKeyStr string 123 | //var KeyStr string 124 | 125 | var position, eachPosition global.PosRequest 126 | var lastGetPos global.PosRequest 127 | err = _bolt.View(func(tx *bbolt.Tx) error { 128 | c := tx.Bucket(_positionBucket).Cursor() 129 | 130 | //取 last Timestamp 131 | lastKeyByte, lastValueByte = c.Last() 132 | //lastKeyStr = string(lastKeyByte) 133 | msgpack.Unmarshal(lastValueByte, &lastGetPos) 134 | logutil.Infof("获取当前的 position : %s %d %d ", lastGetPos.Name, lastGetPos.Pos, lastGetPos.Timestamp) 135 | 136 | //判断 last 是否是空，是空则报错 137 | if IsFirstNilPos(lastKeyByte) { 138 | return errors.NotFoundf("Wrong!,, Currently acquired position is the first nil position, Run in less than 15 minutes,Not enough position to roll back") 139 | } else { 140 | //不为空，则往前一个 Timestamp 取，每个 Timestamp 都需要判断是否为空，直到 last 比 Timestamp 大 900 以上 141 | for { 142 | keyByte, valueByte = c.Prev() 143 | if IsFirstNilPos(keyByte) { 144 | return errors.NotFoundf("Wrong!, Run in less than 15 minutes,Not enough position to roll back") 145 | } 146 | //KeyStr = string(keyByte) 147 | msgpack.Unmarshal(valueByte, &eachPosition) 148 | if lastGetPos.Timestamp-eachPosition.Timestamp > second { 149 | break 150 | } 151 | } 152 | } 153 | return msgpack.Unmarshal(valueByte, &position) 154 | }) 155 | pos.Name = position.Name 156 | pos.Pos = position.Pos 157 | pos.Timestamp = position.Timestamp 158 | logutil.Infof("当前获取 %d s 前的 position :\n", second, pos.String()) 159 | return pos, err 160 | } 161 | 162 | //为什么减少日志文件大小批量删除日志数据删除时间戳之前的数据 163 | func (s *boltPositionStorage) DeletePositionBySecond(second uint32) (err error) { 164 | var value []byte 165 | var pos *global.PosRequest 166 | return _bolt.Update(func(tx *bbolt.Tx) error { 167 | c := tx.Bucket(_positionBucket).Cursor() 168 | _, value = c.First() 169 | msgpack.Unmarshal(value, &pos) 170 | 171 | //fmt.Printf("first timestamp is %d\n",pos.Timestamp) 172 | 173 | if pos.Timestamp < second { 174 | err = c.Delete() 175 | //fmt.Println("delete first timestamp:", pos.Timestamp) 176 | if err != nil { 177 | return err 178 | } 179 | 180 | for { 181 | _, value = c.Next() 182 | msgpack.Unmarshal(value, &pos) 183 | //fmt.Println("get timestamp:", pos.Timestamp) 184 | if pos.Timestamp < second { 185 | err = c.Delete() 186 | //fmt.Println("delete timestamp:", pos.Timestamp) 187 | //time.Sleep(time.Second * 1) 188 | if err != nil { 189 | return err 190 | } 191 | } else { 192 | break 193 | } 194 | } 195 | } 196 | return err 197 | }) 198 | 199 | } 200 | 201 | func IsFirstNilPos(key []byte) bool { 202 | if string(key) == "00000100000000000001" { 203 | return true 204 | } 205 | return false 206 | } 207 | 208 | func ExistsTimestamp(pos global.PosRequest) (bool, error) { 209 | var value []byte 210 | var err error 211 | var str string 212 | str, err = PosToKey(pos) 213 | if err != nil { 214 | return false, err 215 | } 216 | err = _bolt.View(func(tx *bbolt.Tx) error { 217 | bt := tx.Bucket(_positionBucket) 218 | value = bt.Get([]byte(str)) 219 | return nil 220 | }) 221 | if value == nil { 222 | return false, err 223 | } else { 224 | return true, err 225 | } 226 | } 227 | 228 | func PosToKey(position global.PosRequest) (string, error) { 229 | //file 6位补0 pos 14位补0 230 | var fileLeft, posLeft, str string 231 | str = "0" 232 | file := strings.Split(position.Name, ".")[1] 233 | pos := strconv.Itoa(int(position.Pos)) 234 | if len(file) > 6 || len(pos) > 14 { 235 | return "", errors.New("position 数据有误无法解析") 236 | } 237 | if len(file) < 6 { 238 | //fmt.Println(file,"补",6-len(file),"个零") 239 | fileLeft = strings.Repeat(str, 6-len(file)) 240 | //fmt.Println(fileLeft) 241 | } 242 | if len(pos) < 14 { 243 | //fmt.Println(pos,"补",14-len(pos),"个零") 244 | posLeft = strings.Repeat(str, 14-len(pos)) 245 | //fmt.Println(posLeft) 246 | } 247 | return fileLeft + file + posLeft + pos, nil 248 | } 249 | -------------------------------------------------------------------------------- /util/stringutil/string_util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package stringutil 19 | 20 | import ( 21 | "bytes" 22 | "crypto/md5" 23 | "encoding/hex" 24 | "encoding/json" 25 | "net/url" 26 | "regexp" 27 | "strconv" 28 | "strings" 29 | "unicode" 30 | 31 | "github.com/pquerna/ffjson/ffjson" 32 | "github.com/satori/go.uuid" 33 | 34 | "go-mysql-sync/util/logutil" 35 | ) 36 | 37 | // 产生UUID 38 | func UUID() string { 39 | return strings.ReplaceAll(uuid.NewV4().String(), "-", "") 40 | } 41 | 42 | // MD2编码 43 | func MD5(str string) string { 44 | h := md5.New() 45 | h.Write([]byte(str)) 46 | return hex.EncodeToString(h.Sum(nil)) 47 | } 48 | 49 | // 转换为Int 50 | func ToIntSafe(str string) int { 51 | v, e := strconv.Atoi(str) 52 | if nil != e { 53 | return 0 54 | } 55 | return v 56 | } 57 | 58 | // 转换为Uint64 59 | func ToInt64Safe(str string) int64 { 60 | v, e := strconv.ParseInt(str, 10, 64) 61 | if nil != e { 62 | return 0 63 | } 64 | return v 65 | } 66 | 67 | // 转换为Uint64 68 | func ToUint32(str string) (uint32, error) { 69 | v, e := strconv.ParseUint(str, 10, 32) 70 | if nil != e { 71 | return 0, e 72 | } 73 | return uint32(v), nil 74 | } 75 | 76 | // 转换为Uint64 77 | func ToUint32Safe(str string) uint32 { 78 | v, e := strconv.ParseUint(str, 10, 32) 79 | if nil != e { 80 | return 0 81 | } 82 | return uint32(v) 83 | } 84 | 85 | // 转换为Uint64 86 | func ToUint64Safe(str string) uint64 { 87 | v, e := strconv.ParseUint(str, 10, 64) 88 | if nil != e { 89 | return 0 90 | } 91 | return v 92 | } 93 | 94 | // 转换为Uint64 95 | func ToFloat64Safe(str string) float64 { 96 | v, e := strconv.ParseFloat(str, 64) 97 | if nil != e { 98 | return 0 99 | } 100 | return v 101 | } 102 | 103 | // Uint64转换为String 104 | func Uint64ToStr(u uint64) string { 105 | return strconv.FormatUint(u, 10) 106 | } 107 | 108 | // 逗号分隔键值对转MAP,类似"name=wangjie,age=20"或者"name=wangjie|age=20" 109 | func CommasToMap(base string, sep string) map[string]interface{} { 110 | ret := make(map[string]interface{}) 111 | if "" != base && "" != sep { 112 | kvs := strings.Split(base, sep) 113 | for _, kv := range kvs { 114 | temp := strings.Split(kv, "=") 115 | if len(temp) < 2 { 116 | continue 117 | } 118 | if temp[0] == "" { 119 | continue 120 | } 121 | ret[temp[0]] = temp[1] 122 | } 123 | } 124 | return ret 125 | } 126 | 127 | func ToJsonString(v interface{}) string { 128 | bytes, err := ffjson.Marshal(v) 129 | if nil != err { 130 | logutil.GlobalSugar().Errorf("json marshal :%s", err.Error()) 131 | return "" 132 | } 133 | return string(bytes) 134 | } 135 | 136 | func ToJsonIndent(v interface{}) string { 137 | bytes, err := json.MarshalIndent(v, "", "\t") 138 | if nil != err { 139 | logutil.GlobalSugar().Errorf("json marshal :%s", err.Error()) 140 | return "" 141 | } 142 | return string(bytes) 143 | } 144 | 145 | // url.Values转查询字符串 146 | func UrlValuesToQueryString(base string, parameters url.Values) string { 147 | if len(parameters) == 0 { 148 | return base 149 | } 150 | 151 | if !strings.Contains(base, "?") { 152 | base += "?" 153 | } 154 | 155 | if strings.HasSuffix(base, "?") || strings.HasSuffix(base, "&") { 156 | base += parameters.Encode() 157 | } else { 158 | base += "&" + parameters.Encode() 159 | } 160 | 161 | return base 162 | } 163 | 164 | // map转查询字符串 165 | func MapToQueryString(base string, parameters map[string]interface{}) string { 166 | if len(parameters) == 0 { 167 | return base 168 | } 169 | 170 | exist := false 171 | if strings.Contains(base, "?") { 172 | exist = true 173 | } 174 | var buffer bytes.Buffer 175 | buffer.WriteString(base) 176 | for k, v := range parameters { 177 | var temp string 178 | if !exist { 179 | temp = "?" + k + "=" + ToString(v) 180 | exist = true 181 | } else { 182 | temp = "&" + k + "=" + ToString(v) 183 | } 184 | buffer.WriteString(temp) 185 | } 186 | return buffer.String() 187 | } 188 | 189 | // 转换为字符串 190 | func ToString(value interface{}) string { 191 | var key string 192 | if value == nil { 193 | return key 194 | } 195 | 196 | switch value.(type) { 197 | case float64: 198 | ft := value.(float64) 199 | key = strconv.FormatFloat(ft, 'f', -1, 64) 200 | case float32: 201 | ft := value.(float32) 202 | key = strconv.FormatFloat(float64(ft), 'f', -1, 64) 203 | case int: 204 | it := value.(int) 205 | key = strconv.Itoa(it) 206 | case uint: 207 | it := value.(uint) 208 | key = strconv.Itoa(int(it)) 209 | case int8: 210 | it := value.(int8) 211 | key = strconv.Itoa(int(it)) 212 | case uint8: 213 | it := value.(uint8) 214 | key = strconv.Itoa(int(it)) 215 | case int16: 216 | it := value.(int16) 217 | key = strconv.Itoa(int(it)) 218 | case uint16: 219 | it := value.(uint16) 220 | key = strconv.Itoa(int(it)) 221 | case int32: 222 | it := value.(int32) 223 | key = strconv.Itoa(int(it)) 224 | case uint32: 225 | it := value.(uint32) 226 | key = strconv.Itoa(int(it)) 227 | case int64: 228 | it := value.(int64) 229 | key = strconv.FormatInt(it, 10) 230 | case uint64: 231 | it := value.(uint64) 232 | key = strconv.FormatUint(it, 10) 233 | case string: 234 | key = value.(string) 235 | case []byte: 236 | key = string(value.([]byte)) 237 | default: 238 | newValue, _ := ffjson.Marshal(value) 239 | key = string(newValue) 240 | } 241 | 242 | return key 243 | } 244 | 245 | // 是否为邮件格式 246 | func IsEmailFormat(email string) bool { 247 | pattern := `\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*` 248 | reg := regexp.MustCompile(pattern) 249 | return reg.MatchString(email) 250 | } 251 | 252 | // 是否为中文 253 | func IsChineseChar(str string) bool { 254 | for _, r := range str { 255 | if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) { 256 | return true 257 | } 258 | } 259 | return false 260 | } 261 | 262 | // 驼峰式写法转为下划线写法 263 | func Camel2Case(name string) string { 264 | buffer := new(bytes.Buffer) 265 | for i, r := range name { 266 | if unicode.IsUpper(r) { 267 | if i != 0 { 268 | buffer.WriteByte('_') 269 | } 270 | buffer.WriteRune(unicode.ToLower(r)) 271 | } else { 272 | buffer.WriteString(strconv.FormatInt(int64(r), 10)) 273 | } 274 | } 275 | return buffer.String() 276 | } 277 | 278 | // 下划线写法转为驼峰写法 279 | func Case2Camel(name string) string { 280 | name = strings.Replace(name, "_", " ", -1) 281 | name = strings.Title(name) 282 | name = strings.Replace(name, " ", "", -1) 283 | return Lcfirst(name) 284 | } 285 | 286 | // 首字母大写 287 | func Ucfirst(str string) string { 288 | for i, v := range str { 289 | return string(unicode.ToUpper(v)) + str[i+1:] 290 | } 291 | return "" 292 | } 293 | 294 | // 首字母小写 295 | func Lcfirst(str string) string { 296 | for i, v := range str { 297 | return string(unicode.ToLower(v)) + str[i+1:] 298 | } 299 | return "" 300 | } 301 | 302 | //截取字符串开始到倒数位数 303 | func CutLastString(str string, n int) (strLone string) { 304 | strLone = str[:len(str)-n] 305 | return strLone 306 | } 307 | -------------------------------------------------------------------------------- /service/endpoint/kafka.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package endpoint 19 | 20 | import ( 21 | "github.com/Shopify/sarama" 22 | "github.com/juju/errors" 23 | "github.com/pquerna/ffjson/ffjson" 24 | "github.com/vmihailenco/msgpack" 25 | "strings" 26 | 27 | "go-mysql-sync/global" 28 | "go-mysql-sync/service/luaengine" 29 | "go-mysql-sync/storage" 30 | "go-mysql-sync/util/logutil" 31 | ) 32 | 33 | type KafkaEndpoint struct { 34 | config *global.Config 35 | cached *storage.BoltRowStorage 36 | 37 | client sarama.Client 38 | producer sarama.AsyncProducer 39 | } 40 | 41 | func newKafkaEndpoint(c *global.Config) *KafkaEndpoint { 42 | r := &KafkaEndpoint{} 43 | r.config = c 44 | r.cached = &storage.BoltRowStorage{} 45 | return r 46 | } 47 | 48 | func (s *KafkaEndpoint) Start() error { 49 | cfg := sarama.NewConfig() 50 | cfg.Producer.Partitioner = sarama.NewRandomPartitioner 51 | 52 | if s.config.KafkaSASLUser != "" && s.config.KafkaSASLPassword != "" { 53 | cfg.Net.SASL.Enable = true 54 | cfg.Net.SASL.User = s.config.KafkaSASLUser 55 | cfg.Net.SASL.Password = s.config.KafkaSASLPassword 56 | } 57 | 58 | var err error 59 | var client sarama.Client 60 | ls := strings.Split(s.config.KafkaAddr, ",") 61 | client, err = sarama.NewClient(ls, cfg) 62 | if err != nil { 63 | return errors.Errorf("unable to create kafka client: %q", err) 64 | } 65 | 66 | var producer sarama.AsyncProducer 67 | producer, err = sarama.NewAsyncProducerFromClient(client) 68 | if err != nil { 69 | return errors.Errorf("unable to create kafka producer: %q", err) 70 | } 71 | 72 | s.producer = producer 73 | s.client = client 74 | 75 | return nil 76 | } 77 | 78 | func (s *KafkaEndpoint) Ping() error { 79 | return nil 80 | } 81 | 82 | func (s *KafkaEndpoint) Consume(n int, message chan []*global.RowRequest, changeChan global.ChangeChan) { 83 | for rows := range message { 84 | if err := s.doRetryTask(); err != nil { 85 | logutil.Error(err.Error()) 86 | pushFailedRows(rows, s.cached) 87 | return 88 | } 89 | 90 | expect := true 91 | var ms []*sarama.ProducerMessage 92 | for _, row := range rows { 93 | rule, _ := global.RuleIns(row.RuleKey) 94 | if rule.TableColumnSize != len(row.Row) { 95 | logutil.Warnf("%s schema mismatching", row.RuleKey) 96 | continue 97 | } 98 | 99 | exportActionNum(row.Action, row.RuleKey) 100 | 101 | if rule.LuaNecessary() { 102 | ls, err := s.buildMessages(row, rule) 103 | if err != nil { 104 | logutil.Errorf(errors.ErrorStack(err)) 105 | expect = false 106 | break 107 | } 108 | ms = append(ms, ls...) 109 | } else { 110 | m, err := s.buildMessage(row, rule) 111 | if err != nil { 112 | logutil.Errorf(errors.ErrorStack(err)) 113 | expect = false 114 | break 115 | } 116 | ms = append(ms, m) 117 | } 118 | } 119 | 120 | if !expect { 121 | pushFailedRows(rows, s.cached) 122 | return 123 | } 124 | 125 | for _, m := range ms { 126 | s.producer.Input() <- m 127 | select { 128 | case err := <-s.producer.Errors(): 129 | logutil.Error(err.Error()) 130 | expect = false 131 | break 132 | default: 133 | 134 | } 135 | } 136 | 137 | if !expect { 138 | pushFailedRows(rows, s.cached) 139 | } else { 140 | logutil.Infof("处理完成 %d 条数据", len(rows)) 141 | } 142 | } 143 | } 144 | 145 | func (s *KafkaEndpoint) Stock(rows []*global.RowRequest) int64 { 146 | expect := true 147 | for _, row := range rows { 148 | rule, _ := global.RuleIns(row.RuleKey) 149 | if rule.TableColumnSize != len(row.Row) { 150 | logutil.Warnf("%s schema mismatching", row.RuleKey) 151 | continue 152 | } 153 | 154 | if rule.LuaNecessary() { 155 | ls, err := s.buildMessages(row, rule) 156 | if err != nil { 157 | logutil.Errorf(errors.ErrorStack(err)) 158 | expect = false 159 | break 160 | } 161 | for _, m := range ls { 162 | s.producer.Input() <- m 163 | select { 164 | case err := <-s.producer.Errors(): 165 | logutil.Error(err.Error()) 166 | expect = false 167 | break 168 | default: 169 | } 170 | } 171 | if !expect { 172 | break 173 | } 174 | } else { 175 | m, err := s.buildMessage(row, rule) 176 | if err != nil { 177 | logutil.Errorf(errors.ErrorStack(err)) 178 | expect = false 179 | break 180 | } 181 | s.producer.Input() <- m 182 | select { 183 | case err := <-s.producer.Errors(): 184 | logutil.Error(err.Error()) 185 | expect = false 186 | break 187 | default: 188 | 189 | } 190 | } 191 | } 192 | 193 | if !expect { 194 | return 0 195 | } 196 | 197 | return int64(len(rows)) 198 | } 199 | 200 | func (s *KafkaEndpoint) buildMessages(row *global.RowRequest, rule *global.Rule) ([]*sarama.ProducerMessage, error) { 201 | kvm := keyValueMap(row, rule, true) 202 | ls, err := luaengine.DoMQOps(kvm, row.Action, rule) 203 | if err != nil { 204 | return nil, errors.Errorf("lua 脚本执行失败 : %s ", err) 205 | } 206 | 207 | var ms []*sarama.ProducerMessage 208 | for _, resp := range ls { 209 | m := &sarama.ProducerMessage{ 210 | Topic: resp.Topic, 211 | Value: sarama.ByteEncoder(resp.ByteArray), 212 | } 213 | 214 | logutil.Infof("topic: %s, message: %s", resp.Topic, string(resp.ByteArray)) 215 | 216 | global.MQRespondPool.Put(resp) 217 | ms = append(ms, m) 218 | } 219 | 220 | return ms, nil 221 | } 222 | 223 | func (s *KafkaEndpoint) buildMessage(row *global.RowRequest, rule *global.Rule) (*sarama.ProducerMessage, error) { 224 | kvm := keyValueMap(row, rule, false) 225 | resp := global.MQRespondPool.Get().(*global.MQRespond) 226 | resp.Action = row.Action 227 | if rule.ValueEncoder == global.ValEncoderJson { 228 | resp.Date = kvm 229 | } else { 230 | resp.Date = encodeStringValue(rule, kvm) 231 | } 232 | body, err := ffjson.Marshal(resp) 233 | global.MQRespondPool.Put(resp) 234 | if err != nil { 235 | return nil, err 236 | } 237 | m := &sarama.ProducerMessage{ 238 | Topic: rule.KafkaTopic, 239 | Value: sarama.ByteEncoder(body), 240 | } 241 | 242 | logutil.Infof("topic: %s, message: %s", rule.KafkaTopic, string(body)) 243 | 244 | return m, nil 245 | } 246 | 247 | func (s *KafkaEndpoint) DoRetryRow() { 248 | //TODO 249 | } 250 | 251 | func (s *KafkaEndpoint) doRetryTask() error { 252 | if s.cached.Size() == 0 { 253 | return nil 254 | } 255 | 256 | if err := s.Ping(); err != nil { 257 | return err 258 | } 259 | 260 | logutil.Infof("当前重试队列有%d 条数据", s.cached.Size()) 261 | 262 | var data []byte 263 | ids := s.cached.IdList() 264 | for _, id := range ids { 265 | var err error 266 | data, err = s.cached.Get(id) 267 | if err != nil { 268 | logutil.Warn(err.Error()) 269 | s.cached.Delete(id) 270 | continue 271 | } 272 | 273 | var row global.RowRequest 274 | err = msgpack.Unmarshal(data, row) 275 | if err != nil { 276 | logutil.Errorf(err.Error()) 277 | s.cached.Delete(id) 278 | continue 279 | } 280 | 281 | rule, _ := global.RuleIns(row.RuleKey) 282 | if rule.LuaNecessary() { 283 | ls, err := s.buildMessages(&row, rule) 284 | if err != nil { 285 | return err 286 | } 287 | for _, msg := range ls { 288 | s.producer.Input() <- msg 289 | select { 290 | case err := <-s.producer.Errors(): 291 | logutil.Error(err.Error()) 292 | return err 293 | default: 294 | 295 | } 296 | } 297 | } else { 298 | msg, err := s.buildMessage(&row, rule) 299 | if err != nil { 300 | return err 301 | } 302 | s.producer.Input() <- msg 303 | select { 304 | case err := <-s.producer.Errors(): 305 | logutil.Error(err.Error()) 306 | return err 307 | default: 308 | 309 | } 310 | } 311 | 312 | logutil.Infof("cached id :%d , 数据重试成功", id) 313 | s.cached.Delete(id) 314 | } 315 | 316 | return nil 317 | } 318 | 319 | func (s *KafkaEndpoint) Close() { 320 | if s.producer != nil { 321 | s.producer.Close() 322 | } 323 | if s.client != nil { 324 | s.client.Close() 325 | } 326 | } 327 | 328 | func (s *KafkaEndpoint) StockExecSql(sql string, valuesList []interface{}) (int64, error) { 329 | // TODO 330 | return 0, nil 331 | } 332 | -------------------------------------------------------------------------------- /service/endpoint/rocket.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package endpoint 19 | 20 | import ( 21 | "context" 22 | "strings" 23 | "sync" 24 | 25 | "github.com/apache/rocketmq-client-go/v2" 26 | "github.com/apache/rocketmq-client-go/v2/primitive" 27 | "github.com/apache/rocketmq-client-go/v2/producer" 28 | "github.com/apache/rocketmq-client-go/v2/rlog" 29 | "github.com/juju/errors" 30 | "github.com/pquerna/ffjson/ffjson" 31 | "github.com/vmihailenco/msgpack" 32 | 33 | "go-mysql-sync/global" 34 | "go-mysql-sync/service/luaengine" 35 | "go-mysql-sync/storage" 36 | "go-mysql-sync/util/logutil" 37 | ) 38 | 39 | const _rocketRetry = 2 40 | 41 | type RocketEndpoint struct { 42 | config *global.Config 43 | cached *storage.BoltRowStorage 44 | 45 | client rocketmq.Producer 46 | } 47 | 48 | func newRocketEndpoint(c *global.Config) *RocketEndpoint { 49 | rlog.SetLogger(logutil.NewRocketmqLoggerAgent()) 50 | 51 | options := make([]producer.Option, 0) 52 | serverList := strings.Split(c.RocketmqNameServers, ",") 53 | options = append(options, producer.WithNameServer(serverList)) 54 | options = append(options, producer.WithRetry(_rocketRetry)) 55 | if c.RocketmqGroupName != "" { 56 | options = append(options, producer.WithGroupName(c.RocketmqGroupName)) 57 | } 58 | if c.RocketmqInstanceName != "" { 59 | options = append(options, producer.WithInstanceName(c.RocketmqInstanceName)) 60 | } 61 | if c.RocketmqAccessKey != "" && c.RocketmqSecretKey != "" { 62 | options = append(options, producer.WithCredentials(primitive.Credentials{ 63 | AccessKey: c.RocketmqAccessKey, 64 | SecretKey: c.RocketmqSecretKey, 65 | })) 66 | } 67 | 68 | producer, _ := rocketmq.NewProducer(options...) 69 | r := &RocketEndpoint{} 70 | r.config = c 71 | r.client = producer 72 | r.cached = &storage.BoltRowStorage{} 73 | return r 74 | } 75 | 76 | func (s *RocketEndpoint) Start() error { 77 | return s.client.Start() 78 | } 79 | 80 | func (s *RocketEndpoint) Ping() error { 81 | return nil 82 | } 83 | 84 | func (s *RocketEndpoint) Consume(n int, message chan []*global.RowRequest, changeChan global.ChangeChan) { 85 | for rows := range message { 86 | if err := s.doRetryTask(); err != nil { 87 | logutil.Error(err.Error()) 88 | pushFailedRows(rows, s.cached) 89 | return 90 | } 91 | 92 | expect := true 93 | var ms []*primitive.Message 94 | for _, row := range rows { 95 | rule, _ := global.RuleIns(row.RuleKey) 96 | if rule.TableColumnSize != len(row.Row) { 97 | logutil.Warnf("%s schema mismatching", row.RuleKey) 98 | continue 99 | } 100 | 101 | exportActionNum(row.Action, row.RuleKey) 102 | 103 | if rule.LuaNecessary() { 104 | ls, err := s.buildMessages(row, rule) 105 | if err != nil { 106 | logutil.Errorf(errors.ErrorStack(err)) 107 | expect = false 108 | break 109 | } 110 | ms = append(ms, ls...) 111 | } else { 112 | m, err := s.buildMessage(row, rule) 113 | if err != nil { 114 | logutil.Errorf(errors.ErrorStack(err)) 115 | expect = false 116 | break 117 | } 118 | ms = append(ms, m) 119 | } 120 | } 121 | 122 | if !expect { 123 | pushFailedRows(rows, s.cached) 124 | return 125 | } 126 | 127 | var wg sync.WaitGroup 128 | wg.Add(1) 129 | err := s.client.SendAsync(context.Background(), 130 | func(ctx context.Context, result *primitive.SendResult, e error) { 131 | if e != nil { 132 | logutil.Error(e.Error()) 133 | expect = false 134 | } 135 | wg.Done() 136 | }, ms...) 137 | 138 | if err != nil { 139 | logutil.Error(err.Error()) 140 | expect = false 141 | } else { 142 | wg.Wait() 143 | } 144 | 145 | if !expect { 146 | pushFailedRows(rows, s.cached) 147 | } else { 148 | logutil.Infof("处理完成 %d 条数据", len(rows)) 149 | } 150 | } 151 | } 152 | 153 | func (s *RocketEndpoint) Stock(rows []*global.RowRequest) int64 { 154 | expect := true 155 | var ms []*primitive.Message 156 | for _, row := range rows { 157 | rule, _ := global.RuleIns(row.RuleKey) 158 | if rule.TableColumnSize != len(row.Row) { 159 | logutil.Warnf("%s schema mismatching", row.RuleKey) 160 | continue 161 | } 162 | 163 | if rule.LuaNecessary() { 164 | ls, err := s.buildMessages(row, rule) 165 | if err != nil { 166 | logutil.Errorf(errors.ErrorStack(err)) 167 | expect = false 168 | break 169 | } 170 | ms = append(ms, ls...) 171 | } else { 172 | m, err := s.buildMessage(row, rule) 173 | if err != nil { 174 | logutil.Errorf(errors.ErrorStack(err)) 175 | expect = false 176 | break 177 | } 178 | ms = append(ms, m) 179 | } 180 | } 181 | 182 | if !expect { 183 | return 0 184 | } 185 | 186 | if len(ms) == 0 { 187 | return 0 188 | } 189 | 190 | var wg sync.WaitGroup 191 | wg.Add(1) 192 | err := s.client.SendAsync(context.Background(), 193 | func(ctx context.Context, result *primitive.SendResult, e error) { 194 | if e != nil { 195 | logutil.Error(errors.ErrorStack(e)) 196 | expect = false 197 | } 198 | wg.Done() 199 | }, ms...) 200 | 201 | if err != nil { 202 | logutil.Error(errors.ErrorStack(err)) 203 | return 0 204 | } 205 | 206 | wg.Wait() 207 | 208 | if expect { 209 | return int64(len(ms)) 210 | } 211 | return 0 212 | } 213 | 214 | func (s *RocketEndpoint) buildMessages(row *global.RowRequest, rule *global.Rule) ([]*primitive.Message, error) { 215 | kvm := keyValueMap(row, rule, true) 216 | ls, err := luaengine.DoMQOps(kvm, row.Action, rule) 217 | if err != nil { 218 | return nil, errors.Errorf("lua 脚本执行失败 : %s ", err) 219 | } 220 | 221 | var ms []*primitive.Message 222 | for _, resp := range ls { 223 | m := &primitive.Message{ 224 | Topic: resp.Topic, 225 | Body: resp.ByteArray, 226 | } 227 | global.MQRespondPool.Put(resp) 228 | 229 | logutil.Infof("topic: %s, message: %s", m.Topic, string(m.Body)) 230 | 231 | ms = append(ms, m) 232 | } 233 | 234 | return ms, nil 235 | } 236 | 237 | func (s *RocketEndpoint) buildMessage(row *global.RowRequest, rule *global.Rule) (*primitive.Message, error) { 238 | kvm := keyValueMap(row, rule, false) 239 | resp := global.MQRespondPool.Get().(*global.MQRespond) 240 | resp.Action = row.Action 241 | if rule.ValueEncoder == global.ValEncoderJson { 242 | resp.Date = kvm 243 | } else { 244 | resp.Date = encodeStringValue(rule, kvm) 245 | } 246 | body, err := ffjson.Marshal(resp) 247 | global.MQRespondPool.Put(resp) 248 | if err != nil { 249 | return nil, err 250 | } 251 | m := &primitive.Message{ 252 | Topic: rule.RocketmqTopic, 253 | Body: body, 254 | } 255 | 256 | logutil.Infof("topic: %s, message: %s", m.Topic, string(m.Body)) 257 | 258 | return m, nil 259 | } 260 | 261 | func (s *RocketEndpoint) DoRetryRow() { 262 | //TODO 263 | } 264 | 265 | func (s *RocketEndpoint) doRetryTask() error { 266 | if s.cached.Size() == 0 { 267 | return nil 268 | } 269 | 270 | if err := s.Ping(); err != nil { 271 | return err 272 | } 273 | 274 | logutil.Infof("当前重试队列有%d 条数据", s.cached.Size()) 275 | 276 | var data []byte 277 | ids := s.cached.IdList() 278 | for _, id := range ids { 279 | var err error 280 | data, err = s.cached.Get(id) 281 | if err != nil { 282 | logutil.Warn(err.Error()) 283 | s.cached.Delete(id) 284 | continue 285 | } 286 | 287 | var row global.RowRequest 288 | err = msgpack.Unmarshal(data, row) 289 | if err != nil { 290 | logutil.Errorf(err.Error()) 291 | s.cached.Delete(id) 292 | continue 293 | } 294 | 295 | rule, _ := global.RuleIns(row.RuleKey) 296 | if rule.LuaNecessary() { 297 | ls, err := s.buildMessages(&row, rule) 298 | if err != nil { 299 | return err 300 | } 301 | for _, msg := range ls { 302 | _, err = s.client.SendSync(context.Background(), msg) 303 | if err != nil { 304 | return err 305 | } 306 | logutil.Infof("retry: topic:%s, message:%s", msg.Topic, string(msg.Body)) 307 | } 308 | } else { 309 | msg, err := s.buildMessage(&row, rule) 310 | if err != nil { 311 | return err 312 | } 313 | _, err = s.client.SendSync(context.Background(), msg) 314 | if err != nil { 315 | return err 316 | } 317 | logutil.Infof("retry: topic:%s, message:%s", msg.Topic, string(msg.Body)) 318 | } 319 | 320 | logutil.Infof("cached id :%d , 数据重试成功", id) 321 | s.cached.Delete(id) 322 | } 323 | 324 | return nil 325 | } 326 | 327 | func (s *RocketEndpoint) Close() { 328 | if s.client != nil { 329 | s.client.Shutdown() 330 | } 331 | } 332 | 333 | func (s *RocketEndpoint) StockExecSql(sql string, valuesList []interface{}) (int64, error) { 334 | // TODO 335 | return 0, nil 336 | } 337 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 16 | 17 | 22 | 23 | 24 | 25 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 38 | 39 | 41 | 42 | 43 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 160 | 161 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 191 | 192 | true 193 | 194 | 195 | 196 | 197 | 198 | 199 | -------------------------------------------------------------------------------- /util/httputil/perform.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "mime/multipart" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "strings" 14 | "time" 15 | 16 | "github.com/pkg/errors" 17 | 18 | "go-mysql-sync/util/logutil" 19 | "go-mysql-sync/util/stringutil" 20 | ) 21 | 22 | // 执行器 23 | type perform struct { 24 | client *HttpClient 25 | requisite *httpRequisite 26 | 27 | addr string 28 | method string 29 | expectStatus int 30 | } 31 | 32 | // 执行Request 33 | func (s *perform) execute(request *http.Request) (*http.Response, error) { 34 | s.requisite.refactorIfNecessary(s.client.globals) 35 | for k, v := range s.requisite.headers { 36 | request.Header.Add(k, stringutil.ToString(v)) 37 | } 38 | 39 | startTime := time.Now().UnixNano() 40 | res, err := s.client.cli.Do(request) 41 | latency := (time.Now().UnixNano() - startTime) / int64(time.Millisecond) 42 | 43 | if nil != err { 44 | logutil.GlobalLogger().Error(err.Error()) 45 | } else { 46 | logutil.GlobalSugar().Infof("请求成功, %s | %s | %d | %d(毫秒)", request.Method, request.URL.String(), res.StatusCode, latency) 47 | } 48 | 49 | if s.requisite.retryCount > 0 && s.requisite.retryNecessary(res) { 50 | for i := 0; i < s.requisite.retryCount; i++ { 51 | logutil.GlobalSugar().Infof("第%d次重试： %s | %s )", i+1, request.Method, request.URL.String()) 52 | 53 | res, err = s.client.cli.Do(request) 54 | if err != nil { 55 | logutil.GlobalLogger().Error(err.Error()) 56 | } 57 | if !s.requisite.retryNecessary(res) || (i+1) == s.requisite.retryCount { 58 | return res, err 59 | } 60 | <-time.After(time.Duration(s.requisite.retryInterval) * time.Second) 61 | } 62 | } 63 | 64 | if s.expectStatus != 0 && s.expectStatus != res.StatusCode { 65 | defer res.Body.Close() 66 | return nil, errors.Errorf("Response status code : %d (%s)", res.StatusCode, http.StatusText(res.StatusCode)) 67 | } 68 | 69 | return res, err 70 | } 71 | 72 | // 转换Response为string 73 | func (s *perform) toString(response *http.Response) (string, error) { 74 | defer response.Body.Close() 75 | 76 | if data, err := ioutil.ReadAll(response.Body); err == nil { 77 | return string(data), nil 78 | } 79 | 80 | return "", nil 81 | } 82 | 83 | // 转换Response为string 84 | func (s *perform) toEntity(response *http.Response) (*RespondEntity, error) { 85 | defer response.Body.Close() 86 | 87 | data, err := ioutil.ReadAll(response.Body) 88 | if nil != err { 89 | return nil, err 90 | } 91 | 92 | return &RespondEntity{ 93 | statusCode: response.StatusCode, 94 | data: data, 95 | }, nil 96 | } 97 | 98 | type NobodyPerform struct { 99 | super *perform 100 | parameters H 101 | } 102 | 103 | func newNobodyPerform(_addr, _method string, _client *HttpClient) *NobodyPerform { 104 | return &NobodyPerform{ 105 | super: &perform{ 106 | requisite: newHttpRequisite(), 107 | client: _client, 108 | addr: _addr, 109 | method: _method, 110 | }, 111 | parameters: make(H), 112 | } 113 | } 114 | 115 | // 设置请求头 116 | func (r *NobodyPerform) AddHeader(name string, value interface{}) *NobodyPerform { 117 | r.super.requisite.AddHeader(name, value) 118 | return r 119 | } 120 | 121 | // 设置请求头 122 | func (r *NobodyPerform) SetHeaders(values H) *NobodyPerform { 123 | r.super.requisite.SetHeaders(values) 124 | return r 125 | } 126 | 127 | // 设置重试次数 128 | func (r *NobodyPerform) SetRetryCount(_retryCount int) *NobodyPerform { 129 | r.super.requisite.SetRetryCount(_retryCount) 130 | return r 131 | } 132 | 133 | // 设置重试间隔时间，单位为秒 134 | func (r *NobodyPerform) SetRetryInterval(_retryInterval int) *NobodyPerform { 135 | r.super.requisite.SetRetryInterval(_retryInterval) 136 | return r 137 | } 138 | 139 | // 添加重试条件 140 | func (r *NobodyPerform) AddRetryCondition(_retryCondition RetryConditionFunc) *NobodyPerform { 141 | r.super.requisite.AddRetryCondition(_retryCondition) 142 | return r 143 | } 144 | 145 | // 设置查询参数 146 | func (r *NobodyPerform) AddParameter(name string, value interface{}) *NobodyPerform { 147 | r.parameters[name] = value 148 | return r 149 | } 150 | 151 | // 设置查询参数 152 | func (r *NobodyPerform) SetParameters(values H) *NobodyPerform { 153 | r.parameters = values 154 | return r 155 | } 156 | 157 | // 设置预期请求状态 158 | func (r *NobodyPerform) SetExpectStatus(expect int) *NobodyPerform { 159 | r.super.expectStatus = expect 160 | return r 161 | } 162 | 163 | // 执行请求 164 | func (r *NobodyPerform) Do() (*http.Response, error) { 165 | values := make(url.Values) 166 | for k, v := range r.parameters { 167 | values.Set(k, stringutil.ToString(v)) 168 | } 169 | 170 | url := stringutil.UrlValuesToQueryString(r.super.addr, values) 171 | req, err := http.NewRequest(r.super.method, url, nil) 172 | if nil != err { 173 | return nil, err 174 | } 175 | 176 | return r.super.execute(req) 177 | } 178 | 179 | func (r *NobodyPerform) DoForString() (string, error) { 180 | res, err := r.Do() 181 | if nil != err { 182 | return "", err 183 | } 184 | 185 | return r.super.toString(res) 186 | } 187 | 188 | func (r *NobodyPerform) DoForEntity() (*RespondEntity, error) { 189 | res, err := r.Do() 190 | if nil != err { 191 | return nil, err 192 | } 193 | 194 | return r.super.toEntity(res) 195 | } 196 | 197 | type FormFile string 198 | 199 | type BodyPerform struct { 200 | super *perform 201 | } 202 | 203 | // Form请求 204 | type FormBodyPerform struct { 205 | base *BodyPerform 206 | body H 207 | } 208 | 209 | // Json请求 210 | type JsonBodyPerform struct { 211 | base *BodyPerform 212 | body interface{} 213 | } 214 | 215 | func newBodyPerform(_addr, _method string, _client *HttpClient) *BodyPerform { 216 | return &BodyPerform{ 217 | super: &perform{ 218 | requisite: newHttpRequisite(), 219 | client: _client, 220 | addr: _addr, 221 | method: _method, 222 | }, 223 | } 224 | } 225 | 226 | // 设置请求头 227 | func (r *BodyPerform) AddHeader(name string, value interface{}) *BodyPerform { 228 | r.super.requisite.AddHeader(name, value) 229 | return r 230 | } 231 | 232 | // 设置请求头 233 | func (r *BodyPerform) SetHeaders(values H) *BodyPerform { 234 | r.super.requisite.SetHeaders(values) 235 | return r 236 | } 237 | 238 | // 设置超时时间，单位为秒 239 | func (r *BodyPerform) SetTimeout(_timeout int) *BodyPerform { 240 | r.super.requisite.SetTimeout(_timeout) 241 | return r 242 | } 243 | 244 | // 设置重试次数 245 | func (r *BodyPerform) SetRetryCount(_retryCount int) *BodyPerform { 246 | r.super.requisite.SetRetryCount(_retryCount) 247 | return r 248 | } 249 | 250 | // 设置重试间隔时间，单位为秒 251 | func (r *BodyPerform) SetRetryInterval(_retryInterval int) *BodyPerform { 252 | r.super.requisite.SetRetryInterval(_retryInterval) 253 | return r 254 | } 255 | 256 | // 添加重试条件 257 | func (r *BodyPerform) AddRetryCondition(_retryCondition RetryConditionFunc) *BodyPerform { 258 | r.super.requisite.AddRetryCondition(_retryCondition) 259 | return r 260 | } 261 | 262 | // 设置预期请求状态 263 | func (r *BodyPerform) SetExpectStatus(expect int) *BodyPerform { 264 | r.super.expectStatus = expect 265 | return r 266 | } 267 | 268 | // 设置Form 269 | // 请求的contentType为: "application/x-www-form-urlencoded" 或 "multipart/form-data" 270 | func (r *BodyPerform) SetForm(form H) *FormBodyPerform { 271 | fr := &FormBodyPerform{ 272 | base: r, 273 | body: form, 274 | } 275 | return fr 276 | } 277 | 278 | // 执行请求 279 | func (r *FormBodyPerform) Do() (*http.Response, error) { 280 | var multipart bool 281 | for _, v := range r.body { 282 | switch v.(type) { 283 | case FormFile: 284 | multipart = true 285 | break 286 | default: 287 | } 288 | } 289 | 290 | if !multipart { 291 | return r.do() 292 | } else { 293 | return r.doMultipart() 294 | } 295 | } 296 | 297 | func (r *FormBodyPerform) DoForString() (string, error) { 298 | res, err := r.Do() 299 | if nil != err { 300 | return "", err 301 | } 302 | 303 | return r.base.super.toString(res) 304 | } 305 | 306 | func (r *FormBodyPerform) DoForEntity() (*RespondEntity, error) { 307 | res, err := r.Do() 308 | if nil != err { 309 | return nil, err 310 | } 311 | 312 | return r.base.super.toEntity(res) 313 | } 314 | 315 | func (r *FormBodyPerform) do() (*http.Response, error) { 316 | values := make(url.Values) 317 | for k, v := range r.body { 318 | values.Set(k, stringutil.ToString(v)) 319 | } 320 | 321 | req, err := http.NewRequest(r.base.super.method, r.base.super.addr, strings.NewReader(values.Encode())) 322 | if nil != err { 323 | return nil, err 324 | } 325 | 326 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 327 | return r.base.super.execute(req) 328 | } 329 | 330 | func (r *FormBodyPerform) doMultipart() (*http.Response, error) { 331 | bodyBuffer := &bytes.Buffer{} 332 | bodyWriter := multipart.NewWriter(bodyBuffer) 333 | 334 | var closings []*os.File 335 | defer func() { 336 | fmt.Println("closings") 337 | for _, closing := range closings { 338 | closing.Close() 339 | } 340 | }() 341 | 342 | var err error 343 | for k, v := range r.body { 344 | switch v.(type) { 345 | case FormFile: 346 | vv := v.(FormFile) 347 | var fw io.Writer 348 | fw, err = bodyWriter.CreateFormFile(k, string(vv)) 349 | if err != nil { 350 | break 351 | } 352 | 353 | var file *os.File 354 | file, err = os.Open(string(vv)) 355 | if err != nil { 356 | break 357 | } 358 | closings = append(closings, file) 359 | 360 | _, err = io.Copy(fw, file) 361 | if err != nil { 362 | break 363 | } 364 | default: 365 | if err := bodyWriter.WriteField(k, stringutil.ToString(v)); err != nil { 366 | return nil, err 367 | } 368 | } 369 | } 370 | 371 | if err != nil { 372 | return nil, err 373 | } 374 | 375 | bodyWriter.Close() 376 | 377 | req, err := http.NewRequest(r.base.super.method, r.base.super.addr, bodyBuffer) 378 | if nil != err { 379 | return nil, err 380 | } 381 | 382 | req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) 383 | return r.base.super.execute(req) 384 | } 385 | 386 | // 设置Json 387 | // 请求的contentType为: "application/json" 388 | func (r *BodyPerform) SetJson(data interface{}) *JsonBodyPerform { 389 | return &JsonBodyPerform{ 390 | base: r, 391 | body: data, 392 | } 393 | } 394 | 395 | // 执行请求 396 | func (r *JsonBodyPerform) Do() (*http.Response, error) { 397 | var data []byte 398 | switch r.body.(type) { 399 | case string: 400 | ft := r.body.(string) 401 | data = []byte(ft) 402 | case []byte: 403 | data = r.body.([]byte) 404 | default: 405 | temp, err := json.Marshal(r.body) 406 | if err != nil { 407 | return nil, err 408 | } 409 | data = temp 410 | } 411 | 412 | req, err := http.NewRequest(r.base.super.method, r.base.super.addr, bytes.NewReader(data)) 413 | if nil != err { 414 | return nil, err 415 | } 416 | 417 | req.Header.Add("Content-Type", "application/json") 418 | return r.base.super.execute(req) 419 | } 420 | 421 | func (r *JsonBodyPerform) DoForString() (string, error) { 422 | res, err := r.Do() 423 | if nil != err { 424 | return "", err 425 | } 426 | 427 | return r.base.super.toString(res) 428 | } 429 | 430 | func (r *JsonBodyPerform) DoForEntity() (*RespondEntity, error) { 431 | res, err := r.Do() 432 | if nil != err { 433 | return nil, err 434 | } 435 | 436 | return r.base.super.toEntity(res) 437 | } 438 | -------------------------------------------------------------------------------- /global/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2021 the original author(https://github.com/wj596) 3 | * 4 | *

17 | */ 18 | package global 19 | 20 | import ( 21 | "fmt" 22 | "github.com/prometheus/client_golang/prometheus" 23 | "github.com/prometheus/client_golang/prometheus/promauto" 24 | "github.com/prometheus/client_golang/prometheus/promhttp" 25 | "github.com/shirou/gopsutil/cpu" 26 | "github.com/shirou/gopsutil/disk" 27 | "github.com/shirou/gopsutil/mem" 28 | "github.com/shirou/gopsutil/net" 29 | nethttp "net/http" 30 | "strconv" 31 | "time" 32 | ) 33 | 34 | const ( 35 | MetricsStateOK = 1 36 | MetricsStateNO = 0 37 | ) 38 | 39 | var ( 40 | ExecuteSQLTimestamp uint32 41 | CurrentMachineState MachineState 42 | ) 43 | var ( 44 | leaderState = promauto.NewGaugeVec( 45 | prometheus.GaugeOpts{ 46 | Name: "transfer_leader_state", 47 | Help: "The cluster leader state: 0=false, 1=true", 48 | }, []string{"instance"}, 49 | ) 50 | 51 | //目的实例连接状态 52 | //destinationState = promauto.NewGauge( 53 | // prometheus.GaugeOpts{ 54 | // Name: "transfer_destination_state", 55 | // Help: "The destination running state: 0=stopped, 1=ok", 56 | // }, 57 | //) 58 | 59 | transferDelay = promauto.NewGaugeVec( 60 | prometheus.GaugeOpts{ 61 | Name: "transfer_delay", 62 | Help: "The transfer slave lag", 63 | }, []string{"instance"}, 64 | ) 65 | 66 | //源实例连接状态 67 | //sourceState = promauto.NewGauge( 68 | // prometheus.GaugeOpts{ 69 | // Name: "source_state", 70 | // Help: "The source state: 0=false, 1=true", 71 | // }, 72 | //) 73 | 74 | //目的实例连接状态 75 | destinationState = promauto.NewGaugeVec( 76 | prometheus.GaugeOpts{ 77 | Name: "destination_state", 78 | Help: "The destination state: 0=stopped, 1=ok", 79 | }, []string{"instance"}, 80 | ) 81 | ////当前执行sql的时间戳，也就是数据延迟 82 | //executeSQLTimestamp = promauto.NewGauge( 83 | // prometheus.GaugeOpts{ 84 | // Name: "executeSQL_timestamp", 85 | // Help: "The timestamp of the current SQL", 86 | // }, 87 | //) 88 | 89 | //当前时间戳-执行sql的时间戳，也就是数据延迟 90 | delay = promauto.NewGaugeVec( 91 | prometheus.GaugeOpts{ 92 | Name: "delay", 93 | Help: "delay of sync", 94 | }, []string{"instance"}, 95 | ) 96 | 97 | //插入行数 98 | insertCounter = promauto.NewCounterVec( 99 | prometheus.CounterOpts{ 100 | Name: "insert_counter", 101 | Help: "The number of data inserted to destination", 102 | }, []string{"instance"}, 103 | ) 104 | //更新行数 105 | updateCounter = promauto.NewCounterVec( 106 | prometheus.CounterOpts{ 107 | Name: "update_counter", 108 | Help: "The number of data updated to destination", 109 | }, []string{"instance"}, 110 | ) 111 | //删除行数 112 | deleteCounter = promauto.NewCounterVec( 113 | prometheus.CounterOpts{ 114 | Name: "delete_counter", 115 | Help: "The number of data deleted from destination", 116 | }, []string{"instance"}, 117 | ) 118 | //DDl SQL的数量 119 | ddlCounter = promauto.NewCounterVec( 120 | prometheus.CounterOpts{ 121 | Name: "ddl_counter", 122 | Help: "The number of data DDL from destination", 123 | }, []string{"instance"}, 124 | ) 125 | 126 | //insertNum = promauto.NewCounterVec( 127 | // prometheus.CounterOpts{ 128 | // Name: "transfer_inserted_num", 129 | // Help: "The number of data inserted to destination", 130 | // }, []string{"table"}, 131 | //) 132 | // 133 | //updateNum = promauto.NewCounterVec( 134 | // prometheus.CounterOpts{ 135 | // Name: "transfer_updated_num", 136 | // Help: "The number of data updated to destination", 137 | // }, []string{"table"}, 138 | //) 139 | // 140 | //deleteNum = promauto.NewCounterVec( 141 | // prometheus.CounterOpts{ 142 | // Name: "transfer_deleted_num", 143 | // Help: "The number of data deleted from destination", 144 | // }, []string{"table"}, 145 | //) 146 | // 147 | //ddlNum = promauto.NewCounterVec( 148 | // prometheus.CounterOpts{ 149 | // Name: "transfer_DDL_num", 150 | // Help: "The number of data DDL from destination", 151 | // }, []string{"table"}, 152 | //) 153 | 154 | diskTotal = promauto.NewGaugeVec( 155 | prometheus.GaugeOpts{ 156 | Name: "diskTotal", 157 | Help: "diskTotal(G)", 158 | }, []string{"instance"}, 159 | ) 160 | 161 | diskFree = promauto.NewGaugeVec( 162 | prometheus.GaugeOpts{ 163 | Name: "diskFree", 164 | Help: "diskFree(G)", 165 | }, []string{"instance"}, 166 | ) 167 | 168 | diskUsed = promauto.NewGaugeVec( 169 | prometheus.GaugeOpts{ 170 | Name: "diskUsed", 171 | Help: "diskUsed(G)", 172 | }, []string{"instance"}, 173 | ) 174 | 175 | diskUsage = promauto.NewGaugeVec( 176 | prometheus.GaugeOpts{ 177 | Name: "diskUsage", 178 | Help: "diskUsage", 179 | }, []string{"instance"}, 180 | ) 181 | 182 | memTotal = promauto.NewGaugeVec( 183 | prometheus.GaugeOpts{ 184 | Name: "memTotal", 185 | Help: "memTotal(M)", 186 | }, []string{"instance"}, 187 | ) 188 | 189 | memFree = promauto.NewGaugeVec( 190 | prometheus.GaugeOpts{ 191 | Name: "memFree", 192 | Help: "memFree(M)", 193 | }, []string{"instance"}, 194 | ) 195 | 196 | memUsed = promauto.NewGaugeVec( 197 | prometheus.GaugeOpts{ 198 | Name: "memUsed", 199 | Help: "memUsed(M)", 200 | }, []string{"instance"}, 201 | ) 202 | memUsage = promauto.NewGaugeVec( 203 | prometheus.GaugeOpts{ 204 | Name: "menUsage", 205 | Help: "内存使用率", 206 | }, []string{"instance"}, 207 | ) 208 | 209 | cpuUsage = promauto.NewGaugeVec( 210 | prometheus.GaugeOpts{ 211 | Name: "cpuUsage", 212 | Help: "cpuUsage", 213 | }, []string{"instance"}, 214 | ) 215 | 216 | valueBytesRecv = promauto.NewGaugeVec( 217 | prometheus.GaugeOpts{ 218 | Name: "BytesRecv", 219 | Help: "接收流量BytesRecv(bit)", 220 | }, []string{"instance"}, 221 | ) 222 | 223 | valueBytesSent = promauto.NewGaugeVec( 224 | prometheus.GaugeOpts{ 225 | Name: "BytesSent", 226 | Help: "发送流量BytesSent(bit)", 227 | }, []string{"instance"}, 228 | ) 229 | ) 230 | 231 | //func SetSourceState(state int) { 232 | // if _config.EnableExporter { 233 | // sourceState.Set(float64(state)) 234 | // } 235 | //} 236 | 237 | func SetDestinationState(state int) { 238 | destinationState.WithLabelValues(GlobalInstance).Set(float64(state)) 239 | } 240 | 241 | func SetDelay(ExecuteSQLTimestamp uint32, current uint32) { 242 | delay.WithLabelValues(GlobalInstance).Set(float64(current) - float64(ExecuteSQLTimestamp)) 243 | } 244 | 245 | func IncInsertCounter() { 246 | insertCounter.WithLabelValues(GlobalInstance).Inc() 247 | } 248 | 249 | func IncInsertCounterAdd(n int) { 250 | insertCounter.WithLabelValues(GlobalInstance).Add(float64(n)) 251 | } 252 | 253 | func IncUpdateCounter() { 254 | updateCounter.WithLabelValues(GlobalInstance).Inc() 255 | } 256 | 257 | func IncUpdateCounterAdd(n int) { 258 | updateCounter.WithLabelValues(GlobalInstance).Add(float64(n)) 259 | } 260 | 261 | func IncDeleteCounter() { 262 | deleteCounter.WithLabelValues(GlobalInstance).Inc() 263 | } 264 | 265 | func IncDeleteCounterAdd(n int) { 266 | deleteCounter.WithLabelValues(GlobalInstance).Add(float64(n)) 267 | } 268 | 269 | func IncDDLCounter() { 270 | ddlCounter.WithLabelValues(GlobalInstance).Inc() 271 | } 272 | 273 | func SetTransferDelay(delay uint32) { 274 | transferDelay.WithLabelValues(GlobalInstance).Set(float64(delay)) 275 | } 276 | 277 | func SetLeaderState(state int) { 278 | if _config.EnableExporter { 279 | leaderState.WithLabelValues(GlobalInstance).Set(float64(state)) 280 | } 281 | } 282 | 283 | //----------------------------------------------------- 284 | func StartMonitor() { 285 | if _config.EnableExporter { 286 | go func() { 287 | nethttp.Handle("/metrics", promhttp.Handler()) 288 | nethttp.ListenAndServe(fmt.Sprintf(":%d", _config.ExporterPort), nil) 289 | }() 290 | } 291 | } 292 | 293 | //func TickerExecuteSQLDelay(ExecuteSQLTimestamp uint32) { 294 | // ticker := time.NewTicker(time.Millisecond * 1000) 295 | // defer ticker.Stop() 296 | // for { 297 | // select { 298 | // case <-ticker.C: 299 | // SetExecuteSQLTimestamp(ExecuteSQLTimestamp) 300 | // logutil.Info("sql的时间戳为:" + string(ExecuteSQLTimestamp)) 301 | // } 302 | // } 303 | //} 304 | 305 | type MachineState struct { 306 | diskTotal float64 307 | diskFree float64 308 | diskUsed float64 309 | diskUsage float64 310 | memTotal float64 311 | memFree float64 312 | memUsed float64 313 | memUsage float64 314 | cpuUsage float64 315 | valueBytesRecv float64 316 | valueBytesSent float64 317 | } 318 | 319 | func GetMachineState() { 320 | d, _ := disk.Usage("/") 321 | v, _ := mem.VirtualMemory() 322 | cc, _ := cpu.Percent(time.Second, false) 323 | nv, _ := net.IOCounters(true) 324 | 325 | diskTotal := float64(d.Total / 1024 / 1024 / 1024) 326 | //fmt.Println("总磁盘:",diskTotal) 327 | diskFree := float64(d.Free / 1024 / 1024 / 1024) 328 | //fmt.Println("磁盘空闲:",diskFree) 329 | diskUsed := float64(d.Used / 1024 / 1024 / 1024) 330 | //fmt.Println("磁盘空闲:",diskFree) 331 | diskUsageNo := float64(d.UsedPercent) 332 | diskUsage, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", diskUsageNo), 64) 333 | //fmt.Println("磁盘使用率:",diskUsage) 334 | memTotal := float64(v.Total / 1024 / 1024) 335 | //fmt.Println("总内存:",memTotal) 336 | memFree := float64(v.Available / 1024 / 1024) 337 | //fmt.Println("内存空闲:",memFree) 338 | memUsed := float64(v.Used / 1024 / 1024) 339 | //fmt.Println("内存使用:",memUsed) 340 | memUsageNo := float64(v.UsedPercent) 341 | memUsage, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", memUsageNo), 64) 342 | //fmt.Println("内存使用率:",memUsage) 343 | cpuUsageNo := float64(cc[0]) 344 | cpuUsage, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", cpuUsageNo), 64) 345 | //fmt.Println("CPU使用率:",cpuUsage) 346 | bytesRecv := float64(nv[0].BytesRecv) * 8 347 | valueBytesRecv, _ := strconv.ParseFloat(fmt.Sprintf("%.3f", bytesRecv), 64) 348 | //fmt.Println("接收流量:",valueBytesRecv,"M") 349 | bytesSent := float64(nv[0].BytesSent) * 8 350 | valueBytesSent, _ := strconv.ParseFloat(fmt.Sprintf("%.3f", bytesSent), 64) 351 | //fmt.Println("发送流量:",valueBytesSent,"M") 352 | CurrentMachineState = MachineState{diskTotal, diskFree, diskUsed, diskUsage, memTotal, memFree, memUsed, memUsage, cpuUsage, valueBytesRecv, valueBytesSent} 353 | SetMachineState(CurrentMachineState) 354 | } 355 | 356 | func SetMachineState(CurrentMachineState MachineState) { 357 | diskTotal.WithLabelValues(GlobalInstance).Set(CurrentMachineState.diskTotal) 358 | diskFree.WithLabelValues(GlobalInstance).Set(CurrentMachineState.diskFree) 359 | diskUsed.WithLabelValues(GlobalInstance).Set(CurrentMachineState.diskUsed) 360 | diskUsage.WithLabelValues(GlobalInstance).Set(CurrentMachineState.diskUsage) 361 | memTotal.WithLabelValues(GlobalInstance).Set(CurrentMachineState.memTotal) 362 | memUsed.WithLabelValues(GlobalInstance).Set(CurrentMachineState.memUsed) 363 | memUsage.WithLabelValues(GlobalInstance).Set(CurrentMachineState.memUsage) 364 | cpuUsage.WithLabelValues(GlobalInstance).Set(CurrentMachineState.cpuUsage) 365 | valueBytesRecv.WithLabelValues(GlobalInstance).Set(CurrentMachineState.valueBytesRecv) 366 | valueBytesSent.WithLabelValues(GlobalInstance).Set(CurrentMachineState.valueBytesSent) 367 | } 368 | --------------------------------------------------------------------------------