├── rebuild ├── fixture │ ├── TestOnlyTable.golden │ └── TestQueryRollback.golden ├── delete_test.go ├── insert_test.go ├── update_test.go ├── doc.go ├── common_test.go ├── query_test.go ├── schema_test.go ├── delete.go ├── insert.go ├── query.go ├── update.go ├── schema.go └── common.go ├── common ├── fixture │ ├── TestTimeOffset.golden │ ├── master.info │ ├── TestListPlugin.golden │ └── TestPrintConfiguration.golden ├── doc.go ├── log_test.go ├── golden.go ├── config_test.go ├── log.go └── config.go ├── test ├── keyring ├── binlog.000002 ├── binlog.decrypted ├── binlog.encrypted ├── init.sql ├── schema.sh ├── schema.sql └── binlog_decrypt.py ├── doc ├── qq_group.png ├── global.md ├── lightning_exporter.py ├── mysql.md ├── summarize_binlogs.sh ├── lua.md ├── cmd.md ├── FAQ.md ├── reference.md ├── filters.md └── rebuild.md ├── .gitignore ├── plugin ├── mymod.lua ├── demo.redis.lua ├── demo.mod.lua ├── basement.sh ├── demo.mysql.lua ├── demo.sql.lua └── demo.flashback.lua ├── etc ├── lightning.stream.yaml ├── master.info └── lightning.yaml ├── cmd └── lightning │ ├── doc.go │ ├── lightning_test.go │ └── lightning.go ├── event ├── doc.go ├── decrypt_test.go ├── filter_test.go ├── parser_test.go ├── decrypt.go ├── filter.go └── parser.go ├── genver.sh ├── go.mod ├── README.md ├── README_EN.md ├── Makefile └── LICENSE /rebuild/fixture/TestOnlyTable.golden: -------------------------------------------------------------------------------- 1 | `tb` 2 | `tb` 3 | -------------------------------------------------------------------------------- /common/fixture/TestTimeOffset.golden: -------------------------------------------------------------------------------- 1 | 0 2 | 28800 3 | 3600 4 | -25200 5 | -------------------------------------------------------------------------------- /test/keyring: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LianjiaTech/lightning/HEAD/test/keyring -------------------------------------------------------------------------------- /doc/qq_group.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LianjiaTech/lightning/HEAD/doc/qq_group.png -------------------------------------------------------------------------------- /test/binlog.000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LianjiaTech/lightning/HEAD/test/binlog.000002 -------------------------------------------------------------------------------- /test/binlog.decrypted: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LianjiaTech/lightning/HEAD/test/binlog.decrypted -------------------------------------------------------------------------------- /test/binlog.encrypted: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LianjiaTech/lightning/HEAD/test/binlog.encrypted -------------------------------------------------------------------------------- /rebuild/fixture/TestQueryRollback.golden: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `tb`; 2 | DROP DATABASE IF EXISTS `db`; 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | release/ 2 | 3 | lightning 4 | 5 | VERSION 6 | version.go 7 | coverage.data 8 | coverage.html 9 | coverage.txt 10 | *.log 11 | -------------------------------------------------------------------------------- /plugin/mymod.lua: -------------------------------------------------------------------------------- 1 | local mymod = {} 2 | 3 | function mymod.myfunc() 4 | print("-- hi! this is my first mod in lua.") 5 | end 6 | 7 | return mymod 8 | -------------------------------------------------------------------------------- /etc/lightning.stream.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | log-level: 3 3 | log-output: lightning.log 4 | charset: "utf8mb4" 5 | time-zone: "UTC" 6 | 7 | mysql: 8 | master-info: "etc/master.info" 9 | -------------------------------------------------------------------------------- /etc/master.info: -------------------------------------------------------------------------------- 1 | master_host: 127.0.0.1 2 | master_user: root 3 | master_password: '******' 4 | master_port: 3306 5 | master_log_file: binlog.000007 6 | master_log_pos: 4 7 | executed_gtid_set: 376b1ae7-39a1-11e9-a253-14187759814e:1, 3b0075c9-39a1-11e9-a250-f86eee9113c6:1-98 8 | auto_position: false 9 | seconds_behind_master: 105 10 | server-id: 33061 11 | server-type: mysql 12 | -------------------------------------------------------------------------------- /common/fixture/master.info: -------------------------------------------------------------------------------- 1 | master_host: 127.0.0.1 2 | master_user: root 3 | master_password: '******' 4 | master_port: 3306 5 | master_log_file: binlog.000007 6 | master_log_pos: 4 7 | executed_gtid_set: 376b1ae7-39a1-11e9-a253-14187759814e:1, 3b0075c9-39a1-11e9-a250-f86eee9113c6:1-98 8 | auto_position: false 9 | seconds_behind_master: 84 10 | server-id: 33061 11 | server-type: mysql 12 | -------------------------------------------------------------------------------- /common/fixture/TestListPlugin.golden: -------------------------------------------------------------------------------- 1 | lightning -plugin support following type 2 | sql(default): parse ROW format binlog into SQL. 3 | flashback: generate flashback query from ROW format binlog 4 | stat: statistic ROW format binlog table update|insert|delete query count 5 | lua: self define lua scripts 6 | find: find binlog file name by event time 7 | decrypt: decrypt binlog file using keyring 8 | -------------------------------------------------------------------------------- /test/init.sql: -------------------------------------------------------------------------------- 1 | FLUSH LOGS; 2 | SET GLOBAL BINLOG_FORMAT=ROW; 3 | SET GLOBAL BINLOG_ROW_IMAGE=FULL; 4 | SET GLOBAL ENFORCE_GTID_CONSISTENCY = ON; 5 | SET GLOBAL GTID_MODE=OFF_PERMISSIVE; 6 | SET GLOBAL GTID_MODE=ON_PERMISSIVE; 7 | SET GLOBAL GTID_MODE=ON; 8 | SET GLOBAL binlog_rows_query_log_events=on; 9 | 10 | /*!80000 ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY '******' */; 11 | 12 | FLUSH LOGS; 13 | -------------------------------------------------------------------------------- /test/schema.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # audo dump lightning compatible schema file 4 | 5 | HOST=$1 6 | PORT=$2 7 | DATABASES=$3 8 | 9 | mysqldump --defaults-extra-file=my.cnf \ 10 | -h "${HOST:-127.0.0.1}" -P "${PORT:-3306}" \ 11 | --skip-triggers \ 12 | --skip-lock-tables \ 13 | --compact \ 14 | --no-data \ 15 | --set-gtid-purged=OFF \ 16 | --databases ${DATABASES} | \ 17 | grep -v '^CREATE DATABASE\|^DROP TABLE \|50001 DROP VIEW IF EXISTS\|^SET \|50001 SET @\|50001 SET c\|^--\|^/\*!4' > schema.sql 18 | -------------------------------------------------------------------------------- /doc/global.md: -------------------------------------------------------------------------------- 1 | # 全局配置 2 | 3 | ```yaml 4 | # 全局配置 5 | global: 6 | # 日志级别 7 | log-level: 3 8 | # 日志文件名 9 | log-output: lightning.log 10 | # 是否开启守护进程模式 11 | demonize: false 12 | # 数据库字符集 13 | charset: utf8mb4 14 | # CPU 使用限制 15 | cpu: 0 16 | # Verbose 模式，打印更多信息 17 | verbose: false 18 | # 比 Verbose 还 Verbose 19 | verbose-verbose: false 20 | # 设置时区，默认 UTC 21 | time-zone: UTC 22 | ``` 23 | 24 | ## time-zone 25 | 26 | 该参数需要与 MySQL 服务器指定的时区相同，否则会导致 `start-datetime`, `stop-datetime` 等参数无法生效。 27 | 28 | * UTC：世界标准时间。协调世界时，又称世界标准时间或世界协调时间，其以原子时秒长为基础，在时刻上尽量接近于格林尼治标准时间。 29 | * Asia/Shanghai：为本地时间，一个国家或地区使用时间，中国为东八区。 30 | -------------------------------------------------------------------------------- /rebuild/delete_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | -------------------------------------------------------------------------------- /rebuild/insert_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | -------------------------------------------------------------------------------- /rebuild/update_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | -------------------------------------------------------------------------------- /cmd/lightning/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | // Package main the main package of lightning 15 | package main 16 | -------------------------------------------------------------------------------- /common/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | // Package common common library func for lightning 15 | package common 16 | -------------------------------------------------------------------------------- /event/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | // Package event binlog event parse library functions 15 | package event 16 | -------------------------------------------------------------------------------- /rebuild/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | // Package rebuild binlog event rebuild library functions 15 | package rebuild 16 | -------------------------------------------------------------------------------- /cmd/lightning/lightning_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package main 15 | 16 | import ( 17 | "testing" 18 | ) 19 | 20 | func Test_Main(t *testing.T) { 21 | main() 22 | } 23 | -------------------------------------------------------------------------------- /event/decrypt_test.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/LianjiaTech/lightning/common" 9 | ) 10 | 11 | func TestParseKeyRing(t *testing.T) { 12 | keys, err := parseKeyRing(common.DevPath + "/test/keyring") 13 | if err != nil { 14 | t.Error(err.Error()) 15 | } 16 | for _, key := range keys { 17 | fmt.Println(string(key.KeyID), string(key.UserID)) 18 | } 19 | } 20 | 21 | func TestDecryptBinlog(t *testing.T) { 22 | // stdout redirect 23 | std := os.Stdout 24 | fd, err := os.Create(common.DevPath + "/test/binlog.decrypted") 25 | if err != nil { 26 | t.Error(err.Error()) 27 | } 28 | os.Stdout = fd 29 | 30 | err = DecryptBinlog(common.DevPath+"/test/binlog.encrypted", common.DevPath+"/test/keyring") 31 | if err != nil { 32 | t.Error(err.Error()) 33 | } 34 | os.Stdout = std 35 | } 36 | -------------------------------------------------------------------------------- /plugin/demo.redis.lua: -------------------------------------------------------------------------------- 1 | -- GoPrimaryKeys 2 | -- GoColumns 3 | -- GoValues 4 | 5 | -- GoValuesWhere 6 | -- GoValuesSet 7 | 8 | redis = require "redis" 9 | red = redis:new() 10 | 11 | -- Init inital funcation 12 | function Init() 13 | local ok, err = red:connect("127.0.0.1", 6379) 14 | ok, err = red:set("dog", "an animal") 15 | local res, err = red:get("dog") 16 | print("redis status: ",res) 17 | end 18 | 19 | -- InsertRewrite insert rewrite logic 20 | function InsertRewrite (tab) 21 | 22 | end 23 | 24 | -- DeleteRewrite delete rewrite logic 25 | function DeleteRewrite (tab) 26 | 27 | end 28 | 29 | -- UpdateRewrite update rewrite logic 30 | function UpdateRewrite (tab) 31 | 32 | end 33 | 34 | -- QueryRewrite query rewrite logic 35 | function QueryRewrite (sql) 36 | 37 | end 38 | 39 | -- Finalizer final destructor function 40 | function Finalizer () 41 | red:close() 42 | end -------------------------------------------------------------------------------- /plugin/demo.mod.lua: -------------------------------------------------------------------------------- 1 | -- GoPrimaryKeys 2 | -- GoColumns 3 | -- GoValues 4 | 5 | -- GoValuesWhere 6 | -- GoValuesSet 7 | 8 | -- Init inital funcation 9 | function Init() 10 | -- append your package.path 11 | -- use `mymod = require('mymod')` import third party lua library 12 | lfs = require("lfs") 13 | package.path = package.path .. lfs.currentdir() .. [[/?.lua]] 14 | local mymod = require("plugin/mymod") 15 | mymod.myfunc() 16 | end 17 | 18 | -- InsertRewrite insert rewrite logic 19 | function InsertRewrite (tab) 20 | end 21 | 22 | -- DeleteRewrite delete rewrite logic 23 | function DeleteRewrite (tab) 24 | end 25 | 26 | -- UpdateRewrite update rewrite logic 27 | function UpdateRewrite (tab) 28 | end 29 | 30 | -- QueryRewrite query rewrite logic 31 | function QueryRewrite (sql) 32 | end 33 | 34 | -- Finalizer final destructor function 35 | function Finalizer () 36 | end 37 | -------------------------------------------------------------------------------- /genver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## Generate Repository Version 4 | tag="$(git describe --tags --always)" 5 | version="$(git log --date=iso --pretty=format:"%cd" -1) ${tag}" 6 | if [ "X${version}" == "X" ]; then 7 | version="not a git repo" 8 | tag="not a git repo" 9 | fi 10 | 11 | git_dirty=$(git diff --no-ext-diff 2>/dev/null | wc -l) 12 | 13 | compile="$(date +"%F %T %z") by $(go version)" 14 | 15 | branch=$(git rev-parse --abbrev-ref HEAD) 16 | 17 | dev_path=$( 18 | cd "$(dirname "$0")" || exit 19 | pwd 20 | ) 21 | 22 | cat <common/version.go 23 | package common 24 | 25 | // -version输出信息 26 | const ( 27 | Version = "${version}" 28 | Compile = "${compile}" 29 | Branch = "${branch}" 30 | GitDirty= ${git_dirty} 31 | DevPath = "${dev_path}" 32 | ) 33 | EOF 34 | 35 | LIANJIA=$(git ls-remote --get-url | grep -i lianjia) 36 | if [ "x${LIANJIA}" != "x" ]; then 37 | echo "${tag}" | awk -F '-' '{print $1}' > VERSION 38 | fi 39 | -------------------------------------------------------------------------------- /common/fixture/TestPrintConfiguration.golden: -------------------------------------------------------------------------------- 1 | global: 2 | log-level: 3 3 | log-output: lightning.log 4 | daemon: false 5 | charset: utf8mb4 6 | hex-string: false 7 | cpu: 0 8 | verbose: false 9 | verbose-verbose: false 10 | time-zone: Asia/Shanghai 11 | mysql: 12 | binlog-file: [] 13 | schema-file: "" 14 | master-info: "" 15 | replicate-from-current-position: false 16 | sync-interval: 1s 17 | read-timeout: 3s 18 | retry-count: 100 19 | keyring: "" 20 | filters: 21 | tables: [] 22 | ignore-tables: 23 | - mysql.% 24 | - percona.% 25 | event-types: [] 26 | thread-id: 0 27 | server-id: 0 28 | start-position: 0 29 | stop-position: 0 30 | start-datetime: "" 31 | stop-datetime: "" 32 | include-gtid-set: "" 33 | exclude-gtid-set: "" 34 | rebuild: 35 | plugin: sql 36 | complete-insert: false 37 | extended-insert-count: 0 38 | ignore-columns: [] 39 | replace: false 40 | sleep-interval: 0s 41 | lua-script: "" 42 | without-db-name: false 43 | -------------------------------------------------------------------------------- /doc/lightning_exporter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | 4 | import yaml 5 | import time 6 | from prometheus_client import start_http_server, Gauge, CollectorRegistry 7 | 8 | # Create a metric to track time spent and requests made. 9 | master_log_pos = Gauge('lightning_master_log_pos', 'replication master log position', ['server_id', 'port']) 10 | seconds_behind_master = Gauge('lightning_seconds_behind_master', 'seconds behind master', ['server_id', 'port']) 11 | 12 | def snap_metrics(): 13 | with open('master.info') as f: 14 | metrics = yaml.load(f, Loader=yaml.Loader) 15 | master_log_pos.labels(metrics["server-id"], metrics["master_port"]).set(metrics["master_log_pos"]) 16 | seconds_behind_master.labels(metrics["server-id"], metrics["master_port"]).set(metrics["seconds_behind_master"]) 17 | 18 | if __name__ == '__main__': 19 | # Start up the server to expose the metrics. 20 | start_http_server(8100, '0.0.0.0') 21 | # Generate some requests. 22 | while True: 23 | snap_metrics() 24 | time.sleep(10) 25 | -------------------------------------------------------------------------------- /plugin/basement.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB=$1 4 | TABLE=$2 5 | INTERVAL=100 6 | 7 | MYSQL_USER="root" 8 | MYSQL_PASS='******' 9 | MYSQL_PORT=3306 10 | MYSQL_HOST="127.0.0.1" 11 | MYSQL="mysql -A -u${MYSQL_USER} -p${MYSQL_PASS} -h${MYSQL_HOST} -P${MYSQL_PORT} --connect-timeout=5 " 12 | 13 | ${MYSQL} "${DB}" -NBe "CREATE TABLE _${TABLE}_new LIKE ${TABLE}" 14 | 15 | MIN_MAX=$(${MYSQL} "${DB}" -NBe "SELECT MIN(id) min_id, MAX(id) max_id FROM ${TABLE}") 16 | 17 | MIN_ID=$(echo "${MIN_MAX}" | awk '{print $1}') 18 | MAX_ID=$(echo "${MIN_MAX}" | awk '{print $2}') 19 | 20 | LAST_ID=${MIN_ID} 21 | TOTAL_CHUNK=$(echo "${MAX_ID}/100 + 1" | bc) 22 | CHUNK=1 23 | 24 | while true; do 25 | ${MYSQL} "${DB}" -NBe "INSERT LOW_PRIORITY IGNORE INTO _${TABLE}_new SELECT * FROM ${TABLE} WHERE id >= ${LAST_ID} AND id < ${LAST_ID} + ${INTERVAL} LOCK IN SHARE MODE /*lightning coping table, ${CHUNK} of ${TOTAL_CHUNK} */" 26 | LAST_ID=$(echo "${LAST_ID} + 100" | bc) 27 | if [ ${LAST_ID} -gt ${MAX_ID} ]; then 28 | break 29 | else 30 | CHUNK=$(echo "${CHUNK} + 1" | bc) 31 | fi 32 | done 33 | -------------------------------------------------------------------------------- /event/filter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package event 15 | 16 | import ( 17 | "fmt" 18 | "testing" 19 | ) 20 | 21 | func TestTableFilterMatch(t *testing.T) { 22 | tables := []string{ 23 | "`db`.`tb`", 24 | "db.tb", 25 | } 26 | 27 | filters := []string{ 28 | "db.%", 29 | "db.tb%", 30 | "db%.tb%", 31 | "db%.%", 32 | } 33 | for _, table := range tables { 34 | for _, filter := range filters { 35 | fmt.Println(table, filter, tableFilterMatch(table, filter)) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rebuild/common_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "fmt" 18 | "strconv" 19 | "testing" 20 | ) 21 | 22 | func TestLastStatus(t *testing.T) { 23 | LastStatus() 24 | } 25 | 26 | func TestPrintBinlogStat(t *testing.T) { 27 | printBinlogStat() 28 | } 29 | 30 | func TestLoadLuaScript(t *testing.T) { 31 | LoadLuaScript() 32 | } 33 | 34 | func TestStrconvQuote(t *testing.T) { 35 | fmt.Println(strconv.Quote(`'"space `)) 36 | fmt.Printf(`"%s"`, escape(`'"space `)) 37 | } 38 | -------------------------------------------------------------------------------- /rebuild/query_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "flag" 18 | "testing" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | ) 22 | 23 | var update = flag.Bool("update", false, "update .golden files") 24 | 25 | func TestQueryRollback(t *testing.T) { 26 | sqls := []string{ 27 | `CREATE TABLE tb (a int)`, 28 | `create database db`, 29 | // "create index on tb idx_col (`col`)", 30 | } 31 | 32 | err := common.GoldenDiff(func() { 33 | for _, sql := range sqls { 34 | QueryRollback(sql) 35 | } 36 | }, t.Name(), update) 37 | if nil != err { 38 | t.Fatal(err) 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /doc/mysql.md: -------------------------------------------------------------------------------- 1 | # 数据库相关配置 2 | 3 | ## 数据库授权 4 | 5 | ```sql 6 | CREATE USER 'user'@'%' IDENTIFIED /*!80000 WITH mysql_native_password */ BY 'xxx'; 7 | GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'user'@'%'; 8 | ``` 9 | 10 | ## mysql 段配置 11 | 12 | ```yaml 13 | # 日志源 14 | mysql: 15 | # 从文件读取二进制日志 16 | binlog-file: test/binlog.000002 17 | # 建表语句文件 18 | schema-file: test/schema.sql 19 | # MySQL 源 20 | master-info: etc/master.info 21 | # master-info sync interval，默认 1s sync 一次，配置为 0 后，每解析完成一个事务都会更新 master.info 22 | sync-interval: 1s 23 | # Binlog Dump I/O read timeout. 24 | read-timeout: 3s 25 | ``` 26 | 27 | ## master.info 28 | 29 | ```yaml 30 | # 主库 IP 31 | master_host: 127.0.0.1 32 | # 同步账号，用于查建表语句，发起 Binlog Dump 指令 33 | master_user: root 34 | # 同步账号密码 35 | master_password: ****** 36 | # 数据库端口 37 | master_port: 3306 38 | # 起始同步文件 39 | master_log_file: binlog.000002 40 | # 起始同步点位 41 | master_log_pos: 4 42 | # 是否开启 GTID 43 | auto_position: false 44 | # 同步延迟时间 45 | seconds_behind_master: 0 46 | # GTID 模式下起始同步点位 47 | gtid_next: "" 48 | # 模拟从库 server-id 49 | server-id: 33061 50 | # 服务类型，暂未使用不建议修改 51 | server-type: mysql 52 | ``` 53 | 54 | 注意：如不指定 master_log_file 和 master_log_pos 默认从首个 binlog 文件开始同步，而不是当前最新的同步点位。如需从当前最新点位同步需指定 `replicate-from-current-position` 参数。 55 | -------------------------------------------------------------------------------- /cmd/lightning/lightning.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package main 15 | 16 | import ( 17 | "github.com/LianjiaTech/lightning/common" 18 | "github.com/LianjiaTech/lightning/event" 19 | "github.com/LianjiaTech/lightning/rebuild" 20 | // "github.com/pkg/profile" 21 | ) 22 | 23 | func main() { 24 | // defer profile.Start(profile.CPUProfile).Stop() 25 | 26 | // load config from lightning.yaml, master.info, relay.info, command lines 27 | common.ParseConfig() 28 | 29 | // load table schema info from mysql or create table SQL file 30 | rebuild.LoadSchemaInfo() 31 | 32 | // load lua script 33 | rebuild.LoadLuaScript() 34 | 35 | go common.SyncReplicationInfo() 36 | 37 | // binlog parser file || stream 38 | event.BinlogParser() 39 | 40 | // query stat info which need print at last 41 | rebuild.LastStatus() 42 | } 43 | -------------------------------------------------------------------------------- /doc/summarize_binlogs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Identifying useful info from MySQL row-based binary logs 4 | # https://www.percona.com/blog/2015/01/20/identifying-useful-information-mysql-row-based-binary-logs/ 5 | 6 | BINLOG_FILE="mysqld-bin.000035" 7 | START_TIME="2015-01-16 13:30:00" 8 | STOP_TIME="2015-01-16 14:00:00" 9 | 10 | mysqlbinlog --base64-output=decode-rows -vv --start-datetime="${START_TIME}" --stop-datetime="${STOP_TIME}" ${BINLOG_FILE} | awk \ 11 | 'BEGIN {s_type=""; s_count=0;count=0;insert_count=0;update_count=0;delete_count=0;flag=0;} \ 12 | {if(match($0, /#15.*Table_map:.*mapped to number/)) {printf "Timestamp : " $1 " " $2 " Table : " $(NF-4); flag=1} \ 13 | else if (match($0, /(### INSERT INTO .*..*)/)) {count=count+1;insert_count=insert_count+1;s_type="INSERT"; s_count=s_count+1;} \ 14 | else if (match($0, /(### UPDATE .*..*)/)) {count=count+1;update_count=update_count+1;s_type="UPDATE"; s_count=s_count+1;} \ 15 | else if (match($0, /(### DELETE FROM .*..*)/)) {count=count+1;delete_count=delete_count+1;s_type="DELETE"; s_count=s_count+1;} \ 16 | else if (match($0, /^(# at) /) && flag==1 && s_count>0) {print " Query Type : "s_type " " s_count " row(s) affected" ;s_type=""; s_count=0; } \ 17 | else if (match($0, /^(COMMIT)/)) {print "[Transaction total : " count " Insert(s) : " insert_count " Update(s) : " update_count " Delete(s) : " \ 18 | delete_count "] \n+----------------------+----------------------+----------------------+----------------------+"; \ 19 | count=0;insert_count=0;update_count=0; delete_count=0;s_type=""; s_count=0; flag=0} } ' 20 | -------------------------------------------------------------------------------- /common/log_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package common 15 | 16 | import ( 17 | "errors" 18 | "testing" 19 | ) 20 | 21 | func TestLogger(t *testing.T) { 22 | Log.Info("TestLogger_Info") 23 | Log.Debug("TestLogger_Debug") 24 | Log.Warning("TestLogger_Warning") 25 | Log.Error("Warning_Error") 26 | } 27 | 28 | func TestCaller(t *testing.T) { 29 | caller := Caller() 30 | if caller != "testing.tRunner" { 31 | t.Error("get caller failed") 32 | } 33 | } 34 | 35 | func TestGetFunctionName(t *testing.T) { 36 | f := GetFunctionName() 37 | if f != "TestGetFunctionName" { 38 | t.Error("get functionname failed") 39 | } 40 | } 41 | 42 | func TestIfError(t *testing.T) { 43 | err := errors.New("TestIfError") 44 | LogIfError(err, "") 45 | LogIfError(err, "func %s", "func_test") 46 | } 47 | 48 | func TestIfWarn(t *testing.T) { 49 | err := errors.New("test") 50 | LogIfWarn(err, "") 51 | LogIfWarn(err, "func %s", "func_test") 52 | } 53 | -------------------------------------------------------------------------------- /plugin/demo.mysql.lua: -------------------------------------------------------------------------------- 1 | -- GoPrimaryKeys 2 | -- GoColumns 3 | -- GoValues 4 | 5 | -- GoValuesWhere 6 | -- GoValuesSet 7 | 8 | mysql = require "mysql" 9 | db = mysql:new() 10 | 11 | -- Init inital funcation 12 | function Init() 13 | connect() 14 | end 15 | 16 | -- InsertRewrite insert rewrite logic 17 | function InsertRewrite (tab) 18 | 19 | end 20 | 21 | -- DeleteRewrite delete rewrite logic 22 | function DeleteRewrite (tab) 23 | 24 | end 25 | 26 | -- UpdateRewrite update rewrite logic 27 | function UpdateRewrite (tab) 28 | 29 | end 30 | 31 | -- QueryRewrite query rewrite logic 32 | function QueryRewrite (sql) 33 | 34 | end 35 | 36 | -- Finalizer final destructor function 37 | function Finalizer () 38 | db:close() 39 | end 40 | 41 | function connect () 42 | -- lua not support MySQL 8.0 caching_sha2_password authentication method yet 43 | local ok, err, errcode, sqlstate = db:connect{ 44 | host = "127.0.0.1", 45 | port = 3306, 46 | database = "mysql", 47 | user = "root", 48 | password = "******", 49 | charset = "utf8", 50 | max_packet_size = 64 * 1024 * 1024, -- 64MB 51 | } 52 | if err then 53 | print("MySQL connected failed: ", err) 54 | return 55 | end 56 | end 57 | 58 | -- MySQL query 59 | function query (sql) 60 | if db.state ~= STATE_CONNECTED then 61 | db:close() 62 | connect() 63 | end 64 | 65 | local res, err, errcode, sqlstate = db:query(sql, 0) 66 | if err then 67 | print("-- ", sql) 68 | print("MySQL query error: ", errcode, err) 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /etc/lightning.yaml: -------------------------------------------------------------------------------- 1 | # 全局配置 2 | global: 3 | # 日志级别 4 | log-level: 3 5 | # 日志文件名 6 | log-output: lightning.log 7 | # 是否开启守护进程模式 8 | demonize: false 9 | # 数据库字符集 10 | charset: utf8mb4 11 | # CPU 使用限制 12 | cpu: 0 13 | # Verbose 模式，打印更多信息 14 | verbose: false 15 | # 设置时区，默认 Asia/Shanghai 16 | # time-zone: UTC 17 | time-zone: Asia/Shanghai 18 | # 是否将 VARCHAR, CHAR, STRING 等数据类型转成 HEX, 防止不必要转换 19 | hex-string: false 20 | # 日志源 21 | mysql: 22 | # 从文件读取二进制日志 23 | binlog-file: ["test/binlog.000002"] 24 | # 建表语句文件 25 | schema-file: test/schema.sql 26 | # MySQL 源 27 | master-info: etc/master.info 28 | # master-info sync interval 29 | sync-interval: 1s 30 | # Binlog Dump I/O read timeout. 31 | read-timeout: 60s 32 | # 过滤器 33 | filters: 34 | # 表过滤器，只处理特定表 35 | tables: 36 | - test.% 37 | # 表过滤器，忽略特殊定表 38 | ignore-tables: 39 | - test.ignore 40 | # 事件过滤器，只处理特定类型 SQL 41 | event-types: 42 | - insert 43 | - update 44 | - delete 45 | - create 46 | # 线程过滤器，只处理某个线程的更新 47 | thread-id: 0 48 | # 主库过滤器，只处理特定主库的更新 49 | server-id: 0 50 | # 日志起始位点 51 | start-position: 0 52 | # 日志结束位点 53 | stop-position: 0 54 | # 日志开始时间 55 | start-datetime: "" 56 | # 日志结束时间 57 | stop-datetime: "" 58 | # GTID 过滤器，只处理特定 gtid_set 的事件 59 | include-gtid-set: "" 60 | # GTID 过滤器，忽略特殊 gtid_set 的事件 61 | exclude-gtid-set: "" 62 | # 重建规则 63 | rebuild: 64 | # 插件：sql, flashback, stat, lua 65 | plugin: sql 66 | # INSERT 语句是否补全列 67 | complete-insert: false 68 | # 生成 SQL 语句省略某些列，如： INSERT 忽略主键 69 | ignore-columns: [] 70 | # INSERT 语句多个 VALUES 合并 71 | extended-insert-count: 0 72 | # 使用 REPLACE INTO 替代 INSERT INTO 73 | replace: false 74 | # 两条 SQL 语句之前添加 sleep 间隔，最小精度 us 75 | sleep-interval: 0s 76 | # lua 插件脚本位置 77 | lua-script: plugin/demo.flashback.lua 78 | -------------------------------------------------------------------------------- /rebuild/schema_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "fmt" 18 | "testing" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | "github.com/kr/pretty" 22 | ) 23 | 24 | func TestLoadSchemaInfo(t *testing.T) { 25 | TestLoadSchemaFromFile(t) 26 | 27 | TestLoadSchemaFromMySQL(t) 28 | } 29 | 30 | func TestLoadSchemaFromFile(t *testing.T) { 31 | schemaFileOrg := common.Config.MySQL.SchemaFile 32 | 33 | common.Config.MySQL.SchemaFile = common.DevPath + "/test/schema.sql" 34 | err := loadSchemaFromFile() 35 | pretty.Println(err, Schemas) 36 | 37 | common.Config.MySQL.SchemaFile = schemaFileOrg 38 | } 39 | 40 | func TestLoadSchemaFromMySQL(t *testing.T) { 41 | master := common.Config.MySQL.MasterInfo 42 | 43 | common.Config.MySQL.MasterInfo = common.DevPath + "/etc/master.info" 44 | common.LoadMasterInfo() 45 | err := loadSchemaFromMySQL() 46 | pretty.Println(err, Schemas) 47 | 48 | common.Config.MySQL.MasterInfo = master 49 | } 50 | 51 | func TestOnlyTable(t *testing.T) { 52 | tables := []string{ 53 | "`db`.`tb`", 54 | "tb", 55 | } 56 | 57 | err := common.GoldenDiff(func() { 58 | for _, table := range tables { 59 | fmt.Println(onlyTable(table)) 60 | } 61 | }, t.Name(), update) 62 | if nil != err { 63 | t.Fatal(err) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /doc/lua.md: -------------------------------------------------------------------------------- 1 | # Lua 插件 2 | 3 | ## 简介 4 | 5 | lightning 使用 [gopher-lua](https://github.com/yuin/gopher-lua) 作为 [Lua](http://www.lua.org/) 解析执行引擎，可以根据用户的需求对 binlog 的转写方式进行定制化修改，而又不必修改重新编译 Go 代码。为了方便访问数据库 lightning 默认加载了 [gluadb](https://github.com/zhu327/gluadb) 库，用于连接 MySQL, Redis。 6 | 7 | ## 限制 8 | 9 | 当使用 lua 插件做数据同步或双写时要注意数据库写流量不可过大，一般超过 1K qps 就很难保证同步的实时性了。从原理上讲 lightning 写入的速度因为依赖连接 MySQL 的执行速度，所以不可能比 MySQL 自己的同步快，大部分开销在网络上。即使转为使用本地 IP 访问或 SOCKET 连接由于单线程同步，在更新量过大时也无法和 MySQL 原生的同步效率媲美。但如果表与表之间的更新互无相关性时可以考虑为每张表启动一个 lightning 进程实现并行复制。 10 | 11 | ## 配置 12 | 13 | 配置文件 14 | 15 | ```yaml 16 | rebuild: 17 | plugin: lua 18 | lua-script: plugin/demo.flashback.lua 19 | ``` 20 | 21 | 命令行参数 22 | 23 | ```bash 24 | lightning -plugin lua -lua-script plugin/demo.flashback.lua 25 | ``` 26 | 27 | ## 示例脚本 28 | 29 | * [demo.flashback.lua](http://github.com/LianjiaTech/lightning/tree/master/plugin/demo.flashback.lua) 数据闪回示例 30 | * [demo.mysql.lua](http://github.com/LianjiaTech/lightning/tree/master/plugin/demo.mysql.lua) 连接 MySQL 示例 31 | * [demo.redis.lua](http://github.com/LianjiaTech/lightning/tree/master/plugin/demo.redis.lua) 连接 Redis 示例 32 | * [demo.sql.lua](http://github.com/LianjiaTech/lightning/tree/master/plugin/demo.sql.lua) 转写 SQL 示例 33 | * [demo.mode.lua](http://github.com/LianjiaTech/lightning/tree/master/plugin/demo.mod.lua) 引入第三方 Lua 库示例 34 | 35 | ## 全局变量 36 | 37 | 库表元数据 38 | 39 | * GoPrimaryKeys map[string][]string 40 | * GoColumns map[string][]string 41 | 42 | 记录值 43 | 44 | * GoValues [][]string 45 | * GoValuesWhere []string 46 | * GoValuesSet []string 47 | 48 | ## 接口函数 49 | 50 | 以下接口函数必须在 lua 脚本中存在，不需要的函数可以将函数体留空。 51 | 52 | ### Init 53 | 54 | 全局初始化函数 55 | 56 | ### InsertRewrite 57 | 58 | WRITE_ROWS_EVENT 转写函数。 59 | 60 | ### DeleteRewrite 61 | 62 | DELETE_ROWS_EVENT 转写函数。 63 | 64 | ### UpdateRewrite 65 | 66 | UPDATE_ROWS_EVENT 转写函数。 67 | 68 | ### QueryRewrite 69 | 70 | QUERY_EVENT 转写函数。 71 | 72 | ### Finalizer 73 | 74 | 全局析构函数。 -------------------------------------------------------------------------------- /doc/cmd.md: -------------------------------------------------------------------------------- 1 | # 常用命令 2 | 3 | ## ROW 格式转换为 STATEMENT 格式 4 | 5 | ```bash 6 | lightning -user xxx -password xxx -host xxx -port xxx binlog.00000[123] 7 | 或 8 | lightning -schema-file schema.sql -plugin sql binlog.00000[123] 9 | 10 | cat schema.sql 11 | use test; 12 | create table tb ( 13 | a int, 14 | b varchar(10), 15 | primary key (a) 16 | ) 17 | ``` 18 | 19 | ## 生成回滚语句 20 | 21 | ```bash 22 | lightning -schema-file schema.sql -plugin flashback binlog.000001 23 | ``` 24 | 25 | ## 统计各表更新量 26 | 27 | ```bash 28 | lightning -no-defaults -plugin stat -event-types insert,update,delete binlog.000001 | jq -r '.TableStats | keys[] as $k | "\($k) \(.[$k] | .insert + .delete + .update)"' | sort -k 2 -nr | column -t | head 29 | ``` 30 | 31 | ## 大事务、长事务分析 32 | 33 | verbose 模式中可以看到很多 binlog event 的信息，其中 TransactionSizeBytes 表示事务的 binlog event 大小。主库 binlog 和从库的 relay-log 中 ExecutionTime 显示的是事务执行时间，从库的 binlog 中 ExecutionTime 为从库同步延迟时间并不是事务执行耗时。 34 | 35 | ```bash 36 | lightning -no-defaults -verbose -schema-file test/schema.sql test/binlog.000002 | grep "DEBUG" | grep "TransactionSizeBytes\|ExecutionTime" 37 | ``` 38 | 39 | ## 查找指定时间的 event 在哪个 binlog 文件？ 40 | 41 | ```bash 42 | # 要注意命令行支持的最大长度，直接 $(ls mysql-bin.0*) 可能导致参数过长无法获取结果 43 | lightning -binlog-file "$(ls mysql-bin.0*)" -start-datetime "2021-01-13 07:00:00" -stop-datetime "2021-01-13 18:00:00" -plugin find 44 | ``` 45 | 46 | ## 通过 keyring 解密 MySQL 8.0 加密的 binlog 47 | 48 | MySQL 8.0 支持通过 keyring 对 binlog 进行加密，已经加密的 binlog lightning 解密需要提供 keyring 文件路径。只是解密不需要提取 SQL 时可以通过如下命令进行解密。 49 | 50 | ```bash 51 | lightning -plugin decrypt -keyring keyring binlog.encrypted > binlog.decrypted 52 | ``` 53 | 54 | 注意：lightning 解密会将结果打印到标准输出，需要添加输出重定向，不然会满屏乱码。lightning 的解密方式是流式的，不用担心大文件导致内存使用过多。 55 | 56 | lightning 还支持直接分析加密的 binlog，只需要添加 `-keyring` 配置即可，会根据 binlog 文件头的 magic header 类型自动识别。 57 | 58 | ```bash 59 | # 回滚所有删除事件 60 | lightning -plugin flashback -keyring keyring -event-types delete binlog.encrypted 61 | ``` 62 | 63 | ## 从标准输入读取 binlog 64 | 65 | 有时候需要分析的 binlog 并不在本机，可能存储在 s3 或其他服务器上。分析远程 binlog 文件时不想占用本地磁盘空间可以选择从标准输入读取 binlog。添加 `-` 表示从标准输入读取文件内容，示例如下： 66 | 67 | ```bash 68 | # 仅以 binlog 文件存储在 minio 为例，使用 minio client 读取远程经过压缩的 binlog 文件 69 | mc cat path/to/binlog_file.gz | gunzip | ./lightning - > decrypt_log.sql 70 | ``` 71 | -------------------------------------------------------------------------------- /event/parser_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package event 15 | 16 | import ( 17 | "fmt" 18 | "testing" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | "github.com/LianjiaTech/lightning/rebuild" 22 | ) 23 | 24 | func init() { 25 | common.Config.MySQL.SchemaFile = common.DevPath + "/test/schema.sql" 26 | rebuild.LoadSchemaInfo() 27 | } 28 | 29 | func ExampleBinlogFileValidator() { 30 | headers := [][]byte{ 31 | {0xfe, 'b', 'i', 'n'}, // not encrypted 32 | {0xfd, 'b', 'i', 'n'}, // encrypted 33 | {0xfe, 'g', 'i', 'f'}, // wrong file header 34 | } 35 | for _, header := range headers { 36 | fmt.Println("CheckBinlogFileHeader", header, CheckBinlogFileHeader(header)) 37 | fmt.Println("CheckBinlogFileEncrypt", header, CheckBinlogFileEncrypt(header)) 38 | } 39 | // Output: 40 | // CheckBinlogFileHeader [254 98 105 110] true 41 | // CheckBinlogFileEncrypt [254 98 105 110] false 42 | // CheckBinlogFileHeader [253 98 105 110] true 43 | // CheckBinlogFileEncrypt [253 98 105 110] true 44 | // CheckBinlogFileHeader [254 103 105 102] false 45 | // CheckBinlogFileEncrypt [254 103 105 102] false 46 | } 47 | 48 | func TestBinlogFileParser(t *testing.T) { 49 | err := BinlogFileParser([]string{common.DevPath + "/test/binlog.000002"}) 50 | if err != nil { 51 | t.Error(err.Error()) 52 | } 53 | } 54 | 55 | func TestBinlogStreamParser(t *testing.T) { 56 | masterInfoOrg := common.Config.MySQL.MasterInfo 57 | stopPositionOrg := common.Config.Filters.StopPosition 58 | common.Config.MySQL.MasterInfo = common.DevPath + "/etc/master.info" 59 | common.Config.Filters.StopPosition = 190 60 | common.LoadMasterInfo() 61 | err := BinlogStreamParser() 62 | if err != nil { 63 | t.Error(err.Error()) 64 | } 65 | common.Config.MySQL.MasterInfo = masterInfoOrg 66 | common.Config.Filters.StopPosition = stopPositionOrg 67 | } 68 | -------------------------------------------------------------------------------- /common/golden.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package common 15 | 16 | import ( 17 | "bufio" 18 | "bytes" 19 | "fmt" 20 | "io/ioutil" 21 | "os" 22 | "path/filepath" 23 | ) 24 | 25 | // GoldenDiff 从 gofmt 学来的测试方法 26 | // https://medium.com/soon-london/testing-with-golden-files-in-go-7fccc71c43d3 27 | func GoldenDiff(f func(), name string, update *bool) error { 28 | var b bytes.Buffer 29 | w := bufio.NewWriter(&b) 30 | str := captureOutput(f) 31 | _, err := w.WriteString(str) 32 | if err != nil { 33 | Log.Warning(err.Error()) 34 | } 35 | err = w.Flush() 36 | if err != nil { 37 | Log.Warning(err.Error()) 38 | } 39 | 40 | gp := filepath.Join("fixture", name+".golden") 41 | if *update { 42 | if err = ioutil.WriteFile(gp, b.Bytes(), 0644); err != nil { 43 | err = fmt.Errorf("%s failed to update golden file: %s", name, err) 44 | return err 45 | } 46 | } 47 | g, err := ioutil.ReadFile(gp) 48 | if err != nil { 49 | err = fmt.Errorf("%s failed reading .golden: %s", name, err) 50 | } 51 | if !bytes.Equal(b.Bytes(), g) { 52 | err = fmt.Errorf("%s does not match .golden file", name) 53 | } 54 | return err 55 | } 56 | 57 | // captureOutput 获取函数标准输出 58 | func captureOutput(f func()) string { 59 | // keep backup of the real stdout 60 | oldStdout := os.Stdout 61 | r, w, _ := os.Pipe() 62 | os.Stdout = w 63 | 64 | // copy the output in a separate goroutine so printing can't block indefinitely 65 | outC := make(chan string) 66 | go func() { 67 | buf, err := ioutil.ReadAll(r) 68 | if err != nil { 69 | panic(err) 70 | } 71 | outC <- string(buf) 72 | }() 73 | 74 | // execute function 75 | f() 76 | 77 | // back to normal state 78 | if err := w.Close(); err != nil { 79 | panic(err) 80 | } 81 | os.Stdout = oldStdout // restoring the real stdout 82 | out := <-outC 83 | os.Stdout = oldStdout 84 | return out 85 | } 86 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/LianjiaTech/lightning 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/BixData/gluabit32 v0.0.0-20190708203852-cb1e79982fc9 7 | github.com/BixData/gluasocket v0.0.0-20191219185455-6c63b949f5b0 8 | github.com/astaxie/beego v1.12.3 9 | github.com/go-mysql-org/go-mysql v1.1.2 10 | github.com/go-sql-driver/mysql v1.5.0 11 | github.com/juju/errors v0.0.0-20210818161939-5560c4c073ff 12 | github.com/kr/pretty v0.1.0 13 | github.com/montanaflynn/stats v0.5.0 14 | github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb 15 | github.com/satori/go.uuid v1.2.0 16 | github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed 17 | github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 18 | github.com/zhu327/gluadb v0.0.0-20180630095703-9586fc6945a0 19 | gopkg.in/yaml.v2 v2.2.8 20 | layeh.com/gopher-lfs v0.0.0-20201124131141-d5fb28581d14 21 | ) 22 | 23 | require ( 24 | github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect 25 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect 26 | github.com/go-ole/go-ole v1.2.4 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/golang/protobuf v1.4.2 // indirect 29 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect 30 | github.com/kr/text v0.1.0 // indirect 31 | github.com/nubix-io/gluabit32 v0.0.0-20190708203852-cb1e79982fc9 // indirect 32 | github.com/nubix-io/gluasocket v0.0.0-20191219185455-6c63b949f5b0 // indirect 33 | github.com/opentracing/opentracing-go v1.1.0 // indirect 34 | github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect 35 | github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect 36 | github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c // indirect 37 | github.com/pingcap/tipb v0.0.0-20200522051215-f31a15d98fce // indirect 38 | github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect 39 | github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect 40 | github.com/shirou/gopsutil v2.19.10+incompatible // indirect 41 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect 42 | github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect 43 | github.com/sirupsen/logrus v1.6.0 // indirect 44 | github.com/wumansgy/goEncrypt v0.0.0-20210730092718-e359121aa81e // indirect 45 | go.uber.org/atomic v1.7.0 // indirect 46 | go.uber.org/multierr v1.6.0 // indirect 47 | go.uber.org/zap v1.16.0 // indirect 48 | golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect 49 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect 50 | golang.org/x/text v0.3.6 // indirect 51 | google.golang.org/protobuf v1.23.0 // indirect 52 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 53 | ) 54 | -------------------------------------------------------------------------------- /test/schema.sql: -------------------------------------------------------------------------------- 1 | SET NAMES UTF8MB4; 2 | SET SQL_MODE = ""; 3 | 4 | CREATE DATABASE IF NOT EXISTS test; 5 | USE test; 6 | 7 | DROP TABLE IF EXISTS `tb`; 8 | CREATE TABLE `tb` ( 9 | `a` int(11) NOT NULL, 10 | `b` varchar(10) DEFAULT NULL, 11 | PRIMARY KEY (`a`) 12 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 13 | 14 | INSERT INTO `tb` VALUES (1, "abc"); 15 | INSERT INTO `tb` VALUES (2, "ghi"); 16 | INSERT INTO `tb` VALUES (3, 'space '); 17 | UPDATE `tb` SET b = "中文" WHERE a = 2; 18 | UPDATE `tb` SET b = "'abc'" WHERE a = 2; 19 | UPDATE `tb` SET b = '"abc' WHERE a = 2; 20 | DELETE FROM `tb` WHERE a = 1; 21 | 22 | DROP TABLE IF EXISTS `setTest`; 23 | CREATE TABLE `setTest` ( 24 | id int(11) AUTO_INCREMENT, 25 | attrib SET('bold','italic','underline'), 26 | PRIMARY KEY (`id`) 27 | ); 28 | 29 | INSERT INTO setTest (attrib) VALUES ('bold'); 30 | INSERT INTO setTest (attrib) VALUES ('bold,italic'); 31 | INSERT INTO setTest (attrib) VALUES ('bold,italic,underline'); 32 | 33 | DROP TABLE IF EXISTS `enumTest`; 34 | CREATE TABLE `enumTest` ( 35 | id int(11) AUTO_INCREMENT, 36 | color ENUM('red','green','blue'), 37 | PRIMARY KEY (`id`) 38 | ); 39 | 40 | INSERT INTO `enumTest` (color) VALUES ('red'); 41 | 42 | DROP TABLE IF EXISTS `bitTest`; 43 | CREATE TABLE `bitTest` ( 44 | id int(11) AUTO_INCREMENT, 45 | days BIT(7), 46 | PRIMARY KEY(id) 47 | ); 48 | 49 | INSERT INTO `bitTest` (`days`) VALUES (B'1111100'); 50 | 51 | CREATE TABLE testNoPRI ( 52 | `a` int, 53 | `b` varchar(10) 54 | ); 55 | 56 | INSERT INTO testNoPRI VALUES (1, 'abc'); 57 | 58 | CREATE TABLE `timeTest` ( 59 | `a` timestamp NULL DEFAULT NULL, 60 | `b` datetime DEFAULT NULL, 61 | `c` timestamp(3) DEFAULT NULL, 62 | `d` datetime(3) DEFAULT NULL 63 | ) ENGINE=InnoDB; 64 | 65 | INSERT INTO timeTest VALUES ("2016-06-01 23:55:29", "2016-06-01 23:55:29", "2016-06-01 23:55:29.123", "2016-06-01 23:55:29.123"); 66 | INSERT INTO timeTest VALUES (0, 0, 0, 0); 67 | 68 | CREATE TABLE testDecimal( 69 | `d1` DECIMAL(38,2) NOT NULL DEFAULT 0, 70 | `d2` DECIMAL(6,2) NOT NULL DEFAULT 0 71 | ) ENGINE=InnoDB; 72 | 73 | INSERT INTO testDecimal VALUES (2011202301003814000564, 2.4); 74 | INSERT INTO testDecimal VALUES (2011202301003814000564.2, 24); 75 | 76 | CREATE TABLE `test_int_max` ( 77 | `a` TINYINT unsigned NOT NULL, 78 | `b` SMALLINT unsigned NOT NULL, 79 | `c` MEDIUMINT unsigned NOT NULL, 80 | `d` INT unsigned NOT NULL, 81 | `e` BIGINT unsigned NOT NULL, 82 | 83 | `f` TINYINT NOT NULL, 84 | `g` SMALLINT NOT NULL, 85 | `h` MEDIUMINT NOT NULL, 86 | `i` INT NOT NULL, 87 | `j` BIGINT NOT NULL, 88 | PRIMARY KEY (`a`) 89 | ) ENGINE=InnoDB; 90 | 91 | INSERT INTO `test_int_max` VALUES (255, 65535, 16777215, 4294967295, 18446744073709551615, -1, -1, -1, -1, -1); 92 | 93 | CREATE TABLE `city` ( 94 | `ID` int(11) NOT NULL AUTO_INCREMENT, 95 | `Name` char(35) NOT NULL DEFAULT '', 96 | `CountryCode` char(3) NOT NULL DEFAULT '', 97 | `District` char(20) NOT NULL DEFAULT '', 98 | `Info` json DEFAULT NULL, 99 | PRIMARY KEY (`ID`) 100 | ) ENGINE=InnoDB; 101 | 102 | INSERT INTO `city` VALUES 103 | (1,'Kabul','AFG','Kabol','{"Population": 1780000}'), 104 | (2,'Qandahar','AFG','Qandahar','{"Population": 237500}'); 105 | -------------------------------------------------------------------------------- /common/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package common 15 | 16 | import ( 17 | "flag" 18 | "fmt" 19 | "testing" 20 | "time" 21 | 22 | "github.com/kr/pretty" 23 | ) 24 | 25 | var update = flag.Bool("update", false, "update .golden files") 26 | 27 | func TestLoadReplicationInfo(t *testing.T) { 28 | masterInfoOrg := Config.MySQL.MasterInfo 29 | Config.MySQL.MasterInfo = DevPath + "/etc/master.info" 30 | LoadMasterInfo() 31 | pretty.Println(MasterInfo) 32 | Config.MySQL.MasterInfo = masterInfoOrg 33 | } 34 | 35 | func TestPrintConfiguration(t *testing.T) { 36 | err := GoldenDiff(func() { 37 | PrintConfiguration() 38 | }, t.Name(), update) 39 | if nil != err { 40 | t.Fatal(err) 41 | } 42 | } 43 | 44 | func TestListPlugin(t *testing.T) { 45 | err := GoldenDiff(func() { 46 | ListPlugin() 47 | }, t.Name(), update) 48 | if nil != err { 49 | t.Fatal(err) 50 | } 51 | } 52 | 53 | // go test github.com/LianjiaTech/lightning/common -v -update -run TestTimeOffset 54 | func TestTimeOffset(t *testing.T) { 55 | tzs := []string{ 56 | "UTC", 57 | "Asia/Shanghai", 58 | "Europe/London", 59 | "America/Los_Angeles", 60 | } 61 | 62 | err := GoldenDiff(func() { 63 | for _, tz := range tzs { 64 | fmt.Println(TimeOffset(tz)) 65 | } 66 | }, t.Name(), update) 67 | if nil != err { 68 | t.Fatal(err) 69 | } 70 | } 71 | 72 | func TestFlushReplicationInfo(t *testing.T) { 73 | masterInfoOrg := Config.MySQL.MasterInfo 74 | Config.MySQL.MasterInfo = DevPath + "/common/fixture/master.info" 75 | FlushReplicationInfo() 76 | Config.MySQL.MasterInfo = masterInfoOrg 77 | } 78 | 79 | func TestSyncReplicationInfo(t *testing.T) { 80 | durationOrg := Config.MySQL.SyncDuration 81 | Config.MySQL.SyncDuration = time.Duration(0 * time.Second) 82 | SyncReplicationInfo() 83 | Config.MySQL.SyncDuration = durationOrg 84 | } 85 | 86 | func TestParseConfig(t *testing.T) { 87 | ParseConfig() 88 | pretty.Println(Config, MasterInfo) 89 | } 90 | 91 | func TestParseConfigFile(t *testing.T) { 92 | var cfg *Configuration 93 | cfg.parseConfigFile(DevPath + "/etc/lightning.yaml") 94 | pretty.Println(Config) 95 | } 96 | 97 | func TestVersion(t *testing.T) { 98 | version() 99 | } 100 | 101 | func TestPrintMasterInfo(t *testing.T) { 102 | PrintMasterInfo() 103 | } 104 | 105 | func TestShowMasterStatus(t *testing.T) { 106 | // dsn := `root:******@tcp(127.0.0.1:3306)/` 107 | masterInfo := ChangeMaster{ 108 | MasterUser: "root", 109 | MasterPassword: "******", 110 | MasterHost: "127.0.0.1", 111 | MasterPort: 3306, 112 | } 113 | pretty.Println(ShowMasterStatus(masterInfo)) 114 | } 115 | -------------------------------------------------------------------------------- /doc/FAQ.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | ## 数据闪回实现的原理？ 4 | 5 | 参考这篇文章 [MySQL 下实现闪回的设计思路 (MySQL Flashback Feature)](http://www.penglixun.com/tech/database/mysql_flashback_feature.html) 6 | 7 | ## 是否支持 GTID？ 8 | 9 | lightning 支持 GTID event 的分析，可以分析含有 GTID 的 binlog 文件，也可以实时分析开启 GTID 的 MySQL 二进制日志。 10 | 11 | ## 能否用于 relay-log 的解析？ 12 | 13 | MySQL relay-log 的格式与 binlog 相同，但对应 event 中结构体内成员代表的含义会略有不同。解析 relay-log 可以生成 SQL 及对应 SQL 的回滚语句，但如果用于统计分析，需要参考 MySQL [文档](https://dev.mysql.com/doc/internals/en/binary-log-structure-and-contents.html) 明确每个 event 成员的真实含义。 14 | 15 | ## 待恢复表有外键关联 16 | 17 | ```text 18 | ERROR 1451 (23000): Connot delete or update a parent row: a foreign key constraint fails 19 | ``` 20 | 21 | 存在外键的表应先恢复父表记录，再恢复子表，故报错。 22 | 23 | ```sql 24 | SET SESSION FOREIGN_KEY_CHECKS = 0; -- 数据恢复前禁用外键检查 25 | 26 | SOURCE rollback.sql 27 | 28 | SET SESSION FOREIGN_KEY_CHECKS = 1; -- 数据恢复完成后开启外键检查 29 | ``` 30 | 31 | ## GTID 同步报错 32 | 33 | 使用 Binlog Dump GTID 方式同步数据时如果报以下错误，很可能是 master.info 中的 executed_gtid_set 配置错了。正确的配置格式参考 MySQL [官方文档](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) GTID Sets。 34 | 35 | ```text 36 | ERROR 1236 (HY000): The slave is connecting using CHANGE MASTER TO MASTER_AUTO_POSITION = 1, but the master has purged binary logs containing GTIDs that the slave requires. 37 | ``` 38 | 39 | 查看当前主库 master status。 40 | 41 | ```sql 42 | mysql > show master status; 43 | +------------------+----------+--------------+------------------+-----------------------------------------------------------------------------------+ 44 | | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set | 45 | +------------------+----------+--------------+------------------+-----------------------------------------------------------------------------------+ 46 | | mysql-bin.000020 | 234 | | | 376b1ae7-39a1-11e9-a253-14187759814e:1, 47 | 3b0075c9-39a1-11e9-a250-f86eee9113c6:1-98 | 48 | +------------------+----------+--------------+------------------+-----------------------------------------------------------------------------------+ 49 | 1 row in set (0.00 sec) 50 | ``` 51 | 52 | ## zombie Binlog Dump 53 | 54 | 当 lightning 异常退出未断开与主库的同步又重新启动后，主库错误日志会打印如下信息，这种情况不需要处理。 55 | 56 | 多个 lightning 同步同一个主库且 master.info 中 server-id 配置相同进也会出现类型错误日志，此时需要手工指定不同的 server-id 解决。 57 | 58 | ```text 59 | 2019-05-23T10:29:27.082175+08:00 631113 [Note] Start binlog_dump to master_thread_id(631113) slave_server(33061), pos(mysql-bin.000018, 2316) 60 | 2019-05-23T10:30:32.574203+08:00 631125 [Note] While initializing dump thread for slave with server_id <33061>, found a zombie dump thread with the same server_id. Master is killing the zombie dump thread(631113). 61 | ``` 62 | 63 | 参考文章： 64 | 65 | * [MySQL 多个 Slave 同一 server_id 的冲突原因分析](http://www.penglixun.com/tech/database/mysql_multi_slave_same_serverid.html) 66 | * [两台备库设置 server_id 一致的问题](https://win-man.github.io/2017/07/11/%E4%B8%A4%E5%8F%B0%E5%A4%87%E5%BA%93%E8%AE%BE%E7%BD%AEserver-id%E4%B8%80%E8%87%B4%E7%9A%84%E9%97%AE%E9%A2%98/) 67 | 68 | ## 使用 lua 插件实现数据双写同步效率问题 69 | 70 | 尝试用 lua 插件来实现将某些表数据写两份时，我们发现写入速度不够快，在高 QPS(>1K) 情况下很难保证数据低延迟。 71 | 72 | 从实现机制上分析，lua 插件是通过 TCP 协议远程写对端 MySQL，执行一条写操作一来一回网络相对较大，而且更新线程只有一个，不像 MySQL 是进程内本地多线程复制。 73 | 74 | 优化思路： 75 | 76 | 1. 将 lightning 与待更新的 MySQL 本地部署，通过 127.0.0.1 或 socket 方式写数据库。 77 | 2. 对一定时间的 binlog 合并更新，减少更新中间态。 78 | 3. 模仿多线程同步，通过配置表过滤规则，不同表开启不同的 lightning 进程同步。 79 | -------------------------------------------------------------------------------- /doc/reference.md: -------------------------------------------------------------------------------- 1 | # 参考 2 | 3 | ## 同类产品 4 | 5 | * [MariaDB mysqlbinlog](https://mariadb.com/kb/en/library/flashback/) 6 | * [binlog2sql](https://github.com/danfengcao/binlog2sql) 7 | * [MyFlash](https://github.com/Meituan-Dianping/MyFlash) 8 | * [mysqlbinlog_flashback](https://github.com/58daojia-dba/mysqlbinlog_flashback) 9 | * [pt-query-digest](https://www.percona.com/doc/percona-toolkit/LATEST/pt-query-digest.html) 10 | 11 | ## 技术文章 12 | 13 | 数据闪回 14 | 15 | * [MySQL Internals Manual / The Binary Log](https://dev.mysql.com/doc/internals/en/binary-log.html) 16 | * [ROWS_EVENT](https://dev.mysql.com/doc/internals/en/rows-event.html) 17 | * [MySQL 下实现闪回的设计思路 (MySQL Flashback Feature)](http://www.penglixun.com/tech/database/mysql_flashback_feature.html) 18 | * [Provide the flashback feature by binlog](https://bugs.mysql.com/bug.php?id=65178) 19 | * [MySQL 闪回方案讨论及实现](https://dinglin.iteye.com/blog/1539167) 20 | * [mysqlbinlog flashback 5.6 完全使用手册与原理](http://www.cnblogs.com/youge-OneSQL/p/5249736.html) 21 | * [拿走不谢，Flashback for MySQL 5.7](http://t.cn/E9ZH8sU) 22 | * [AliSQL and some features that have made it into MariaDB Server](https://mariadb.com/resources/blog/alisql-and-some-features-that-have-made-it-into-mariadb-server/) 23 | * [MyFlash—— 美团点评的开源 MySQL 闪回工具](http://t.cn/RjRbjpM) 24 | * [Identifying useful info from MySQL row-based binary logs](https://www.percona.com/blog/2015/01/20/identifying-useful-information-mysql-row-based-binary-logs/) 25 | * [MySQL 多个 Slave 同一 server_id 的冲突原因分析](http://www.penglixun.com/tech/database/mysql_multi_slave_same_serverid.html) 26 | * [两台备库设置 server_id 一致的问题](https://win-man.github.io/2017/07/11/两台备库设置server-id一致的问题/) 27 | * [pt-query-digest 解析 MySQL Binlog 日志文件](https://blog.csdn.net/dba_waterbin/article/details/14453255) 28 | * [Binary Log Options and Variables](https://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html) 29 | * [Read MySQL Binlogs better with rows query log events](https://mydbops.wordpress.com/2017/08/02/read-mysql-binlogs-better-with-binlog_rows_query_log_events/) 30 | 31 | Lua 32 | 33 | * [Lua 教程](https://www.runoob.com/lua/lua-tutorial.html) 34 | * [gopher-lua](https://github.com/yuin/gopher-lua) 35 | * [Embedding Lua in Go](https://otm.github.io/2015/07/embedding-lua-in-go/) 36 | 37 | ## 第三方包引用 38 | 39 | * [go-mysql](https://github.com/go-mysql-org/go-mysql) 40 | * [pingcap/parser](https://github.com/pingcap/parser) 41 | * [gopher-lua](https://github.com/yuin/gopher-lua) 42 | 43 | ## 性能对比 44 | 45 | lightning 与 mysqlbinlog 原生工具比较在文件解析速度上存在差距。目前分析主要的瓶颈点在于 lightning 需要识别不同的库、表、列、值等数据，并按照正确的语法逻辑进行拼接，但 mysqlbinlog 并不需要这样做，即使是 verbose 模式生成的 SQL 也并不能真正执行。 46 | 47 | ```bash 48 | ls -lh mysql-bin.001287 49 | -rw-rw---- 1 mysql mysql 513M May 22 01:12 mysql-bin.001287 50 | 51 | time lightning mysql-bin.001287 > mysql-bin.001287.lightning.sql 52 | 53 | real 0m20.563s 54 | user 0m23.280s 55 | sys 0m6.806s 56 | 57 | time mysqlbinlog -vv mysql-bin.001287 > mysql-bin.001287.mysqlbinlog.sql 58 | 59 | real 0m7.892s 60 | user 0m5.636s 61 | sys 0m2.163s 62 | 63 | time mysqlbinlog mysql-bin.001287 > mysql-bin.001287.mysqlbinlog.raw.sql 64 | 65 | real 0m3.492s 66 | user 0m1.571s 67 | sys 0m1.871s 68 | 69 | ls -lh mysql-bin.001287.* 70 | -rw-r--r-- 1 mysql mysql 758M May 23 11:32 mysql-bin.001287.lightning.sql 71 | -rw-r--r-- 1 mysql mysql 1.4G May 23 11:32 mysql-bin.001287.mysqlbinlog.sql 72 | -rw-r--r-- 1 mysql mysql 635M May 23 12:37 mysql-bin.001287.mysqlbinlog.raw.sql 73 | ``` 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 简介 2 | 3 | [文档](http://github.com/LianjiaTech/lightning/blob/master/doc/) | [English Readme](http://github.com/LianjiaTech/lightning/blob/master/README_EN.md) 4 | 5 | ![lightning](https://github.githubassets.com/images/icons/emoji/unicode/26a1.png) lightning 是由贝壳找房 DBA 团队开发和维护的一个 MySQL binlog 转换工具。该工具可以将 MySQL ROW 格式的 binlog 转换为想要的 SQL，如：原始 SQL，闪回 SQL等。也可以对 binlog 进行统计分析，用于数据库异常分析。甚至可以通过定制 lua 插件进行二次开发，发挥无限的想象力。 6 | 7 | ## 应用 8 | 9 | * 数据修改错误，需要快速回滚 (闪回) 10 | * DELETE, UPDATE 未指定 WHERE 条件 11 | * UPDATE SET 误用 AND 连接 12 | * 数据异常， 从 binlog 中找特定表某些数据是什么时间修的 13 | * 业务流量异常或从库同步延迟，需要统计排查是哪些表在频繁更新 14 | * 需要把指定表，指定时间的更新提供给开发定位服务异常问题 15 | * 主从切换后新主库丢失数据的修复 16 | * 从 binlog 生成标准 SQL，带来的衍生功能 17 | * 找出某个时间点数据库是否有大事务 (Size) 或者长事务 (Time) 18 | 19 | ## 优点 20 | 21 | * 跨平台支持，二进制文件即下即用，无其他依赖。 22 | * 支持 lua 定制化插件，发挥无限的想象力，二次开发周期短。 23 | * 支持从 SQL 文件加载库表信息，不必连接 MySQL 便于历史变更恢复。 24 | * SQL 进行多行合并，相比 mysqlbinlog ROW 格式，更好过滤。 25 | 26 | ## 安装 27 | 28 | ### 二进制安装 29 | 30 | lightning 使用 Go 1.11+ 开发，可以直接下载编译好的二进制文件在命令行下使用。由于 Go 原生对跨平台支持较好，在 Windows, Linux, Mac 下均可使用。 31 | 32 | [下载地址](https://github.com/LianjiaTech/lightning/releases) 33 | 34 | ### 源码安装 35 | 36 | ```bash 37 | go get -d github.com/LianjiaTech/lightning 38 | cd ${PATH_TO_SOURCE}/lightning # 进入源码路径，PATH_TO_SOURCE 需要人为具体指定。 39 | make 40 | ``` 41 | 42 | ## 测试示例 43 | 44 | [常用命令](http://github.com/LianjiaTech/lightning/blob/master/doc/cmd.md) 45 | 46 | 直接读取文件生成回滚语句 47 | 48 | ```bash 49 | lightning -no-defaults \ 50 | -plugin flashback \ 51 | -start-datetime "2019-01-01 00:00:00" \ 52 | -stop-datetime "2019-01-01 00:01:00" \ 53 | -event-types delete,update \ 54 | -tables test.tb \ 55 | -schema-file schema.sql \ 56 | -binlog-file binlog.0000001 > flashback.sql 57 | ``` 58 | 59 | 使用 `Binlog Dump` 方式读取日志生成回滚语句 60 | 61 | ```bash 62 | cat > master.info 63 | master_host: 127.0.0.1 64 | master_user: root 65 | master_password: ****** 66 | master_port: 3306 67 | master_log_file: binlog.000002 68 | master_log_pos: 4 69 | +D 70 | 71 | lightning -no-defaults \ 72 | -plugin flashback \ 73 | -start-datetime "2019-01-01 00:00:00" \ 74 | -stop-datetime "2019-01-01 00:01:00" \ 75 | -event-types delete,update \ 76 | -tables test.tb \ 77 | -master-info master.info > flashback.sql 78 | ``` 79 | 80 | ## 配置 81 | 82 | lightning 使用 YAML 格式的配置文件。使用 `-config` 参数指定配置文件路径，如不指定默认按 /etc/lightning.yaml -> ./etc/lightning.yaml -> ./lightning.yaml 的顺序加载配置文件。如果不想使用默认路径下的配置文件还可以通过 `-no-defaults` 参数屏蔽所有默认配置文件。 83 | 84 | * [全局配置](http://github.com/LianjiaTech/lightning/blob/master/doc/global.md) 85 | * [MySQL 日志源](http://github.com/LianjiaTech/lightning/blob/master/doc/mysql.md) 86 | * [过滤器](http://github.com/LianjiaTech/lightning/blob/master/doc/filters.md) 87 | * [SQL 重建规则](http://github.com/LianjiaTech/lightning/blob/master/doc/rebuild.md) 88 | 89 | ## 限制/局限 90 | 91 | * 仅测试了 v4 版本 (MySQL 5.1+) 的 binlog，更早版本未做测试。 92 | * BINLOG_FORMAT = ROW 93 | * 参数 BINLOG_ROW_IMAGE 必须为 FULL，暂不支持 MINIMAL 94 | * 由于添加了更多的处理逻辑，解析速度不如 mysqlbinlog 快 95 | * 当 binlog 中的 DDL 语句变更表结构时，lightning 中的表结构原数据并不随之改变（TODO） 96 | 97 | ## 沟通交流 98 | 99 | * [常见问题(FAQ)](http://github.com/LianjiaTech/lightning/blob/master/doc/FAQ.md) 100 | * 欢迎通过 Github Issues 提交问题报告与建议 101 | * QQ 群： 573877257 102 | 103 | ![QQ](https://github.com/LianjiaTech/lightning/raw/master/doc/qq_group.png) 104 | 105 | ## License 106 | 107 | [Apache License 2.0](http://github.com/LianjiaTech/lightning/blob/master/LICENSE) 108 | -------------------------------------------------------------------------------- /plugin/demo.sql.lua: -------------------------------------------------------------------------------- 1 | -- GoPrimaryKeys 2 | -- GoColumns 3 | -- GoValues 4 | 5 | -- GoValuesWhere 6 | -- GoValuesSet 7 | 8 | -- Init inital funcation 9 | function Init() 10 | 11 | end 12 | 13 | -- InsertRewrite insert rewrite logic 14 | function InsertRewrite (tab) 15 | local columnStr = "" 16 | -- columns 17 | for k, values in pairs(GoColumns) do 18 | if k == tab 19 | then 20 | columnStr = table.concat(values, ", ") 21 | end 22 | end 23 | print(string.format("INSERT INTO %s (%s) VALUES (%s);", tab, columnStr, table.concat(GoValues, ", "))) 24 | end 25 | 26 | -- DeleteRewrite delete rewrite logic 27 | function DeleteRewrite (tab) 28 | local whereStr = {} 29 | -- primary-keys 30 | for k, keys in pairs(GoPrimaryKeys) do 31 | if k == tab 32 | then 33 | for _, key in ipairs(keys) do 34 | for t, cols in pairs(GoColumns) do 35 | if k == t 36 | then 37 | for i, col in ipairs(cols) do 38 | if col == key 39 | then 40 | if GoValues[i] == "NULL" 41 | then 42 | table.insert(whereStr, string.format("%s IS %s", col, GoValues[i])) 43 | else 44 | table.insert(whereStr, string.format("%s = %s", col, GoValues[i])) 45 | end 46 | end 47 | end 48 | end 49 | end 50 | end 51 | end 52 | end 53 | print(string.format("DELETE FROM %s WHERE %s;", tab, table.concat(whereStr, " AND "))) 54 | end 55 | 56 | -- UpdateRewrite update rewrite logic 57 | function UpdateRewrite (tab) 58 | local whereStr = {} 59 | local setStr = {} 60 | for k, keys in pairs(GoPrimaryKeys) do 61 | if k == tab 62 | then 63 | for _, key in ipairs(keys) do 64 | for t, cols in pairs(GoColumns) do 65 | if k == t 66 | then 67 | for i, col in ipairs(cols) do 68 | if col == key 69 | then 70 | if GoValuesWhere[i] == "NULL" 71 | then 72 | table.insert(whereStr, string.format("%s IS %s", col, GoValuesWhere[i])) 73 | else 74 | table.insert(whereStr, string.format("%s = %s", col, GoValuesWhere[i])) 75 | end 76 | end 77 | end 78 | end 79 | end 80 | end 81 | end 82 | end 83 | 84 | for t, cols in pairs(GoColumns) do 85 | if t == tab 86 | then 87 | for i, col in ipairs(cols) do 88 | table.insert(setStr, string.format("%s = %s", col, GoValuesSet[i])) 89 | end 90 | end 91 | end 92 | 93 | print(string.format("UPDATE %s SET %s WHERE %s;", tab, table.concat(setStr, ", "), table.concat(whereStr, " AND "))) 94 | end 95 | 96 | -- QueryRewrite query rewrite logic 97 | function QueryRewrite (sql) 98 | print(string.format("%s", sql)) 99 | end 100 | 101 | -- Finalizer final destructor function 102 | function Finalizer () 103 | end 104 | -------------------------------------------------------------------------------- /plugin/demo.flashback.lua: -------------------------------------------------------------------------------- 1 | -- GoPrimaryKeys 2 | -- GoColumns 3 | -- GoValues 4 | 5 | -- GoValuesWhere 6 | -- GoValuesSet 7 | 8 | -- Init inital funcation 9 | function Init() 10 | -- print("Init ...") 11 | end 12 | 13 | -- InsertRewrite insert rewrite logic 14 | function InsertRewrite (tab) 15 | local whereStr = {} 16 | -- primary-keys 17 | for k, keys in pairs(GoPrimaryKeys) do 18 | if k == tab 19 | then 20 | for _, key in ipairs(keys) do 21 | for t, cols in pairs(GoColumns) do 22 | if k == t 23 | then 24 | for i, col in ipairs(cols) do 25 | if col == key 26 | then 27 | if GoValues[i] == "NULL" 28 | then 29 | table.insert(whereStr, string.format("%s IS %s", col, GoValues[i])) 30 | else 31 | table.insert(whereStr, string.format("%s = %s", col, GoValues[i])) 32 | end 33 | end 34 | end 35 | end 36 | end 37 | end 38 | end 39 | end 40 | print(string.format("DELETE FROM %s WHERE %s;", tab, table.concat(whereStr, " AND "))) 41 | end 42 | 43 | -- DeleteRewrite delete rewrite logic 44 | function DeleteRewrite (tab) 45 | local columnStr = "" 46 | -- columns 47 | for k, values in pairs(GoColumns) do 48 | if k == tab 49 | then 50 | columnStr = table.concat(values, ", ") 51 | end 52 | end 53 | print(string.format("INSERT INTO %s (%s) VALUES (%s);", tab, columnStr, table.concat(GoValues, ", "))) 54 | end 55 | 56 | -- UpdateRewrite update rewrite logic 57 | function UpdateRewrite (tab) 58 | local whereStr = {} 59 | local setStr = {} 60 | for k, keys in pairs(GoPrimaryKeys) do 61 | if k == tab 62 | then 63 | for _, key in ipairs(keys) do 64 | for t, cols in pairs(GoColumns) do 65 | if k == t 66 | then 67 | for i, col in ipairs(cols) do 68 | if col == key 69 | then 70 | if GoValuesWhere[i] == "NULL" 71 | then 72 | table.insert(setStr, string.format("%s IS %s", col, GoValuesWhere[i])) 73 | else 74 | table.insert(setStr, string.format("%s = %s", col, GoValuesWhere[i])) 75 | end 76 | end 77 | end 78 | end 79 | end 80 | end 81 | end 82 | end 83 | 84 | for t, cols in pairs(GoColumns) do 85 | if t == tab 86 | then 87 | for i, col in ipairs(cols) do 88 | table.insert(whereStr, string.format("%s = %s", col, GoValuesSet[i])) 89 | end 90 | end 91 | end 92 | 93 | print(string.format("UPDATE %s SET %s WHERE %s;", tab, table.concat(setStr, ", "), table.concat(whereStr, " AND "))) 94 | end 95 | 96 | -- QueryRewrite query rewrite logic 97 | function QueryRewrite (sql) 98 | print(string.format("%s", sql)) 99 | end 100 | 101 | -- Finalizer final destructor function 102 | function Finalizer () 103 | -- print("Finalizer ...") 104 | end 105 | -------------------------------------------------------------------------------- /doc/filters.md: -------------------------------------------------------------------------------- 1 | # 过滤器 2 | 3 | ## 表过滤器 4 | 5 | 可以配置只同步某些表 `tables` 或不同步某些表 `ignore-tables`。需要注意的是必需指定库名，不可以只指定表名，如需匹配所有库的某张表可以写 %.tb。 6 | 7 | ### 命令行 8 | 9 | 用逗号作为分隔符，格式为：{db}.{tb}，使用 `%` 作为通配符。 10 | 11 | ```bash 12 | -tables db1.tb1,db1.tb2,db2.%,%.tb -ignore-tables db3.ignore_tb 13 | ``` 14 | 15 | ### 配置文件 16 | 17 | ```yaml 18 | filters: 19 | tables: 20 | - db1.tb1 21 | - db1.tb2 22 | - db2.% 23 | ignore-tables: 24 | - db2.ignore 25 | ``` 26 | 27 | ## 事件过滤器 28 | 29 | 只匹配特殊的事件类型，如：insert, delete, update, delete, create, drop 等，主意中间不要有空格，使用小写字母。 30 | 31 | ### 命令行 32 | 33 | ```bash 34 | -event-types delete,insert 35 | ``` 36 | 37 | ### 配置文件 38 | 39 | ```yaml 40 | filters: 41 | event-types: 42 | - insert 43 | - update 44 | ``` 45 | 46 | ## 时间过滤器 47 | 48 | 像 `mysqlbinlog` 一样可以指定开始时间 `start-datetime` 和结束时间 `stop-datetime` 。如不指定 `stop-datetime` 又未配置 `demonize` 时 `stop-datetime` 使用当前时间为默认值。时间格式： `2006-01-02 15:04:05`。注意要配合时区使用，如不配置 lightning 使用 `UTC` 作为默认时区。 49 | 50 | ### 命令行 51 | 52 | ```bash 53 | -time-zone Asia/Shanghai -start-datetime "2019-10-01 00:00:00" -stop-datetime "2019-10-01 01:00:00" 54 | ``` 55 | 56 | ### 配置文件 57 | 58 | ```yaml 59 | global: 60 | time-zone: Asia/shanghai 61 | filters: 62 | start-datetime: "" 63 | stop-datetime: "2019-10-01 01:00:00" 64 | ``` 65 | 66 | ## 文件及位点过滤器 67 | 68 | lightning 可以从文件读取日志，也可以向 MySQL 发送 `Binlog Dump` 命令模拟从库读取日志。如需从文件读取日志使用 `-binlog-file` 指定启始读取的日志文件名，如使用 `Binlog Dump` 读取日志需使用 `-master-info` 指定 MySQL 同步源。只解析 ROW 格式的 binlog 无得到库表结构，因为无法直接还原为 SQL 语句，需要结合 `schema-file` 或 `master-info` 来获取库表结构。与 `mysqlbinlog` 使用方式相同，使用 `start-position` 和 `stop-position` 两个参数指定起始点和终止点位。 69 | 70 | ### 命令行 71 | 72 | ```bash 73 | lightning -binlog-file binlog.000001 74 | 或将文件名置于最后一个参数 75 | lightning binlog.000001 76 | ``` 77 | 78 | ### 配置文件 79 | 80 | ```yaml 81 | mysql: 82 | binlog-file: binlog.000002 83 | schema-file: schema.sql 84 | master-info: etc/master.info 85 | filters: 86 | start-position: 0 87 | stop-position: 0 88 | ``` 89 | 90 | 使用 `schema-file` 来读取库表结构的处是可以使用表结修改前的信息来复原 SQL 。 91 | 92 | ```sql 93 | use test; 94 | create table tb ( 95 | `a` int, 96 | `b` varchar(10), 97 | PRIMARY KEY (`a`) 98 | ) ENGINE = InnoDB; 99 | ``` 100 | 101 | master.info 文件格式如下，如从文件读取日志不需要指定 `master_log_file` 和 `master_log_pos` 等信息，只会连接 MySQL 获取库表结构。如使用 `Binlog Dump` 方式读取日志，需要指定 `master_log_file`, `master_log_pos` 及 `server-id`。`server-id` 如不指定使用 3306+RAND(3306) 作为 `server-id`，为避免冲突建议手工指定 `server-id`。 102 | 103 | ```yaml 104 | master_host: 127.0.0.1 105 | master_user: root 106 | master_password: ****** 107 | master_port: 3306 108 | master_log_file: binlog.000002 109 | master_log_pos: 4 110 | gtid_next: "" 111 | server-id: 3307 112 | server-type: mysql 113 | ``` 114 | 115 | ## 线程过滤器 116 | 117 | 通过 `thread-id` 过滤指定线程，可用于单次 SQL 上线的快速回滚。 118 | 119 | ### 命令行 120 | 121 | ```bash 122 | -thread-id 10086 123 | ``` 124 | 125 | ### 配置文件 126 | 127 | ```yaml 128 | filters: 129 | thread-id: 10086 130 | ``` 131 | 132 | ## 主库 server-id 过滤器 133 | 134 | 通过 `server-id` 过滤指定的主库，当存在多源级联同步或从库有数据写入时使用该参数。可以使用 `SELECT @@server_id;` 查看主库的 `server-id`。 135 | 136 | ### 命令行 137 | 138 | ```bash 139 | -server-id 1 140 | ``` 141 | 142 | ### 配置文件 143 | 144 | ```yaml 145 | filers: 146 | server-id: 1 147 | ``` 148 | 149 | ## GTID 过滤器 150 | 151 | 对于开启了 GTID 的 MySQL 实例也可以通过 `include-gtids` 和 `exclude-gtids` 来做过滤。以上两个参数可以配多个 `gtid_set`， 格式为 {uuid}:N-M，多个 `gtid_set` 使用逗号连接。 152 | 153 | ### 命令行 154 | 155 | ```bash 156 | -include-gtids 376b1ae7-39a1-11e9-a253-14187759814e:1-100 -exclude-gtids 376b1ae7-39a1-11e9-a253-14187759814e:1-20 157 | ``` 158 | 159 | ### 配置文件 160 | 161 | ```yaml 162 | filters: 163 | include-gtids: 376b1ae7-39a1-11e9-a253-14187759814e:1-100 164 | exclude-gtids: 376b1ae7-39a1-11e9-a253-14187759814e:1-20,376b1ae7-39a1-11e9-a253-14187759814e:40-60 165 | ``` 166 | -------------------------------------------------------------------------------- /README_EN.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | ![lightning](https://github.githubassets.com/images/icons/emoji/unicode/26a1.png) lightning is developed and maintained by Ke's DBA Team. It's a tool for binlog parsing. It can generate rollback SQL if BINLOG_FORMAT=ROW, and also binlog statistics. Lua self develop plugin is also supportted, which can do what you can imagine. 4 | 5 | ## Scenario 6 | 7 | * Data modification error, need to quickly rollback (flashback). 8 | * DELETE, UPDATE with no WHERE condition. 9 | * UPDATE SET connect with AND. 10 | * Data is abnormal, find at which time it changed? 11 | * binlog statistic, find which table update most. 12 | * filter specified table's query. 13 | * Repair of lost data from master failure. 14 | * Generate self define format SQL from binlog. 15 | * Find out if the database has a large transaction (Size) or a long transaction (Time) at certain time. 16 | 17 | ## Advance 18 | 19 | * Cross operation system support. 20 | * Lua plugin supported, self develop friendly. 21 | * Schema info can load from file, can parse binlog file offline. 22 | * SQL format self definable, can be filtered line by line. 23 | 24 | ## Installation 25 | 26 | ### Binary Install 27 | 28 | lightning developed with Go 1.11+，no matter you are using with Windows, Linux, MySQL, just downloading the released binary file and happy to work with it. 29 | 30 | [Download](https://github.com/LianjiaTech/lightning/releases) 31 | 32 | ### Source Code Install 33 | 34 | ```bash 35 | go get -d github.com/LianjiaTech/lightning 36 | cd ${PATH_TO_SOURCE}/lightning 37 | make 38 | ``` 39 | 40 | ## Try it 41 | 42 | [Useful command](http://github.com/LianjiaTech/lightning/blob/master/doc/cmd.md) 43 | 44 | Read binlog event from file and generate rollback SQL. 45 | 46 | ```bash 47 | lightning -no-defaults \ 48 | -plugin flashback \ 49 | -start-datetime "2019-01-01 00:00:00" \ 50 | -stop-datetime "2019-01-01 00:01:00" \ 51 | -event-types delete,update \ 52 | -tables test.tb \ 53 | -schema-file schema.sql \ 54 | -binlog-file binlog.0000001 > flashback.sql 55 | ``` 56 | 57 | Send `Binlog Dump` command and simulate as slave genearate rollback SQL. 58 | 59 | ```bash 60 | cat > master.info 61 | master_host: 127.0.0.1 62 | master_user: root 63 | master_password: ****** 64 | master_port: 3306 65 | master_log_file: binlog.000002 66 | master_log_pos: 4 67 | +D 68 | 69 | lightning -no-defaults \ 70 | -plugin flashback \ 71 | -start-datetime "2019-01-01 00:00:00" \ 72 | -stop-datetime "2019-01-01 00:01:00" \ 73 | -event-types delete,update \ 74 | -tables test.tb \ 75 | -master-info master.info > flashback.sql 76 | ``` 77 | 78 | ## Configuration 79 | 80 | lightning's config file is YAML formated. Configure file load sequence: /etc/lightning.yaml -> ./etc/lightning.yaml -> ./lightning.yaml. With `-config` argument can find new config file, with `-no-defaults` can disable all default config. 81 | 82 | * [Global Config](http://github.com/LianjiaTech/lightning/blob/master/doc/global.md) 83 | * [MySQL Config](http://github.com/LianjiaTech/lightning/blob/master/doc/mysql.md) 84 | * [Filter Config](http://github.com/LianjiaTech/lightning/blob/master/doc/filters.md) 85 | * [Rebuild Config](http://github.com/LianjiaTech/lightning/blob/master/doc/rebuild.md) 86 | 87 | ## Limitation 88 | 89 | * binlog version only support v4 (MySQL 5.1+), no test with(<= MySQL 5.0) 90 | * BINLOG_FORMAT = ROW 91 | * BINLOG_ROW_IMAGE = FULL 92 | * for binlog parsing performance not better than mysqlbinlog itself. 93 | 94 | ## Communication 95 | 96 | * [FAQ](http://github.com/LianjiaTech/lightning/blob/master/doc/FAQ.md) 97 | * Welcome feed back with Github Issues. 98 | * QQ Group： 573877257 99 | 100 | ![QQ](http://github.com/LianjiaTech/lightning/raw/master/doc/qq_group.png) 101 | 102 | ## License 103 | 104 | [Apache License 2.0](http://github.com/LianjiaTech/lightning/blob/master/LICENSE) 105 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # colors compatible setting 2 | CRED:=$(shell tput setaf 1 2>/dev/null) 3 | CGREEN:=$(shell tput setaf 2 2>/dev/null) 4 | CYELLOW:=$(shell tput setaf 3 2>/dev/null) 5 | CEND:=$(shell tput sgr0 2>/dev/null) 6 | 7 | # Add mysql version for testing `MYSQL_RELEASE=percona MYSQL_VERSION=5.7 make docker` 8 | # MySQL 5.1 `MYSQL_RELEASE=vsamov/mysql-5.1.73 make docker` 9 | # MYSQL_RELEASE: mysql, percona, mariadb ... 10 | # MYSQL_VERSION: latest, 8.0, 5.7, 5.6, 5.5 ... 11 | # use mysql:latest as default 12 | MYSQL_RELEASE := $(or ${MYSQL_RELEASE}, ${MYSQL_RELEASE}, mysql) 13 | MYSQL_VERSION := $(or ${MYSQL_VERSION}, ${MYSQL_VERSION}, latest) 14 | 15 | # Build the project 16 | .PHONY: build 17 | build: 18 | @echo "$(CGREEN)Building ...$(CEND)" 19 | @bash ./genver.sh 20 | @ret=0 && for d in $$(go list -f '{{if (eq .Name "main")}}{{.ImportPath}}{{end}}' ./...); do \ 21 | b=$$(basename $${d}) ; \ 22 | go build ${GCFLAGS} -o $${b} $$d || ret=$$? ; \ 23 | done ; exit $$ret 24 | @echo "build Success!" 25 | 26 | .PHONY: release 27 | release: build 28 | @echo "$(CGREEN)Cross platform building for release ...$(CEND)" 29 | @mkdir -p release 30 | @for GOOS in darwin linux; do \ 31 | for GOARCH in amd64; do \ 32 | for d in $$(go list -f '{{if (eq .Name "main")}}{{.ImportPath}}{{end}}' ./...); do \ 33 | b=$$(basename $${d}) ; \ 34 | echo "Building $${b}.$${GOOS}-$${GOARCH} ..."; \ 35 | GOOS=$${GOOS} GOARCH=$${GOARCH} go build ${GCFLAGS} ${LDFLAGS} -v -o release/$${b}.$${GOOS}-$${GOARCH} $$d 2>/dev/null ; \ 36 | done ; \ 37 | done ;\ 38 | done 39 | 40 | # Code format 41 | .PHONY: fmt 42 | fmt: 43 | @echo "$(CGREEN)Run gofmt on all source files ...$(CEND)" 44 | @echo "gofmt -l -s -w ..." 45 | @ret=0 && for d in $$(go list -f '{{.Dir}}' ./... | grep -v /vendor/); do \ 46 | gofmt -l -s -w $$d/*.go || ret=$$? ; \ 47 | done ; exit $$ret 48 | 49 | # Run golang test cases 50 | .PHONY: test 51 | test: 52 | @echo "$(CGREEN)Run all test cases ...$(CEND)" 53 | go test -timeout 10m -race ./... 54 | @git checkout common/fixture/master.info etc/master.info 55 | @echo "test Success!" 56 | 57 | # Code Coverage 58 | # colorful coverage numerical >=90% GREEN, <80% RED, Other YELLOW 59 | .PHONY: cover 60 | cover: test 61 | @echo "$(CGREEN)Run test cover check ...$(CEND)" 62 | go test -coverpkg=./... -coverprofile=coverage.data ./... | column -t 63 | go tool cover -html=coverage.data -o coverage.html 64 | go tool cover -func=coverage.data -o coverage.txt 65 | @tail -n 1 coverage.txt | awk '{sub(/%/, "", $$NF); \ 66 | if($$NF < 80) \ 67 | {print "$(CRED)"$$0"%$(CEND)"} \ 68 | else if ($$NF >= 90) \ 69 | {print "$(CGREEN)"$$0"%$(CEND)"} \ 70 | else \ 71 | {print "$(CYELLOW)"$$0"%$(CEND)"}}' 72 | 73 | .PHONY: docker 74 | docker: 75 | @echo "$(CGREEN)Build mysql test environment ...$(CEND)" 76 | @docker stop lightning-mysql 2>/dev/null || true 77 | @docker wait lightning-mysql 2>/dev/null >/dev/null || true 78 | @echo "docker run --name lightning-mysql $(MYSQL_RELEASE):$(MYSQL_VERSION)" 79 | @docker run --name lightning-mysql --rm -d \ 80 | -e MYSQL_ROOT_PASSWORD='******' \ 81 | -e MYSQL_DATABASE=test \ 82 | -p 3306:3306 \ 83 | -v `pwd`/test/schema.sql:/docker-entrypoint-initdb.d/schema.sql \ 84 | -v `pwd`/test/init.sql:/docker-entrypoint-initdb.d/init.sql \ 85 | $(MYSQL_RELEASE):$(MYSQL_VERSION) 86 | 87 | @echo "waiting for test database initializing " 88 | @timeout=180; while [ $${timeout} -gt 0 ] ; do \ 89 | if ! docker exec lightning-mysql mysql --user=root --password='******' --host "127.0.0.1" --silent -NBe "do 1" >/dev/null 2>&1 ; then \ 90 | timeout=`expr $$timeout - 1`; \ 91 | printf '.' ; sleep 1 ; \ 92 | else \ 93 | echo "." ; echo "mysql test environment is ready!" ; break ; \ 94 | fi ; \ 95 | if [ $$timeout = 0 ] ; then \ 96 | echo "." ; echo "$(CRED)docker lightning-mysql start timeout(180 s)!$(CEND)" ; exit 1 ; \ 97 | fi ; \ 98 | done 99 | 100 | .PHONY: docker-connect 101 | docker-connect: 102 | @docker exec -it lightning-mysql env LANG=C.UTF-8 mysql --user=root --password='******' --host "127.0.0.1" test 103 | 104 | # attach docker container with bash interactive mode 105 | .PHONY: docker-it 106 | docker-it: 107 | docker exec -it lightning-mysql /bin/bash 108 | 109 | # Installs our project: copies binaries 110 | install: build 111 | @echo "$(CGREEN)Install ...$(CEND)" 112 | go install ./... 113 | @echo "install Success!" 114 | -------------------------------------------------------------------------------- /common/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package common 15 | 16 | import ( 17 | "encoding/json" 18 | "fmt" 19 | "regexp" 20 | "runtime" 21 | "strings" 22 | 23 | "github.com/astaxie/beego/logs" 24 | disablelog "github.com/siddontang/go-log/log" 25 | ) 26 | 27 | // Log 使用 beego 的 log 库 28 | var Log *logs.BeeLogger 29 | 30 | func init() { 31 | Log = logs.NewLogger(0) 32 | Log.EnableFuncCallDepth(true) 33 | disablelog.SetLevel(disablelog.LevelFatal) 34 | } 35 | 36 | // loggerInit Log配置初始化 37 | func loggerInit() { 38 | Log.SetLevel(Config.Global.LogLevel) 39 | func() { _ = Log.DelLogger(logs.AdapterFile) }() 40 | logConfig := map[string]interface{}{ 41 | "filename": Config.Global.LogOutput, 42 | "level": 7, 43 | "maxlines": 0, 44 | "maxsize": 0, 45 | "daily": false, 46 | "maxdays": 0, 47 | } 48 | logConfigJSON, _ := json.Marshal(logConfig) 49 | err := Log.SetLogger(logs.AdapterFile, string(logConfigJSON)) 50 | if err != nil { 51 | fmt.Println(err.Error()) 52 | } 53 | } 54 | 55 | // Caller returns the caller of the function that called it :) 56 | // https://stackoverflow.com/questions/35212985/is-it-possible-get-information-about-caller-function-in-golang 57 | func Caller() string { 58 | // we get the callers as uintptrs - but we just need 1 59 | fpcs := make([]uintptr, 1) 60 | 61 | // skip 3 levels to get to the caller of whoever called Caller() 62 | n := runtime.Callers(3, fpcs) 63 | if n == 0 { 64 | return "n/a" // proper error her would be better 65 | } 66 | 67 | // get the info of the actual function that's in the pointer 68 | fun := runtime.FuncForPC(fpcs[0] - 1) 69 | if fun == nil { 70 | return "n/a" 71 | } 72 | 73 | // return its name 74 | return fun.Name() 75 | } 76 | 77 | // GetFunctionName 获取调当前函数名 78 | func GetFunctionName() string { 79 | // Skip this function, and fetch the PC and file for its parent 80 | pc, _, _, _ := runtime.Caller(1) 81 | // Retrieve a Function object this functions parent 82 | functionObject := runtime.FuncForPC(pc) 83 | // Regex to extract just the function name (and not the module path) 84 | extractFnName := regexp.MustCompile(`^.*\.(.*)$`) 85 | fnName := extractFnName.ReplaceAllString(functionObject.Name(), "$1") 86 | return fnName 87 | } 88 | 89 | // fileName get filename from path 90 | func fileName(original string) string { 91 | i := strings.LastIndex(original, "/") 92 | if i == -1 { 93 | return original 94 | } 95 | return original[i+1:] 96 | } 97 | 98 | // LogIfError 简化if err != nil 打 Error 日志代码长度 99 | func LogIfError(err error, format string, v ...interface{}) { 100 | if err != nil { 101 | _, fn, line, _ := runtime.Caller(1) 102 | if format == "" { 103 | format = "[%s:%d] %s" 104 | Log.Error(format, fileName(fn), line, err.Error()) 105 | } else { 106 | format = "[%s:%d] " + format + " Error: %s" 107 | Log.Error(format, fileName(fn), line, v, err.Error()) 108 | } 109 | } 110 | } 111 | 112 | // LogIfWarn 简化if err != nil 打 Warn 日志代码长度 113 | func LogIfWarn(err error, format string, v ...interface{}) { 114 | if err != nil { 115 | _, fn, line, _ := runtime.Caller(1) 116 | if format == "" { 117 | format = "[%s:%d] %s" 118 | Log.Warn(format, fileName(fn), line, err.Error()) 119 | } else { 120 | format = "[%s:%d] " + format + " Error: %s" 121 | Log.Warn(format, fileName(fn), line, v, err.Error()) 122 | } 123 | } 124 | } 125 | 126 | // Verbose ... 127 | func Verbose(format string, a ...interface{}) { 128 | if Config.Global.Verbose || Config.Global.VerboseVerbose { 129 | if !strings.HasSuffix(format, "\n") { 130 | format += "\n" 131 | } 132 | fmt.Printf(format, a...) 133 | } 134 | } 135 | 136 | // VerboseVerbose ... 137 | func VerboseVerbose(format string, a ...interface{}) { 138 | if Config.Global.VerboseVerbose { 139 | if !strings.HasSuffix(format, "\n") { 140 | format += "\n" 141 | } 142 | fmt.Printf(format, a...) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /doc/rebuild.md: -------------------------------------------------------------------------------- 1 | # 重建 SQL 2 | 3 | lightning 内建支持两种重建规则以及 SQL 类型统计功能，同时支持是 lua 插件形式进行自定义二次开发。 4 | 5 | * sql: 生成 ROW 格式对应的原始 SQL 6 | * flashback: 生成数据闪回 SQL，即：INSERT -> DELETE, DELETE -> INSERT, UPDATE WHERE 和 SET 互换。 7 | * stat: 按表统计各表的请求类型 8 | 9 | 注：MySQL 高版本（5.6.2+）支持 `binlog_rows_query_log_events` 参数，该参数默认是关闭的，开启后可以在 binlog 中记录原始 SQL 请求，不需要使用其他工具进行复原效果更好。 10 | 11 | ## 差异 12 | 13 | * ENUM, SET, BIT 使用整型替代，不影响数据一致性。 14 | * DECIMAL 使用 float 替代，不影响精度。 15 | 16 | ## 配置文件 17 | 18 | ```yaml 19 | # 重建规则 20 | rebuild: 21 | # 插件：sql, flashback, stat, lua 22 | plugin: sql 23 | # INSERT 语句是否补全列 24 | complete-insert: false 25 | # INSERT 语句多个 VALUES 合并 26 | extended-insert-count: 0 27 | # 使用 REPLACE INTO 替代 INSERT INTO 28 | replace: false 29 | # 两条 SQL 语句之前添加 sleep 间隔，，最小精度 us 30 | sleep-interval: 0s 31 | # 生成 SQL 语句省略某些列，如： INSERT 忽略主键 32 | ignore-columns: 33 | - id 34 | # lua 插件脚本位置 35 | lua-script: plugin/demo.flashback.lua 36 | # 对表名进行简写，如：`db`.`tb` -> `tb`，可以用在测试库做预恢复的场景 37 | without-db-name: false 38 | ``` 39 | 40 | ## 示例 41 | 42 | ROW 格式 binlog 生成 SQL 更新语句。不指定 `-plugin` 默认即为该模式。 43 | 44 | ```bash 45 | lightning -no-defaults -schema-file test/schema.sql -binlog-file test/binlog.000002 46 | ``` 47 | 48 | ROW 格式 binlog 生成回滚语句。 49 | 50 | ```bash 51 | lightning -no-defaults -plugin flashback -schema-file test/schema.sql -binlog-file test/binlog.000002 52 | ``` 53 | 54 | ## 统计分析 55 | 56 | ### 统计各库表更新语句数量 57 | 58 | ```bash 59 | lightning -no-defaults -plugin stat -binlog-file test/binlog.000002 60 | 61 | { 62 | "TableStats": { 63 | "`test`.`bitTest`": { 64 | "insert": 1 65 | }, 66 | "`test`.`enumTest`": { 67 | "insert": 1 68 | }, 69 | "`test`.`setTest`": { 70 | "insert": 3 71 | }, 72 | "`test`.`tb`": { 73 | "delete": 1, 74 | "insert": 1, 75 | "update": 1 76 | }, 77 | "`test`.`testNoPRI`": { 78 | "insert": 1 79 | } 80 | }, 81 | "QueryStats": { 82 | "ALTER": 1, 83 | "BEGIN": 9, 84 | "CREATE": 6, 85 | "DROP": 4 86 | }, 87 | "TransactionStats": { 88 | "SizeBytes": { 89 | "Max": "141.0", 90 | "MaxTransactionPos": "1292", 91 | "Mean": "131.3", 92 | "Median": "129.0", 93 | "P95": "139.5", 94 | "P99": "139.5" 95 | }, 96 | "TimeSeconds": { 97 | "Max": "0.00", 98 | "MaxTransactionPos": "0", 99 | "Mean": "0.00", 100 | "Median": "0.00", 101 | "P95": "0.00", 102 | "P99": "0.00" 103 | } 104 | } 105 | } 106 | ``` 107 | 108 | ### 使用 mysqlbinlog + awk 分析 109 | 110 | 参考: [Identifying useful info from MySQL row-based binary logs](https://www.percona.com/blog/2015/01/20/identifying-useful-information-mysql-row-based-binary-logs/) 111 | 112 | Q1: Which tables received highest number of insert/update/delete statements? 113 | 114 | ```bash 115 | ./summarize_binlogs.sh | grep Table |cut -d':' -f5| cut -d' ' -f2 | sort | uniq -c | sort -nr 116 | ``` 117 | 118 | Q2: Which table received the highest number of DELETE queries? 119 | 120 | ```bash 121 | ./summarize_binlogs.sh | grep -E 'DELETE' |cut -d':' -f5| cut -d' ' -f2 | sort | uniq -c | sort -nr 122 | ``` 123 | 124 | Q3: How many insert/update/delete queries executed against sakila.country table? 125 | 126 | ```bash 127 | ./summarize_binlogs.sh | grep -i '`sakila`.`country`' | awk '{print $7 " " $11}' | sort -k1,2 | uniq -c 128 | ``` 129 | 130 | Q4: Give me the top 3 statements which affected maximum number of rows. 131 | 132 | ```bash 133 | ./summarize_binlogs.sh | grep Table | sort -nr -k 12 | head -n 3 134 | ``` 135 | 136 | Q5: Find DELETE queries that affected more than 1000 rows. 137 | 138 | ```bash 139 | ./summarize_binlogs.sh | grep -E 'DELETE' | awk '{if($12>1000) print $0}' 140 | 141 | ./summarize_binlogs.sh | grep -E 'Table' | awk '{if($12>1000) print $0}' 142 | ``` 143 | 144 | ### 使用 mysqlbinlog + pt-query-digest 分析 145 | 146 | 参考： 147 | 148 | * [pt-query-digest](https://www.percona.com/doc/percona-toolkit/LATEST/pt-query-digest.html) 149 | * [pt-query-digest 解析 MySQL Binlog 日志文件](https://blog.csdn.net/dba_waterbin/article/details/14453255) 150 | 151 | ```bash 152 | mysqlbinlog mysql-bin.000441 > mysql-bin.000441.txt 153 | 154 | pt-query-digest --type binlog mysql-bin.000441.txt 155 | 156 | pt-query-digest --type binlog --since "2019-01-06 20:55:00" --until "2019-01-06 21:00:00" mysql-bin.000441.txt 157 | 158 | pt-query-digest --type binlog --group-by fingerprint --limit "100%" --order-by "Query_time:cnt" --output report --report-format profile mysql-bin.000441.txt 159 | ``` 160 | -------------------------------------------------------------------------------- /rebuild/delete.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "fmt" 18 | "strings" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | 22 | "github.com/go-mysql-org/go-mysql/replication" 23 | lua "github.com/yuin/gopher-lua" 24 | ) 25 | 26 | // DeleteRebuild ... 27 | func DeleteRebuild(event *replication.BinlogEvent) string { 28 | switch common.Config.Rebuild.Plugin { 29 | case "sql": 30 | DeleteQuery(event) 31 | case "flashback": 32 | DeleteRollbackQuery(event) 33 | case "stat": 34 | DeleteStat(event) 35 | case "lua": 36 | DeleteLua(event) 37 | default: 38 | } 39 | return "" 40 | } 41 | 42 | // DeleteQuery build original delete SQL 43 | func DeleteQuery(event *replication.BinlogEvent) { 44 | var table string 45 | defer func() { 46 | if r := recover(); r != nil { 47 | fmt.Printf("-- Table: %s, Error: %s\n", table, strings.Split(fmt.Sprint(r), "\n")[0]) 48 | } 49 | }() 50 | table = RowEventTable(event) 51 | ev := event.Event.(*replication.RowsEvent) 52 | values := BuildValues(ev) 53 | 54 | common.Verbose("-- [DEBUG] event: delete, table: %s, rows: %d\n", table, len(values)) 55 | 56 | deleteQuery(table, values) 57 | } 58 | 59 | func deleteQuery(table string, values [][]string) { 60 | // for common.Config.Rebuild.WithoutDBName 61 | shortTableName := onlyTable(table) 62 | 63 | var deletePrefix = "DELETE FROM" 64 | if common.Config.Rebuild.ForeachTime && common.Config.Rebuild.CurrentEventTime != "" { 65 | deletePrefix = fmt.Sprintf(`/* %s */%s`, common.Config.Rebuild.CurrentEventTime, deletePrefix) 66 | } 67 | 68 | if ok := PrimaryKeys[table]; ok != nil { 69 | for _, value := range values { 70 | var where []string 71 | for _, col := range PrimaryKeys[table] { 72 | for i, c := range Columns[table] { 73 | if c == col { 74 | if value[i] == "NULL" { 75 | where = append(where, fmt.Sprintf("%s IS NULL", col)) 76 | } else { 77 | where = append(where, fmt.Sprintf("%s = %s", col, value[i])) 78 | } 79 | } 80 | } 81 | } 82 | 83 | if common.Config.Rebuild.WithoutDBName { 84 | fmt.Printf("%s %s WHERE %s LIMIT 1;\n", deletePrefix, shortTableName, strings.Join(where, " AND ")) 85 | } else { 86 | fmt.Printf("%s %s WHERE %s LIMIT 1;\n", deletePrefix, table, strings.Join(where, " AND ")) 87 | } 88 | } 89 | } else { 90 | for _, value := range values { 91 | var where []string 92 | for i, v := range value { 93 | col := fmt.Sprintf("@%d", i) 94 | if v == "NULL" { 95 | where = append(where, fmt.Sprintf("%s IS NULL", col)) 96 | } else { 97 | where = append(where, fmt.Sprintf("%s = %s", col, v)) 98 | } 99 | } 100 | if common.Config.Rebuild.WithoutDBName { 101 | fmt.Printf("-- %s %s WHERE %s LIMIT 1;\n", deletePrefix, shortTableName, strings.Join(where, " AND ")) 102 | } else { 103 | fmt.Printf("-- %s %s WHERE %s LIMIT 1;\n", deletePrefix, table, strings.Join(where, " AND ")) 104 | } 105 | } 106 | } 107 | } 108 | 109 | // DeleteRollbackQuery build rollback insert SQL 110 | func DeleteRollbackQuery(event *replication.BinlogEvent) { 111 | var table string 112 | defer func() { 113 | if r := recover(); r != nil { 114 | fmt.Printf("-- Table: %s, Error: %s\n", table, strings.Split(fmt.Sprint(r), "\n")[0]) 115 | } 116 | }() 117 | table = RowEventTable(event) 118 | ev := event.Event.(*replication.RowsEvent) 119 | values := BuildValues(ev) 120 | 121 | insertQuery(table, values) 122 | } 123 | 124 | // DeleteStat ... 125 | func DeleteStat(event *replication.BinlogEvent) { 126 | table := RowEventTable(event) 127 | if TableStats[table] != nil { 128 | TableStats[table]["delete"]++ 129 | } else { 130 | TableStats[table] = map[string]int64{"delete": 1} 131 | } 132 | 133 | ev := event.Event.(*replication.RowsEvent) 134 | values := BuildValues(ev) 135 | if RowsStats[table] != nil { 136 | RowsStats[table]["delete"] += int64(len(values)) 137 | } else { 138 | RowsStats[table] = map[string]int64{"delete": int64(len(values))} 139 | } 140 | } 141 | 142 | // DeleteLua ... 143 | func DeleteLua(event *replication.BinlogEvent) { 144 | if common.Config.Rebuild.LuaScript == "" || event == nil { 145 | return 146 | } 147 | 148 | table := RowEventTable(event) 149 | ev := event.Event.(*replication.RowsEvent) 150 | values := BuildValues(ev) 151 | 152 | // lua function 153 | f := lua.P{ 154 | Fn: Lua.GetGlobal("DeleteRewrite"), 155 | NRet: 0, 156 | Protect: true, 157 | } 158 | // lua value 159 | v := lua.LString(table) 160 | for _, value := range values { 161 | LuaStringList("GoValues", value) 162 | if err := Lua.CallByParam(f, v); err != nil { 163 | common.Log.Error(err.Error()) 164 | return 165 | } 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /rebuild/insert.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "fmt" 18 | "strings" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | 22 | "github.com/go-mysql-org/go-mysql/replication" 23 | lua "github.com/yuin/gopher-lua" 24 | ) 25 | 26 | // InsertRebuild ... 27 | func InsertRebuild(event *replication.BinlogEvent) string { 28 | switch common.Config.Rebuild.Plugin { 29 | case "sql": 30 | InsertQuery(event) 31 | case "flashback": 32 | InsertRollbackQuery(event) 33 | case "stat": 34 | InsertStat(event) 35 | case "lua": 36 | InsertLua(event) 37 | default: 38 | } 39 | return "" 40 | } 41 | 42 | // InsertQuery ... 43 | func InsertQuery(event *replication.BinlogEvent) { 44 | var table string 45 | defer func() { 46 | if r := recover(); r != nil { 47 | fmt.Printf("-- Table: %s, Error: %s\n", table, strings.Split(fmt.Sprint(r), "\n")[0]) 48 | } 49 | }() 50 | table = RowEventTable(event) 51 | ev := event.Event.(*replication.RowsEvent) 52 | values := BuildValues(ev) 53 | insertQuery(table, values) 54 | } 55 | 56 | func insertQuery(table string, values [][]string) { 57 | var insertPrefix string 58 | if common.Config.Rebuild.Replace { 59 | insertPrefix = "REPLACE INTO" 60 | } else { 61 | insertPrefix = "INSERT INTO" 62 | } 63 | if common.Config.Rebuild.ForeachTime && common.Config.Rebuild.CurrentEventTime != "" { 64 | insertPrefix = fmt.Sprintf(`/* %s */%s`, common.Config.Rebuild.CurrentEventTime, insertPrefix) 65 | } 66 | 67 | // for common.Config.Rebuild.WithoutDBName 68 | shortTableName := onlyTable(table) 69 | 70 | colStr := "" 71 | for row, v := range values { 72 | valStr := "" 73 | if common.Config.Rebuild.CompleteInsert { 74 | if ok := Columns[table]; ok != nil { 75 | if len(common.Config.Rebuild.IgnoreColumns) > 0 { 76 | var truncValues, truncColumns []string 77 | for i, col := range Columns[table] { 78 | ignore := false 79 | for _, c := range common.Config.Rebuild.IgnoreColumns { 80 | if c == strings.Trim(col, "`") { 81 | ignore = true 82 | } 83 | } 84 | if !ignore { 85 | truncColumns = append(truncColumns, col) 86 | truncValues = append(truncValues, v[i]) 87 | } 88 | } 89 | colStr = fmt.Sprintf("(%s)", strings.Join(truncColumns, ", ")) 90 | valStr = strings.Join(truncValues, ", ") 91 | } else { 92 | colStr = fmt.Sprintf("(%s)", strings.Join(Columns[table], ", ")) 93 | valStr = strings.Join(v, ", ") 94 | } 95 | } else { 96 | valStr = strings.Join(v, ", ") 97 | } 98 | } else { 99 | valStr = strings.Join(v, ", ") 100 | } 101 | 102 | if common.Config.Rebuild.ExtendedInsertCount > 1 { 103 | InsertValuesMerge = append(InsertValuesMerge, fmt.Sprintf("(%s)", valStr)) 104 | } else { 105 | if common.Config.Rebuild.WithoutDBName { 106 | fmt.Printf("%s %s %s VALUES (%s);\n", insertPrefix, shortTableName, colStr, valStr) 107 | } else { 108 | fmt.Printf("%s %s %s VALUES (%s);\n", insertPrefix, table, colStr, valStr) 109 | } 110 | } 111 | 112 | // INSERT VALUES merge 113 | if row != 0 && common.Config.Rebuild.ExtendedInsertCount > 1 && 114 | (row+1)%common.Config.Rebuild.ExtendedInsertCount == 0 { 115 | if common.Config.Rebuild.WithoutDBName { 116 | fmt.Printf("%s %s %s VALUES %s;\n", insertPrefix, shortTableName, colStr, strings.Join(InsertValuesMerge, ", ")) 117 | } else { 118 | fmt.Printf("%s %s %s VALUES %s;\n", insertPrefix, table, colStr, strings.Join(InsertValuesMerge, ", ")) 119 | } 120 | InsertValuesMerge = []string{} 121 | } 122 | } 123 | if len(InsertValuesMerge) > 0 { 124 | if common.Config.Rebuild.WithoutDBName { 125 | fmt.Printf("%s %s %s VALUES %s;\n", insertPrefix, shortTableName, colStr, strings.Join(InsertValuesMerge, ", ")) 126 | } else { 127 | fmt.Printf("%s %s %s VALUES %s;\n", insertPrefix, table, colStr, strings.Join(InsertValuesMerge, ", ")) 128 | } 129 | InsertValuesMerge = []string{} 130 | } 131 | } 132 | 133 | // InsertRollbackQuery ... 134 | func InsertRollbackQuery(event *replication.BinlogEvent) { 135 | var table string 136 | defer func() { 137 | if r := recover(); r != nil { 138 | fmt.Printf("-- Table: %s, Error: %s\n", table, strings.Split(fmt.Sprint(r), "\n")[0]) 139 | } 140 | }() 141 | table = RowEventTable(event) 142 | ev := event.Event.(*replication.RowsEvent) 143 | values := BuildValues(ev) 144 | 145 | common.Verbose("-- [DEBUG] event: insert, table: %s, rows: %d\n", table, len(values)) 146 | 147 | deleteQuery(table, values) 148 | } 149 | 150 | // InsertStat ... 151 | func InsertStat(event *replication.BinlogEvent) { 152 | table := RowEventTable(event) 153 | if TableStats[table] != nil { 154 | TableStats[table]["insert"]++ 155 | } else { 156 | TableStats[table] = map[string]int64{"insert": 1} 157 | } 158 | 159 | ev := event.Event.(*replication.RowsEvent) 160 | values := BuildValues(ev) 161 | if RowsStats[table] != nil { 162 | RowsStats[table]["insert"] += int64(len(values)) 163 | } else { 164 | RowsStats[table] = map[string]int64{"insert": int64(len(values))} 165 | } 166 | } 167 | 168 | // InsertLua ... 169 | func InsertLua(event *replication.BinlogEvent) { 170 | if common.Config.Rebuild.LuaScript == "" || event == nil { 171 | return 172 | } 173 | 174 | table := RowEventTable(event) 175 | ev := event.Event.(*replication.RowsEvent) 176 | values := BuildValues(ev) 177 | 178 | // lua function 179 | f := lua.P{ 180 | Fn: Lua.GetGlobal("InsertRewrite"), 181 | NRet: 0, 182 | Protect: true, 183 | } 184 | // lua value 185 | v := lua.LString(table) 186 | for _, value := range values { 187 | LuaStringList("GoValues", value) 188 | if err := Lua.CallByParam(f, v); err != nil { 189 | common.Log.Error(err.Error()) 190 | return 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /rebuild/query.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "fmt" 18 | "strings" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | "github.com/go-mysql-org/go-mysql/replication" 22 | lua "github.com/yuin/gopher-lua" 23 | 24 | "github.com/pingcap/parser" 25 | "github.com/pingcap/parser/ast" 26 | "github.com/pingcap/parser/mysql" 27 | 28 | // pingcap/parser 29 | _ "github.com/pingcap/tidb/types/parser_driver" 30 | ) 31 | 32 | // QueryRebuild rebuild sql, need pingcap/parser 33 | func QueryRebuild(queryEvent *replication.BinlogEvent) string { 34 | switch queryEvent.Header.EventType { 35 | case replication.QUERY_EVENT: 36 | default: 37 | common.Log.Error("QueryRebuild get wrong event: %s", queryEvent.Header.EventType.String()) 38 | return "" 39 | } 40 | 41 | event := queryEvent.Event.(*replication.QueryEvent) 42 | 43 | common.Verbose("-- [DEBUG] ThreadID: %d, Schema: %s, ErrorCode: %d, ExecutionTime: %d, GSet: %v\n", 44 | event.SlaveProxyID, event.Schema, event.ErrorCode, event.ExecutionTime, event.GSet) 45 | 46 | sql := string(event.Query) 47 | switch common.Config.Rebuild.Plugin { 48 | case "sql": 49 | // fmt.Printf("SET TIMESTAMP=%d;\n", queryEvent.Header.Timestamp) 50 | QueryFormat(sql) 51 | case "flashback": 52 | QueryRollback(sql) 53 | case "stat": 54 | if sql == "BEGIN" { 55 | TransactionStartPos = float64(queryEvent.Header.LogPos) 56 | TransactionStartTimeStamp = float64(queryEvent.Header.Timestamp) 57 | } 58 | QueryStat(sql) 59 | case "lua": 60 | QueryLua(sql) 61 | default: 62 | } 63 | 64 | // stat transaction time, exec_time on slave it's replication lag time. 65 | // https://dev.mysql.com/doc/refman/5.6/en/mysqlbinlog.html 66 | transactionTime := float64(event.ExecutionTime) 67 | if transactionTime > MaxTransactionTime { 68 | MaxTransactionTime = transactionTime 69 | MaxTransactionTimeStartPos = TransactionStartPos 70 | } 71 | TransactionTimeStats = append(TransactionTimeStats, transactionTime) 72 | 73 | return "" 74 | } 75 | 76 | // RowsQueryRebuild ... 77 | func RowsQueryRebuild(rowsQueryEvent *replication.BinlogEvent) string { 78 | switch rowsQueryEvent.Header.EventType { 79 | case replication.ROWS_QUERY_EVENT: 80 | default: 81 | common.Log.Error("RowsQueryRebuild get wrong event: %s", rowsQueryEvent.Header.EventType.String()) 82 | return "" 83 | } 84 | 85 | event := rowsQueryEvent.Event.(*replication.RowsQueryEvent) 86 | common.VerboseVerbose("-- [DEBUG] RowsQuery Event, Query: %s\n", string(event.Query)) 87 | return "" 88 | } 89 | 90 | // XidRebuild ... 91 | func XidRebuild(event *replication.BinlogEvent) string { 92 | // stat transaction size 93 | transactionSize := float64(event.Header.LogPos) - TransactionStartPos 94 | if transactionSize > MaxTransactionSize { 95 | MaxTransactionSizeStartPos = TransactionStartPos 96 | MaxTransactionSize = transactionSize 97 | } 98 | TransactionSizeStats = append(TransactionSizeStats, transactionSize) 99 | 100 | if MaxTransactionTimeStartPos == TransactionStartPos { 101 | MaxTransactionTimeStopPos = float64(event.Header.LogPos) 102 | } 103 | 104 | common.Verbose("-- [DEBUG] XID_EVENT TransactionSizeBytes: %s, Xid: %d, GSet: %v\n", 105 | fmt.Sprintf("%0.0f", transactionSize), event.Event.(*replication.XIDEvent).XID, event.Event.(*replication.XIDEvent).GSet) 106 | return "" 107 | } 108 | 109 | // TiParse TiDB 语法解析 110 | func TiParse(sql, charset, collation string) ([]ast.StmtNode, error) { 111 | p := parser.New() 112 | stmt, _, err := p.Parse(sql, charset, collation) 113 | return stmt, err 114 | } 115 | 116 | // QueryFormat ... 117 | func QueryFormat(sql string) { 118 | if strings.HasPrefix(sql, "BEGIN") { 119 | common.Verbose("-- [DEBUG] BEGIN;") 120 | return 121 | } 122 | 123 | if strings.HasSuffix(sql, ";") { 124 | fmt.Println(sql) 125 | } else { 126 | fmt.Println(sql, ";") 127 | } 128 | } 129 | 130 | func QueryRollback(sql string) { 131 | stmts, err := TiParse(sql, common.Config.Global.Charset, mysql.Charsets[common.Config.Global.Charset]) 132 | if err == nil { 133 | for _, stmt := range stmts { 134 | switch node := stmt.(type) { 135 | case *ast.CreateTableStmt: 136 | CreateTableRollback(node) 137 | case *ast.CreateDatabaseStmt: 138 | CreateDatabaseRollback(node) 139 | case *ast.CreateIndexStmt: 140 | CreateIndexRollback(node) 141 | case *ast.CreateViewStmt: 142 | CreateViewRollback(node) 143 | // case *ast.AlterTableStmt: 144 | // TODO: ALTER TABLE tb ADD col int; 145 | case *ast.BeginStmt: 146 | common.Verbose("-- [DEBUG] BEGIN;") 147 | default: 148 | common.VerboseVerbose("-- [DEBUG] can't rollback: %s;", sql) 149 | } 150 | } 151 | } else { 152 | common.Log.Error(err.Error()) 153 | } 154 | } 155 | 156 | // CreateTableRollback ... 157 | func CreateTableRollback(stmt *ast.CreateTableStmt) { 158 | if stmt.Table.Schema.String() == "" { 159 | fmt.Printf("DROP TABLE IF EXISTS `%s`;\n", stmt.Table.Name) 160 | } else { 161 | fmt.Printf("DROP TABLE IF EXISTS `%s`.`%s`;\n", stmt.Table.Schema, stmt.Table.Name) 162 | } 163 | } 164 | 165 | // CreateDatabaseRollback ... 166 | func CreateDatabaseRollback(stmt *ast.CreateDatabaseStmt) { 167 | fmt.Printf("DROP DATABASE IF EXISTS `%s`;\n", stmt.Name) 168 | } 169 | 170 | // CreateIndexRollback ... 171 | func CreateIndexRollback(stmt *ast.CreateIndexStmt) { 172 | if stmt.Table.Schema.String() == "" { 173 | fmt.Printf("DROP INDEX `%s` ON `%s`;\n", stmt.IndexName, stmt.Table.Name) 174 | } else { 175 | fmt.Printf("DROP INDEX `%s` ON `%s`.`%s`;\n", stmt.IndexName, stmt.Table.Schema, stmt.Table.Name) 176 | } 177 | } 178 | 179 | // CreateViewRollback ... 180 | func CreateViewRollback(stmt *ast.CreateViewStmt) { 181 | if stmt.ViewName.Schema.String() == "" { 182 | fmt.Printf("DROP VIEW IF EXISTS `%s`;\n", stmt.ViewName.Name) 183 | } else { 184 | fmt.Printf("DROP VIEW IF EXISTS `%s`.`%s`;\n", stmt.ViewName.Schema, stmt.ViewName.Name) 185 | } 186 | } 187 | 188 | // QueryStat ... 189 | func QueryStat(sql string) { 190 | // TODO: statement base table stat 191 | // stmt, err := TiParse(sql, common.Config.Global.Charset, mysql.Charsets[common.Config.Global.Charset]) 192 | t := strings.ToLower(strings.Fields(sql)[0]) 193 | 194 | // ignore comment in Query Event 195 | if t == "--" || strings.HasPrefix(t, "#") || strings.HasPrefix(t, "/*") { 196 | return 197 | } 198 | 199 | if QueryStats != nil { 200 | QueryStats[t]++ 201 | } else { 202 | QueryStats = map[string]int64{t: 1} 203 | } 204 | } 205 | 206 | // QueryLua ... 207 | func QueryLua(sql string) { 208 | if common.Config.Rebuild.LuaScript == "" || sql == "" { 209 | return 210 | } 211 | 212 | if err := Lua.CallByParam(lua.P{ 213 | Fn: Lua.GetGlobal("QueryRewrite"), 214 | NRet: 1, 215 | Protect: true, 216 | }, lua.LString(sql)); err != nil { 217 | common.Log.Error(err.Error()) 218 | return 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /event/decrypt.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "crypto/aes" 7 | "crypto/cipher" 8 | "crypto/sha512" 9 | "encoding/binary" 10 | "fmt" 11 | "io" 12 | "io/ioutil" 13 | "os" 14 | ) 15 | 16 | // Reference: 17 | // https://dev.mysql.com/blog-archive/how-to-manually-decrypt-an-encrypted-binary-log-file/ 18 | // https://mysql.wisborg.dk/2019/01/28/automatic-decryption-of-mysql-binary-logs-using-python/ 19 | 20 | // KeyRingXORStr mysql-server: plugin/keyring/common/keyring_key.cc obfuscate_str 21 | const KeyRingXORStr = `*305=Ljt0*!@$Hnm(*-9-w;:` 22 | 23 | // EncryptFileHeaderOffset encrypt file header size 24 | const EncryptFileHeaderOffset = 512 25 | 26 | type EncryptHeader struct { 27 | Version int 28 | KeyRingID []byte // keyring id 29 | Key []byte // key raw bytes 30 | IV []byte // 16 bytes 31 | Password []byte // 32 bytes 32 | } 33 | 34 | /* 35 | KeyRing ID Format: 36 | |<-----Keyword------|<---------------UUID--------------->|SEQ_NO| 37 | Keyring key ID for 'binlog.000003' is 'MySQLReplicationKey_d1deace2-24cc-11ea-a1db-0800270a0142_2' 38 | */ 39 | 40 | type KeyRing struct { 41 | Type []byte 42 | KeyID []byte 43 | UserID []byte 44 | Key []byte 45 | } 46 | 47 | // parseEncryptHeader parse encrypt binlog header 48 | func parseEncryptHeader(binlog string, keys []KeyRing) (enc EncryptHeader, err error) { 49 | fd, err := os.Open(binlog) 50 | if err != nil { 51 | return enc, err 52 | } 53 | defer fd.Close() 54 | 55 | // Check is encrypted binlog file 56 | bufFileHeader := make([]byte, FileHeaderLength) 57 | if _, err := io.ReadFull(fd, bufFileHeader); err != nil { 58 | return enc, err 59 | } 60 | if !CheckBinlogFileEncrypt(bufFileHeader) { 61 | return enc, fmt.Errorf("file not encrypted") 62 | } 63 | 64 | oneByte := make([]byte, 1) 65 | // Get the encrypted version 66 | if _, err = io.ReadFull(fd, oneByte); err != nil { 67 | return enc, err 68 | } 69 | enc.Version = int(oneByte[0]) 70 | if enc.Version != 1 { 71 | return enc, fmt.Errorf("wrong version: %d", enc.Version) 72 | } 73 | 74 | // First header field is a TLV: the keyring key ID 75 | if _, err = io.ReadFull(fd, oneByte); err != nil { 76 | return enc, err 77 | } 78 | fieldType := int(oneByte[0]) 79 | if fieldType != 1 { // First header 80 | return enc, fmt.Errorf("wrong field type: %d", fieldType) 81 | } 82 | if _, err = io.ReadFull(fd, oneByte); err != nil { 83 | return enc, err 84 | } 85 | id := make([]byte, int(oneByte[0])) 86 | if _, err = io.ReadFull(fd, id); err != nil { 87 | return enc, err 88 | } 89 | enc.KeyRingID = id 90 | 91 | // get tmp key from keyring 92 | tk := make([]byte, 32) 93 | for _, key := range keys { 94 | if bytes.EqualFold(key.KeyID, enc.KeyRingID) { 95 | tk = key.Key 96 | break 97 | } 98 | } 99 | 100 | // Second header is a TV: the encrypted file password 101 | if _, err = io.ReadFull(fd, oneByte); err != nil { 102 | return enc, err 103 | } 104 | fieldType = int(oneByte[0]) 105 | if fieldType != 2 { // Second header 106 | return enc, fmt.Errorf("wrong field type: %d", fieldType) 107 | } 108 | pass := make([]byte, 32) 109 | if _, err = io.ReadFull(fd, pass); err != nil { 110 | return enc, err 111 | } 112 | enc.Password = pass 113 | 114 | // Third header field is a TV: the IV to decrypt the file password 115 | if _, err = io.ReadFull(fd, oneByte); err != nil { 116 | return enc, err 117 | } 118 | fieldType = int(oneByte[0]) 119 | if fieldType != 3 { // Third header 120 | return enc, fmt.Errorf("wrong field type: %d", fieldType) 121 | } 122 | tiv := make([]byte, 16) // tmp iv 123 | if _, err = io.ReadFull(fd, tiv); err != nil { 124 | return enc, err 125 | } 126 | 127 | // generate key & iv 128 | enc.Password, err = decryptAESCBC(pass, tk, tiv) 129 | if err != nil { 130 | return enc, err 131 | } 132 | 133 | keyAndIV := sha512.Sum512(enc.Password) 134 | if len(keyAndIV) < 48 { 135 | return enc, fmt.Errorf("generate key & iv failed") 136 | } 137 | enc.Key = keyAndIV[:32] 138 | enc.IV = keyAndIV[32:48] 139 | for i := 8; i < len(enc.IV); i++ { 140 | enc.IV[i] = 0 141 | } 142 | 143 | return enc, err 144 | } 145 | 146 | func decryptAESCBC(src, key, iv []byte) ([]byte, error) { 147 | block, err := aes.NewCipher(key) 148 | mode := cipher.NewCBCDecrypter(block, iv) 149 | dst := make([]byte, len(src)) 150 | mode.CryptBlocks(dst, src) 151 | return dst, err 152 | } 153 | 154 | // parseKeyRing parse keyring file 155 | func parseKeyRing(keyring string) ([]KeyRing, error) { 156 | 157 | var keys []KeyRing 158 | fd, err := os.Open(keyring) 159 | if err != nil { 160 | return keys, err 161 | } 162 | defer fd.Close() 163 | 164 | body, err := ioutil.ReadAll(fd) 165 | if err != nil { 166 | return keys, err 167 | } 168 | 169 | // Verify the start of the file is "Keyring file version:" 170 | if string(body[:21]) != `Keyring file version:` { 171 | return keys, fmt.Errorf("wrong file format, not keyring") 172 | } 173 | 174 | // Get the keyring version - currently only 2.0 is supported 175 | if string(body[21:24]) != "2.0" { 176 | return keys, fmt.Errorf("wrong version of keyring") 177 | } 178 | 179 | // get keys from keyring 180 | for offset := 24; offset < len(body); { 181 | if string(body[offset:offset+3]) == "EOF" { 182 | break 183 | } 184 | 185 | k, l := getKeyFromKeyring(body[offset:]) 186 | keys = append(keys, k) 187 | offset += l 188 | } 189 | 190 | return keys, err 191 | } 192 | 193 | func getKeyFromKeyring(data []byte) (KeyRing, int) { 194 | var key KeyRing 195 | totalLength := int(binary.LittleEndian.Uint32(data[:8])) 196 | keyIDLength := data[8:16] 197 | keyTypeLength := data[16:24] 198 | userIDLength := data[24:32] 199 | keyLength := data[32:40] 200 | 201 | keyIDStart := 40 202 | keyTypeStart := keyIDStart + int(binary.LittleEndian.Uint32(keyIDLength)) 203 | userIDStart := keyTypeStart + int(binary.LittleEndian.Uint32(keyTypeLength)) 204 | keyStart := userIDStart + int(binary.LittleEndian.Uint32(userIDLength)) 205 | keyEnd := keyStart + int(binary.LittleEndian.Uint32(keyLength)) 206 | 207 | key.KeyID = data[keyIDStart:keyTypeStart] 208 | key.Type = data[keyTypeStart:userIDStart] 209 | // User ID may be blank in which case the length is zero 210 | key.UserID = data[userIDStart:keyStart] 211 | key.Key = data[keyStart:keyEnd] 212 | 213 | // XOR_STR 214 | for i, v := range key.Key { 215 | key.Key[i] = v ^ KeyRingXORStr[i%len(KeyRingXORStr)] 216 | } 217 | return key, totalLength 218 | } 219 | 220 | func initAESCTRStream(binlog, keyring string) (cipher.Stream, error) { 221 | var stream cipher.Stream 222 | keys, err := parseKeyRing(keyring) 223 | if err != nil { 224 | return stream, err 225 | } 226 | 227 | header, err := parseEncryptHeader(binlog, keys) 228 | if err != nil { 229 | return stream, err 230 | } 231 | 232 | block, err := aes.NewCipher(header.Key) 233 | if err != nil { 234 | return stream, err 235 | } 236 | stream = cipher.NewCTR(block, header.IV) 237 | return stream, err 238 | } 239 | 240 | // decryptAESCTR decrypt single encrypted block 241 | func decryptAESCTR(stream cipher.Stream, src []byte) []byte { 242 | dst := make([]byte, len(src)) 243 | stream.XORKeyStream(dst, src) 244 | return dst 245 | } 246 | 247 | // DecryptBinlog decrypt binlog file with keyring 248 | func DecryptBinlog(orgBinlog, keyring string) error { 249 | stream, err := initAESCTRStream(orgBinlog, keyring) 250 | if err != nil { 251 | return err 252 | } 253 | 254 | ofd, err := os.Open(orgBinlog) 255 | if err != nil { 256 | return err 257 | } 258 | defer ofd.Close() 259 | r := bufio.NewReader(ofd) 260 | w := bufio.NewWriter(os.Stdout) 261 | 262 | var offset int64 263 | for { 264 | data := make([]byte, 32) // BlockSize 32 265 | n, err := r.Read(data) 266 | if err == io.EOF { 267 | break 268 | } 269 | if err != nil { 270 | return err 271 | } 272 | // The encrypted binary log headers are 512, so skip those 273 | offset += int64(n) 274 | if offset <= EncryptFileHeaderOffset { 275 | continue 276 | } 277 | w.Write(decryptAESCTR(stream, data[:n])) 278 | } 279 | return w.Flush() 280 | } 281 | -------------------------------------------------------------------------------- /rebuild/update.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "fmt" 18 | "strings" 19 | 20 | "github.com/LianjiaTech/lightning/common" 21 | 22 | "github.com/go-mysql-org/go-mysql/replication" 23 | lua "github.com/yuin/gopher-lua" 24 | ) 25 | 26 | // UpdateRebuild ... 27 | func UpdateRebuild(event *replication.BinlogEvent) string { 28 | switch common.Config.Rebuild.Plugin { 29 | case "sql": 30 | UpdateQuery(event) 31 | case "flashback": 32 | UpdateRollbackQuery(event) 33 | case "stat": 34 | UpdateStat(event) 35 | case "lua": 36 | UpdateLua(event) 37 | default: 38 | } 39 | return "" 40 | } 41 | 42 | // UpdateQuery ... 43 | func UpdateQuery(event *replication.BinlogEvent) { 44 | var table string 45 | defer func() { 46 | if r := recover(); r != nil { 47 | fmt.Printf("-- Table: %s, Error: %s\n", table, strings.Split(fmt.Sprint(r), "\n")[0]) 48 | } 49 | }() 50 | table = RowEventTable(event) 51 | ev := event.Event.(*replication.RowsEvent) 52 | values := BuildValues(ev) 53 | 54 | common.Verbose("-- [DEBUG] event: update, table: %s, rows: %d\n", table, len(values)) 55 | 56 | if common.Config.Rebuild.Replace { 57 | var insertValues [][]string 58 | for odd, value := range values { 59 | if odd%2 == 1 { 60 | insertValues = append(insertValues, value) 61 | } 62 | } 63 | insertQuery(table, insertValues) 64 | } else { 65 | updateQuery(table, values) 66 | } 67 | } 68 | 69 | func updateQuery(table string, values [][]string) { 70 | var where []string 71 | var set []string 72 | 73 | var updatePrefix = "UPDATE" 74 | if common.Config.Rebuild.ForeachTime && common.Config.Rebuild.CurrentEventTime != "" { 75 | updatePrefix = fmt.Sprintf(`/* %s */%s`, common.Config.Rebuild.CurrentEventTime, updatePrefix) 76 | } 77 | 78 | // for common.Config.Rebuild.WithoutDBName 79 | shortTableName := onlyTable(table) 80 | 81 | if ok := PrimaryKeys[table]; ok != nil { 82 | // 0 是 where 条件， 1 是 set 值 83 | for odd, value := range values { 84 | if odd%2 == 0 { 85 | where = []string{} 86 | set = []string{} 87 | for _, col := range PrimaryKeys[table] { 88 | for i, c := range Columns[table] { 89 | if c == col { 90 | if value[i] == "NULL" { 91 | where = append(where, fmt.Sprintf("%s IS NULL", col)) 92 | } else { 93 | where = append(where, fmt.Sprintf("%s = %s", col, value[i])) 94 | } 95 | } 96 | } 97 | } 98 | } else { 99 | if len(common.Config.Rebuild.IgnoreColumns) > 0 { 100 | for i, col := range Columns[table] { 101 | ignore := false 102 | for _, c := range common.Config.Rebuild.IgnoreColumns { 103 | if c == strings.Trim(col, "`") { 104 | ignore = true 105 | } 106 | } 107 | if !ignore { 108 | set = append(set, fmt.Sprintf("%s = %s", col, value[i])) 109 | } 110 | } 111 | } else { 112 | for i, c := range Columns[table] { 113 | set = append(set, fmt.Sprintf("%s = %s", c, value[i])) 114 | } 115 | } 116 | 117 | if common.Config.Rebuild.WithoutDBName { 118 | fmt.Printf("%s %s SET %s WHERE %s LIMIT 1;\n", updatePrefix, shortTableName, strings.Join(set, ", "), strings.Join(where, " AND ")) 119 | } else { 120 | fmt.Printf("%s %s SET %s WHERE %s LIMIT 1;\n", updatePrefix, table, strings.Join(set, ", "), strings.Join(where, " AND ")) 121 | } 122 | } 123 | } 124 | } else { 125 | for odd, value := range values { 126 | if odd%2 == 0 { 127 | where = []string{} 128 | set = []string{} 129 | for i, v := range value { 130 | if v == "NULL" { 131 | where = append(where, fmt.Sprintf("@%d IS NULL", i)) 132 | } else { 133 | where = append(where, fmt.Sprintf("@%d = %s", i, v)) 134 | } 135 | } 136 | } else { 137 | for i, v := range value { 138 | set = append(set, fmt.Sprintf("@%d = %s", i, v)) 139 | } 140 | if common.Config.Rebuild.WithoutDBName { 141 | fmt.Printf("-- %s %s SET %s WHERE %s LIMIT 1;\n", updatePrefix, shortTableName, strings.Join(set, ", "), strings.Join(where, " AND ")) 142 | } else { 143 | fmt.Printf("-- %s %s SET %s WHERE %s LIMIT 1;\n", updatePrefix, table, strings.Join(set, ", "), strings.Join(where, " AND ")) 144 | } 145 | } 146 | } 147 | } 148 | } 149 | 150 | // UpdateRollbackQuery ... 151 | func UpdateRollbackQuery(event *replication.BinlogEvent) { 152 | var table string 153 | defer func() { 154 | if r := recover(); r != nil { 155 | fmt.Printf("-- Table: %s, Error: %s\n", table, strings.Split(fmt.Sprint(r), "\n")[0]) 156 | } 157 | }() 158 | table = RowEventTable(event) 159 | ev := event.Event.(*replication.RowsEvent) 160 | values := BuildValues(ev) 161 | 162 | if common.Config.Rebuild.Replace { 163 | var insertValues [][]string 164 | for odd, value := range values { 165 | if odd%2 == 0 { 166 | insertValues = append(insertValues, value) 167 | } 168 | } 169 | insertQuery(table, insertValues) 170 | } else { 171 | updateRollbackQuery(table, values) 172 | } 173 | } 174 | 175 | func updateRollbackQuery(table string, values [][]string) { 176 | var where []string 177 | var set []string 178 | 179 | if ok := PrimaryKeys[table]; ok != nil { 180 | for odd, value := range values { 181 | if odd%2 == 0 { 182 | where = []string{} 183 | set = []string{} 184 | if len(common.Config.Rebuild.IgnoreColumns) > 0 { 185 | for i, col := range Columns[table] { 186 | ignore := false 187 | for _, c := range common.Config.Rebuild.IgnoreColumns { 188 | if c == strings.Trim(col, "`") { 189 | ignore = true 190 | } 191 | } 192 | if !ignore { 193 | set = append(set, fmt.Sprintf("%s = %s", col, value[i])) 194 | } 195 | } 196 | } else { 197 | for i, c := range Columns[table] { 198 | set = append(set, fmt.Sprintf("%s = %s", c, value[i])) 199 | } 200 | } 201 | } else { 202 | for _, col := range PrimaryKeys[table] { 203 | for i, c := range Columns[table] { 204 | if c == col { 205 | if value[i] == "NULL" { 206 | where = append(where, fmt.Sprintf("%s IS NULL", col)) 207 | } else { 208 | where = append(where, fmt.Sprintf("%s = %s", col, value[i])) 209 | } 210 | } 211 | } 212 | } 213 | fmt.Printf("UPDATE %s SET %s WHERE %s LIMIT 1;\n", table, strings.Join(set, ", "), strings.Join(where, " AND ")) 214 | } 215 | } 216 | } else { 217 | for odd, value := range values { 218 | if odd%2 == 0 { 219 | where = []string{} 220 | set = []string{} 221 | for i, v := range value { 222 | set = append(set, fmt.Sprintf("@%d = %s", i, v)) 223 | } 224 | } else { 225 | for i, v := range value { 226 | if v == "NULL" { 227 | where = append(where, fmt.Sprintf("@%d IS NULL", i)) 228 | } else { 229 | where = append(where, fmt.Sprintf("@%d = %s", i, v)) 230 | } 231 | } 232 | fmt.Printf("-- UPDATE %s SET %s WHERE %s LIMIT 1;\n", table, strings.Join(set, ", "), strings.Join(where, " AND ")) 233 | } 234 | } 235 | } 236 | } 237 | 238 | // UpdateStat ... 239 | func UpdateStat(event *replication.BinlogEvent) { 240 | table := RowEventTable(event) 241 | if TableStats[table] != nil { 242 | TableStats[table]["update"]++ 243 | } else { 244 | TableStats[table] = map[string]int64{"update": 1} 245 | } 246 | 247 | ev := event.Event.(*replication.RowsEvent) 248 | values := BuildValues(ev) 249 | if RowsStats[table] != nil { 250 | RowsStats[table]["update"] += int64(len(values)) 251 | } else { 252 | RowsStats[table] = map[string]int64{"update": int64(len(values))} 253 | } 254 | } 255 | 256 | // UpdateLua ... 257 | func UpdateLua(event *replication.BinlogEvent) { 258 | if common.Config.Rebuild.LuaScript == "" || event == nil { 259 | return 260 | } 261 | 262 | table := RowEventTable(event) 263 | ev := event.Event.(*replication.RowsEvent) 264 | values := BuildValues(ev) 265 | 266 | // lua function 267 | f := lua.P{ 268 | Fn: Lua.GetGlobal("UpdateRewrite"), 269 | NRet: 0, 270 | Protect: true, 271 | } 272 | // lua value 273 | v := lua.LString(table) 274 | var where, set []string 275 | for odd, value := range values { 276 | if odd%2 == 0 { 277 | where = []string{} 278 | set = []string{} 279 | for _, v := range value { 280 | where = append(where, v) 281 | } 282 | } else { 283 | for _, v := range value { 284 | set = append(set, v) 285 | } 286 | 287 | LuaStringList("GoValuesWhere", where) 288 | LuaStringList("GoValuesSet", set) 289 | 290 | if err := Lua.CallByParam(f, v); err != nil { 291 | common.Log.Error(err.Error()) 292 | return 293 | } 294 | } 295 | } 296 | } 297 | -------------------------------------------------------------------------------- /rebuild/schema.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package rebuild 15 | 16 | import ( 17 | "database/sql" 18 | "fmt" 19 | "io/ioutil" 20 | "os" 21 | "regexp" 22 | "strings" 23 | 24 | "github.com/juju/errors" 25 | 26 | "github.com/LianjiaTech/lightning/common" 27 | 28 | // database/sql 29 | _ "github.com/go-sql-driver/mysql" 30 | "github.com/pingcap/parser/ast" 31 | "github.com/pingcap/parser/model" 32 | "github.com/pingcap/parser/mysql" 33 | ) 34 | 35 | // Schemas ... 36 | var Schemas map[string]*ast.CreateTableStmt 37 | 38 | // Columns ... 39 | var Columns map[string][]string 40 | 41 | // PrimaryKeys ... 42 | var PrimaryKeys map[string][]string 43 | 44 | // LoadSchemaInfo load schema info from file or mysql 45 | func LoadSchemaInfo() { 46 | if common.Config.MySQL.SchemaFile != "" { 47 | // load from file 48 | err := loadSchemaFromFile() 49 | if err != nil { 50 | common.Log.Error(errors.Trace(err).Error()) 51 | } 52 | return 53 | } else { 54 | // load from mysql server 55 | err := loadSchemaFromMySQL() 56 | if err != nil { 57 | common.Log.Error(errors.Trace(err).Error()) 58 | } 59 | } 60 | } 61 | 62 | func loadSchemaFromFile() error { 63 | common.Log.Debug("loadSchemaFromFile %s", common.Config.MySQL.SchemaFile) 64 | if _, err := os.Stat(common.Config.MySQL.SchemaFile); err != nil { 65 | return err 66 | } 67 | buf, err := ioutil.ReadFile(common.Config.MySQL.SchemaFile) 68 | if err != nil { 69 | return err 70 | } 71 | err = schemaAppend("", string(buf)) 72 | buildColumns() 73 | buildPrimaryKeys() 74 | return err 75 | } 76 | 77 | func loadSchemaFromMySQL() error { 78 | var databases []string 79 | dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=%s&timeout=5s", 80 | common.MasterInfo.MasterUser, 81 | common.MasterInfo.MasterPassword, 82 | common.MasterInfo.MasterHost, 83 | common.MasterInfo.MasterPort, 84 | common.Config.Global.Charset, 85 | ) 86 | common.Log.Debug("loadSchemaFromMySQL %s", dsn) 87 | db, err := sql.Open("mysql", dsn) 88 | if err != nil { 89 | return err 90 | } 91 | defer db.Close() 92 | 93 | res, err := db.Query("SHOW DATABASES;") 94 | if err != nil { 95 | return err 96 | } 97 | for res.Next() { 98 | var database string 99 | res.Scan(&database) 100 | switch database { 101 | case "information_schema", "sys", "mysql", "performance_schema": 102 | default: 103 | databases = append(databases, database) 104 | } 105 | } 106 | res.Close() 107 | 108 | for _, database := range databases { 109 | res, err := db.Query(fmt.Sprintf("SHOW TABLES FROM `%s`", database)) 110 | if err != nil { 111 | common.Log.Error(errors.Trace(err).Error()) 112 | continue 113 | } 114 | 115 | // SHOW TABLES 116 | var tables []string 117 | for res.Next() { 118 | var table string 119 | err = res.Scan(&table) 120 | if err != nil { 121 | common.Log.Error(errors.Trace(err).Error()) 122 | continue 123 | } 124 | tables = append(tables, table) 125 | } 126 | 127 | // SHOW CREATE TABLE 128 | for _, table := range tables { 129 | var ignore bool 130 | if len(common.Config.Filters.Tables) > 0 { 131 | ignore = true 132 | } 133 | for _, tb := range common.Config.Filters.Tables { 134 | // TODO: % 匹配有点复杂，这里暂且加载所有表结构 135 | if strings.Contains(tb, "%") && strings.HasPrefix(strings.Replace(tb, "`", "", -1), database+".") { 136 | ignore = false 137 | break 138 | } 139 | // 对于表较多的情况，只加载需要的表将极大加速表结构加载速度 140 | if strings.Replace(tb, "`", "", -1) == fmt.Sprintf("%s.%s", database, table) { 141 | ignore = false 142 | break 143 | } 144 | } 145 | if ignore { 146 | continue 147 | } 148 | 149 | tableRes, err := db.Query(fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`;", database, table)) 150 | if err != nil { 151 | common.Log.Error(errors.Trace(err).Error()) 152 | continue 153 | } 154 | 155 | cols, err := tableRes.Columns() 156 | if err != nil { 157 | common.Log.Error(errors.Trace(err).Error()) 158 | continue 159 | } 160 | // SHOW CREATE VIEW WILL GET 4 COLUMNS 161 | if len(cols) != 2 { 162 | common.Log.Info("by pass host: %s, port: %d, database: %s, table: %s", 163 | common.MasterInfo.MasterHost, 164 | common.MasterInfo.MasterPort, 165 | database, table) 166 | continue 167 | } 168 | 169 | for tableRes.Next() { 170 | var name, schema string 171 | err = tableRes.Scan(&name, &schema) 172 | if err != nil { 173 | common.Log.Error("host: %s, port: %d, database: %s, table: %s, error: %s", 174 | common.MasterInfo.MasterHost, 175 | common.MasterInfo.MasterPort, 176 | database, table, errors.Trace(err).Error()) 177 | continue 178 | } 179 | err = schemaAppend(database, schema) 180 | if err != nil { 181 | common.Log.Error("host: %s, port: %d, database: %s, table: %s, sql: %s, error: %s", 182 | common.MasterInfo.MasterHost, 183 | common.MasterInfo.MasterPort, 184 | database, table, 185 | schema, 186 | errors.Trace(err).Error()) 187 | schemaAppend(database, buildFakeTable(db, fmt.Sprintf("`%s`.`%s`", database, table))) 188 | } 189 | } 190 | tableRes.Close() 191 | } 192 | res.Close() 193 | } 194 | buildColumns() 195 | buildPrimaryKeys() 196 | return nil 197 | } 198 | 199 | func schemaAppend(database, sql string) error { 200 | sql = removeIncompatibleWords(sql) 201 | stmts, err := TiParse(sql, common.Config.Global.Charset, mysql.Charsets[common.Config.Global.Charset]) 202 | if err != nil { 203 | return err 204 | } 205 | if database == "" { 206 | database = "%" 207 | } 208 | for _, stmt := range stmts { 209 | switch node := stmt.(type) { 210 | case *ast.CreateTableStmt: 211 | if node.Table.Schema.String() == "" { 212 | node.Table.Schema = model.NewCIStr(database) 213 | } 214 | Schemas[fmt.Sprintf("`%s`.`%s`", database, node.Table.Name)] = node 215 | case *ast.UseStmt: 216 | database = node.DBName 217 | } 218 | } 219 | return nil 220 | } 221 | 222 | // removeIncompatibleWords remove pingcap/parser not support words from schema 223 | // Note: only for MySQL `SHOW CREATE TABLE` hand-writing SQL not compatible 224 | func removeIncompatibleWords(sql string) string { 225 | // CONSTRAINT col_fk FOREIGN KEY (col) REFERENCES tb (id) ON UPDATE CASCADE 226 | re := regexp.MustCompile(` ON UPDATE CASCADE`) 227 | sql = re.ReplaceAllString(sql, "") 228 | 229 | // FULLTEXT KEY col_fk (col) /*!50100 WITH PARSER `ngram` */ 230 | // /*!50100 PARTITION BY LIST (col) 231 | re = regexp.MustCompile(`/\*!5`) 232 | sql = re.ReplaceAllString(sql, "/* 5") 233 | 234 | // col varchar(10) CHARACTER SET gbk DEFAULT NULL 235 | re = regexp.MustCompile(`CHARACTER SET [a-z_0-9]* `) 236 | sql = re.ReplaceAllString(sql, "") 237 | 238 | // DEFAULT CHARSET=utf8mb3 239 | re = regexp.MustCompile(`DEFAULT CHARSET=[a-z_0-9]*`) 240 | sql = re.ReplaceAllString(sql, "") 241 | 242 | return sql 243 | } 244 | 245 | // buildColumns build column name list 246 | func buildColumns() { 247 | Columns = make(map[string][]string) 248 | for _, schema := range Schemas { 249 | table := fmt.Sprintf("`%s`.`%s`", schema.Table.Schema.String(), schema.Table.Name.String()) 250 | for _, col := range schema.Cols { 251 | Columns[table] = append(Columns[table], fmt.Sprintf("`%s`", col.Name.String())) 252 | } 253 | } 254 | } 255 | 256 | // buildPrimaryKeys build primary key list 257 | func buildPrimaryKeys() { 258 | PrimaryKeys = make(map[string][]string) 259 | for _, schema := range Schemas { 260 | table := fmt.Sprintf("`%s`.`%s`", schema.Table.Schema.String(), schema.Table.Name.String()) 261 | for _, con := range schema.Constraints { 262 | if con.Tp == ast.ConstraintPrimaryKey { 263 | for _, col := range con.Keys { 264 | PrimaryKeys[table] = append(PrimaryKeys[table], fmt.Sprintf("`%s`", col.Column.String())) 265 | } 266 | } 267 | } 268 | // 如果表没有主键，把表的所有列合起来当主键 269 | if len(PrimaryKeys[table]) == 0 { 270 | PrimaryKeys[table] = Columns[table] 271 | } 272 | } 273 | } 274 | 275 | // buildFakeTable ... 276 | func buildFakeTable(db *sql.DB, table string) string { 277 | var col, key string 278 | var t []byte 279 | var columns, primary []string 280 | res, err := db.Query(fmt.Sprintf("SHOW COLUMNS FROM %s", table)) 281 | if err != nil { 282 | common.Log.Error(err.Error()) 283 | return "" 284 | } 285 | defer res.Close() 286 | for res.Next() { 287 | res.Scan(&col, &t, &t, &key, &t, &t) 288 | columns = append(columns, fmt.Sprintf("`%s` INT", col)) 289 | if key == "PRI" { 290 | primary = append(primary, fmt.Sprintf("`%s`", col)) 291 | } 292 | } 293 | return fmt.Sprintf("CREATE TABLE %s (%s %s);", table, strings.Join(columns, ","), fmt.Sprintf(", PRIMARY KEY (%s)", strings.Join(primary, ","))) 294 | } 295 | 296 | func onlyTable(table string) string { 297 | tup := strings.Split(strings.Trim(table, "`"), "`.`") 298 | length := len(tup) 299 | if length <= 0 { 300 | return "" 301 | } 302 | return fmt.Sprint("`", tup[length-1], "`") 303 | } 304 | -------------------------------------------------------------------------------- /event/filter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package event 15 | 16 | import ( 17 | "fmt" 18 | "strings" 19 | "time" 20 | 21 | "github.com/LianjiaTech/lightning/common" 22 | "github.com/LianjiaTech/lightning/rebuild" 23 | 24 | "github.com/go-mysql-org/go-mysql/replication" 25 | uuid "github.com/satori/go.uuid" 26 | ) 27 | 28 | var FollowGTID bool 29 | var FollowThreadID bool 30 | var Ending bool 31 | var Starting bool 32 | 33 | // FilterThreadID ... 34 | func FilterThreadID(event *replication.BinlogEvent) bool { 35 | var do bool 36 | if common.Config.Filters.ThreadID == 0 { 37 | return true 38 | } 39 | var threadId uint32 40 | switch event.Header.EventType { 41 | case replication.QUERY_EVENT: 42 | threadId = event.Event.(*replication.QueryEvent).SlaveProxyID 43 | if threadId == uint32(common.Config.Filters.ThreadID) { 44 | do = true 45 | FollowThreadID = do 46 | } else { 47 | FollowThreadID = false 48 | } 49 | default: 50 | common.VerboseVerbose("-- [DEBUG] FilterThreadID do: %v, Table: %s", FollowThreadID, threadId) 51 | return FollowThreadID 52 | } 53 | common.VerboseVerbose("-- [DEBUG] FilterThreadID do: %v, Table: %s", do, threadId) 54 | return do 55 | } 56 | 57 | // FilterTables ... 58 | func FilterTables(event *replication.BinlogEvent) bool { 59 | var do bool 60 | if len(common.Config.Filters.Tables) == 0 { 61 | do = true 62 | } 63 | table := rebuild.RowEventTable(event) 64 | for _, filter := range common.Config.Filters.Tables { 65 | if tableFilterMatch(table, filter) { 66 | do = true 67 | break 68 | } 69 | } 70 | common.VerboseVerbose("-- [DEBUG] FilterTables do: %v, Table: %s", do, table) 71 | return do 72 | } 73 | 74 | // FilterIgnoreTables ... 75 | func FilterIgnoreTables(event *replication.BinlogEvent) bool { 76 | do := true 77 | if len(common.Config.Filters.IgnoreTables) == 0 { 78 | return true 79 | } 80 | table := rebuild.RowEventTable(event) 81 | for _, filter := range common.Config.Filters.IgnoreTables { 82 | if tableFilterMatch(table, filter) { 83 | do = false 84 | break 85 | } 86 | } 87 | common.VerboseVerbose("-- [DEBUG] FilterIgnoreTables do: %v, Table: %s", do, table) 88 | return do 89 | } 90 | 91 | // FilterStartDatetime ... 92 | func FilterStartDatetime(event *replication.BinlogEvent) bool { 93 | var do bool 94 | if common.Config.Filters.StartTimestamp == 0 { 95 | do = true 96 | } 97 | if int64(event.Header.Timestamp) >= common.Config.Filters.StartTimestamp { 98 | do = true 99 | } 100 | return do 101 | } 102 | 103 | // FilterStopDatetime ... 104 | func FilterStopDatetime(event *replication.BinlogEvent) bool { 105 | var do bool 106 | if common.Config.Filters.StopTimestamp == 0 { 107 | do = true 108 | return do 109 | } 110 | if int64(event.Header.Timestamp) <= common.Config.Filters.StopTimestamp { 111 | do = true 112 | } else { 113 | Ending = true 114 | } 115 | return do 116 | } 117 | 118 | // FilterServerID ... 119 | func FilterServerID(event *replication.BinlogEvent) bool { 120 | var do bool 121 | if common.Config.Filters.ServerID == 0 { 122 | do = true 123 | } 124 | if event.Header.ServerID == uint32(common.Config.Filters.ServerID) { 125 | do = true 126 | } 127 | return do 128 | } 129 | 130 | // FilterIncludeGTIDs ... 131 | func FilterIncludeGTIDs(event *replication.BinlogEvent) bool { 132 | var do bool 133 | if common.Config.Filters.IncludeGTIDSet == "" { 134 | return true 135 | } 136 | switch event.Header.EventType { 137 | case replication.GTID_EVENT: 138 | do = InGTIDSet(event.Event.(*replication.GTIDEvent).SID, event.Event.(*replication.GTIDEvent).GNO, common.Config.Filters.IncludeGTIDSet) 139 | if FollowGTID && !do { 140 | Ending = true 141 | } 142 | FollowGTID = do 143 | default: 144 | do = FollowGTID 145 | } 146 | return do 147 | } 148 | 149 | // FilterExcludeGTIDs ... 150 | func FilterExcludeGTIDs(event *replication.BinlogEvent) bool { 151 | var do bool 152 | if common.Config.Filters.ExcludeGTIDSet == "" { 153 | return true 154 | } 155 | switch event.Header.EventType { 156 | case replication.GTID_EVENT: 157 | do = !InGTIDSet(event.Event.(*replication.GTIDEvent).SID, event.Event.(*replication.GTIDEvent).GNO, common.Config.Filters.ExcludeGTIDSet) 158 | FollowGTID = do 159 | default: 160 | do = FollowGTID 161 | } 162 | return do 163 | } 164 | 165 | // FilterStartPos ... 166 | func FilterStartPos(event *replication.BinlogEvent) bool { 167 | var do bool 168 | if common.Config.Filters.StartPosition == 0 || Starting { 169 | do = true 170 | } 171 | if event.Header.LogPos >= common.Config.Filters.StartPosition { 172 | do = true 173 | Starting = true 174 | } 175 | return do 176 | } 177 | 178 | // FilterStopPos ... 179 | func FilterStopPos(event *replication.BinlogEvent) bool { 180 | var do bool 181 | if common.Config.Filters.StopPosition == 0 { 182 | do = true 183 | return do 184 | } 185 | if event.Header.LogPos <= common.Config.Filters.StopPosition { 186 | do = true 187 | } else { 188 | Ending = true 189 | } 190 | return do 191 | } 192 | 193 | // FilterQueryType ... 194 | func FilterQueryType(event *replication.BinlogEvent) bool { 195 | var do bool 196 | if len(common.Config.Filters.EventType) == 0 { 197 | return true 198 | } 199 | 200 | for _, t := range common.Config.Filters.EventType { 201 | switch event.Header.EventType { 202 | case replication.WRITE_ROWS_EVENTv2, replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv0: 203 | if strings.ToLower(t) == "insert" { 204 | do = true 205 | } 206 | case replication.UPDATE_ROWS_EVENTv2, replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv0: 207 | if strings.ToLower(t) == "update" { 208 | do = true 209 | } 210 | case replication.DELETE_ROWS_EVENTv2, replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv0: 211 | if strings.ToLower(t) == "delete" { 212 | do = true 213 | } 214 | case replication.QUERY_EVENT: 215 | prefix := strings.Fields(string(event.Event.(*replication.QueryEvent).Query))[0] 216 | if strings.ToLower(t) == prefix { 217 | do = true 218 | } 219 | default: 220 | } 221 | if do { 222 | break 223 | } 224 | } 225 | return do 226 | } 227 | 228 | // UpdateMasterInfo ... 229 | func UpdateMasterInfo(event *replication.BinlogEvent) { 230 | switch event.Header.EventType { 231 | case replication.ROTATE_EVENT: 232 | nextFile := string(event.Event.(*replication.RotateEvent).NextLogName) 233 | if nextFile != common.MasterInfo.MasterLogFile { 234 | common.MasterInfo.MasterLogFile = nextFile 235 | common.MasterInfo.MasterLogPos = 4 236 | } 237 | case replication.QUERY_EVENT: 238 | common.MasterInfo.MasterLogPos = int64(event.Header.LogPos) 239 | case replication.XID_EVENT: 240 | common.MasterInfo.MasterLogPos = int64(event.Header.LogPos) 241 | executedGTIDSet := fmt.Sprint(event.Event.(*replication.XIDEvent).GSet) 242 | if executedGTIDSet != "" { 243 | common.MasterInfo.ExecutedGTIDSet = executedGTIDSet 244 | } 245 | default: 246 | } 247 | 248 | // START SLAVE UNTIL MASTER_LOG_FILE = 'log_name', MASTER_LOG_POS = log_pos 249 | if common.MasterInfo.MasterLogFile == common.MasterInfo.UntilLogFile && 250 | common.MasterInfo.UntilLogPos <= common.MasterInfo.MasterLogPos { 251 | Ending = true 252 | } 253 | common.MasterInfo.SecondsBehindMaster = time.Now().Unix() - int64(event.Header.Timestamp) 254 | if common.Config.MySQL.SyncDuration.Seconds() == 0 { 255 | common.FlushReplicationInfo() 256 | } 257 | } 258 | 259 | // BinlogFilter check if event will do 260 | func BinlogFilter(event *replication.BinlogEvent) bool { 261 | if !FilterStopPos(event) { 262 | return false 263 | } 264 | if !FilterStartPos(event) { 265 | return false 266 | } 267 | if !FilterThreadID(event) { 268 | return false 269 | } 270 | if !FilterExcludeGTIDs(event) { 271 | return false 272 | } 273 | if !FilterIncludeGTIDs(event) { 274 | return false 275 | } 276 | if !FilterServerID(event) { 277 | return false 278 | } 279 | if !FilterStopDatetime(event) { 280 | return false 281 | } 282 | if !FilterStartDatetime(event) { 283 | return false 284 | } 285 | if !FilterTables(event) { 286 | return false 287 | } 288 | if !FilterIgnoreTables(event) { 289 | return false 290 | } 291 | if !FilterQueryType(event) { 292 | return false 293 | } 294 | return true 295 | } 296 | 297 | func tableFilterMatch(table, filter string) bool { 298 | var match, dbMatch, tbMatch bool 299 | table = strings.Replace(table, "`", "", -1) 300 | schema := strings.Split(table, ".") 301 | if len(schema) < 2 { 302 | return match 303 | } 304 | sep := strings.Split(filter, ".") 305 | if len(sep) < 2 { 306 | common.Log.Error("tableFilterMatch, -tables: '%s' filter format error", filter) 307 | return match 308 | } 309 | 310 | // 库表名大小写不敏感 311 | if sep[0] == "%" { 312 | dbMatch = true 313 | } 314 | // 当 -schema 指定的文件中只有 CREATE TABLE 忘了写 USE db 的时候，schema[0] 为 % 315 | if schema[0] == "%" { 316 | dbMatch = true 317 | } 318 | if i := strings.Index(sep[0], "%"); i > 0 { 319 | if strings.HasPrefix(schema[0], sep[0][0:i]) { 320 | dbMatch = true 321 | } 322 | } 323 | if strings.ToLower(schema[0]) == strings.ToLower(sep[0]) { 324 | dbMatch = true 325 | } 326 | 327 | if sep[1] == "%" { 328 | tbMatch = true 329 | } 330 | if i := strings.Index(sep[1], "%"); i > 0 { 331 | if strings.HasPrefix(strings.ToLower(schema[1]), strings.ToLower(sep[1][0:i])) { 332 | tbMatch = true 333 | } 334 | } 335 | if schema[1] == sep[1] { 336 | tbMatch = true 337 | } 338 | match = dbMatch && tbMatch 339 | return match 340 | } 341 | 342 | // InGTIDSet ... 343 | func InGTIDSet(sid []byte, gno int64, gtidSet string) bool { 344 | var gtidSets [][]string 345 | for _, set := range strings.Split(gtidSet, ",") { 346 | var couple []string 347 | tmp := strings.Split(strings.TrimSpace(set), ":") 348 | if len(tmp) != 2 { 349 | return true 350 | } 351 | couple = append(couple, tmp[0]) 352 | couple = append(couple, strings.Split(tmp[1], "-")...) 353 | gtidSets = append(gtidSets, couple) 354 | } 355 | for _, set := range gtidSets { 356 | s, _ := uuid.FromBytes(sid) 357 | if len(set) != 3 { 358 | continue 359 | } 360 | if set[0] == s.String() { 361 | if strings.Compare(set[1], fmt.Sprint(gno)) <= 0 && 362 | strings.Compare(set[2], fmt.Sprint(gno)) >= 0 { 363 | return true 364 | } 365 | } 366 | } 367 | return false 368 | } 369 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /test/binlog_decrypt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copy from: https://mysql.wisborg.dk/2019/01/28/automatic-decryption-of-mysql-binary-logs-using-python/ 4 | 5 | import sys 6 | import os 7 | import struct 8 | import collections 9 | import hashlib 10 | from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes 11 | from cryptography.hazmat.backends import default_backend 12 | 13 | def key_and_iv_from_password(password): 14 | # Based on 15 | # https://stackoverflow.com/questions/13907841/implement-openssl-aes-encryption-in-python 16 | 17 | key_length = 32 18 | iv_length = 16 19 | required_length = key_length + iv_length 20 | password = password 21 | 22 | key_iv = hashlib.sha512(password).digest() 23 | tmp = [key_iv] 24 | while len(tmp) < required_length: 25 | tmp.append(hashlib.sha512(tmp[-1] + password).digest()) 26 | key_iv += tmp[-1] 27 | 28 | key = key_iv[:key_length] 29 | iv = key_iv[key_length:required_length] 30 | 31 | return key, iv 32 | 33 | 34 | class Key( 35 | collections.namedtuple( 36 | 'Key', [ 37 | 'key_id', 38 | 'key_type', 39 | 'user_id', 40 | 'key_data', 41 | ] 42 | )): 43 | __slots__ = () 44 | 45 | 46 | class Keyring(object): 47 | _keys = [] 48 | _keyring_file_version = None 49 | _xor_str = '*305=Ljt0*!@$Hnm(*-9-w;:'.encode('utf-8') 50 | 51 | def __init__(self, keyring_filepath): 52 | self.read_keyring(keyring_filepath) 53 | 54 | def _read_key(self, data): 55 | overall_length = struct.unpack(' 0 else None 71 | key_raw = data[key_start:key_end] 72 | xor_str_len = len(self._xor_str) 73 | key_data = bytes([key_raw[i] ^ self._xor_str[i%xor_str_len] 74 | for i in range(len(key_raw))]) 75 | 76 | return Key(key_id, key_type, user_id, key_data) 77 | 78 | def read_keyring(self, filepath): 79 | keyring_data = bytearray() 80 | with open(filepath, 'rb') as keyring_fs: 81 | chunk = keyring_fs.read() 82 | while len(chunk) > 0: 83 | keyring_data.extend(chunk) 84 | chunk = keyring_fs.read() 85 | 86 | keyring_fs.close() 87 | 88 | # Verify the start of the file is "Keyring file version:" 89 | header = keyring_data[0:21] 90 | if header.decode('utf-8') != 'Keyring file version:': 91 | raise ValueError('Invalid header in the keyring file: {0}' 92 | .format(header.hex())) 93 | 94 | # Get the keyring version - currently only 2.0 is supported 95 | version = keyring_data[21:24].decode('utf-8') 96 | if version != '2.0': 97 | raise ValueError('Unsupported keyring version: {0}' 98 | .format(version)) 99 | 100 | self._keyring_file_version = version 101 | keyring_length = len(keyring_data) 102 | offset = 24 103 | keys = [] 104 | while offset < keyring_length and keyring_data[offset:offset+3] != b'EOF': 105 | key_length = struct.unpack(' 0 { 130 | unsigned = true 131 | } 132 | } 133 | switch t { 134 | case mysql.MYSQL_TYPE_DECIMAL, mysql.MYSQL_TYPE_NEWDECIMAL, mysql.MYSQL_TYPE_FLOAT, mysql.MYSQL_TYPE_DOUBLE, mysql.MYSQL_TYPE_NULL, 135 | mysql.MYSQL_TYPE_TIMESTAMP: 136 | columns = append(columns, fmt.Sprint(row[i])) 137 | // binlog use -1 for unsigned int max value 138 | case mysql.MYSQL_TYPE_TINY: 139 | if unsigned && fmt.Sprint(row[i]) == "-1" { 140 | columns = append(columns, "255") 141 | } else { 142 | columns = append(columns, fmt.Sprint(row[i])) 143 | } 144 | case mysql.MYSQL_TYPE_SHORT: 145 | if unsigned && fmt.Sprint(row[i]) == "-1" { 146 | columns = append(columns, "65535") 147 | } else { 148 | columns = append(columns, fmt.Sprint(row[i])) 149 | } 150 | case mysql.MYSQL_TYPE_INT24: 151 | if unsigned && fmt.Sprint(row[i]) == "-1" { 152 | columns = append(columns, "16777215") 153 | } else { 154 | columns = append(columns, fmt.Sprint(row[i])) 155 | } 156 | case mysql.MYSQL_TYPE_LONG: 157 | if unsigned && fmt.Sprint(row[i]) == "-1" { 158 | columns = append(columns, "4294967295") 159 | } else { 160 | columns = append(columns, fmt.Sprint(row[i])) 161 | } 162 | case mysql.MYSQL_TYPE_LONGLONG: 163 | if unsigned && fmt.Sprint(row[i]) == "-1" { 164 | columns = append(columns, "18446744073709551615") 165 | } else { 166 | columns = append(columns, fmt.Sprint(row[i])) 167 | } 168 | case mysql.MYSQL_TYPE_DATE, mysql.MYSQL_TYPE_TIME, mysql.MYSQL_TYPE_DATETIME, mysql.MYSQL_TYPE_YEAR, 169 | mysql.MYSQL_TYPE_NEWDATE, mysql.MYSQL_TYPE_TIMESTAMP2, mysql.MYSQL_TYPE_DATETIME2, mysql.MYSQL_TYPE_TIME2: 170 | columns = append(columns, fmt.Sprint("'", row[i], "'")) 171 | case mysql.MYSQL_TYPE_VARCHAR, mysql.MYSQL_TYPE_VAR_STRING, mysql.MYSQL_TYPE_STRING: 172 | switch row[i].(type) { 173 | case string: 174 | if common.Config.Global.HexString { 175 | columns = append(columns, fmt.Sprintf(`X'%s'`, hex.EncodeToString(row[i].([]byte)))) 176 | } else { 177 | // strconv.Quote will escape unicode \u0100 178 | // escape function maybe not correct with multi byte charset 179 | // columns = append(columns, strconv.Quote(row[i].(string))) 180 | columns = append(columns, fmt.Sprintf(`"%s"`, escape(row[i].(string)))) 181 | } 182 | case int, int64, int32, int16, int8, uint64, uint32, uint16, uint8: 183 | // SET ENUM 184 | columns = append(columns, fmt.Sprint(row[i])) 185 | default: 186 | columns = append(columns, fmt.Sprintf(`'%s'`, fmt.Sprint(row[i]))) 187 | } 188 | 189 | case mysql.MYSQL_TYPE_JSON: 190 | columns = append(columns, fmt.Sprintf(`'%s'`, row[i].([]byte))) 191 | case mysql.MYSQL_TYPE_BIT: 192 | columns = append(columns, fmt.Sprintf(`%d`, row[i].(int64))) 193 | default: 194 | // mysql.MYSQL_TYPE_TINY_BLOB, mysql.MYSQL_TYPE_BLOB, mysql.MYSQL_TYPE_MEDIUM_BLOB, mysql.MYSQL_TYPE_LONG_BLOB 195 | // mysql.MYSQL_TYPE_GEOMETRY 196 | columns = append(columns, fmt.Sprintf(`X'%s'`, hex.EncodeToString(row[i].([]byte)))) 197 | } 198 | } 199 | values = append(values, columns) 200 | } 201 | return values 202 | } 203 | 204 | // GTIDRebuild ... 205 | func GTIDRebuild(event *replication.GTIDEvent) { 206 | serverID, _ := uuid.FromBytes(event.SID) 207 | common.Verbose("-- [DEBUG] GTID_NEXT: %s:%d, LastCommitted: %d, SequenceNumber: %d, CommitFlag: %d\n", serverID, event.GNO, event.LastCommitted, event.SequenceNumber, event.CommitFlag) 208 | } 209 | 210 | // EventHeaderRebuild ... 211 | func EventHeaderRebuild(event *replication.BinlogEvent) { 212 | header := event.Header 213 | common.Verbose("-- [DEBUG] EventType: %s, ServerID: %d, Timestamp: %d, LogPos: %d, EventSize: %d, Flags: %d\n", 214 | header.EventType.String(), header.ServerID, header.Timestamp, header.LogPos, header.EventSize, header.Flags) 215 | 216 | if common.Config.Rebuild.ForeachTime { 217 | common.Config.Rebuild.CurrentEventTime = fmt.Sprint(time.Unix(int64(header.Timestamp), 0).Format("2006-01-02 15:04:05")) 218 | } 219 | } 220 | 221 | // LastStatus ... 222 | func LastStatus() { 223 | switch common.Config.Rebuild.Plugin { 224 | case "stat": 225 | printBinlogStat() 226 | } 227 | if Lua != nil { 228 | if err := Lua.CallByParam(lua.P{ 229 | Fn: Lua.GetGlobal("Finalizer"), 230 | NRet: 1, 231 | Protect: true, 232 | }); err != nil { 233 | common.Log.Error(err.Error()) 234 | return 235 | } 236 | defer Lua.Close() 237 | } 238 | } 239 | 240 | // printBinlogStat ... 241 | func printBinlogStat() { 242 | // TransactionTimeStats 243 | medianTime, _ := stats.Median(TransactionTimeStats) 244 | maxTime, _ := stats.Max(TransactionTimeStats) 245 | meanTime, _ := stats.Mean(TransactionTimeStats) 246 | p99Time, _ := stats.Percentile(TransactionTimeStats, 99) 247 | p95Time, _ := stats.Percentile(TransactionTimeStats, 95) 248 | // TransactionSizeStats 249 | medianSize, _ := stats.Median(TransactionSizeStats) 250 | maxSize, _ := stats.Max(TransactionSizeStats) 251 | meanSize, _ := stats.Mean(TransactionSizeStats) 252 | p99Size, _ := stats.Percentile(TransactionSizeStats, 99) 253 | p95Size, _ := stats.Percentile(TransactionSizeStats, 95) 254 | 255 | BinlogStats = Stats{ 256 | Table: TableStats, 257 | Rows: RowsStats, 258 | Query: QueryStats, 259 | Transaction: map[string]map[string]string{ 260 | "TimeSeconds": { 261 | "MaxTransactionPos": fmt.Sprintf("-start-position %d -stop-position %d", int64(MaxTransactionTimeStartPos), int64(MaxTransactionTimeStopPos)), 262 | "Median": fmt.Sprintf("%0.2f", medianTime), 263 | "Max": fmt.Sprintf("%0.2f", maxTime), 264 | "Mean": fmt.Sprintf("%0.2f", meanTime), 265 | "P99": fmt.Sprintf("%0.2f", p99Time), 266 | "P95": fmt.Sprintf("%0.2f", p95Time), 267 | }, 268 | "SizeBytes": { 269 | "MaxTransactionPos": fmt.Sprintf("-start-position %d -stop-position %d", int64(MaxTransactionSizeStartPos), int64(MaxTransactionSizeStartPos+MaxTransactionSize)), 270 | "Median": fmt.Sprintf("%0.1f", medianSize), 271 | "Max": fmt.Sprintf("%0.1f", maxSize), 272 | "Mean": fmt.Sprintf("%0.1f", meanSize), 273 | "P99": fmt.Sprintf("%0.1f", p99Size), 274 | "P95": fmt.Sprintf("%0.1f", p95Size), 275 | }, 276 | }, 277 | } 278 | 279 | buf, err := json.MarshalIndent(BinlogStats, "", " ") 280 | if err != nil { 281 | fmt.Println(err) 282 | } 283 | fmt.Println(string(buf)) 284 | } 285 | 286 | // LuaStringList ... 287 | func LuaStringList(name string, values []string) { 288 | t := Lua.NewTable() 289 | for k, v := range values { 290 | Lua.SetTable(t, lua.LNumber(k+1), lua.LString(v)) 291 | } 292 | Lua.SetGlobal(name, t) 293 | } 294 | 295 | // LuaMapStringList ... 296 | func LuaMapStringList(name string, values map[string][]string) { 297 | t := Lua.NewTable() 298 | for k, cols := range values { 299 | l := Lua.NewTable() 300 | for i, col := range cols { 301 | Lua.SetTable(l, lua.LNumber(i+1), lua.LString(col)) 302 | } 303 | Lua.SetTable(t, lua.LString(k), l) 304 | } 305 | Lua.SetGlobal(name, t) 306 | } 307 | 308 | // LoadLuaScript ... 309 | func LoadLuaScript() { 310 | if common.Config.Rebuild.LuaScript == "" || common.Config.Rebuild.Plugin != "lua" { 311 | return 312 | } 313 | Lua = lua.NewState() 314 | gluasocket.Preload(Lua) 315 | gluabit32.Preload(Lua) 316 | gluadb.Preload(Lua) // lua package require "mysql", "redis" 317 | lfs.Preload(Lua) // lfs.currentdir() for package loading 318 | 319 | if err := Lua.DoFile(common.Config.Rebuild.LuaScript); err != nil { 320 | common.Log.Error(err.Error()) 321 | return 322 | } 323 | 324 | LuaMapStringList("GoPrimaryKeys", PrimaryKeys) 325 | LuaMapStringList("GoColumns", Columns) 326 | 327 | if err := Lua.CallByParam(lua.P{ 328 | Fn: Lua.GetGlobal("Init"), 329 | NRet: 1, 330 | Protect: true, 331 | }); err != nil { 332 | common.Log.Error(err.Error()) 333 | return 334 | } 335 | } 336 | 337 | func escape(sql string) string { 338 | dest := make([]byte, 0, 2*len(sql)) 339 | var escape byte 340 | for i := 0; i < len(sql); i++ { 341 | c := sql[i] 342 | 343 | escape = 0 344 | 345 | switch c { 346 | case 0: /* Must be escaped for 'mysql' */ 347 | escape = '0' 348 | break 349 | case '\n': /* Must be escaped for logs */ 350 | escape = 'n' 351 | break 352 | case '\r': 353 | escape = 'r' 354 | break 355 | case '\\': 356 | escape = '\\' 357 | break 358 | case '\'': 359 | escape = '\'' 360 | break 361 | case '"': /* Better safe than sorry */ 362 | escape = '"' 363 | break 364 | case '\032': /* This gives problems on Win32 */ 365 | escape = 'Z' 366 | } 367 | 368 | if escape != 0 { 369 | dest = append(dest, '\\', escape) 370 | } else { 371 | dest = append(dest, c) 372 | } 373 | } 374 | 375 | return string(dest) 376 | } 377 | -------------------------------------------------------------------------------- /event/parser.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package event 15 | 16 | import ( 17 | "bytes" 18 | "context" 19 | "crypto/cipher" 20 | "database/sql" 21 | "fmt" 22 | "io" 23 | "os" 24 | "sort" 25 | "time" 26 | 27 | "github.com/LianjiaTech/lightning/common" 28 | "github.com/LianjiaTech/lightning/rebuild" 29 | 30 | // database/sql 31 | "github.com/go-mysql-org/go-mysql/mysql" 32 | "github.com/go-mysql-org/go-mysql/replication" 33 | _ "github.com/go-sql-driver/mysql" 34 | "github.com/juju/errors" 35 | ) 36 | 37 | // https://dev.mysql.com/doc/internals/en/binary-log-structure-and-contents.html 38 | 39 | const ( 40 | FileHeaderLength = 4 // binlog file magic header 0XFE bin 41 | EventHeaderLength = 19 // event header length 42 | ) 43 | 44 | // BinlogParser ... 45 | func BinlogParser() { 46 | if len(common.Config.MySQL.BinlogFile) > 0 { 47 | // check each binlog file start time for event time filter 48 | err := CheckBinlogFileTime(common.Config.MySQL.BinlogFile) 49 | if err != nil { 50 | println(err.Error()) 51 | return 52 | } 53 | switch common.Config.Rebuild.Plugin { 54 | case "find": 55 | fmt.Println(common.Config.MySQL.BinlogFile) 56 | return 57 | case "decrypt": 58 | for _, binlog := range common.Config.MySQL.BinlogFile { 59 | err = DecryptBinlog(binlog, common.Config.MySQL.Keyring) 60 | if err != nil { 61 | println(err.Error()) 62 | } 63 | } 64 | return 65 | } 66 | 67 | // parse each binlog file 68 | err = BinlogFileParser(common.Config.MySQL.BinlogFile) 69 | if err != nil { 70 | fmt.Println(err.Error()) 71 | } 72 | return 73 | } 74 | if common.Config.MySQL.MasterInfo != "" { 75 | err := BinlogStreamParser() 76 | if err != nil { 77 | println(err.Error()) 78 | } 79 | } 80 | } 81 | 82 | // CheckBinlogFileHeader check file is binary log 83 | func CheckBinlogFileHeader(buf []byte) bool { 84 | // 0xFE62696E not encrypted 85 | // 0xFD62696E encrypted 86 | return bytes.Equal(buf, []byte{0xfe, 'b', 'i', 'n'}) || bytes.Equal(buf, []byte{0xfd, 'b', 'i', 'n'}) 87 | } 88 | 89 | // CheckBinlogFileEncrypt check file is encrypted 90 | func CheckBinlogFileEncrypt(buf []byte) bool { 91 | return bytes.Equal(buf, []byte{0xfd, 'b', 'i', 'n'}) 92 | } 93 | 94 | // CheckBinlogFormat check binlog format 95 | func CheckBinlogFormat(dsn string) string { 96 | format := "unknown" 97 | db, err := sql.Open("mysql", dsn) 98 | if err != nil { 99 | return format 100 | } 101 | defer db.Close() 102 | res, err := db.Query("SELECT @@binlog_format") 103 | if err != nil { 104 | fmt.Println("CheckBinlogFormat:", err.Error()) 105 | return format 106 | } 107 | defer res.Close() 108 | for res.Next() { 109 | res.Scan(&format) 110 | } 111 | return format 112 | } 113 | 114 | // CheckBinlogFileTime ... 115 | func CheckBinlogFileTime(files []string) error { 116 | var err error 117 | var filteredBinlogs []string 118 | 119 | // no binlog, or only one, by pass check 120 | if len(files) < 2 { 121 | return err 122 | } 123 | 124 | // if no time filter, no need to check binlog files time 125 | if common.Config.Filters.StartDatetime == "" && 126 | common.Config.Filters.StopDatetime == "" { 127 | return err 128 | } 129 | 130 | // file sort by index 131 | sort.Strings(common.Config.MySQL.BinlogFile) 132 | 133 | // each file only check first event 134 | for idx, filename := range files { 135 | do := true 136 | fd, err := os.Open(filename) 137 | if err != nil { 138 | return err 139 | } 140 | 141 | bufFileHeader := make([]byte, FileHeaderLength) 142 | if _, err := io.ReadFull(fd, bufFileHeader); err != nil { 143 | return errors.Trace(err) 144 | } 145 | if !CheckBinlogFileHeader(bufFileHeader) { 146 | err = errors.Errorf("invalid file type, not binlog") 147 | return err 148 | } 149 | var stream cipher.Stream 150 | if CheckBinlogFileEncrypt(bufFileHeader) { 151 | stream, err = initAESCTRStream(filename, common.Config.MySQL.Keyring) 152 | if err != nil { 153 | return err 154 | } 155 | fd.Seek(EncryptFileHeaderOffset+FileHeaderLength, 0) 156 | } else { 157 | stream = nil 158 | } 159 | 160 | p := replication.NewBinlogParser() 161 | event, err := FileNextEvent(p, fd, stream) 162 | if err == io.EOF { 163 | continue 164 | } 165 | if err != nil { 166 | return errors.Trace(err) 167 | } 168 | 169 | if !FilterStartDatetime(event) { 170 | do = false 171 | } 172 | if !FilterStopDatetime(event) { 173 | if len(filteredBinlogs) == 0 && idx > 0 { 174 | filteredBinlogs = append(filteredBinlogs, files[idx-1]) 175 | } 176 | do = false 177 | } 178 | fd.Close() 179 | if do { 180 | if len(filteredBinlogs) == 0 && idx > 0 { 181 | filteredBinlogs = append(filteredBinlogs, files[idx-1]) 182 | } 183 | filteredBinlogs = append(filteredBinlogs, filename) 184 | } 185 | } 186 | common.Config.MySQL.BinlogFile = filteredBinlogs 187 | return err 188 | } 189 | 190 | // BinlogFileParser parser binary log file 191 | func BinlogFileParser(files []string) error { 192 | for _, filename := range files { 193 | var fd *os.File 194 | var err error 195 | switch filename { 196 | case "-": 197 | fd = os.Stdin 198 | default: 199 | fd, err = os.Open(filename) 200 | } 201 | if err != nil { 202 | return err 203 | } 204 | bufFileHeader := make([]byte, FileHeaderLength) 205 | if _, err := io.ReadFull(fd, bufFileHeader); err != nil { 206 | return errors.Trace(err) 207 | } 208 | if !CheckBinlogFileHeader(bufFileHeader) { 209 | err = errors.Errorf("invalid file type, not binlog") 210 | return err 211 | } 212 | var stream cipher.Stream 213 | if CheckBinlogFileEncrypt(bufFileHeader) { 214 | stream, err = initAESCTRStream(filename, common.Config.MySQL.Keyring) 215 | if err != nil { 216 | return err 217 | } 218 | fd.Seek(EncryptFileHeaderOffset, 0) 219 | if _, err := io.ReadFull(fd, bufFileHeader); err != nil { 220 | return errors.Trace(err) 221 | } 222 | if !CheckBinlogFileHeader(decryptAESCTR(stream, bufFileHeader)) { 223 | err = errors.Errorf("invalid file type, not binlog") 224 | return err 225 | } 226 | } else { 227 | stream = nil 228 | } 229 | 230 | p := replication.NewBinlogParser() 231 | p.SetUseDecimal(true) // support Decimal type 232 | for { 233 | event, err := FileNextEvent(p, fd, stream) 234 | if err == io.EOF { 235 | break 236 | } 237 | if err != nil { 238 | return errors.Trace(err) 239 | } 240 | if BinlogFilter(event) { 241 | TypeSwitcher(event) 242 | } else { 243 | common.VerboseVerbose("-- [DEBUG] BinlogFilter ignore, EventType: %s, Position: %d, ServerID: %d, TimeStamp: %d", 244 | event.Header.EventType.String(), 245 | event.Header.LogPos, 246 | event.Header.ServerID, 247 | event.Header.Timestamp, 248 | ) 249 | } 250 | if Ending { 251 | break 252 | } 253 | } 254 | fd.Close() 255 | } 256 | return nil 257 | } 258 | 259 | // FileNextEvent ... 260 | func FileNextEvent(p *replication.BinlogParser, r io.Reader, stream cipher.Stream) (*replication.BinlogEvent, error) { 261 | var err error 262 | var head *replication.EventHeader 263 | var event *replication.BinlogEvent 264 | 265 | bufHead := make([]byte, EventHeaderLength) 266 | if _, err = io.ReadFull(r, bufHead); err != nil { 267 | return event, err 268 | } 269 | if stream != nil { 270 | bufHead = decryptAESCTR(stream, bufHead) 271 | } 272 | 273 | head, err = ParseEventHeader(bufHead) 274 | if err != nil { 275 | return event, errors.Trace(err) 276 | } 277 | 278 | eventLength := head.EventSize - replication.EventHeaderSize 279 | bufBody := make([]byte, eventLength) 280 | if n, err := io.ReadFull(r, bufBody); err != nil { 281 | err = errors.Errorf("get event body err %v, need %d - %d, but got %d", err, head.EventSize, replication.EventHeaderSize, n) 282 | return event, err 283 | } 284 | if stream != nil { 285 | bufBody = decryptAESCTR(stream, bufBody) 286 | } 287 | 288 | var rawData []byte 289 | rawData = append(rawData, bufHead...) 290 | rawData = append(rawData, bufBody...) 291 | return p.Parse(rawData) 292 | } 293 | 294 | // BinlogStreamParser parser mysql connection replication event 295 | func BinlogStreamParser() error { 296 | readTimeout, err := time.ParseDuration(common.Config.MySQL.ReadTimeout) 297 | if err != nil { 298 | common.Log.Error("BinlogStreamParser Error: %s", err.Error()) 299 | return err 300 | } 301 | 302 | changeMaster := replication.BinlogSyncerConfig{ 303 | ServerID: common.MasterInfo.ServerID, 304 | Flavor: common.MasterInfo.ServerType, 305 | Host: common.MasterInfo.MasterHost, 306 | Port: uint16(common.MasterInfo.MasterPort), 307 | User: common.MasterInfo.MasterUser, 308 | Password: common.MasterInfo.MasterPassword, 309 | Charset: common.Config.Global.Charset, 310 | ReadTimeout: readTimeout, 311 | MaxReconnectAttempts: common.Config.MySQL.RetryCount, 312 | SemiSyncEnabled: false, 313 | 314 | ParseTime: false, // parse mysql datetime/time as string 315 | TimestampStringLocation: common.Config.Global.Location, // If ParseTime is false, convert TIMESTAMP into this specified timezone. 316 | UseDecimal: true, // support Decimal type 317 | } 318 | syncer := replication.NewBinlogSyncer(changeMaster) 319 | defer syncer.Close() 320 | var streamer *replication.BinlogStreamer 321 | if common.MasterInfo.AutoPosition { 322 | streamer, err = binlogDumpGTIDSyncer(syncer) 323 | } else { 324 | streamer, err = binlogDumpSyncer(syncer) 325 | } 326 | if err != nil { 327 | return err 328 | } 329 | 330 | for { 331 | event, err := getEvent(streamer, readTimeout) 332 | if err != nil { 333 | return errors.Trace(err) 334 | } 335 | if BinlogFilter(event) { 336 | TypeSwitcher(event) 337 | } else { 338 | common.VerboseVerbose("-- [DEBUG] BinlogFilter ignore, EventType: %s, Position: %d, ServerID: %d, TimeStamp: %d", 339 | event.Header.EventType.String(), 340 | event.Header.LogPos, 341 | event.Header.ServerID, 342 | event.Header.Timestamp, 343 | ) 344 | } 345 | UpdateMasterInfo(event) 346 | if Ending { 347 | break 348 | } 349 | } 350 | return nil 351 | } 352 | 353 | func getEvent(streamer *replication.BinlogStreamer, readTimeout time.Duration) (*replication.BinlogEvent, error) { 354 | var ctx context.Context 355 | var cancel context.CancelFunc 356 | if common.Config.Global.Daemon { 357 | ctx = context.Background() 358 | } else { 359 | ctx, cancel = context.WithTimeout(context.Background(), readTimeout) 360 | defer cancel() 361 | } 362 | return streamer.GetEvent(ctx) 363 | } 364 | 365 | // TypeSwitcher event router by type 366 | func TypeSwitcher(event *replication.BinlogEvent) { 367 | rebuild.EventHeaderRebuild(event) 368 | switch event.Header.EventType { 369 | case replication.GTID_EVENT: 370 | rebuild.GTIDRebuild(event.Event.(*replication.GTIDEvent)) 371 | case replication.WRITE_ROWS_EVENTv0, replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2: 372 | rebuild.InsertRebuild(event) 373 | case replication.UPDATE_ROWS_EVENTv0, replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2: 374 | rebuild.UpdateRebuild(event) 375 | case replication.DELETE_ROWS_EVENTv0, replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2: 376 | rebuild.DeleteRebuild(event) 377 | case replication.QUERY_EVENT: 378 | rebuild.QueryRebuild(event) 379 | case replication.ROWS_QUERY_EVENT: 380 | rebuild.RowsQueryRebuild(event) 381 | case replication.XID_EVENT: 382 | rebuild.XidRebuild(event) 383 | case replication.ROTATE_EVENT: 384 | common.VerboseVerbose("-- [DEBUG] EventType: %s, NextLogName: %s", event.Header.EventType.String(), string(event.Event.(*replication.RotateEvent).NextLogName)) 385 | // case replication.ANONYMOUS_GTID_EVENT, replication.PREVIOUS_GTIDS_EVENT, replication.TABLE_MAP_EVENT: 386 | default: 387 | common.VerboseVerbose("-- [DEBUG] TypeSwitcher EventType: %s bypass", event.Header.EventType.String()) 388 | } 389 | sleepInterval(event) 390 | } 391 | 392 | func binlogDumpSyncer(syncer *replication.BinlogSyncer) (*replication.BinlogStreamer, error) { 393 | if common.MasterInfo.MasterLogFile == "" && common.Config.MySQL.ReplicateFromCurrentPosition { 394 | masterInfo := common.ShowMasterStatus(common.MasterInfo) 395 | common.MasterInfo.MasterLogFile = masterInfo.MasterLogFile 396 | common.MasterInfo.MasterLogPos = masterInfo.MasterLogPos 397 | } 398 | position := mysql.Position{Name: common.MasterInfo.MasterLogFile, Pos: uint32(common.MasterInfo.MasterLogPos)} 399 | return syncer.StartSync(position) 400 | } 401 | 402 | func binlogDumpGTIDSyncer(syncer *replication.BinlogSyncer) (*replication.BinlogStreamer, error) { 403 | gtid, err := mysql.ParseGTIDSet(common.MasterInfo.ServerType, common.MasterInfo.ExecutedGTIDSet) 404 | if err != nil { 405 | return nil, err 406 | } 407 | return syncer.StartSyncGTID(gtid) 408 | } 409 | 410 | // ParseEventHeader parser event header, in go-mysql it's internal func, make it public 411 | func ParseEventHeader(buf []byte) (*replication.EventHeader, error) { 412 | head := new(replication.EventHeader) 413 | err := head.Decode(buf) 414 | if err != nil { 415 | return nil, err 416 | } 417 | 418 | if head.EventSize <= uint32(replication.EventHeaderSize) { 419 | err = errors.Errorf("invalid event header, event size is %d, too small", head.EventSize) 420 | return nil, err 421 | } 422 | return head, nil 423 | } 424 | 425 | // sleepInterval ... 426 | func sleepInterval(event *replication.BinlogEvent) { 427 | switch common.Config.Rebuild.Plugin { 428 | case "sql", "flashback": 429 | default: 430 | return 431 | } 432 | interval := common.Config.Rebuild.SleepDuration.Seconds() 433 | if interval > 0 { 434 | switch event.Header.EventType { 435 | case replication.WRITE_ROWS_EVENTv0, replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2, 436 | replication.UPDATE_ROWS_EVENTv0, replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2, 437 | replication.DELETE_ROWS_EVENTv0, replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2: 438 | fmt.Printf("SELECT sleep(%f);\n", interval) 439 | case replication.QUERY_EVENT: 440 | switch string(event.Event.(*replication.QueryEvent).Query) { 441 | case "BEGIN", "COMMIT": 442 | default: 443 | fmt.Printf("SELECT sleep(%f);\n", interval) 444 | } 445 | } 446 | } 447 | } 448 | -------------------------------------------------------------------------------- /common/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(c) 2019 Lianjia, Inc. All Rights Reserved 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * Unless required by applicable law or agreed to in writing, software 8 | * distributed under the License is distributed on an "AS IS" BASIS, 9 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | * See the License for the specific language governing permissions and 11 | * limitations under the License. 12 | */ 13 | 14 | package common 15 | 16 | import ( 17 | "database/sql" 18 | "flag" 19 | "fmt" 20 | "io/ioutil" 21 | "math/rand" 22 | "os" 23 | "runtime" 24 | "strconv" 25 | "strings" 26 | "time" 27 | 28 | // database/sql 29 | _ "github.com/go-sql-driver/mysql" 30 | "github.com/juju/errors" 31 | pingcap "github.com/pingcap/parser/mysql" 32 | yaml "gopkg.in/yaml.v2" 33 | ) 34 | 35 | // GlobalConfig global config 36 | type GlobalConfig struct { 37 | // 日志级别，这里使用了 beego 的 log 包 38 | // [0:Emergency, 1:Alert, 2:Critical, 3:Error, 4:Warning, 5:Notice, 6:Informational, 7:Debug] 39 | LogLevel int `yaml:"log-level"` 40 | // 日志输出位置，默认日志输出到控制台 41 | // 目前只支持['console', 'file']两种形式，如非console形式这里需要指定文件的路径，可以是相对路径 42 | LogOutput string `yaml:"log-output"` 43 | Daemon bool `yaml:"daemon"` 44 | Charset string `yaml:"charset"` 45 | HexString bool `yaml:"hex-string"` // string, varchar 等数据是否使用 hex 转义，防止数据转换 46 | CPU int `yaml:"cpu"` // CPU core limit 47 | Verbose bool `yaml:"verbose"` // more info to print 48 | VerboseVerbose bool `yaml:"verbose-verbose"` // more and more info to print 49 | TimeZone string `yaml:"time-zone"` // "UTC", "Asia/Shanghai" 50 | Location *time.Location `yaml:"-"` 51 | } 52 | 53 | var gConfig = GlobalConfig{ 54 | LogLevel: 3, 55 | LogOutput: "lightning.log", 56 | TimeZone: "Asia/Shanghai", 57 | Charset: "utf8mb4", // MySQL 低版本不支持 utf8mb4, 可能会有报错需要通过修改配置文件避免 58 | } 59 | 60 | // MySQL binlog file location or streamer, if streamer use dsn format 61 | type MySQL struct { 62 | BinlogFile []string `yaml:"binlog-file"` 63 | SchemaFile string `yaml:"schema-file"` 64 | MasterInfo string `yaml:"master-info"` 65 | ReplicateFromCurrentPosition bool `yaml:"replicate-from-current-position"` 66 | SyncInterval string `yaml:"sync-interval"` 67 | SyncDuration time.Duration `yaml:"-"` 68 | ReadTimeout string `yaml:"read-timeout"` 69 | RetryCount int `yaml:"retry-count"` 70 | Keyring string `yaml:"keyring"` 71 | } 72 | 73 | var mConfig = MySQL{ 74 | BinlogFile: []string{}, 75 | SchemaFile: "", 76 | Keyring: "", 77 | SyncInterval: "1s", 78 | ReadTimeout: "3s", 79 | RetryCount: 100, 80 | } 81 | 82 | // Filters filters about event 83 | type Filters struct { 84 | Tables []string `yaml:"tables"` // replication_wild_do_tables format 85 | IgnoreTables []string `yaml:"ignore-tables"` // replicate_wild_ignore_tables format 86 | EventType []string `yaml:"event-types"` // insert, update, delete 87 | ThreadID int `yaml:"thread-id"` 88 | ServerID int `yaml:"server-id"` 89 | StartPosition uint32 `yaml:"start-position"` 90 | StopPosition uint32 `yaml:"stop-position"` 91 | StartDatetime string `yaml:"start-datetime"` 92 | StopDatetime string `yaml:"stop-datetime"` 93 | IncludeGTIDSet string `yaml:"include-gtid-set"` 94 | ExcludeGTIDSet string `yaml:"exclude-gtid-set"` 95 | StartTimestamp int64 `yaml:"-"` 96 | StopTimestamp int64 `yaml:"-"` 97 | } 98 | 99 | var fConfig = Filters{ 100 | Tables: []string{}, 101 | IgnoreTables: []string{ 102 | "mysql.%", 103 | "percona.%", 104 | }, 105 | StartDatetime: "", 106 | StopDatetime: "", 107 | } 108 | 109 | // Rebuild rebuild plugins 110 | type Rebuild struct { 111 | Plugin string `yaml:"plugin"` // Plugin name: sql, flashback, stat, lua, find 112 | CompleteInsert bool `yaml:"complete-insert"` 113 | ExtendedInsertCount int `yaml:"extended-insert-count"` 114 | IgnoreColumns []string `yaml:"ignore-columns"` 115 | Replace bool `yaml:"replace"` 116 | SleepInterval string `yaml:"sleep-interval"` 117 | SleepDuration time.Duration `yaml:"-"` 118 | ForeachTime bool `yaml:"foreach-time"` 119 | CurrentEventTime string `yaml:"-"` 120 | LuaScript string `yaml:"lua-script"` 121 | WithoutDBName bool `yaml:"without-db-name"` 122 | } 123 | 124 | var rConfig = Rebuild{ 125 | Plugin: "sql", 126 | SleepInterval: "0s", 127 | WithoutDBName: false, 128 | } 129 | 130 | // Configuration config sections 131 | type Configuration struct { 132 | Global GlobalConfig `yaml:"global"` 133 | MySQL MySQL `yaml:"mysql"` 134 | Filters Filters `yaml:"filters"` 135 | Rebuild Rebuild `yaml:"rebuild"` 136 | } 137 | 138 | // Config global config variable 139 | var Config = Configuration{ 140 | gConfig, 141 | mConfig, 142 | fConfig, 143 | rConfig, 144 | } 145 | 146 | // ChangeMaster change master info 147 | type ChangeMaster struct { 148 | MasterHost string `yaml:"master_host"` 149 | MasterUser string `yaml:"master_user"` 150 | MasterPassword string `yaml:"master_password"` 151 | MasterPort int `yaml:"master_port"` 152 | MasterLogFile string `yaml:"master_log_file"` 153 | MasterLogPos int64 `yaml:"master_log_pos"` 154 | ExecutedGTIDSet string `yaml:"executed_gtid_set"` 155 | AutoPosition bool `yaml:"auto_position"` 156 | 157 | UntilLogFile string `yaml:"until_log_file"` 158 | UntilLogPos int64 `yaml:"until_log_pos"` 159 | UntilBeforeGTIDs string `yaml:"until_before_gtids"` 160 | UntilAfterGTIDs string `yaml:"until_after_gtids"` 161 | 162 | SecondsBehindMaster int64 `yaml:"seconds_behind_master"` // last execute event timestamp 163 | ServerID uint32 `yaml:"server-id"` 164 | ServerType string `yaml:"server-type"` // mysql, mariadb 165 | } 166 | 167 | // MasterInfo replication status info 168 | var MasterInfo = ChangeMaster{ 169 | MasterPort: 3306, 170 | ServerID: 11, 171 | ServerType: "mysql", 172 | } 173 | 174 | // ShowMasterStatus execute `show master status`, get master info 175 | func ShowMasterStatus(masterInfo ChangeMaster) ChangeMaster { 176 | db, err := sql.Open("mysql", 177 | fmt.Sprintf(`%s:%s@tcp(%s:%d)/`, 178 | masterInfo.MasterUser, 179 | masterInfo.MasterPassword, 180 | masterInfo.MasterHost, 181 | masterInfo.MasterPort, 182 | )) 183 | if err != nil { 184 | Log.Error(err.Error()) 185 | return masterInfo 186 | } 187 | defer db.Close() 188 | 189 | rows, err := db.Query("show master status") 190 | if err != nil { 191 | Log.Error(err.Error()) 192 | return masterInfo 193 | } 194 | 195 | columns, err := rows.Columns() 196 | if err != nil { 197 | Log.Error(err.Error()) 198 | return masterInfo 199 | } 200 | values := make([]sql.RawBytes, len(columns)) 201 | scanArgs := make([]interface{}, len(values)) 202 | for i := range values { 203 | scanArgs[i] = &values[i] 204 | } 205 | for rows.Next() { 206 | err = rows.Scan(scanArgs...) 207 | if err != nil { 208 | Log.Error(err.Error()) 209 | break 210 | } 211 | for i, v := range values { 212 | switch columns[i] { 213 | case "File": 214 | masterInfo.MasterLogFile = string(v) 215 | case "Position": 216 | masterInfo.MasterLogPos, _ = strconv.ParseInt(string(v), 10, 64) 217 | case "Binlog_Do_DB": 218 | case "Binlog_Ignore_DB": 219 | case "Executed_Gtid_Set": 220 | masterInfo.ExecutedGTIDSet = string(v) 221 | } 222 | } 223 | } 224 | return masterInfo 225 | } 226 | 227 | // ParseConfig parse configuration 228 | func ParseConfig() { 229 | var err error 230 | 231 | // Not in config flags 232 | noDefaults := flag.Bool("no-defaults", false, "don't load config from default file") 233 | configFile := flag.String("config", "", "load config from specify file") 234 | printConfig := flag.Bool("print-config", false, "print config into stdout") 235 | printMasterInfo := flag.Bool("print-master-info", false, "print master.info into stdout") 236 | checkConfig := flag.Bool("check-config", false, "check config file format") 237 | printVersion := flag.Bool("version", false, "print version info into stdout") 238 | listPlugin := flag.Bool("list-plugin", false, "list support plugins") 239 | 240 | // Global section config 241 | globalLogLevel := flag.Int("log-level", 0, "log level") 242 | globalLogOutput := flag.String("log-output", "", "log output file name") 243 | globalTimeZone := flag.String("time-zone", "", "time zone info") 244 | globalCharset := flag.String("charset", "", "charset use for binlog parsing") 245 | globalCPU := flag.Int("cpu", 0, "cpu cores limit") 246 | globalVerbose := flag.Bool("verbose", false, "verbose mode, more info will print") 247 | globalVerboseVerbose := flag.Bool("vv", false, "verbose verbose mode, more and more info will print") 248 | globalDaemon := flag.Bool("daemon", false, "replication run as daemon") 249 | globalHexString := flag.Bool("hex-string", false, "convert string to hex format") 250 | 251 | // MySQL section config 252 | mysqlUser := flag.String("user", "", "mysql user") 253 | mysqlHost := flag.String("host", "", "mysql host") 254 | mysqlPort := flag.Int("port", 0, "mysql port") 255 | mysqlPassword := flag.String("password", "", "mysql password") 256 | mysqlBinlogFile := flag.String("binlog-file", "", "binlog files separate with space, eg. --binlog-file='binlog.000001 binlog.000002'") 257 | mysqlSchemaFile := flag.String("schema-file", "", "schema load from file") 258 | mysqlKeyring := flag.String("keyring", "", "mysql keyring file path") 259 | mysqlMasterInfo := flag.String("master-info", "", "master.info file") 260 | mysqlReplicateFromCurrent := flag.Bool("replicate-from-current-position", false, "binlog dump from current `show master status`") 261 | mysqlSyncInterval := flag.String("sync-interval", "", "sync master.info interval") 262 | mysqlReadTimeout := flag.String("read-timeout", "", "I/O read timeout. The value must be a decimal number with a unit suffix ('ms', 's', 'm', 'h'), such as '30s', '0.5m' or '1m30s'.") 263 | mysqlRetryCount := flag.Int("retry-count", 0, "maximum number of attempts to re-establish a broken connection") 264 | 265 | // Filters section config 266 | filterThreadID := flag.Int("thread-id", 0, "binlog filter thread-id") 267 | filterServerID := flag.Int("server-id", 0, "binlog filter server-id") 268 | filterIncludeGTID := flag.String("include-gtids", "", "like mysqlbinlog include-gtids") 269 | filterExcludeGTID := flag.String("exclude-gtids", "", "like mysqlbinlog exclude-gtids") 270 | filterStartPosition := flag.Uint("start-position", 0, "binlog start-position") 271 | filterStopPosition := flag.Uint("stop-position", 0, "binlog stop-position") 272 | filterStartDatetime := flag.String("start-datetime", "", "binlog filter start-datetime") 273 | filterStopDatetime := flag.String("stop-datetime", "", "binlog filter stop-datetime") 274 | filterTables := flag.String("tables", "", "binlog filter tables. eg. -tables db1.tb1,db1.tb2,db2.%") 275 | filterIgnoreTables := flag.String("ignore-tables", "", "binlog filter ignore tables") 276 | filterEventTypes := flag.String("event-types", "", "binlog filter event types") 277 | 278 | // Rebuild section config 279 | rebuildPlugin := flag.String("plugin", "", "plugin name, use --list-plugin check all supported plugins") 280 | rebuildCompleteInsert := flag.Bool("complete-insert", false, "complete column info, like 'INSERT INTO tb (col) VALUES (1)'") 281 | rebuildExtendedInsertCount := flag.Int("extended-insert-count", 0, "use multiple-row INSERT syntax that include several VALUES") 282 | rebuildReplace := flag.Bool("replace", false, "use REPLACE INTO instead of INSERT INTO, UPDATE") 283 | rebuildSleepInterval := flag.String("sleep-interval", "", "execute commands repeatedly with a sleep between") 284 | rebuildIgnoreColumns := flag.String("ignore-columns", "", "query rebuild ignore columns") 285 | rebuildLuaScript := flag.String("lua-script", "", "lua plugin script file") 286 | rebuildWithoutDBName := flag.Bool("without-db-name", false, "insert/delete/update query without database name, only table name") 287 | rebuildForeachTime := flag.Bool("foreach-time", false, "add time foreach sql") 288 | 289 | // master.info config 290 | masterHost := flag.String("master-host", "", "master.info master_host") 291 | masterUser := flag.String("master-user", "", "master.info master_user") 292 | masterPassword := flag.String("master-password", "", "master.info master_password") 293 | masterPort := flag.Int("master-port", 0, "master.info master_port") 294 | masterLogFile := flag.String("master-log-file", "", "master.info master_log_file") 295 | masterLogPos := flag.Int64("master-log-pos", 0, "master.info master_log_pos") 296 | executedGtidSet := flag.String("executed-gtid-set", "", "master.info executed_gtid_set") 297 | autoPosition := flag.Bool("auto-position", false, "master.info auto_position") 298 | serverId := flag.Uint("slave-server-id", 0, "master.info server-id") 299 | serverType := flag.String("server-type", "", "master.info server-type") 300 | masterUntilLogFile := flag.String("master-until-log-file", "", "start slave until master-log-file") 301 | masterUntilLogPos := flag.Int64("master-until-log-pos", 0, "start slave until master-log-pos") 302 | // masterUntilBeforeGTIDs := flag.String("master-until-before-gtids", "", "start slave SQL_BEFORE_GTIDS") 303 | // masterUntilAfterGTIDs := flag.String("master-until-after-gtids", "", "start slave SQL_AFTER_GTIDS") 304 | 305 | flag.CommandLine.SetOutput(os.Stdout) 306 | flag.Parse() 307 | 308 | // Not in config flags 309 | if !*noDefaults { 310 | err = Config.parseConfigFile(*configFile) 311 | } 312 | if *configFile != "" { 313 | err = Config.parseConfigFile(*configFile) 314 | } 315 | if *checkConfig { 316 | if err != nil { 317 | fmt.Println(err.Error()) 318 | os.Exit(1) 319 | } else { 320 | fmt.Println("OK") 321 | os.Exit(0) 322 | } 323 | } 324 | if *printVersion { 325 | version() 326 | os.Exit(0) 327 | } 328 | if *listPlugin { 329 | ListPlugin() 330 | os.Exit(0) 331 | } 332 | 333 | // Global config 334 | if *globalLogLevel > 0 { 335 | Config.Global.LogLevel = *globalLogLevel 336 | } 337 | if *globalLogOutput != "" { 338 | Config.Global.LogOutput = *globalLogOutput 339 | } 340 | if *globalTimeZone != "" { 341 | Config.Global.TimeZone = *globalTimeZone 342 | } 343 | Config.Global.Location, err = time.LoadLocation(Config.Global.TimeZone) 344 | if err != nil { 345 | Log.Error(errors.Trace(err).Error()) 346 | Config.Global.Location = time.Now().Location() 347 | } 348 | if *globalCharset != "" { 349 | Config.Global.Charset = *globalCharset 350 | } 351 | if ok := pingcap.Charsets[Config.Global.Charset]; ok == "" { 352 | Log.Warn("Config.Global.Charset: %s not exist", Config.Global.Charset) 353 | Config.Global.Charset = "utf8mb4" 354 | } 355 | if *globalCPU > 0 { 356 | Config.Global.CPU = *globalCPU 357 | runtime.GOMAXPROCS(*globalCPU) 358 | } 359 | if *globalVerbose { 360 | Config.Global.Verbose = *globalVerbose 361 | } 362 | if *globalVerboseVerbose { 363 | Config.Global.VerboseVerbose = *globalVerboseVerbose 364 | } 365 | if *globalDaemon { 366 | Config.Global.Daemon = *globalDaemon 367 | } 368 | if *globalHexString { 369 | Config.Global.HexString = *globalHexString 370 | } 371 | 372 | // MySQL config 373 | if *mysqlBinlogFile != "" { 374 | Config.MySQL.BinlogFile = strings.Fields(*mysqlBinlogFile) 375 | } else { 376 | // Only parse first not flags file 377 | files := flag.Args() 378 | if len(files) >= 1 { 379 | Config.MySQL.BinlogFile = files 380 | } 381 | } 382 | if *mysqlSchemaFile != "" { 383 | Config.MySQL.SchemaFile = *mysqlSchemaFile 384 | } 385 | if *mysqlKeyring != "" { 386 | Config.MySQL.Keyring = *mysqlKeyring 387 | } 388 | if *mysqlMasterInfo != "" { 389 | Config.MySQL.MasterInfo = *mysqlMasterInfo 390 | } 391 | if *mysqlReplicateFromCurrent { 392 | Config.MySQL.ReplicateFromCurrentPosition = *mysqlReplicateFromCurrent 393 | } 394 | if *mysqlSyncInterval != "" { 395 | _, err = time.ParseDuration(*mysqlSyncInterval) 396 | if err != nil { 397 | Log.Warn("-sync-interval '%s' Error: %s", *mysqlSyncInterval, err.Error()) 398 | } else { 399 | Config.MySQL.SyncInterval = *mysqlSyncInterval 400 | } 401 | } 402 | Config.MySQL.SyncDuration, err = time.ParseDuration(Config.MySQL.SyncInterval) 403 | if err != nil { 404 | Log.Warn("sync-interval '%s' Error: %s", Config.MySQL.SyncInterval, err.Error()) 405 | Config.MySQL.SyncDuration = time.Duration(0 * time.Second) 406 | } 407 | if *mysqlReadTimeout != "" { 408 | _, err = time.ParseDuration(*mysqlReadTimeout) 409 | if err != nil { 410 | Log.Warn("-read-timeout '%s' Error: %s", *mysqlReadTimeout, err.Error()) 411 | } else { 412 | Config.MySQL.ReadTimeout = *mysqlReadTimeout 413 | } 414 | } 415 | if *mysqlRetryCount > 0 { 416 | Config.MySQL.RetryCount = *mysqlRetryCount 417 | } 418 | 419 | // Filters Config 420 | if *filterThreadID > 0 { 421 | Config.Filters.ThreadID = *filterThreadID 422 | } 423 | if *filterServerID > 0 { 424 | Config.Filters.ServerID = *filterServerID 425 | } 426 | if *filterIncludeGTID != "" { 427 | Config.Filters.IncludeGTIDSet = *filterIncludeGTID 428 | } 429 | if *filterExcludeGTID != "" { 430 | Config.Filters.ExcludeGTIDSet = *filterExcludeGTID 431 | } 432 | if *filterStartDatetime != "" { 433 | Config.Filters.StartDatetime = *filterStartDatetime 434 | } 435 | layout := "2006-01-02 15:04:05" 436 | if Config.Filters.StartDatetime != "" { 437 | t, err := time.ParseInLocation(layout, Config.Filters.StartDatetime, Config.Global.Location) 438 | if err != nil { 439 | fmt.Println(err.Error()) 440 | os.Exit(1) 441 | } else { 442 | Config.Filters.StartTimestamp = t.Unix() 443 | } 444 | } 445 | VerboseVerbose("-- [DEBUG] Config.Filters.StartTimestamp: %d", Config.Filters.StartTimestamp) 446 | if *filterStopDatetime != "" { 447 | Config.Filters.StopDatetime = *filterStopDatetime 448 | } 449 | if Config.Filters.StopDatetime == "" && Config.Global.Daemon == false { 450 | Config.Filters.StopDatetime = time.Now().Format(layout) 451 | } 452 | 453 | if Config.Filters.StopDatetime != "" { 454 | t, err := time.ParseInLocation(layout, Config.Filters.StopDatetime, Config.Global.Location) 455 | if err != nil { 456 | fmt.Println(err.Error()) 457 | os.Exit(1) 458 | } else { 459 | Config.Filters.StopTimestamp = t.Unix() 460 | } 461 | } 462 | VerboseVerbose("-- [DEBUG] Config.Filters.StopTimestamp: %d", Config.Filters.StopTimestamp) 463 | if *filterStartPosition > 0 { 464 | if *filterStartPosition >= 1<<32 { 465 | fmt.Println("binlog --start-position overflow, should be uint32") 466 | os.Exit(1) 467 | } else { 468 | Config.Filters.StartPosition = uint32(*filterStartPosition) 469 | } 470 | } 471 | if *filterStopPosition > 0 { 472 | if *filterStopPosition >= 1<<32 { 473 | fmt.Println("binlog --stop-position overflow, should be uint32") 474 | os.Exit(1) 475 | } else { 476 | Config.Filters.StopPosition = uint32(*filterStopPosition) 477 | } 478 | } 479 | if *filterTables != "" { 480 | Config.Filters.Tables = strings.Split(*filterTables, ",") 481 | } 482 | for _, t := range Config.Filters.Tables { 483 | if !strings.Contains(t, ".") { 484 | fmt.Println("filter -tables format should be db.tb") 485 | os.Exit(1) 486 | } 487 | } 488 | if *filterIgnoreTables != "" { 489 | Config.Filters.IgnoreTables = strings.Split(*filterIgnoreTables, ",") 490 | } 491 | for _, t := range Config.Filters.IgnoreTables { 492 | if !strings.Contains(t, ".") { 493 | fmt.Println("filter -ignore-tables format should be db.tb") 494 | os.Exit(1) 495 | } 496 | } 497 | if *filterEventTypes != "" { 498 | Config.Filters.EventType = strings.Split(*filterEventTypes, ",") 499 | } 500 | 501 | // Rebuild config 502 | if *rebuildPlugin != "" { 503 | Config.Rebuild.Plugin = *rebuildPlugin 504 | } 505 | switch Config.Rebuild.Plugin { 506 | case "": 507 | Config.Rebuild.Plugin = "sql" 508 | case "lua", "sql", "flashback", "stat", "find", "decrypt": 509 | default: 510 | ListPlugin() 511 | os.Exit(1) 512 | } 513 | if *rebuildCompleteInsert { 514 | Config.Rebuild.CompleteInsert = *rebuildCompleteInsert 515 | } 516 | if *rebuildExtendedInsertCount > 0 { 517 | Config.Rebuild.ExtendedInsertCount = *rebuildExtendedInsertCount 518 | } 519 | if *rebuildReplace { 520 | Config.Rebuild.Replace = *rebuildReplace 521 | } 522 | if *rebuildSleepInterval != "" { 523 | _, err = time.ParseDuration(*rebuildSleepInterval) 524 | if err != nil { 525 | Log.Warn("-sleep-interval '%s' Error: %s", *rebuildSleepInterval, err.Error()) 526 | } else { 527 | Config.Rebuild.SleepInterval = *rebuildSleepInterval 528 | } 529 | } 530 | Config.Rebuild.SleepDuration, err = time.ParseDuration(Config.Rebuild.SleepInterval) 531 | if err != nil { 532 | Log.Warn("sleep-interval '%s' Error: %s", Config.Rebuild.SleepInterval, err.Error()) 533 | Config.Rebuild.SleepDuration = time.Duration(0 * time.Second) 534 | } 535 | if *rebuildIgnoreColumns != "" { 536 | Config.Rebuild.IgnoreColumns = strings.Split(*rebuildIgnoreColumns, ",") 537 | } 538 | if len(Config.Rebuild.IgnoreColumns) > 0 { 539 | Config.Rebuild.CompleteInsert = true 540 | } 541 | if *rebuildLuaScript != "" { 542 | Config.Rebuild.LuaScript = *rebuildLuaScript 543 | } 544 | if *rebuildWithoutDBName { 545 | Config.Rebuild.WithoutDBName = *rebuildWithoutDBName 546 | } 547 | if *rebuildForeachTime { 548 | Config.Rebuild.ForeachTime = *rebuildForeachTime 549 | } 550 | 551 | LoadMasterInfo() 552 | 553 | if len(Config.MySQL.BinlogFile) == 0 && Config.MySQL.MasterInfo == "" { 554 | Config.MySQL.MasterInfo = "master.info" 555 | } 556 | 557 | if *printConfig { 558 | PrintConfiguration() 559 | os.Exit(0) 560 | } 561 | 562 | // master.info config 563 | if *mysqlUser != "" { 564 | MasterInfo.MasterUser = *mysqlUser 565 | } 566 | if *mysqlHost != "" { 567 | MasterInfo.MasterHost = *mysqlHost 568 | } 569 | if *mysqlPassword != "" { 570 | MasterInfo.MasterPassword = *mysqlPassword 571 | } 572 | if *mysqlPort != 0 { 573 | MasterInfo.MasterPort = *mysqlPort 574 | } 575 | if *masterHost != "" { 576 | MasterInfo.MasterHost = *masterHost 577 | } 578 | if *masterUser != "" { 579 | MasterInfo.MasterUser = *masterUser 580 | } 581 | if *masterPassword != "" { 582 | MasterInfo.MasterPassword = *masterPassword 583 | } 584 | if *masterPort != 0 { 585 | MasterInfo.MasterPort = *masterPort 586 | } 587 | if *masterLogFile != "" { 588 | MasterInfo.MasterLogFile = *masterLogFile 589 | } 590 | if *masterLogPos != 0 { 591 | MasterInfo.MasterLogPos = *masterLogPos 592 | } 593 | if *executedGtidSet != "" { 594 | MasterInfo.ExecutedGTIDSet = *executedGtidSet 595 | } 596 | if *autoPosition { 597 | MasterInfo.AutoPosition = *autoPosition 598 | } 599 | if *serverId != 0 { 600 | MasterInfo.ServerID = uint32(*serverId) 601 | } 602 | if *serverType != "" { 603 | MasterInfo.ServerType = *serverType 604 | } 605 | if *masterUntilLogFile != "" { 606 | MasterInfo.UntilLogFile = *masterUntilLogFile 607 | } 608 | if *masterUntilLogPos != 0 { 609 | MasterInfo.UntilLogPos = *masterUntilLogPos 610 | } 611 | // TODO: 612 | // if *masterUntilBeforeGTIDs != "" { 613 | // MasterInfo.UntilBeforeGTIDs = *masterUntilBeforeGTIDs 614 | // } 615 | // if *masterUntilAfterGTIDs != "" { 616 | // MasterInfo.UntilAfterGTIDs = *masterUntilAfterGTIDs 617 | // } 618 | 619 | if *printMasterInfo { 620 | PrintMasterInfo() 621 | os.Exit(0) 622 | } 623 | 624 | loggerInit() 625 | } 626 | 627 | // PrintConfiguration for `-print-config` flag 628 | func PrintConfiguration() { 629 | data, _ := yaml.Marshal(Config) 630 | fmt.Print(string(data)) 631 | } 632 | 633 | // PrintMasterInfo for `-print-master-info` flag 634 | func PrintMasterInfo() { 635 | data, _ := yaml.Marshal(MasterInfo) 636 | fmt.Print(string(data)) 637 | } 638 | 639 | // parseConfigFile load config from file 640 | func (conf *Configuration) parseConfigFile(path string) error { 641 | path = getConfigFile(path) 642 | configFile, err := os.Open(path) 643 | if err != nil { 644 | return err 645 | } 646 | defer configFile.Close() 647 | 648 | content, err := ioutil.ReadAll(configFile) 649 | if err != nil { 650 | return err 651 | } 652 | 653 | err = yaml.Unmarshal(content, &Config) 654 | if err != nil { 655 | return err 656 | } 657 | return nil 658 | } 659 | 660 | // getConfigFile config file load sequence 661 | func getConfigFile(path string) string { 662 | if path == "" { 663 | path = "/etc/lightning.yaml" 664 | _, err := os.Stat(path) 665 | if err == nil { 666 | return path 667 | } 668 | 669 | path = "etc/lightning.yaml" 670 | _, err = os.Stat(path) 671 | if err == nil { 672 | return path 673 | } 674 | 675 | path = "lightning.yaml" 676 | _, err = os.Stat(path) 677 | if err == nil { 678 | return path 679 | } 680 | } 681 | return path 682 | } 683 | 684 | // version print version info 685 | func version() { 686 | fmt.Println("Version: ", Version) 687 | fmt.Println("Compiled time: ", Compile) 688 | fmt.Println("Code branch: ", Branch) 689 | fmt.Println("GirDirty: ", GitDirty) 690 | } 691 | 692 | // SyncReplicationInfo sync replication status into master.info 693 | func SyncReplicationInfo() { 694 | if Config.MySQL.SyncDuration.Seconds() == 0 { 695 | return 696 | } 697 | 698 | if Config.MySQL.MasterInfo == "" || len(Config.MySQL.BinlogFile) > 0 { 699 | Log.Info("SyncReplicationInfo -master-info not specified, reading from '%v'", Config.MySQL.BinlogFile) 700 | return 701 | } 702 | 703 | for { 704 | FlushReplicationInfo() 705 | time.Sleep(Config.MySQL.SyncDuration) 706 | } 707 | } 708 | 709 | // FlushReplicationInfo flush master.info 710 | func FlushReplicationInfo() { 711 | if Config.MySQL.MasterInfo == "" { 712 | return 713 | } 714 | f, err := os.OpenFile(Config.MySQL.MasterInfo, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) 715 | if err != nil { 716 | Log.Error(errors.Trace(err).Error()) 717 | return 718 | } 719 | defer f.Close() 720 | info, err := yaml.Marshal(MasterInfo) 721 | if err != nil { 722 | Log.Error(errors.Trace(err).Error()) 723 | return 724 | } 725 | _, err = f.WriteString(string(info)) 726 | if err != nil { 727 | Log.Error(errors.Trace(err).Error()) 728 | } 729 | } 730 | 731 | // LoadMasterInfo get master.info from file 732 | func LoadMasterInfo() { 733 | if Config.MySQL.MasterInfo == "" { 734 | return 735 | } 736 | conf, err := ioutil.ReadFile(Config.MySQL.MasterInfo) 737 | if err != nil { 738 | fmt.Println("-- LoadMasterInfo Error: ", err.Error()) 739 | return 740 | } 741 | err = yaml.Unmarshal(conf, &MasterInfo) 742 | if err != nil { 743 | fmt.Println("-- LoadMasterInfo Error: ", err.Error()) 744 | return 745 | } 746 | if MasterInfo.ServerID == 0 { 747 | s1 := rand.NewSource(time.Now().UnixNano()) 748 | r1 := rand.New(s1) 749 | MasterInfo.ServerID = uint32(r1.Intn(3306) + 3306) 750 | } 751 | } 752 | 753 | // ListPlugin list support plugin name and description 754 | func ListPlugin() { 755 | fmt.Println("lightning -plugin support following type") 756 | fmt.Println(" sql(default): parse ROW format binlog into SQL.") 757 | fmt.Println(" flashback: generate flashback query from ROW format binlog") 758 | fmt.Println(" stat: statistic ROW format binlog table update|insert|delete query count") 759 | fmt.Println(" lua: self define lua scripts") 760 | fmt.Println(" find: find binlog file name by event time") 761 | fmt.Println(" decrypt: decrypt binlog file using keyring") 762 | } 763 | 764 | // TimeOffset timezone offset seconds 765 | func TimeOffset(timezone string) int { 766 | loc, err := time.LoadLocation(timezone) 767 | if err != nil { 768 | Log.Error(err.Error()) 769 | return 0 770 | } 771 | now := time.Now() 772 | _, destOffset := now.In(loc).Zone() 773 | _, localOffset := now.Zone() 774 | return destOffset - localOffset 775 | } 776 | --------------------------------------------------------------------------------